1 package org.unicode.cldr.util; 2 3 import java.io.IOException; 4 import java.util.ArrayList; 5 import java.util.Arrays; 6 import java.util.Collections; 7 import java.util.EnumSet; 8 import java.util.HashMap; 9 import java.util.HashSet; 10 import java.util.LinkedHashSet; 11 import java.util.List; 12 import java.util.Map; 13 import java.util.Map.Entry; 14 import java.util.Objects; 15 import java.util.Set; 16 import java.util.TreeMap; 17 import java.util.TreeSet; 18 import java.util.concurrent.ExecutionException; 19 import java.util.concurrent.ForkJoinPool; 20 import java.util.concurrent.RecursiveAction; 21 22 import org.unicode.cldr.test.CheckCLDR; 23 import org.unicode.cldr.test.CheckCLDR.CheckStatus; 24 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 25 import org.unicode.cldr.test.CheckCLDR.Options; 26 import org.unicode.cldr.test.CheckCoverage; 27 import org.unicode.cldr.test.CheckNew; 28 import org.unicode.cldr.test.CoverageLevel2; 29 import org.unicode.cldr.test.OutdatedPaths; 30 import org.unicode.cldr.test.SubmissionLocales; 31 import org.unicode.cldr.util.CLDRFile.Status; 32 import org.unicode.cldr.util.PathHeader.PageId; 33 import org.unicode.cldr.util.PathHeader.SectionId; 34 import org.unicode.cldr.util.StandardCodes.LocaleCoverageType; 35 36 import com.ibm.icu.impl.Relation; 37 import com.ibm.icu.impl.Row; 38 import com.ibm.icu.impl.Row.R2; 39 import com.ibm.icu.text.NumberFormat; 40 import com.ibm.icu.text.UnicodeSet; 41 import com.ibm.icu.util.ICUUncheckedIOException; 42 import com.ibm.icu.util.Output; 43 import com.ibm.icu.util.ULocale; 44 45 /** 46 * Provides data for the Dashboard, showing the important issues for vetters to review for 47 * a given locale. 48 * 49 * Also provides the Priority Items Summary, which is like a Dashboard that combines multiple locales. 50 * 51 * @author markdavis 52 */ 53 public class VettingViewer<T> { 54 55 private static final boolean DEBUG = false; 56 57 private static final boolean SHOW_SUBTYPES = false; 58 59 private static final String CONNECT_PREFIX = "₍_"; 60 private static final String CONNECT_SUFFIX = "₎"; 61 62 private static final String TH_AND_STYLES = "<th class='tv-th' style='text-align:left'>"; 63 64 private static final String SPLIT_CHAR = "\uFFFE"; 65 66 private static final boolean DEBUG_THREADS = false; 67 68 private static final Set<CheckCLDR.CheckStatus.Subtype> OK_IF_VOTED = EnumSet.of(Subtype.sameAsEnglish); 69 getNeutralOrgForSummary()70 public static Organization getNeutralOrgForSummary() { 71 return Organization.surveytool; 72 } 73 orgIsNeutralForSummary(Organization org)74 private static boolean orgIsNeutralForSummary(Organization org) { 75 return org.equals(getNeutralOrgForSummary()); 76 } 77 78 private LocaleBaselineCount localeBaselineCount = null; 79 setLocaleBaselineCount(LocaleBaselineCount localeBaselineCount)80 public void setLocaleBaselineCount(LocaleBaselineCount localeBaselineCount) { 81 this.localeBaselineCount = localeBaselineCount; 82 } 83 getOutdatedPaths()84 public static OutdatedPaths getOutdatedPaths() { 85 return outdatedPaths; 86 } 87 88 private static PathHeader.Factory pathTransform; 89 private static final OutdatedPaths outdatedPaths = new OutdatedPaths(); 90 91 /** 92 * See VoteResolver getStatusForOrganization to see how this is computed. 93 */ 94 public enum VoteStatus { 95 /** 96 * The value for the path is either contributed or approved, and 97 * the user's organization didn't vote. (see class def for null user) 98 */ 99 ok_novotes, 100 101 /** 102 * The value for the path is either contributed or approved, and 103 * the user's organization chose the winning value. (see class def for null user) 104 */ 105 ok, 106 107 /** 108 * The user's organization chose the winning value for the path, but 109 * that value is neither contributed nor approved. (see class def for null user) 110 */ 111 provisionalOrWorse, 112 113 /** 114 * The user's organization's choice is not winning. There may be 115 * insufficient votes to overcome a previously approved value, or other 116 * organizations may be voting against it. (see class def for null user) 117 */ 118 losing, 119 120 /** 121 * There is a dispute, meaning more than one item with votes, or the item with votes didn't win. 122 */ 123 disputed 124 } 125 126 /** 127 * @author markdavis 128 * 129 * @param <T> 130 */ 131 public interface UsersChoice<T> { 132 /** 133 * Return the value that the user's organization (as a whole) voted for, 134 * or null if none of the users in the organization voted for the path. <br> 135 * NOTE: Would be easier if this were a method on CLDRFile. 136 * NOTE: if organization = null, then it must return the absolute winning value. 137 */ getWinningValueForUsersOrganization(CLDRFile cldrFile, String path, T organization)138 String getWinningValueForUsersOrganization(CLDRFile cldrFile, String path, T organization); 139 140 /** 141 * Return the vote status 142 * NOTE: if organization = null, then it must disregard the organization and never return losing. See VoteStatus. 143 */ getStatusForUsersOrganization(CLDRFile cldrFile, String path, T organization)144 VoteStatus getStatusForUsersOrganization(CLDRFile cldrFile, String path, T organization); 145 146 /** 147 * Has the given user voted for the given path and locale? 148 * @param userId 149 * @param loc 150 * @param path 151 * @return true if that user has voted, else false 152 */ userDidVote(int userId, CLDRLocale loc, String path)153 boolean userDidVote(int userId, CLDRLocale loc, String path); 154 getVoteResolver(CLDRFile baselineFile, CLDRLocale loc, String path)155 VoteResolver<String> getVoteResolver(CLDRFile baselineFile, CLDRLocale loc, String path); 156 } 157 158 public interface ErrorChecker { 159 enum Status { 160 ok, error, warning 161 } 162 163 /** 164 * Initialize an error checker with a cldrFile. MUST be called before 165 * any getErrorStatus. 166 */ initErrorStatus(CLDRFile cldrFile)167 Status initErrorStatus(CLDRFile cldrFile); 168 169 /** 170 * Return the detailed CheckStatus information. 171 */ getErrorCheckStatus(String path, String value)172 List<CheckStatus> getErrorCheckStatus(String path, String value); 173 174 /** 175 * Return the status, and append the error message to the status 176 * message. If there are any errors, then the warnings are not included. 177 */ getErrorStatus(String path, String value, StringBuilder statusMessage)178 Status getErrorStatus(String path, String value, StringBuilder statusMessage); 179 180 /** 181 * Return the status, and append the error message to the status 182 * message, and get the subtypes. If there are any errors, then the warnings are not included. 183 */ getErrorStatus(String path, String value, StringBuilder statusMessage, EnumSet<Subtype> outputSubtypes)184 Status getErrorStatus(String path, String value, StringBuilder statusMessage, 185 EnumSet<Subtype> outputSubtypes); 186 } 187 188 private static class DefaultErrorStatus implements ErrorChecker { 189 190 private CheckCLDR checkCldr; 191 private HashMap<String, String> options = new HashMap<>(); 192 private ArrayList<CheckStatus> result = new ArrayList<>(); 193 private CLDRFile cldrFile; 194 private final Factory factory; 195 DefaultErrorStatus(Factory cldrFactory)196 private DefaultErrorStatus(Factory cldrFactory) { 197 this.factory = cldrFactory; 198 } 199 200 @Override initErrorStatus(CLDRFile cldrFile)201 public Status initErrorStatus(CLDRFile cldrFile) { 202 this.cldrFile = cldrFile; 203 options = new HashMap<>(); 204 result = new ArrayList<>(); 205 checkCldr = CheckCLDR.getCheckAll(factory, ".*"); 206 checkCldr.setCldrFileToCheck(cldrFile, new Options(options), result); 207 return Status.ok; 208 } 209 210 @Override getErrorCheckStatus(String path, String value)211 public List<CheckStatus> getErrorCheckStatus(String path, String value) { 212 String fullPath = cldrFile.getFullXPath(path); 213 ArrayList<CheckStatus> result2 = new ArrayList<>(); 214 checkCldr.check(path, fullPath, value, new CheckCLDR.Options(options), result2); 215 return result2; 216 } 217 218 @Override getErrorStatus(String path, String value, StringBuilder statusMessage)219 public Status getErrorStatus(String path, String value, StringBuilder statusMessage) { 220 return getErrorStatus(path, value, statusMessage, null); 221 } 222 223 @Override getErrorStatus(String path, String value, StringBuilder statusMessage, EnumSet<Subtype> outputSubtypes)224 public Status getErrorStatus(String path, String value, StringBuilder statusMessage, 225 EnumSet<Subtype> outputSubtypes) { 226 Status result0 = Status.ok; 227 StringBuilder errorMessage = new StringBuilder(); 228 String fullPath = cldrFile.getFullXPath(path); 229 checkCldr.check(path, fullPath, value, new CheckCLDR.Options(options), result); 230 for (CheckStatus checkStatus : result) { 231 final CheckCLDR cause = checkStatus.getCause(); 232 /* 233 * CheckCoverage will be shown under Missing, not under Warnings; and 234 * CheckNew will be shown under New, not under Warnings; so skip them here. 235 */ 236 if (cause instanceof CheckCoverage || cause instanceof CheckNew) { 237 continue; 238 } 239 CheckStatus.Type statusType = checkStatus.getType(); 240 if (statusType.equals(CheckStatus.errorType)) { 241 // throw away any accumulated warning messages 242 if (result0 == Status.warning) { 243 errorMessage.setLength(0); 244 if (outputSubtypes != null) { 245 outputSubtypes.clear(); 246 } 247 } 248 result0 = Status.error; 249 if (outputSubtypes != null) { 250 outputSubtypes.add(checkStatus.getSubtype()); 251 } 252 appendToMessage(checkStatus.getMessage(), checkStatus.getSubtype(), errorMessage); 253 } else if (result0 != Status.error && statusType.equals(CheckStatus.warningType)) { 254 result0 = Status.warning; 255 // accumulate all the warning messages 256 if (outputSubtypes != null) { 257 outputSubtypes.add(checkStatus.getSubtype()); 258 } 259 appendToMessage(checkStatus.getMessage(), checkStatus.getSubtype(), errorMessage); 260 } 261 } 262 if (result0 != Status.ok) { 263 appendToMessage(errorMessage, statusMessage); 264 } 265 return result0; 266 } 267 } 268 269 private final Factory cldrFactory; 270 private final CLDRFile englishFile; 271 private final UsersChoice<T> userVoteStatus; 272 private final SupplementalDataInfo supplementalDataInfo; 273 private final Set<String> defaultContentLocales; 274 275 /** 276 * Create the Vetting Viewer. 277 * 278 * @param supplementalDataInfo 279 * @param cldrFactory 280 * @param userVoteStatus 281 */ VettingViewer(SupplementalDataInfo supplementalDataInfo, Factory cldrFactory, UsersChoice<T> userVoteStatus)282 public VettingViewer(SupplementalDataInfo supplementalDataInfo, Factory cldrFactory, 283 UsersChoice<T> userVoteStatus) { 284 285 super(); 286 this.cldrFactory = cldrFactory; 287 englishFile = cldrFactory.make("en", true); 288 if (pathTransform == null) { 289 pathTransform = PathHeader.getFactory(englishFile); 290 } 291 this.userVoteStatus = userVoteStatus; 292 this.supplementalDataInfo = supplementalDataInfo; 293 this.defaultContentLocales = supplementalDataInfo.getDefaultContentLocales(); 294 295 reasonsToPaths = Relation.of(new HashMap<>(), HashSet.class); 296 } 297 298 public class WritingInfo implements Comparable<WritingInfo> { 299 public final PathHeader codeOutput; 300 public final Set<NotificationCategory> problems; 301 public final String htmlMessage; 302 public final Subtype subtype; 303 WritingInfo(PathHeader ph, EnumSet<NotificationCategory> problems, CharSequence htmlMessage, Subtype subtype)304 public WritingInfo(PathHeader ph, EnumSet<NotificationCategory> problems, CharSequence htmlMessage, Subtype subtype) { 305 super(); 306 this.codeOutput = ph; 307 this.problems = Collections.unmodifiableSet(problems.clone()); 308 this.htmlMessage = htmlMessage.toString(); 309 this.subtype = subtype; 310 } 311 312 @Override compareTo(WritingInfo other)313 public int compareTo(WritingInfo other) { 314 return codeOutput.compareTo(other.codeOutput); 315 } 316 } 317 318 public class DashboardData { 319 public Relation<R2<SectionId, PageId>, WritingInfo> sorted = Relation.of( 320 new TreeMap<R2<SectionId, PageId>, Set<WritingInfo>>(), TreeSet.class); 321 322 public VoterProgress voterProgress = new VoterProgress(); 323 } 324 325 /** 326 * Generate the Dashboard 327 * 328 * @param args the DashboardArgs 329 * @return the DashboardData 330 */ generateDashboard(VettingParameters args)331 public DashboardData generateDashboard(VettingParameters args) { 332 333 DashboardData dd = new DashboardData(); 334 335 FileInfo fileInfo = new FileInfo(args.locale.getBaseName(), args.coverageLevel, args.choices, (T) args.organization); 336 if (args.specificSinglePath != null) { 337 fileInfo.setSinglePath(args.specificSinglePath); 338 } 339 fileInfo.setFiles(args.sourceFile, args.baselineFile); 340 fileInfo.setSorted(dd.sorted); 341 fileInfo.setVoterProgressAndId(dd.voterProgress, args.userId); 342 fileInfo.getFileInfo(); 343 344 return dd; 345 } 346 generateLocaleCompletion(VettingParameters args)347 public LocaleCompletionData generateLocaleCompletion(VettingParameters args) { 348 if (!args.sourceFile.isResolved()) { 349 throw new IllegalArgumentException("File must be resolved for locale completion"); 350 } 351 FileInfo fileInfo = new FileInfo(args.locale.getBaseName(), args.coverageLevel, args.choices, (T) args.organization); 352 fileInfo.setFiles(args.sourceFile, args.baselineFile); 353 fileInfo.getFileInfo(); 354 return new LocaleCompletionData(fileInfo.vc.problemCounter); 355 } 356 357 private class VettingCounters { 358 private final Counter<NotificationCategory> problemCounter = new Counter<>(); 359 private final Counter<Subtype> errorSubtypeCounter = new Counter<>(); 360 private final Counter<Subtype> warningSubtypeCounter = new Counter<>(); 361 362 /** 363 * Combine some statistics into this VettingCounters from another VettingCounters 364 * 365 * This is used by Priority Items Summary to combine stats from multiple locales. 366 * 367 * @param other the other VettingCounters object (for a single locale) 368 */ addAll(VettingCounters other)369 private void addAll(VettingCounters other) { 370 problemCounter.addAll(other.problemCounter); 371 errorSubtypeCounter.addAll(other.errorSubtypeCounter); 372 warningSubtypeCounter.addAll(other.warningSubtypeCounter); 373 } 374 } 375 376 /** 377 * A FileInfo contains parameters, results, and methods for gathering information about a locale 378 */ 379 private class FileInfo { 380 private final String localeId; 381 private final CLDRLocale cldrLocale; 382 private final Level usersLevel; 383 private final EnumSet<NotificationCategory> choices; 384 private final T organization; 385 FileInfo(String localeId, Level level, EnumSet<NotificationCategory> choices, T organization)386 private FileInfo(String localeId, Level level, EnumSet<NotificationCategory> choices, T organization) { 387 this.localeId = localeId; 388 this.cldrLocale = CLDRLocale.getInstance(localeId); 389 this.usersLevel = level; 390 this.choices = choices; 391 this.organization = organization; 392 } 393 394 private CLDRFile sourceFile = null; 395 private CLDRFile baselineFile = null; 396 private CLDRFile baselineFileUnresolved = null; 397 private boolean latin = false; 398 setFiles(CLDRFile sourceFile, CLDRFile baselineFile)399 private void setFiles(CLDRFile sourceFile, CLDRFile baselineFile) { 400 this.sourceFile = sourceFile; 401 this.baselineFile = baselineFile; 402 this.baselineFileUnresolved = (baselineFile == null) ? null : baselineFile.getUnresolved(); 403 this.latin = VettingViewer.isLatinScriptLocale(sourceFile); 404 } 405 406 /** 407 * If not null, this object gets filled in with additional information 408 */ 409 private Relation<R2<SectionId, PageId>, WritingInfo> sorted = null; 410 setSorted(Relation<R2<SectionId, PageId>, VettingViewer<T>.WritingInfo> sorted)411 private void setSorted(Relation<R2<SectionId, PageId>, VettingViewer<T>.WritingInfo> sorted) { 412 this.sorted = sorted; 413 } 414 415 /** 416 * If voterId > 0, calculate voterProgress for the indicated user. 417 */ 418 private int voterId = 0; 419 private VoterProgress voterProgress = null; 420 setVoterProgressAndId(VoterProgress voterProgress, int userId)421 private void setVoterProgressAndId(VoterProgress voterProgress, int userId) { 422 this.voterProgress = voterProgress; 423 this.voterId = userId; 424 } 425 426 private final VettingCounters vc = new VettingCounters(); 427 private final EnumSet<NotificationCategory> problems = EnumSet.noneOf(NotificationCategory.class); 428 private final StringBuilder htmlMessage = new StringBuilder(); 429 private final StringBuilder statusMessage = new StringBuilder(); 430 private final EnumSet<Subtype> subtypes = EnumSet.noneOf(Subtype.class); 431 private final DefaultErrorStatus errorChecker = new DefaultErrorStatus(cldrFactory); 432 433 /** 434 * If not null, getFileInfo will skip all paths except this one 435 */ 436 private String specificSinglePath = null; 437 setSinglePath(String path)438 private void setSinglePath(String path) { 439 this.specificSinglePath = path; 440 } 441 442 /** 443 * Loop through paths for the Dashboard or the Priority Items Summary 444 * 445 * @return the FileInfo 446 */ getFileInfo()447 private void getFileInfo() { 448 if (progressCallback.isStopped()) { 449 throw new RuntimeException("Requested to stop"); 450 } 451 errorChecker.initErrorStatus(sourceFile); 452 if (specificSinglePath != null) { 453 handleOnePath(specificSinglePath); 454 return; 455 } 456 Set<String> seenSoFar = new HashSet<>(); 457 for (String path : sourceFile.fullIterable()) { 458 if (seenSoFar.contains(path)) { 459 continue; 460 } 461 seenSoFar.add(path); 462 progressCallback.nudge(); // Let the user know we're moving along 463 handleOnePath(path); 464 } 465 } 466 handleOnePath(String path)467 private void handleOnePath(String path) { 468 PathHeader ph = pathTransform.fromPath(path); 469 if (ph == null || ph.shouldHide()) { 470 return; 471 } 472 String value = sourceFile.getWinningValue(path); 473 statusMessage.setLength(0); 474 subtypes.clear(); 475 ErrorChecker.Status errorStatus = errorChecker.getErrorStatus(path, value, statusMessage, subtypes); 476 477 // note that the value might be missing! 478 Level pathLevel = supplementalDataInfo.getCoverageLevel(path, localeId); 479 480 // skip all but errors above the requested level 481 boolean pathLevelIsTooHigh = pathLevel.compareTo(usersLevel) > 0; 482 boolean onlyRecordErrors = pathLevelIsTooHigh; 483 484 problems.clear(); 485 htmlMessage.setLength(0); 486 487 final String oldValue = (baselineFileUnresolved == null) ? null : baselineFileUnresolved.getWinningValue(path); 488 if (skipForLimitedSubmission(path, errorStatus, oldValue)) { 489 return; 490 } 491 if (!onlyRecordErrors && choices.contains(NotificationCategory.changedOldValue)) { 492 if (oldValue != null && !oldValue.equals(value)) { 493 problems.add(NotificationCategory.changedOldValue); 494 vc.problemCounter.increment(NotificationCategory.changedOldValue); 495 } 496 } 497 VoteStatus voteStatus = userVoteStatus.getStatusForUsersOrganization(sourceFile, path, organization); 498 boolean itemsOkIfVoted = (voteStatus == VoteStatus.ok); 499 MissingStatus missingStatus = onlyRecordErrors ? null : recordMissingChangedEtc(path, itemsOkIfVoted, value, oldValue); 500 recordChoice(errorStatus, itemsOkIfVoted, onlyRecordErrors); 501 if (!onlyRecordErrors) { 502 recordLosingDisputedEtc(path, voteStatus, missingStatus); 503 } 504 if (pathLevelIsTooHigh && problems.isEmpty()) { 505 return; 506 } 507 updateVotedOrAbstained(path); 508 509 if (!problems.isEmpty() && sorted != null) { 510 reasonsToPaths.clear(); 511 R2<SectionId, PageId> group = Row.of(ph.getSectionId(), ph.getPageId()); 512 sorted.put(group, new WritingInfo(ph, problems, htmlMessage, firstSubtype())); 513 } 514 } 515 firstSubtype()516 private Subtype firstSubtype() { 517 for (Subtype subtype : subtypes) { 518 if (subtype != Subtype.none) { 519 return subtype; 520 } 521 } 522 return Subtype.none; 523 } 524 updateVotedOrAbstained(String path)525 private void updateVotedOrAbstained(String path) { 526 if (voterProgress == null || voterId == 0) { 527 return; 528 } 529 voterProgress.incrementVotablePathCount(); 530 if (userVoteStatus.userDidVote(voterId, cldrLocale, path)) { 531 voterProgress.incrementVotedPathCount(); 532 } else if (choices.contains(NotificationCategory.abstained)) { 533 problems.add(NotificationCategory.abstained); 534 vc.problemCounter.increment(NotificationCategory.abstained); 535 } 536 } 537 skipForLimitedSubmission(String path, ErrorChecker.Status errorStatus, String oldValue)538 private boolean skipForLimitedSubmission(String path, ErrorChecker.Status errorStatus, String oldValue) { 539 if (CheckCLDR.LIMITED_SUBMISSION) { 540 boolean isError = (errorStatus == ErrorChecker.Status.error); 541 boolean isMissing = (oldValue == null); 542 if (!SubmissionLocales.allowEvenIfLimited(localeId, path, isError, isMissing)) { 543 return true; 544 } 545 } 546 return false; 547 } 548 recordMissingChangedEtc(String path, boolean itemsOkIfVoted, String value, String oldValue)549 private MissingStatus recordMissingChangedEtc(String path, 550 boolean itemsOkIfVoted, String value, String oldValue) { 551 VoteResolver<String> resolver = userVoteStatus.getVoteResolver(baselineFile, cldrLocale, path); 552 MissingStatus missingStatus; 553 if (resolver.getWinningStatus() == VoteResolver.Status.missing) { 554 missingStatus = getMissingStatus(sourceFile, path, latin); 555 } else { 556 missingStatus = MissingStatus.PRESENT; 557 } 558 if (choices.contains(NotificationCategory.missingCoverage) && missingStatus == MissingStatus.ABSENT) { 559 problems.add(NotificationCategory.missingCoverage); 560 vc.problemCounter.increment(NotificationCategory.missingCoverage); 561 } 562 if (!CheckCLDR.LIMITED_SUBMISSION 563 && !itemsOkIfVoted && outdatedPaths.isOutdated(localeId, path)) { 564 recordEnglishChanged(path, value, oldValue); 565 } 566 return missingStatus; 567 } 568 recordEnglishChanged(String path, String value, String oldValue)569 private void recordEnglishChanged(String path, String value, String oldValue) { 570 if (Objects.equals(value, oldValue) && choices.contains(NotificationCategory.englishChanged)) { 571 String oldEnglishValue = outdatedPaths.getPreviousEnglish(path); 572 if (!OutdatedPaths.NO_VALUE.equals(oldEnglishValue)) { 573 // check to see if we voted 574 problems.add(NotificationCategory.englishChanged); 575 vc.problemCounter.increment(NotificationCategory.englishChanged); 576 } 577 } 578 } 579 recordChoice(ErrorChecker.Status errorStatus, boolean itemsOkIfVoted, boolean onlyRecordErrors)580 private void recordChoice(ErrorChecker.Status errorStatus, boolean itemsOkIfVoted, boolean onlyRecordErrors) { 581 NotificationCategory choice = errorStatus == ErrorChecker.Status.error ? NotificationCategory.error 582 : errorStatus == ErrorChecker.Status.warning ? NotificationCategory.warning 583 : null; 584 585 if (choice == NotificationCategory.error && choices.contains(NotificationCategory.error) 586 && (!itemsOkIfVoted 587 || !OK_IF_VOTED.containsAll(subtypes))) { 588 problems.add(choice); 589 appendToMessage(statusMessage, htmlMessage); 590 vc.problemCounter.increment(choice); 591 for (Subtype subtype : subtypes) { 592 vc.errorSubtypeCounter.increment(subtype); 593 } 594 } else if (!onlyRecordErrors && choice == NotificationCategory.warning && choices.contains(NotificationCategory.warning) 595 && (!itemsOkIfVoted 596 || !OK_IF_VOTED.containsAll(subtypes))) { 597 problems.add(choice); 598 appendToMessage(statusMessage, htmlMessage); 599 vc.problemCounter.increment(choice); 600 for (Subtype subtype : subtypes) { 601 vc.warningSubtypeCounter.increment(subtype); 602 } 603 } 604 } 605 recordLosingDisputedEtc(String path, VoteStatus voteStatus, MissingStatus missingStatus)606 private void recordLosingDisputedEtc(String path, VoteStatus voteStatus, MissingStatus missingStatus) { 607 switch (voteStatus) { 608 case losing: 609 if (choices.contains(NotificationCategory.weLost)) { 610 problems.add(NotificationCategory.weLost); 611 vc.problemCounter.increment(NotificationCategory.weLost); 612 } 613 String usersValue = userVoteStatus.getWinningValueForUsersOrganization(sourceFile, path, organization); 614 if (usersValue != null) { 615 usersValue = "Losing value: <" + TransliteratorUtilities.toHTML.transform(usersValue) + ">"; 616 appendToMessage(usersValue, htmlMessage); 617 } 618 break; 619 case disputed: 620 if (choices.contains(NotificationCategory.hasDispute)) { 621 problems.add(NotificationCategory.hasDispute); 622 vc.problemCounter.increment(NotificationCategory.hasDispute); 623 } 624 break; 625 case provisionalOrWorse: 626 if (missingStatus == MissingStatus.PRESENT && choices.contains(NotificationCategory.notApproved)) { 627 problems.add(NotificationCategory.notApproved); 628 vc.problemCounter.increment(NotificationCategory.notApproved); 629 } 630 break; 631 default: 632 } 633 } 634 } 635 636 public final class LocalesWithExplicitLevel implements Predicate<String> { 637 private final Organization org; 638 private final Level desiredLevel; 639 LocalesWithExplicitLevel(Organization org, Level level)640 public LocalesWithExplicitLevel(Organization org, Level level) { 641 this.org = org; 642 this.desiredLevel = level; 643 } 644 645 @Override is(String localeId)646 public boolean is(String localeId) { 647 StandardCodes sc = StandardCodes.make(); 648 if (orgIsNeutralForSummary(org)) { 649 if (!summarizeAllLocales && !SubmissionLocales.CLDR_LOCALES.contains(localeId)) { 650 return false; 651 } 652 return desiredLevel == sc.getTargetCoverageLevel(localeId); 653 } else { 654 Output<LocaleCoverageType> output = new Output<>(); 655 Level level = sc.getLocaleCoverageLevel(org, localeId, output); 656 return desiredLevel == level && output.value == StandardCodes.LocaleCoverageType.explicit; 657 } 658 } 659 } 660 661 /** 662 * Get the number of locales to be summarized for the given organization 663 * 664 * @param org the organization 665 * @return the number of locales 666 */ getLocaleCount(Organization org)667 public int getLocaleCount(Organization org) { 668 int localeCount = 0; 669 for (Level lv : Level.values()) { 670 Map<String, String> sortedNames = getSortedNames(org, lv); 671 localeCount += sortedNames.size(); 672 } 673 return localeCount; 674 } 675 676 /** 677 * Get the list of locales to be summarized for the given organization 678 * 679 * @param org the organization 680 * @return the list of locale id strings 681 */ getLocaleList(Organization org)682 public ArrayList<String> getLocaleList(Organization org) { 683 final ArrayList<String> list = new ArrayList<>(); 684 for (Level lv : Level.values()) { 685 final Map<String, String> sortedNames = getSortedNames(org, lv); 686 for (Map.Entry<String,String> entry : sortedNames.entrySet()) { 687 list.add(entry.getValue()); 688 } 689 } 690 return list; 691 } 692 generatePriorityItemsSummary(Appendable output, EnumSet<NotificationCategory> choices, T organization)693 public void generatePriorityItemsSummary(Appendable output, EnumSet<NotificationCategory> choices, T organization) throws ExecutionException { 694 try { 695 StringBuilder headerRow = new StringBuilder(); 696 headerRow 697 .append("<tr class='tvs-tr'>") 698 .append(TH_AND_STYLES) 699 .append("Level</th>") 700 .append(TH_AND_STYLES) 701 .append("Locale</th>") 702 .append(TH_AND_STYLES) 703 .append("Codes</th>") 704 .append(TH_AND_STYLES) 705 .append("Progress</th>"); 706 for (NotificationCategory choice : choices) { 707 headerRow.append("<th class='tv-th'>"); 708 appendDisplay(headerRow, choice); 709 headerRow.append("</th>"); 710 } 711 headerRow.append("</tr>\n"); 712 String header = headerRow.toString(); 713 714 for (Level level : Level.values()) { 715 writeSummaryTable(output, header, level, choices, organization); 716 } 717 } catch (IOException e) { 718 throw new ICUUncheckedIOException(e); // dang'ed checked exceptions 719 } 720 } 721 appendDisplay(StringBuilder target, NotificationCategory category)722 private void appendDisplay(StringBuilder target, NotificationCategory category) throws IOException { 723 target.append("<span title='") 724 .append(category.description); 725 target.append("'>") 726 .append(category.buttonLabel) 727 .append("*</span>"); 728 } 729 730 /** 731 * This is a context object for Vetting Viewer parallel writes. 732 * It keeps track of the input locales, other parameters, as well as the output 733 * streams. 734 * 735 * When done, appendTo() is called to append the output to the original requester. 736 * @author srl 737 */ 738 private class WriteContext { 739 740 private final List<String> localeNames = new ArrayList<>(); 741 private final List<String> localeIds = new ArrayList<>(); 742 private final StringBuffer[] outputs; 743 private final EnumSet<NotificationCategory> choices; 744 private final EnumSet<NotificationCategory> ourChoicesThatRequireOldFile; 745 private final T organization; 746 private final VettingViewer<T>.VettingCounters totals; 747 private final Map<String, VettingViewer<T>.FileInfo> localeNameToFileInfo; 748 private final String header; 749 private final int configChunkSize; // Number of locales to process at once, minimum 1 750 WriteContext(Set<Entry<String, String>> entrySet, EnumSet<NotificationCategory> choices, T organization, VettingCounters totals, Map<String, FileInfo> localeNameToFileInfo, String header)751 private WriteContext(Set<Entry<String, String>> entrySet, EnumSet<NotificationCategory> choices, T organization, VettingCounters totals, 752 Map<String, FileInfo> localeNameToFileInfo, String header) { 753 for (Entry<String, String> e : entrySet) { 754 localeNames.add(e.getKey()); 755 localeIds.add(e.getValue()); 756 } 757 int count = localeNames.size(); 758 this.outputs = new StringBuffer[count]; 759 for (int i = 0; i < count; i++) { 760 outputs[i] = new StringBuffer(); 761 } 762 if (DEBUG_THREADS) { 763 System.out.println("Initted " + this.outputs.length + " outputs"); 764 } 765 766 // other data 767 this.choices = choices; 768 769 EnumSet<NotificationCategory> thingsThatRequireOldFile = EnumSet.of(NotificationCategory.englishChanged, NotificationCategory.missingCoverage, NotificationCategory.changedOldValue); 770 ourChoicesThatRequireOldFile = choices.clone(); 771 ourChoicesThatRequireOldFile.retainAll(thingsThatRequireOldFile); 772 773 this.organization = organization; 774 this.totals = totals; 775 this.localeNameToFileInfo = localeNameToFileInfo; 776 this.header = header; 777 778 if (DEBUG_THREADS) { 779 System.out.println("writeContext for " + organization.toString() + " booted with " + count + " locales"); 780 } 781 782 // setup env 783 CLDRConfig config = CLDRConfig.getInstance(); 784 785 // parallelism. 0 means "let Java decide" 786 int configParallel = Math.max(config.getProperty("CLDR_VETTINGVIEWER_PARALLEL", 0), 0); 787 if (configParallel < 1) { 788 configParallel = java.lang.Runtime.getRuntime().availableProcessors(); // matches ForkJoinPool() behavior 789 } 790 this.configChunkSize = Math.max(config.getProperty("CLDR_VETTINGVIEWER_CHUNKSIZE", 1), 1); 791 if (DEBUG) { 792 System.out.println("vv: CLDR_VETTINGVIEWER_PARALLEL=" + configParallel + 793 ", CLDR_VETTINGVIEWER_CHUNKSIZE=" + configChunkSize); 794 } 795 } 796 797 /** 798 * Append all of the results (one stream per locale) to the output parameter. 799 * Insert the "header" as needed. 800 * @param output 801 * @throws IOException 802 */ appendTo(Appendable output)803 private void appendTo(Appendable output) throws IOException { 804 // all done, append all 805 char lastChar = ' '; 806 807 for (int n = 0; n < outputs.length; n++) { 808 final String name = localeNames.get(n); 809 if (DEBUG_THREADS) { 810 System.out.println("Appending " + name + " - " + outputs[n].length()); 811 } 812 char nextChar = name.charAt(0); 813 if (lastChar != nextChar) { 814 output.append(this.header); 815 lastChar = nextChar; 816 } 817 output.append(outputs[n]); 818 } 819 } 820 821 /** 822 * How many locales are represented in this context? 823 * @return 824 */ size()825 private int size() { 826 return localeNames.size(); 827 } 828 } 829 830 /** 831 * Worker action to implement parallel Vetting Viewer writes. 832 * This takes a WriteContext as a parameter, as well as a subset of the locales 833 * to operate on. 834 * 835 * @author srl 836 */ 837 private class WriteAction extends RecursiveAction { 838 private final int length; 839 private final int start; 840 private final WriteContext context; 841 WriteAction(WriteContext context)842 public WriteAction(WriteContext context) { 843 this(context, 0, context.size()); 844 } 845 WriteAction(WriteContext context, int start, int length)846 public WriteAction(WriteContext context, int start, int length) { 847 this.context = context; 848 this.start = start; 849 this.length = length; 850 if (DEBUG_THREADS) { 851 System.out.println("writeAction(…," + start + ", " + length + ") of " + context.size() + 852 " with outputCount:" + context.outputs.length); 853 } 854 } 855 856 private static final long serialVersionUID = 1L; 857 858 @Override compute()859 protected void compute() { 860 if (length == 0) { 861 return; 862 } else if (length <= context.configChunkSize) { 863 computeAll(); 864 } else { 865 int split = length / 2; 866 // subdivide 867 invokeAll(new WriteAction(context, start, split), 868 new WriteAction(context, start + split, length - split)); 869 } 870 } 871 872 /** 873 * Compute this entire task. 874 * Can call this to run this step as a single thread. 875 */ computeAll()876 private void computeAll() { 877 // do this many at once 878 for (int n = start; n < (start + length); n++) { 879 computeOne(n); 880 } 881 } 882 883 /** 884 * Calculate the Priority Items Summary output for one locale 885 * @param n 886 */ computeOne(int n)887 private void computeOne(int n) { 888 if (progressCallback.isStopped()) { 889 throw new RuntimeException("Requested to stop"); 890 } 891 if (DEBUG) { 892 MemoryHelper.availableMemory("VettingViewer.WriteAction.computeOne", true); 893 } 894 final String name = context.localeNames.get(n); 895 final String localeID = context.localeIds.get(n); 896 if (DEBUG_THREADS) { 897 System.out.println("writeAction.compute(" + n + ") - " + name + ": " + localeID); 898 } 899 EnumSet<NotificationCategory> choices = context.choices; 900 Appendable output = context.outputs[n]; 901 if (output == null) { 902 throw new NullPointerException("output " + n + " null"); 903 } 904 // Initialize 905 CLDRFile sourceFile = cldrFactory.make(localeID, true); 906 CLDRFile baselineFile = null; 907 if (!context.ourChoicesThatRequireOldFile.isEmpty()) { 908 try { 909 Factory baselineFactory = CLDRConfig.getInstance().getCommonAndSeedAndMainAndAnnotationsFactory(); 910 baselineFile = baselineFactory.make(localeID, true); 911 } catch (Exception e) { 912 } 913 } 914 Level level = Level.MODERN; 915 if (context.organization != null) { 916 StandardCodes sc = StandardCodes.make(); 917 if (orgIsNeutralForSummary((Organization) context.organization)) { 918 level = sc.getTargetCoverageLevel(localeID); 919 } else { 920 level = sc.getLocaleCoverageLevel(context.organization.toString(), localeID); 921 } 922 } 923 FileInfo fileInfo = new FileInfo(localeID, level, choices, context.organization); 924 fileInfo.setFiles(sourceFile, baselineFile); 925 fileInfo.getFileInfo(); 926 927 context.localeNameToFileInfo.put(name, fileInfo); 928 context.totals.addAll(fileInfo.vc); 929 if (DEBUG_THREADS) { 930 System.out.println("writeAction.compute(" + n + ") - got fileinfo " + name + ": " + localeID); 931 } 932 try { 933 writeSummaryRow(output, choices, fileInfo.vc.problemCounter, name, localeID, level); 934 if (DEBUG_THREADS) { 935 System.out.println("writeAction.compute(" + n + ") - wrote " + name + ": " + localeID); 936 } 937 } catch (IOException | ExecutionException e) { 938 System.err.println("writeAction.compute(" + n + ") - writeexc " + name + ": " + localeID); 939 this.completeExceptionally(new RuntimeException("While writing " + localeID, e)); 940 } 941 if (DEBUG) { 942 System.out.println("writeAction.compute(" + n + ") - DONE " + name + ": " + localeID); 943 } 944 } 945 } 946 947 /** 948 * Write the table for the Priority Items Summary 949 * @param output 950 * @param header 951 * @param desiredLevel 952 * @param choices 953 * @param organization 954 * @throws IOException 955 */ writeSummaryTable(Appendable output, String header, Level desiredLevel, EnumSet<NotificationCategory> choices, T organization)956 private void writeSummaryTable(Appendable output, String header, Level desiredLevel, 957 EnumSet<NotificationCategory> choices, T organization) throws IOException, ExecutionException { 958 Map<String, String> sortedNames = getSortedNames((Organization) organization, desiredLevel); 959 if (sortedNames.isEmpty()) { 960 return; 961 } 962 output.append("<h2>Level: ").append(desiredLevel.toString()).append("</h2>"); 963 output.append("<table class='tvs-table'>\n"); 964 Map<String, FileInfo> localeNameToFileInfo = new TreeMap<>(); 965 966 VettingCounters totals = new VettingCounters(); 967 968 Set<Entry<String, String>> entrySet = sortedNames.entrySet(); 969 970 WriteContext context = this.new WriteContext(entrySet, choices, organization, totals, localeNameToFileInfo, header); 971 972 WriteAction writeAction = this.new WriteAction(context); 973 if (USE_FORKJOIN) { 974 ForkJoinPool.commonPool().invoke(writeAction); 975 } else { 976 if (DEBUG) { 977 System.out.println("WARNING: calling writeAction.computeAll(), as the ForkJoinPool is disabled."); 978 } 979 writeAction.computeAll(); 980 } 981 context.appendTo(output); // write all of the results together 982 output.append(header); // add one header at the bottom before the Total row 983 writeSummaryRow(output, choices, totals.problemCounter, "Total", null, desiredLevel); 984 output.append("</table>"); 985 if (SHOW_SUBTYPES) { 986 showSubtypes(output, sortedNames, localeNameToFileInfo, totals, true); 987 showSubtypes(output, sortedNames, localeNameToFileInfo, totals, false); 988 } 989 } 990 getSortedNames(Organization org, Level desiredLevel)991 private Map<String, String> getSortedNames(Organization org, Level desiredLevel) { 992 Map<String, String> sortedNames = new TreeMap<>(CLDRConfig.getInstance().getCollator()); 993 // TODO Fix HACK 994 // We are going to ignore the predicate for now, just using the locales that have explicit coverage. 995 // in that locale, or allow all locales for admin@ 996 LocalesWithExplicitLevel includeLocale = new LocalesWithExplicitLevel(org, desiredLevel); 997 998 for (String localeID : cldrFactory.getAvailable()) { 999 if (defaultContentLocales.contains(localeID) 1000 || localeID.equals("en") 1001 || !includeLocale.is(localeID)) { 1002 continue; 1003 } 1004 sortedNames.put(getName(localeID), localeID); 1005 } 1006 return sortedNames; 1007 } 1008 1009 private final boolean USE_FORKJOIN = false; 1010 showSubtypes(Appendable output, Map<String, String> sortedNames, Map<String, FileInfo> localeNameToFileInfo, VettingCounters totals, boolean errors)1011 private void showSubtypes(Appendable output, Map<String, String> sortedNames, 1012 Map<String, FileInfo> localeNameToFileInfo, 1013 VettingCounters totals, boolean errors) throws IOException { 1014 1015 output.append("<h3>Details: ").append(errors ? "Error Types" : "Warning Types").append("</h3>"); 1016 output.append("<table class='tvs-table'>"); 1017 Counter<Subtype> subtypeCounterTotals = errors ? totals.errorSubtypeCounter : totals.warningSubtypeCounter; 1018 Set<Subtype> sortedBySize = subtypeCounterTotals.getKeysetSortedByCount(false); 1019 1020 // header 1021 writeDetailHeader(sortedBySize, output); 1022 1023 // items 1024 for (Entry<String, FileInfo> entry : localeNameToFileInfo.entrySet()) { 1025 Counter<Subtype> counter = errors ? entry.getValue().vc.errorSubtypeCounter : entry.getValue().vc.warningSubtypeCounter; 1026 if (counter.getTotal() == 0) { 1027 continue; 1028 } 1029 String name = entry.getKey(); 1030 String localeID = sortedNames.get(name); 1031 output.append("<tr>").append(TH_AND_STYLES); 1032 appendNameAndCode(name, localeID, output); 1033 output.append("</th>"); 1034 for (Subtype subtype : sortedBySize) { 1035 long count = counter.get(subtype); 1036 output.append("<td class='tvs-count'>"); 1037 if (count != 0) { 1038 output.append(nf.format(count)); 1039 } 1040 output.append("</td>"); 1041 } 1042 } 1043 1044 // subtotals 1045 writeDetailHeader(sortedBySize, output); 1046 output.append("<tr>").append(TH_AND_STYLES).append("<i>Total</i>").append("</th>").append(TH_AND_STYLES).append("</th>"); 1047 for (Subtype subtype : sortedBySize) { 1048 long count = subtypeCounterTotals.get(subtype); 1049 output.append("<td class='tvs-count'>"); 1050 if (count != 0) { 1051 output.append("<b>").append(nf.format(count)).append("</b>"); 1052 } 1053 output.append("</td>"); 1054 } 1055 output.append("</table>"); 1056 } 1057 writeDetailHeader(Set<Subtype> sortedBySize, Appendable output)1058 private void writeDetailHeader(Set<Subtype> sortedBySize, Appendable output) throws IOException { 1059 output.append("<tr>") 1060 .append(TH_AND_STYLES).append("Name").append("</th>") 1061 .append(TH_AND_STYLES).append("ID").append("</th>"); 1062 for (Subtype subtype : sortedBySize) { 1063 output.append(TH_AND_STYLES).append(subtype.toString()).append("</th>"); 1064 } 1065 } 1066 1067 /** 1068 * Write one row of the Priority Items Summary 1069 * 1070 * @param output 1071 * @param choices 1072 * @param problemCounter 1073 * @param name 1074 * @param localeID if null, this is a "Total" row to be shown at the bottom of the table 1075 * @param level 1076 * @throws IOException 1077 * 1078 * CAUTION: this method not only uses "th" for "table header" in the usual sense, it also 1079 * uses "th" for cells that contain data, including locale names like "Kashmiri (Devanagari)" 1080 * and code values like "<code>ks_Deva₍_IN₎</code>". The same row may have both "th" and "td" cells. 1081 */ writeSummaryRow(Appendable output, EnumSet<NotificationCategory> choices, Counter<NotificationCategory> problemCounter, String name, String localeID, Level level)1082 private void writeSummaryRow(Appendable output, EnumSet<NotificationCategory> choices, Counter<NotificationCategory> problemCounter, 1083 String name, String localeID, Level level) throws IOException, ExecutionException { 1084 output 1085 .append("<tr>") 1086 .append(TH_AND_STYLES) 1087 .append(level.toString()) 1088 .append("</th>") 1089 .append(TH_AND_STYLES); 1090 if (localeID == null) { 1091 output 1092 .append("<i>") 1093 .append(name) // here always name = "Total" 1094 .append("</i>") 1095 .append("</th>") 1096 .append(TH_AND_STYLES); // empty cell for Codes 1097 } else { 1098 appendNameAndCode(name, localeID, output); 1099 } 1100 output.append("</th>\n"); 1101 final String progPerc = (localeID == null) ? "" : getLocaleProgressPercent(localeID, problemCounter); 1102 output.append("<td class='tvs-count'>").append(progPerc).append("</td>\n"); 1103 for (NotificationCategory choice : choices) { 1104 long count = problemCounter.get(choice); 1105 output.append("<td class='tvs-count'>"); 1106 if (localeID == null) { 1107 output.append("<b>"); 1108 } 1109 output.append(nf.format(count)); 1110 if (localeID == null) { 1111 output.append("</b>"); 1112 } 1113 output.append("</td>\n"); 1114 } 1115 output.append("</tr>\n"); 1116 } 1117 getLocaleProgressPercent(String localeId, Counter<NotificationCategory> problemCounter)1118 private String getLocaleProgressPercent(String localeId, Counter<NotificationCategory> problemCounter) throws ExecutionException { 1119 final LocaleCompletionData lcd = new LocaleCompletionData(problemCounter); 1120 final int problemCount = lcd.problemCount(); 1121 final int total = localeBaselineCount.getBaselineProblemCount(CLDRLocale.getInstance(localeId)); 1122 final int done = (problemCount >= total) ? 0 : total - problemCount; 1123 // return CompletionPercent.calculate(done, total) + "%"; 1124 1125 // Adjust according to https://unicode-org.atlassian.net/browse/CLDR-15785 1126 // This is NOT a logical long-term solution 1127 int perc = CompletionPercent.calculate(done, total); 1128 if (perc == 100 && problemCount > 0) { 1129 perc = 99; 1130 } 1131 return perc + "%"; 1132 } 1133 appendNameAndCode(String name, String localeID, Appendable output)1134 private void appendNameAndCode(String name, String localeID, Appendable output) throws IOException { 1135 // See https://unicode-org.atlassian.net/browse/CLDR-15279 1136 String url = "v#/" + localeID + "//"; 1137 String[] names = name.split(SPLIT_CHAR); 1138 output 1139 .append("<a href='" + url) 1140 .append("'>") 1141 .append(TransliteratorUtilities.toHTML.transform(names[0])) 1142 .append("</a>") 1143 .append("</th>") 1144 .append(TH_AND_STYLES) 1145 .append("<code>") 1146 .append(names[1]) 1147 .append("</code>"); 1148 } 1149 getName(String localeID)1150 private String getName(String localeID) { 1151 Set<String> contents = supplementalDataInfo.getEquivalentsForLocale(localeID); 1152 // put in special character that can be split on later 1153 return englishFile.getName(localeID, true, CLDRFile.SHORT_ALTS) + SPLIT_CHAR + gatherCodes(contents); 1154 } 1155 1156 /** 1157 * Collapse the names 1158 {en_Cyrl, en_Cyrl_US} => en_Cyrl(_US) 1159 {en_GB, en_Latn_GB} => en(_Latn)_GB 1160 {en, en_US, en_Latn, en_Latn_US} => en(_Latn)(_US) 1161 {az_IR, az_Arab, az_Arab_IR} => az_IR, az_Arab(_IR) 1162 */ gatherCodes(Set<String> contents)1163 private static String gatherCodes(Set<String> contents) { 1164 Set<Set<String>> source = new LinkedHashSet<>(); 1165 for (String s : contents) { 1166 source.add(new LinkedHashSet<>(Arrays.asList(s.split("_")))); 1167 } 1168 Set<Set<String>> oldSource = new LinkedHashSet<>(); 1169 1170 do { 1171 // exchange source/target 1172 oldSource.clear(); 1173 oldSource.addAll(source); 1174 source.clear(); 1175 Set<String> last = null; 1176 for (Set<String> ss : oldSource) { 1177 if (last == null) { 1178 last = ss; 1179 } else { 1180 if (ss.containsAll(last)) { 1181 last = combine(last, ss); 1182 } else { 1183 source.add(last); 1184 last = ss; 1185 } 1186 } 1187 } 1188 source.add(last); 1189 } while (oldSource.size() != source.size()); 1190 1191 StringBuilder b = new StringBuilder(); 1192 for (Set<String> stringSet : source) { 1193 if (b.length() != 0) { 1194 b.append(", "); 1195 } 1196 String sep = ""; 1197 for (String string : stringSet) { 1198 if (string.startsWith(CONNECT_PREFIX)) { 1199 b.append(string + CONNECT_SUFFIX); 1200 } else { 1201 b.append(sep + string); 1202 } 1203 sep = "_"; 1204 } 1205 } 1206 return b.toString(); 1207 } 1208 combine(Set<String> last, Set<String> ss)1209 private static Set<String> combine(Set<String> last, Set<String> ss) { 1210 LinkedHashSet<String> result = new LinkedHashSet<>(); 1211 for (String s : ss) { 1212 if (last.contains(s)) { 1213 result.add(s); 1214 } else { 1215 result.add(CONNECT_PREFIX + s); 1216 } 1217 } 1218 return result; 1219 } 1220 1221 /** 1222 * Used to determine what the status of a particular path's value is in a given locale. 1223 */ 1224 public enum MissingStatus { 1225 /** 1226 * There is an explicit value for the path, including ↑↑↑, 1227 * or there is an inherited value (but not including the ABSENT conditions, e.g. not from root). 1228 */ 1229 PRESENT, 1230 1231 /** 1232 * The value is inherited from a different path. Only applies if the parent is not root. 1233 * That path might be in the same locale or from a parent (but not root or CODE_FALLBACK). 1234 */ 1235 ALIASED, 1236 1237 /** 1238 * See ABSENT 1239 */ 1240 MISSING_OK, 1241 1242 /** 1243 * See ABSENT 1244 */ 1245 ROOT_OK, 1246 1247 /** 1248 * The supplied CLDRFile is null, or the value is null, or the value is inherited from root or CODE_FALLBACK. 1249 * A special ValuePathStatus.isMissingOk method allows for some exceptions, changing the result to MISSING_OK or ROOT_OK. 1250 */ 1251 ABSENT 1252 } 1253 1254 /** 1255 * Get the MissingStatus: for details see the javadoc for MissingStatus. 1256 * 1257 * @param sourceFile the CLDRFile 1258 * @param path the path 1259 * @param latin boolean from isLatinScriptLocale, passed to isMissingOk 1260 * @return the MissingStatus 1261 */ getMissingStatus(CLDRFile sourceFile, String path, boolean latin)1262 public static MissingStatus getMissingStatus(CLDRFile sourceFile, String path, boolean latin) { 1263 if (sourceFile == null) { 1264 return MissingStatus.ABSENT; 1265 } 1266 final String sourceLocaleID = sourceFile.getLocaleID(); 1267 if ("root".equals(sourceLocaleID)) { 1268 return MissingStatus.MISSING_OK; 1269 } 1270 MissingStatus result; 1271 1272 String value = sourceFile.getStringValue(path); 1273 Status status = new Status(); 1274 String sourceLocale = sourceFile.getSourceLocaleIdExtended(path, status, false); // does not skip inheritance marker 1275 1276 boolean isAliased = !path.equals(status.pathWhereFound); 1277 if (DEBUG) { 1278 if (path.equals("//ldml/characterLabels/characterLabelPattern[@type=\"subscript\"]")) { 1279 int debug = 0; 1280 } 1281 if (!isAliased && !sourceLocale.equals(sourceLocaleID)) { 1282 int debug = 0; 1283 } 1284 } 1285 1286 if (value == null) { 1287 result = ValuePathStatus.isMissingOk(sourceFile, path, latin, isAliased) ? MissingStatus.MISSING_OK 1288 : MissingStatus.ABSENT; 1289 } else { 1290 /* 1291 * skipInheritanceMarker must be false for getSourceLocaleIdExtended here, since INHERITANCE_MARKER 1292 * may be found if there are votes for inheritance, in which case we must not skip up to "root" and 1293 * treat the item as missing. Reference: https://unicode.org/cldr/trac/ticket/11765 1294 */ 1295 String localeFound = sourceFile.getSourceLocaleIdExtended(path, status, false /* skipInheritanceMarker */); 1296 final boolean localeFoundIsRootOrCodeFallback = localeFound.equals("root") 1297 || localeFound.equals(XMLSource.CODE_FALLBACK_ID); 1298 final boolean isParentRoot = CLDRLocale.getInstance(sourceFile.getLocaleID()).isParentRoot(); 1299 /* 1300 * Only count it as missing IF the (localeFound is root or codeFallback) 1301 * AND the aliasing didn't change the path. 1302 * Note that localeFound will be where an item with ↑↑↑ was found even though 1303 * the resolved value is actually inherited from somewhere else. 1304 */ 1305 1306 if (localeFoundIsRootOrCodeFallback) { 1307 result = ValuePathStatus.isMissingOk(sourceFile, path, latin, isAliased) ? MissingStatus.ROOT_OK 1308 : isParentRoot ? MissingStatus.ABSENT 1309 : MissingStatus.ALIASED; 1310 } else if (!isAliased) { 1311 result = MissingStatus.PRESENT; 1312 } else if (isParentRoot) { // We handle ALIASED specially, depending on whether the parent is root or not. 1313 result = ValuePathStatus.isMissingOk(sourceFile, path, latin, isAliased) ? MissingStatus.MISSING_OK 1314 : MissingStatus.ABSENT; 1315 } else { 1316 result = MissingStatus.ALIASED; 1317 } 1318 } 1319 return result; 1320 } 1321 1322 public static final UnicodeSet LATIN = ValuePathStatus.LATIN; 1323 isLatinScriptLocale(CLDRFile sourceFile)1324 public static boolean isLatinScriptLocale(CLDRFile sourceFile) { 1325 return ValuePathStatus.isLatinScriptLocale(sourceFile); 1326 } 1327 appendToMessage(CharSequence usersValue, Subtype subtype, StringBuilder testMessage)1328 private static void appendToMessage(CharSequence usersValue, Subtype subtype, StringBuilder testMessage) { 1329 if (subtype != null) { 1330 usersValue = "<" + subtype + "> " + usersValue; 1331 } 1332 appendToMessage(usersValue, testMessage); 1333 } 1334 appendToMessage(CharSequence usersValue, StringBuilder testMessage)1335 private static void appendToMessage(CharSequence usersValue, StringBuilder testMessage) { 1336 if (usersValue.length() == 0) { 1337 return; 1338 } 1339 if (testMessage.length() != 0) { 1340 testMessage.append("<br>"); 1341 } 1342 testMessage.append(usersValue); 1343 } 1344 1345 static final NumberFormat nf = NumberFormat.getIntegerInstance(ULocale.ENGLISH); 1346 private final Relation<String, String> reasonsToPaths; 1347 1348 static { 1349 nf.setGroupingUsed(true); 1350 } 1351 1352 /** 1353 * Class that allows the relaying of progress information 1354 * 1355 * @author srl 1356 * 1357 */ 1358 public static class ProgressCallback { 1359 /** 1360 * Note any progress. This will be called before any output is printed. 1361 * It will be called approximately once per xpath. 1362 */ nudge()1363 public void nudge() { 1364 } 1365 1366 /** 1367 * Called when all operations are complete. 1368 */ done()1369 public void done() { 1370 } 1371 1372 /** 1373 * Return true to cause an early stop. 1374 * @return 1375 */ isStopped()1376 public boolean isStopped() { 1377 return false; 1378 } 1379 } 1380 1381 /* 1382 * null instance by default 1383 */ 1384 private ProgressCallback progressCallback = new ProgressCallback(); 1385 1386 /** 1387 * Select a new callback. Must be set before running. 1388 * 1389 * @return 1390 * 1391 */ setProgressCallback(ProgressCallback newCallback)1392 public VettingViewer<T> setProgressCallback(ProgressCallback newCallback) { 1393 progressCallback = newCallback; 1394 return this; 1395 } 1396 1397 /** 1398 * Provide the styles for inclusion into the ST <head> element. 1399 * 1400 * @return 1401 */ getHeaderStyles()1402 public static String getHeaderStyles() { 1403 return "<style>\n" 1404 + ".hide {display:none}\n" 1405 + ".vve {}\n" 1406 + ".vvn {}\n" 1407 + ".vvp {}\n" 1408 + ".vvl {}\n" 1409 + ".vvm {}\n" 1410 + ".vvu {}\n" 1411 + ".vvw {}\n" 1412 + ".vvd {}\n" 1413 + ".vvo {}\n" 1414 + "</style>"; 1415 } 1416 1417 /** 1418 * Find the status of all the paths in the input file. See the full getStatus for more information. 1419 * @param file the source. Must be a resolved file, made with minimalDraftStatus = unconfirmed 1420 * @param pathHeaderFactory PathHeaderFactory. 1421 * @param foundCounter output counter of the number of paths with values having contributed or approved status 1422 * @param unconfirmedCounter output counter of the number of paths with values, but neither contributed nor approved status 1423 * @param missingCounter output counter of the number of paths without values 1424 * @param missingPaths output if not null, the specific paths that are missing. 1425 * @param unconfirmedPaths TODO 1426 */ getStatus(CLDRFile file, PathHeader.Factory pathHeaderFactory, Counter<Level> foundCounter, Counter<Level> unconfirmedCounter, Counter<Level> missingCounter, Relation<MissingStatus, String> missingPaths, Set<String> unconfirmedPaths)1427 public static void getStatus(CLDRFile file, PathHeader.Factory pathHeaderFactory, 1428 Counter<Level> foundCounter, Counter<Level> unconfirmedCounter, 1429 Counter<Level> missingCounter, 1430 Relation<MissingStatus, String> missingPaths, 1431 Set<String> unconfirmedPaths) { 1432 getStatus(file.fullIterable(), file, pathHeaderFactory, foundCounter, unconfirmedCounter, missingCounter, missingPaths, unconfirmedPaths); 1433 } 1434 1435 /** 1436 * Find the status of an input set of paths in the input file. 1437 * It partitions the returned data according to the Coverage levels. 1438 * NOTE: MissingStatus.ALIASED is handled specially; it is mapped to ABSENT if the parent is root, and otherwise mapped to PRESENT. 1439 * @param allPaths manual list of paths 1440 * @param file the source. Must be a resolved file, made with minimalDraftStatus = unconfirmed 1441 * @param pathHeaderFactory PathHeaderFactory. 1442 * @param foundCounter output counter of the number of paths with values having contributed or approved status 1443 * @param unconfirmedCounter output counter of the number of paths with values, but neither contributed nor approved status 1444 * @param missingCounter output counter of the number of paths without values 1445 * @param missingPaths output if not null, the specific paths that are missing. 1446 * @param unconfirmedPaths TODO 1447 */ getStatus(Iterable<String> allPaths, CLDRFile file, PathHeader.Factory pathHeaderFactory, Counter<Level> foundCounter, Counter<Level> unconfirmedCounter, Counter<Level> missingCounter, Relation<MissingStatus, String> missingPaths, Set<String> unconfirmedPaths)1448 public static void getStatus(Iterable<String> allPaths, CLDRFile file, 1449 PathHeader.Factory pathHeaderFactory, Counter<Level> foundCounter, 1450 Counter<Level> unconfirmedCounter, 1451 Counter<Level> missingCounter, 1452 Relation<MissingStatus, String> missingPaths, Set<String> unconfirmedPaths) { 1453 1454 if (!file.isResolved()) { 1455 throw new IllegalArgumentException("File must be resolved, no minimal draft status"); 1456 } 1457 foundCounter.clear(); 1458 unconfirmedCounter.clear(); 1459 missingCounter.clear(); 1460 1461 boolean latin = VettingViewer.isLatinScriptLocale(file); 1462 CoverageLevel2 coverageLevel2 = CoverageLevel2.getInstance(SupplementalDataInfo.getInstance(), file.getLocaleID()); 1463 1464 for (String path : allPaths) { 1465 1466 PathHeader ph = pathHeaderFactory.fromPath(path); 1467 if (ph.getSectionId() == SectionId.Special) { 1468 continue; 1469 } 1470 1471 Level level = coverageLevel2.getLevel(path); 1472 if (level.compareTo(Level.MODERN) > 0) { 1473 continue; 1474 } 1475 MissingStatus missingStatus = VettingViewer.getMissingStatus(file, path, latin); 1476 1477 switch (missingStatus) { 1478 case ABSENT: 1479 missingCounter.add(level, 1); 1480 if (missingPaths != null) { 1481 missingPaths.put(missingStatus, path); 1482 } 1483 break; 1484 case ALIASED: 1485 case PRESENT: 1486 String fullPath = file.getFullXPath(path); 1487 if (fullPath.contains("unconfirmed") 1488 || fullPath.contains("provisional")) { 1489 unconfirmedCounter.add(level, 1); 1490 if (unconfirmedPaths != null) { 1491 unconfirmedPaths.add(path); 1492 } 1493 } else { 1494 foundCounter.add(level, 1); 1495 } 1496 break; 1497 case MISSING_OK: 1498 case ROOT_OK: 1499 break; 1500 default: 1501 throw new IllegalArgumentException(); 1502 } 1503 } 1504 } 1505 1506 final private static EnumSet<NotificationCategory> localeCompletionCategories = EnumSet.of( 1507 NotificationCategory.error, 1508 NotificationCategory.hasDispute, 1509 NotificationCategory.notApproved, 1510 NotificationCategory.missingCoverage 1511 ); 1512 getDashboardNotificationCategories(Organization usersOrg)1513 public static EnumSet<NotificationCategory> getDashboardNotificationCategories(Organization usersOrg) { 1514 EnumSet<NotificationCategory> choiceSet = EnumSet.allOf(NotificationCategory.class); 1515 if (orgIsNeutralForSummary(usersOrg)) { 1516 choiceSet = EnumSet.of( 1517 NotificationCategory.error, 1518 NotificationCategory.warning, 1519 NotificationCategory.hasDispute, 1520 NotificationCategory.notApproved, 1521 NotificationCategory.missingCoverage 1522 ); 1523 // skip weLost, englishChanged, changedOldValue, abstained 1524 } 1525 return choiceSet; 1526 } 1527 getPriorityItemsSummaryCategories(Organization org)1528 public static EnumSet<NotificationCategory> getPriorityItemsSummaryCategories(Organization org) { 1529 EnumSet<NotificationCategory> set = getDashboardNotificationCategories(org); 1530 set.remove(NotificationCategory.abstained); 1531 return set; 1532 } 1533 getLocaleCompletionCategories()1534 public static EnumSet<NotificationCategory> getLocaleCompletionCategories() { 1535 return localeCompletionCategories; 1536 } 1537 1538 public interface LocaleBaselineCount { getBaselineProblemCount(CLDRLocale cldrLocale)1539 int getBaselineProblemCount(CLDRLocale cldrLocale) throws ExecutionException; 1540 } 1541 1542 private boolean summarizeAllLocales = false; 1543 setSummarizeAllLocales(boolean b)1544 public void setSummarizeAllLocales(boolean b) { 1545 summarizeAllLocales = b; 1546 } 1547 } 1548