1 package org.unicode.cldr.util; 2 3 import com.google.common.base.Objects; 4 import com.google.common.collect.ImmutableSet; 5 import com.ibm.icu.text.Collator; 6 import com.ibm.icu.util.Output; 7 import com.ibm.icu.util.ULocale; 8 import java.sql.Timestamp; 9 import java.util.*; 10 import java.util.Map.Entry; 11 import java.util.regex.Matcher; 12 import java.util.regex.Pattern; 13 import org.unicode.cldr.icu.LDMLConstants; 14 import org.unicode.cldr.test.CheckCLDR; 15 import org.unicode.cldr.test.CheckCLDR.Phase; 16 import org.unicode.cldr.test.CheckWidths; 17 import org.unicode.cldr.test.DisplayAndInputProcessor; 18 19 /** 20 * This class implements the vote resolution process agreed to by the CLDR committee. Here is an 21 * example of usage: 22 * 23 * <pre> 24 * // before doing anything, initialize the voter data (who are the voters at what levels) with setVoterToInfo. 25 * // We assume this doesn't change often 26 * // here is some fake data: 27 * VoteResolver.setVoterToInfo(Utility.asMap(new Object[][] { 28 * { 666, new VoterInfo(Organization.google, Level.vetter, "J. Smith") }, 29 * { 555, new VoterInfo(Organization.google, Level.street, "S. Jones") }, 30 * { 444, new VoterInfo(Organization.google, Level.vetter, "S. Samuels") }, 31 * { 333, new VoterInfo(Organization.apple, Level.vetter, "A. Mutton") }, 32 * { 222, new VoterInfo(Organization.adobe, Level.expert, "A. Aldus") }, 33 * { 111, new VoterInfo(Organization.ibm, Level.street, "J. Henry") }, })); 34 * 35 * // you can create a resolver and keep it around. It isn't thread-safe, so either have a separate one per thread (they 36 * // are small), or synchronize. 37 * VoteResolver resolver = new VoteResolver(); 38 * 39 * // For any particular base path, set the values 40 * // set the 1.5 status (if we're working on 1.6). This <b>must</b> be done for each new base path 41 * resolver.newPath(oldValue, oldStatus); 42 * [TODO: function newPath doesn't exist, revise this documentation] 43 * 44 * // now add some values, with who voted for them 45 * resolver.add(value1, voter1); 46 * resolver.add(value1, voter2); 47 * resolver.add(value2, voter3); 48 * 49 * // Once you've done that, you can get the results for the base path 50 * winner = resolver.getWinningValue(); 51 * status = resolver.getWinningStatus(); 52 * conflicts = resolver.getConflictedOrganizations(); 53 * </pre> 54 */ 55 public class VoteResolver<T> { 56 public static final boolean DROP_HARD_INHERITANCE = true; 57 58 private final VoterInfoList voterInfoList; 59 VoteResolver(VoterInfoList vil)60 public VoteResolver(VoterInfoList vil) { 61 voterInfoList = vil; 62 } 63 64 private static final boolean DEBUG = false; 65 66 /** This enables a prose discussion of the voting process. */ 67 private DeferredTranscript transcript = null; 68 enableTranscript()69 public void enableTranscript() { 70 if (transcript == null) { 71 transcript = new DeferredTranscript(); 72 } 73 } 74 disableTranscript()75 public void disableTranscript() { 76 transcript = null; 77 } 78 getTranscript()79 public String getTranscript() { 80 if (transcript == null) { 81 return null; 82 } else { 83 return transcript.get(); 84 } 85 } 86 87 /** 88 * Add an annotation 89 * 90 * @param fmt 91 * @param args 92 */ annotateTranscript(String fmt, Object... args)93 private final void annotateTranscript(String fmt, Object... args) { 94 if (transcript != null) { 95 transcript.add(fmt, args); 96 } 97 } 98 99 /** 100 * A placeholder for winningValue when it would otherwise be null. It must match 101 * NO_WINNING_VALUE in the client JavaScript code. 102 */ 103 private static final String NO_WINNING_VALUE = "no-winning-value"; 104 105 /** 106 * The status levels according to the committee, in ascending order 107 * 108 * <p>Status corresponds to icons as follows: A checkmark means it’s approved and is slated to 109 * be used. A cross means it’s a missing value. Green/orange check: The item has enough votes to 110 * be used in CLDR. Red/orange/black X: The item does not have enough votes to be used in CLDR, 111 * by most implementations (or is completely missing). Reference: <a 112 * href="https://cldr.unicode.org/translation/getting-started/guide">guide</a> 113 * 114 * <p>When the item is inherited, i.e., winningValue is INHERITANCE_MARKER (↑↑↑), then 115 * orange/red X are replaced by orange/red up-arrow. That change is made only on the client. 116 * 117 * <p>Status.approved: green check Status.contributed: orange check Status.provisional: orange X 118 * (or orange up-arrow if inherited) Status.unconfirmed: red X (or red up-arrow if inherited 119 * Status.missing: black X 120 * 121 * <p>Not to be confused with VoteResolver.VoteStatus 122 */ 123 public enum Status { 124 missing, 125 unconfirmed, 126 provisional, 127 contributed, 128 approved; 129 fromString(String source)130 public static Status fromString(String source) { 131 return source == null ? missing : Status.valueOf(source); 132 } 133 } 134 135 /** 136 * This is the "high bar" level where flagging is required. 137 * 138 * @see #getRequiredVotes() 139 */ 140 public static final int HIGH_BAR = Level.tc.votes; 141 142 public static final int LOWER_BAR = (2 * Level.vetter.votes); 143 144 /** 145 * This is the level at which a vote counts. Each level also contains the weight. 146 * 147 * <p>Code related to Level.expert removed 2021-05-18 per CLDR-14597 148 */ 149 public enum Level { 150 locked(0 /* votes */, 999 /* stlevel */), 151 guest(1 /* votes */, 10 /* stlevel */), 152 anonymous(0 /* votes */, 8 /* stlevel */), 153 vetter(4 /* votes */, 5 /* stlevel */, /* tcorgvotes */ 6), // org dependent- see getVotes() 154 // Manager and below can manage users 155 manager(4 /* votes */, 2 /* stlevel */), 156 tc(50 /* votes */, 1 /* stlevel */), 157 admin(100 /* votes */, 0 /* stlevel */); 158 159 /** 160 * PERMANENT_VOTES is used by TC voters to "lock" locale+path permanently (including future 161 * versions, until unlocked), in the current VOTE_VALUE table. It is public for 162 * STFactory.java and PermanentVote.java. 163 */ 164 public static final int PERMANENT_VOTES = 1000; 165 166 /** 167 * LOCKING_VOTES is used (nominally by ADMIN voter, but not really by someone logged in as 168 * ADMIN, instead by combination of two PERMANENT_VOTES) to "lock" locale+path permanently 169 * in the LOCKED_XPATHS table. It is public for STFactory.PerLocaleData.loadVoteValues. 170 */ 171 public static final int LOCKING_VOTES = 2000; 172 173 /** The vote count a user of this level normally votes with */ 174 private final int votes; 175 176 /** The vote count a user of this level normally votes with if a tc org */ 177 private final int tcorgvotes; 178 179 /** The level as an integer, where 0 = admin, ..., 999 = locked */ 180 private final int stlevel; 181 Level(int votes, int stlevel, int tcorgvotes)182 Level(int votes, int stlevel, int tcorgvotes) { 183 this.votes = votes; 184 this.stlevel = stlevel; 185 this.tcorgvotes = tcorgvotes; 186 } 187 Level(int votes, int stlevel)188 Level(int votes, int stlevel) { 189 this(votes, stlevel, votes); 190 } 191 192 /** 193 * Get the votes for each level and organization 194 * 195 * @param o the given organization 196 */ getVotes(Organization o)197 public int getVotes(Organization o) { 198 if (this == vetter && o.isTCOrg()) { 199 return tcorgvotes; 200 } 201 return votes; 202 } 203 204 /** Get the Survey Tool userlevel for each level. (0=admin, 999=locked) */ getSTLevel()205 public int getSTLevel() { 206 return stlevel; 207 } 208 209 /** 210 * Find the Level, given ST Level 211 * 212 * @param stlevel 213 * @return the Level corresponding to the integer 214 */ fromSTLevel(int stlevel)215 public static Level fromSTLevel(int stlevel) { 216 for (Level l : Level.values()) { 217 if (l.getSTLevel() == stlevel) { 218 return l; 219 } 220 } 221 return null; 222 } 223 224 /** 225 * Policy: can this user manage the "other" user's settings? 226 * 227 * @param myOrg the current organization 228 * @param otherLevel the other user's level 229 * @param otherOrg the other user's organization 230 * @return 231 */ isManagerFor(Organization myOrg, Level otherLevel, Organization otherOrg)232 public boolean isManagerFor(Organization myOrg, Level otherLevel, Organization otherOrg) { 233 return (this == admin 234 || (canManageSomeUsers() 235 && (myOrg == otherOrg) 236 && atLeastAsPowerfulAs(otherLevel))); 237 } 238 239 /** 240 * Policy: Can this user manage any users? 241 * 242 * @return 243 */ canManageSomeUsers()244 public boolean canManageSomeUsers() { 245 return atLeastAsPowerfulAs(manager); 246 } 247 248 /** Internal: uses the ST Level as a measure of 'power' */ morePowerfulThan(Level other)249 boolean morePowerfulThan(Level other) { 250 return getSTLevel() < other.getSTLevel(); 251 } 252 253 /** Internal: uses the ST Level as a measure of 'power' */ atLeastAsPowerfulAs(Level other)254 boolean atLeastAsPowerfulAs(Level other) { 255 return getSTLevel() <= other.getSTLevel(); 256 } 257 258 /** 259 * Policy: can this user create or set a user to the specified level? 260 * 261 * @param otherLevel the desired new level for the other user 262 * <p>Note: UserRegistry.canSetUserLevel enforces additional limitations depending on 263 * more than this user's level and the other user's desired new level 264 */ canCreateOrSetLevelTo(Level otherLevel)265 public boolean canCreateOrSetLevelTo(Level otherLevel) { 266 // Must be a manager at all 267 if (!canManageSomeUsers()) return false; 268 // Cannot elevate privilege 269 return !otherLevel.morePowerfulThan(this); 270 } 271 272 /** 273 * Can a user with this level and organization vote with the given vote count? 274 * 275 * @param org the given organization 276 * @param withVotes the given vote count 277 * @return true if the user can vote with the given vote count, else false 278 */ canVoteWithCount(Organization org, int withVotes)279 public boolean canVoteWithCount(Organization org, int withVotes) { 280 /* 281 * ADMIN is allowed to vote with LOCKING_VOTES, but not directly in the GUI, only 282 * by two TC voting together with PERMANENT_VOTES. Therefore LOCKING_VOTES is omitted 283 * from the GUI menu (voteCountMenu), but included in canVoteWithCount. 284 */ 285 if (withVotes == LOCKING_VOTES && this == admin) { 286 return true; 287 } 288 Set<Integer> menu = getVoteCountMenu(org); 289 return menu == null ? withVotes == getVotes(org) : menu.contains(withVotes); 290 } 291 292 /** 293 * If not null, an array of different vote counts from which a user of this level is allowed 294 * to choose. 295 */ 296 private ImmutableSet<Integer> voteCountMenu = null; 297 298 /** 299 * Get the ordered immutable set of different vote counts a user of this level can vote with 300 * 301 * @param ignoredOrg the given organization 302 * @return the set, or null if the user has no choice of vote count 303 */ getVoteCountMenu(Organization ignoredOrg)304 public ImmutableSet<Integer> getVoteCountMenu(Organization ignoredOrg) { 305 // Right now, the organization does not affect the menu. 306 // but update the API to future proof. 307 return voteCountMenu; 308 } 309 310 /* 311 * Set voteCountMenu for admin and tc in this static block, which will be run after 312 * all the constructors have run, rather than in the constructor itself. For example, 313 * vetter.votes needs to be defined before we can set admin.voteCountMenu. 314 */ 315 static { 316 admin.voteCountMenu = 317 ImmutableSet.of( 318 guest.votes, 319 vetter.votes, 320 vetter.tcorgvotes, 321 tc.votes, 322 admin.votes, 323 PERMANENT_VOTES); 324 /* Not LOCKING_VOTES; see canVoteWithCount */ 325 tc.voteCountMenu = 326 ImmutableSet.of( 327 guest.votes, 328 vetter.votes, 329 vetter.tcorgvotes, 330 tc.votes, 331 PERMANENT_VOTES); 332 } 333 334 // The following methods were moved here from UserRegistry 335 // TODO: remove this todo notice 336 isAdmin()337 public boolean isAdmin() { 338 return stlevel <= admin.stlevel; 339 } 340 isTC()341 public boolean isTC() { 342 return stlevel <= tc.stlevel; 343 } 344 isExactlyManager()345 public boolean isExactlyManager() { 346 return stlevel == manager.stlevel; 347 } 348 isManagerOrStronger()349 public boolean isManagerOrStronger() { 350 return stlevel <= manager.stlevel; 351 } 352 isVetter()353 public boolean isVetter() { 354 return stlevel <= vetter.stlevel; 355 } 356 isGuest()357 public boolean isGuest() { 358 return stlevel <= guest.stlevel; 359 } 360 isLocked()361 public boolean isLocked() { 362 return stlevel == locked.stlevel; 363 } 364 isExactlyAnonymous()365 public boolean isExactlyAnonymous() { 366 return stlevel == anonymous.stlevel; 367 } 368 369 /** 370 * Is this user an administrator 'over' this user? Always true if admin, or if TC in same 371 * org. 372 * 373 * @param myOrg 374 */ isAdminForOrg(Organization myOrg, Organization target)375 public boolean isAdminForOrg(Organization myOrg, Organization target) { 376 return isAdmin() || ((isTC() || stlevel == manager.stlevel) && (myOrg == target)); 377 } 378 canImportOldVotes(CheckCLDR.Phase inPhase)379 public boolean canImportOldVotes(CheckCLDR.Phase inPhase) { 380 return isVetter() && (inPhase == Phase.SUBMISSION); 381 } 382 canListUsers()383 public boolean canListUsers() { 384 return isManagerOrStronger(); 385 } 386 canCreateUsers()387 public boolean canCreateUsers() { 388 return isTC() || isExactlyManager(); 389 } 390 canEmailUsers()391 public boolean canEmailUsers() { 392 return isTC() || isExactlyManager(); 393 } 394 canModifyUsers()395 public boolean canModifyUsers() { 396 return isTC() || isExactlyManager(); 397 } 398 canCreateOtherOrgs()399 public boolean canCreateOtherOrgs() { 400 return isAdmin(); 401 } 402 canUseVettingSummary()403 public boolean canUseVettingSummary() { 404 return isManagerOrStronger(); 405 } 406 canSubmit(CheckCLDR.Phase inPhase)407 public boolean canSubmit(CheckCLDR.Phase inPhase) { 408 if (inPhase == Phase.FINAL_TESTING) { 409 return false; 410 // TODO: Note, this will mean not just READONLY, but VETTING_CLOSED will return 411 // false here. 412 // This is probably desired! 413 } 414 return isGuest(); 415 } 416 canCreateSummarySnapshot()417 public boolean canCreateSummarySnapshot() { 418 return isAdmin(); 419 } 420 canMonitorForum()421 public boolean canMonitorForum() { 422 return isTC() || isExactlyManager(); 423 } 424 canSetInterestLocales()425 public boolean canSetInterestLocales() { 426 return isManagerOrStronger(); 427 } 428 canGetEmailList()429 public boolean canGetEmailList() { 430 return isManagerOrStronger(); 431 } 432 433 /** If true, can delete users at their user level or lower. */ canDeleteUsers()434 public boolean canDeleteUsers() { 435 return isAdmin(); 436 } 437 canUseVettingParticipation()438 public boolean canUseVettingParticipation() { 439 return isManagerOrStronger(); 440 } 441 } 442 443 /** 444 * See getStatusForOrganization to see how this is computed. 445 * 446 * <p>Not to be confused with VoteResolver.Status 447 */ 448 public enum VoteStatus { 449 /** 450 * The value for the path is either contributed or approved, and the user's organization 451 * didn't vote. 452 */ 453 ok_novotes, 454 455 /** 456 * The value for the path is either contributed or approved, and the user's organization 457 * chose the winning value. 458 */ 459 ok, 460 461 /** The winning value is neither contributed nor approved. */ 462 provisionalOrWorse, 463 464 /** 465 * The user's organization's choice is not winning, and the winning value is either 466 * contributed or approved. There may be insufficient votes to overcome a previously 467 * approved value, or other organizations may be voting against it. 468 */ 469 losing, 470 471 /** 472 * There is a dispute, meaning more than one item with votes, or the item with votes didn't 473 * win. 474 */ 475 disputed 476 } 477 478 /** Internal class for voter information. It is public for testing only */ 479 public static class VoterInfo { 480 private Organization organization; 481 private Level level; 482 private String name; 483 /** 484 * A set of locales associated with this voter; it is often empty (as when the user has "*" 485 * for their set of locales); it may not serve any purpose in ordinary operation of Survey 486 * Tool; its main (only?) purpose seems to be for computeMaxVotes, whose only purpose seems 487 * to be creation of localeToOrganizationToMaxVote, which is used only by ConsoleCheckCLDR 488 * (for obscure reason), not by Survey Tool itself. 489 */ 490 private final Set<CLDRLocale> locales = new TreeSet<>(); 491 getLocales()492 public Iterable<CLDRLocale> getLocales() { 493 return locales; 494 } 495 VoterInfo(Organization organization, Level level, String name, LocaleSet localeSet)496 public VoterInfo(Organization organization, Level level, String name, LocaleSet localeSet) { 497 this.setOrganization(organization); 498 this.setLevel(level); 499 this.setName(name); 500 if (!localeSet.isAllLocales()) { 501 this.locales.addAll(localeSet.getSet()); 502 } 503 } 504 VoterInfo(Organization organization, Level level, String name)505 public VoterInfo(Organization organization, Level level, String name) { 506 this.setOrganization(organization); 507 this.setLevel(level); 508 this.setName(name); 509 } 510 VoterInfo()511 public VoterInfo() {} 512 513 @Override toString()514 public String toString() { 515 return "{" + getName() + ", " + getLevel() + ", " + getOrganization() + "}"; 516 } 517 setOrganization(Organization organization)518 public void setOrganization(Organization organization) { 519 this.organization = organization; 520 } 521 getOrganization()522 public Organization getOrganization() { 523 return organization; 524 } 525 setLevel(Level level)526 public void setLevel(Level level) { 527 this.level = level; 528 } 529 getLevel()530 public Level getLevel() { 531 return level; 532 } 533 setName(String name)534 public void setName(String name) { 535 this.name = name; 536 } 537 getName()538 public String getName() { 539 return name; 540 } 541 addLocale(CLDRLocale locale)542 void addLocale(CLDRLocale locale) { 543 this.locales.add(locale); 544 } 545 546 @Override equals(Object obj)547 public boolean equals(Object obj) { 548 if (obj == null) { 549 return false; 550 } 551 VoterInfo other = (VoterInfo) obj; 552 return organization.equals(other.organization) 553 && level.equals(other.level) 554 && name.equals(other.name) 555 && Objects.equal(locales, other.locales); 556 } 557 558 @Override hashCode()559 public int hashCode() { 560 return organization.hashCode() ^ level.hashCode() ^ name.hashCode(); 561 } 562 } 563 564 /** 565 * MaxCounter: make sure that we are always only getting the maximum of the values. 566 * 567 * @author markdavis 568 * @param <T> 569 */ 570 static class MaxCounter<T> extends Counter<T> { MaxCounter(boolean b)571 public MaxCounter(boolean b) { 572 super(b); 573 } 574 575 /** Add, but only to bring up to the maximum value. */ 576 @Override add(T obj, long countValue, long time)577 public MaxCounter<T> add(T obj, long countValue, long time) { 578 long value = getCount(obj); 579 if ((value <= countValue)) { 580 super.add(obj, countValue - value, time); // only add the difference! 581 } 582 return this; 583 } 584 } 585 586 /** Internal class for getting from an organization to its vote. */ 587 private class OrganizationToValueAndVote<T> { 588 private final Map<Organization, MaxCounter<T>> orgToVotes = 589 new EnumMap<>(Organization.class); 590 /** 591 * All votes, even those that aren't any org's vote because they lost an intra-org dispute 592 */ 593 private final Counter<T> allVotesIncludingIntraOrgDispute = new Counter<>(); 594 595 private final Map<Organization, Integer> orgToMax = new EnumMap<>(Organization.class); 596 /** The result of {@link #getTotals(EnumSet)} */ 597 private final Counter<T> totals = new Counter<>(true); 598 599 private final Map<String, Long> nameTime = new LinkedHashMap<>(); 600 601 /** map an organization to the value it voted for. */ 602 private final Map<Organization, T> orgToAdd = new EnumMap<>(Organization.class); 603 604 private T baileyValue; 605 private boolean baileySet; // was the bailey value set 606 OrganizationToValueAndVote()607 OrganizationToValueAndVote() { 608 for (Organization org : Organization.values()) { 609 orgToVotes.put(org, new MaxCounter<>(true)); 610 } 611 } 612 613 /** Call clear before considering each new path */ clear()614 public void clear() { 615 for (Map.Entry<Organization, MaxCounter<T>> entry : orgToVotes.entrySet()) { 616 entry.getValue().clear(); 617 } 618 orgToAdd.clear(); 619 orgToMax.clear(); 620 allVotesIncludingIntraOrgDispute.clear(); 621 baileyValue = null; 622 baileySet = false; 623 if (transcript != null) { 624 // there was a transcript before, so retain it 625 transcript = new DeferredTranscript(); 626 } 627 } 628 getNameTime()629 public Map<String, Long> getNameTime() { 630 return nameTime; 631 } 632 633 /** 634 * Call this to add votes 635 * 636 * @param value 637 * @param voter 638 * @param withVotes optionally, vote at a non-typical voting level. May not exceed voter's 639 * maximum allowed level. null = use default level. 640 * @param date 641 */ add(T value, int voter, Integer withVotes, Date date)642 public void add(T value, int voter, Integer withVotes, Date date) { 643 final VoterInfo info = voterInfoList.get(voter); 644 if (info == null) { 645 throw new UnknownVoterException(voter); 646 } 647 Level level = info.getLevel(); 648 if (withVotes == null || !level.canVoteWithCount(info.organization, withVotes)) { 649 withVotes = level.getVotes(info.organization); 650 } 651 addInternal(value, info, withVotes, date); // do the add 652 } 653 654 /** 655 * Called by add(T,int,Integer) to actually add a value. 656 * 657 * @param value 658 * @param info 659 * @param votes 660 * @param time 661 * @see #add(Object, int, Integer) 662 */ addInternal(T value, final VoterInfo info, final int votes, Date time)663 private void addInternal(T value, final VoterInfo info, final int votes, Date time) { 664 if (DROP_HARD_INHERITANCE) { 665 value = changeBaileyToInheritance(value); 666 } 667 /* All votes are added here, even if they will later lose an intra-org dispute. */ 668 allVotesIncludingIntraOrgDispute.add(value, votes, time.getTime()); 669 nameTime.put(info.getName(), time.getTime()); 670 if (DEBUG) { 671 System.out.println( 672 "allVotesIncludingIntraOrgDispute Info: " 673 + allVotesIncludingIntraOrgDispute); 674 } 675 if (DEBUG) { 676 System.out.println("VoteInfo: " + info.getName() + info.getOrganization()); 677 } 678 Organization organization = info.getOrganization(); 679 orgToVotes.get(organization).add(value, votes, time.getTime()); 680 if (DEBUG) { 681 System.out.println( 682 "Adding now Info: " 683 + organization.getDisplayName() 684 + info.getName() 685 + " is adding: " 686 + votes 687 + value 688 + new Timestamp(time.getTime())); 689 } 690 691 if (DEBUG) { 692 System.out.println( 693 "addInternal: " 694 + organization.getDisplayName() 695 + " : " 696 + orgToVotes.get(organization).toString()); 697 } 698 699 // add the new votes to orgToMax, if they are greater that what was there 700 Integer max = orgToMax.get(info.getOrganization()); 701 if (max == null || max < votes) { 702 orgToMax.put(organization, votes); 703 } 704 } 705 706 /** 707 * Return the overall vote for each organization. It is the max for each value. When the 708 * organization is conflicted (the top two values have the same vote), the organization is 709 * also added to disputed. 710 * 711 * @param conflictedOrganizations if not null, to be filled in with the set of conflicted 712 * organizations. 713 */ getTotals(EnumSet<Organization> conflictedOrganizations)714 public Counter<T> getTotals(EnumSet<Organization> conflictedOrganizations) { 715 if (conflictedOrganizations != null) { 716 conflictedOrganizations.clear(); 717 } 718 totals.clear(); 719 720 annotateTranscript("- Getting all totals by organization:"); 721 for (Map.Entry<Organization, MaxCounter<T>> entry : orgToVotes.entrySet()) { 722 Counter<T> items = entry.getValue(); 723 if (items.size() == 0) { 724 continue; 725 } 726 Iterator<T> iterator = items.getKeysetSortedByCount(false).iterator(); 727 T value = iterator.next(); 728 long weight = items.getCount(value); 729 if (weight == 0) { 730 continue; 731 } 732 annotateTranscript( 733 "-- Considering %s which has %d item(s)", 734 entry.getKey().getDisplayName(), items.size()); 735 Organization org = entry.getKey(); 736 if (DEBUG) { 737 System.out.println("sortedKeys?? " + value + " " + org.getDisplayName()); 738 } 739 // if there is more than one item, check that it is less 740 if (iterator.hasNext()) { 741 T value2 = iterator.next(); 742 long weight2 = items.getCount(value2); 743 // if the votes for #1 are not better than #2, we have a dispute 744 if (weight == weight2) { 745 if (conflictedOrganizations != null) { 746 annotateTranscript( 747 "--- There are conflicts due to different values by users of this organization."); 748 conflictedOrganizations.add(org); 749 } 750 } 751 } 752 // This is deprecated, but preserve it until the method is removed. 753 /* 754 * TODO: explain the above comment, and follow through. What is deprecated (orgToAdd, or getOrgVote)? 755 * Preserve until which method is removed (getOrgVote)? 756 */ 757 orgToAdd.put(org, value); 758 759 // We add the max vote for each of the organizations choices 760 long maxCount = 0; 761 T considerItem = null; 762 long considerCount = 0; 763 long maxtime = 0; 764 long considerTime = 0; 765 for (T item : items.keySet()) { 766 if (DEBUG) { 767 System.out.println( 768 "Items in order: " 769 + item.toString() 770 + new Timestamp(items.getTime(item))); 771 } 772 long count = items.getCount(item); 773 long time = items.getTime(item); 774 if (count > maxCount) { 775 maxCount = count; 776 maxtime = time; 777 // tell the 'losing' item 778 if (considerItem != null) { 779 annotateTranscript( 780 "---- Org is not voting for '%s': there is a higher ranked vote", 781 considerItem); 782 } 783 considerItem = item; 784 if (DEBUG) { 785 System.out.println( 786 "count>maxCount: " 787 + considerItem 788 + ":" 789 + new Timestamp(considerTime) 790 + " COUNT: " 791 + considerCount 792 + "MAXCOUNT: " 793 + maxCount); 794 } 795 considerCount = items.getCount(considerItem); 796 considerTime = items.getTime(considerItem); 797 } else if ((time > maxtime) && (count == maxCount)) { 798 maxtime = time; 799 // tell the 'losing' item 800 if (considerItem != null) { 801 annotateTranscript( 802 "---- Org is not voting for '%s': there is a later vote", 803 considerItem); 804 } 805 considerItem = item; 806 considerCount = items.getCount(considerItem); 807 considerTime = items.getTime(considerItem); 808 if (DEBUG) { 809 System.out.println( 810 "time>maxTime: " 811 + considerItem 812 + ":" 813 + new Timestamp(considerTime)); 814 } 815 } 816 } 817 annotateTranscript( 818 "--- %s vote is for '%s' with strength %d", 819 org.getDisplayName(), considerItem, considerCount); 820 orgToAdd.put(org, considerItem); 821 totals.add(considerItem, considerCount, considerTime); 822 823 if (DEBUG) { 824 System.out.println("Totals: " + totals + " : " + new Timestamp(considerTime)); 825 } 826 } 827 828 if (DEBUG) { 829 System.out.println("FINALTotals: " + totals); 830 } 831 return totals; 832 } 833 getOrgCount(T winningValue)834 public int getOrgCount(T winningValue) { 835 int orgCount = 0; 836 for (Map.Entry<Organization, MaxCounter<T>> entry : orgToVotes.entrySet()) { 837 Counter<T> counter = entry.getValue(); 838 long count = counter.getCount(winningValue); 839 if (count > 0) { 840 orgCount++; 841 } 842 } 843 return orgCount; 844 } 845 getBestPossibleVote()846 private int getBestPossibleVote() { 847 int total = 0; 848 for (Map.Entry<Organization, Integer> entry : orgToMax.entrySet()) { 849 total += entry.getValue(); 850 } 851 return total; 852 } 853 854 @Override toString()855 public String toString() { 856 String orgToVotesString = ""; 857 for (Entry<Organization, MaxCounter<T>> entry : orgToVotes.entrySet()) { 858 Counter<T> counter = entry.getValue(); 859 if (counter.size() != 0) { 860 if (orgToVotesString.length() != 0) { 861 orgToVotesString += ", "; 862 } 863 Organization org = entry.getKey(); 864 orgToVotesString += org.toString() + "=" + counter; 865 } 866 } 867 EnumSet<Organization> conflicted = EnumSet.noneOf(Organization.class); 868 return "{orgToVotes: " 869 + orgToVotesString 870 + ", totals: " 871 + getTotals(conflicted) 872 + ", conflicted: " 873 + conflicted 874 + "}"; 875 } 876 877 /** 878 * This is now deprecated, since the organization may have multiple votes. 879 * 880 * @param org 881 * @return 882 * @deprecated 883 */ 884 @Deprecated getOrgVote(Organization org)885 public T getOrgVote(Organization org) { 886 return orgToAdd.get(org); 887 } 888 getOrgVoteRaw(Organization orgOfUser)889 public T getOrgVoteRaw(Organization orgOfUser) { 890 return orgToAdd.get(orgOfUser); 891 } 892 getOrgToVotes(Organization org)893 public Map<T, Long> getOrgToVotes(Organization org) { 894 Map<T, Long> result = new LinkedHashMap<>(); 895 MaxCounter<T> counter = orgToVotes.get(org); 896 for (T item : counter) { 897 result.put(item, counter.getCount(item)); 898 } 899 return result; 900 } 901 } 902 903 /** Data built internally */ 904 private T winningValue; 905 906 private T oValue; // optimal value; winning if better approval status than old 907 private T nValue; // next to optimal value 908 private final List<T> valuesWithSameVotes = new ArrayList<>(); 909 private Counter<T> totals = null; 910 911 private Status winningStatus; 912 private final EnumSet<Organization> conflictedOrganizations = 913 EnumSet.noneOf(Organization.class); 914 private final OrganizationToValueAndVote<T> organizationToValueAndVote = 915 new OrganizationToValueAndVote<>(); 916 private T baselineValue; 917 private Status baselineStatus; 918 919 private boolean resolved; 920 private boolean valueIsLocked; 921 private int requiredVotes = 0; 922 private final SupplementalDataInfo supplementalDataInfo = SupplementalDataInfo.getInstance(); 923 private CLDRLocale locale; 924 private PathHeader pathHeader; 925 926 private static final Collator englishCollator = Collator.getInstance(ULocale.ENGLISH).freeze(); 927 928 /** Used for comparing objects of type T */ 929 private final Comparator<T> objectCollator = 930 (o1, o2) -> englishCollator.compare(String.valueOf(o1), String.valueOf(o2)); 931 932 /** 933 * Set the baseline (or "trunk") value and status for this VoteResolver. 934 * 935 * @param baselineValue the baseline value 936 * @param baselineStatus the baseline status 937 */ setBaseline(T baselineValue, Status baselineStatus)938 public void setBaseline(T baselineValue, Status baselineStatus) { 939 this.baselineValue = baselineValue; 940 this.baselineStatus = baselineValue == null ? Status.missing : baselineStatus; 941 } 942 getBaselineValue()943 public T getBaselineValue() { 944 return baselineValue; 945 } 946 getBaselineStatus()947 public Status getBaselineStatus() { 948 return baselineStatus; 949 } 950 951 /** 952 * Set the locale and PathHeader for this VoteResolver 953 * 954 * <p>You must call this whenever you are using a VoteResolver with a new locale or a new 955 * PathHeader 956 * 957 * @param locale the CLDRLocale 958 * @param pathHeader the PathHeader 959 */ setLocale(CLDRLocale locale, PathHeader pathHeader)960 public void setLocale(CLDRLocale locale, PathHeader pathHeader) { 961 this.locale = locale; 962 this.pathHeader = pathHeader; 963 } 964 965 /** 966 * What are the required votes for this item? 967 * 968 * @return the number of votes (as of this writing: usually 4, 8 for established locales) 969 */ getRequiredVotes()970 public int getRequiredVotes() { 971 if (requiredVotes == 0) { 972 int preliminaryRequiredVotes = 973 supplementalDataInfo.getRequiredVotes(locale, pathHeader); 974 if (preliminaryRequiredVotes == HIGH_BAR && baselineStatus != Status.approved) { 975 requiredVotes = LOWER_BAR; 976 } else { 977 requiredVotes = preliminaryRequiredVotes; 978 } 979 } 980 return requiredVotes; 981 } 982 983 /** 984 * Call this method first, for a new base path. You'll then call add for each value associated 985 * with that base path. 986 */ clear()987 public void clear() { 988 baselineValue = null; 989 baselineStatus = Status.missing; 990 requiredVotes = 0; 991 locale = null; 992 pathHeader = null; 993 organizationToValueAndVote.clear(); 994 resolved = valueIsLocked = false; 995 values.clear(); 996 997 // TODO: clear these out between reuse 998 // Are there other values that should be cleared? 999 oValue = null; 1000 setWinningValue(null); 1001 nValue = null; 1002 1003 if (transcript != null) { 1004 transcript.clear(); 1005 } 1006 } 1007 1008 /** 1009 * Get the bailey value (what the inherited value would be if there were no explicit value) for 1010 * this VoteResolver. 1011 * 1012 * <p>Throw an exception if !baileySet. 1013 * 1014 * @return the bailey value. 1015 * <p>Called by STFactory.PerLocaleData.getResolverInternal in the special circumstance 1016 * where getWinningValue has returned INHERITANCE_MARKER. 1017 */ getBaileyValue()1018 public T getBaileyValue() { 1019 if (!organizationToValueAndVote.baileySet) { 1020 throw new IllegalArgumentException( 1021 "setBaileyValue must be called before getBaileyValue"); 1022 } 1023 return organizationToValueAndVote.baileyValue; 1024 } 1025 1026 /** 1027 * Set the Bailey value (what the inherited value would be if there were no explicit value). 1028 * This value is used in handling any CldrUtility.INHERITANCE_MARKER. This value must be set 1029 * <i>before</i> adding values. Usually by calling CLDRFile.getBaileyValue(). 1030 */ setBaileyValue(T baileyValue)1031 public void setBaileyValue(T baileyValue) { 1032 organizationToValueAndVote.baileySet = true; 1033 organizationToValueAndVote.baileyValue = baileyValue; 1034 } 1035 1036 /** 1037 * Call once for each voter for a value. If there are no voters for an item, then call 1038 * add(value); 1039 * 1040 * @param value 1041 * @param voter 1042 * @param withVotes override to lower the user's voting permission. May be null for default. 1043 * @param date 1044 * <p>Called by getResolverInternal in STFactory, and elsewhere 1045 */ add(T value, int voter, Integer withVotes, Date date)1046 public void add(T value, int voter, Integer withVotes, Date date) { 1047 if (DROP_HARD_INHERITANCE) { 1048 value = changeBaileyToInheritance(value); 1049 } 1050 if (resolved) { 1051 throw new IllegalArgumentException( 1052 "Must be called after clear, and before any getters."); 1053 } 1054 if (withVotes != null && withVotes == Level.LOCKING_VOTES) { 1055 valueIsLocked = true; 1056 } 1057 organizationToValueAndVote.add(value, voter, withVotes, date); 1058 values.add(value); 1059 } 1060 1061 /** 1062 * Call once for each voter for a value. If there are no voters for an item, then call 1063 * add(value); 1064 * 1065 * @param value 1066 * @param voter 1067 * @param withVotes override to lower the user's voting permission. May be null for default. 1068 * <p>Called only for TestUtilities, not used in Survey Tool. 1069 */ add(T value, int voter, Integer withVotes)1070 public void add(T value, int voter, Integer withVotes) { 1071 if (DROP_HARD_INHERITANCE) { 1072 value = changeBaileyToInheritance(value); 1073 } 1074 if (resolved) { 1075 throw new IllegalArgumentException( 1076 "Must be called after clear, and before any getters."); 1077 } 1078 Date date = new Date(); 1079 organizationToValueAndVote.add(value, voter, withVotes, date); 1080 values.add(value); 1081 } 1082 changeBaileyToInheritance(T value)1083 private <T> T changeBaileyToInheritance(T value) { 1084 if (value != null && value.equals(getBaileyValue())) { 1085 return (T) CldrUtility.INHERITANCE_MARKER; 1086 } 1087 return value; 1088 } 1089 1090 /** Used only in add(value, voter) for making a pseudo-Date */ 1091 private int maxcounter = 100; 1092 1093 /** 1094 * Call once for each voter for a value. If there are no voters for an item, then call 1095 * add(value); 1096 * 1097 * @param value 1098 * @param voter 1099 * <p>Called by ConsoleCheckCLDR and TestUtilities; not used in SurveyTool. 1100 */ add(T value, int voter)1101 public void add(T value, int voter) { 1102 Date date = new Date(++maxcounter); 1103 add(value, voter, null, date); 1104 } 1105 1106 /** 1107 * Call if a value has no voters. It is safe to also call this if there is a voter, just 1108 * unnecessary. 1109 * 1110 * @param value 1111 * <p>Called by getResolverInternal for the baseline (trunk) value; also called for 1112 * ConsoleCheckCLDR. 1113 */ add(T value)1114 public void add(T value) { 1115 if (resolved) { 1116 throw new IllegalArgumentException( 1117 "Must be called after clear, and before any getters."); 1118 } 1119 values.add(value); 1120 } 1121 1122 private final Set<T> values = new TreeSet<>(objectCollator); 1123 1124 private final Comparator<T> votesThenUcaCollator = 1125 new Comparator<>() { 1126 1127 /** 1128 * Compare candidate items by vote count, highest vote first. In the case of ties, 1129 * favor (a) the baseline (trunk) value, then (b) votes for inheritance 1130 * (INHERITANCE_MARKER), then (c) the alphabetical order (as a last resort). 1131 * 1132 * <p>Return negative to favor o1, positive to favor o2. 1133 * 1134 * @see VoteResolver#setBestNextAndSameVoteValues(Set, HashMap) 1135 * @see VoteResolver#annotateNextBestValue(long, long, T, T) 1136 */ 1137 @Override 1138 public int compare(T o1, T o2) { 1139 long v1 = organizationToValueAndVote.allVotesIncludingIntraOrgDispute.get(o1); 1140 long v2 = organizationToValueAndVote.allVotesIncludingIntraOrgDispute.get(o2); 1141 if (v1 != v2) { 1142 return v1 < v2 ? 1 : -1; // highest vote first 1143 } 1144 if (o1.equals(baselineValue)) { 1145 return -1; 1146 } else if (o2.equals(baselineValue)) { 1147 return 1; 1148 } 1149 if (o1.equals(CldrUtility.INHERITANCE_MARKER)) { 1150 return -1; 1151 } else if (o2.equals(CldrUtility.INHERITANCE_MARKER)) { 1152 return 1; 1153 } 1154 return englishCollator.compare(String.valueOf(o1), String.valueOf(o2)); 1155 } 1156 }; 1157 1158 /** 1159 * Annotate why the O (winning) value is winning vs the N (next) value. Assumes that the prior 1160 * annotation mentioned the O value. 1161 * 1162 * @param O optimal value 1163 * @param N next-best value 1164 */ annotateNextBestValue(long O, long N, final T oValue, final T nValue)1165 private void annotateNextBestValue(long O, long N, final T oValue, final T nValue) { 1166 // See the Comparator<> defined immediately above. 1167 1168 // sortedValues.size() >= 2 - explain why O won and N lost. 1169 // We have to perform the function of the votesThenUcaCollator one more time 1170 if (O > N) { 1171 annotateTranscript( 1172 "- This is the optimal value because it has the highest weight (voting score)."); 1173 } else if (winningValue.equals(baselineValue)) { 1174 annotateTranscript( 1175 "- This is the optimal value because it is the same as the baseline value, though the weight was otherwise equal to the next-best."); // aka blue star 1176 } else if (winningValue.equals(CldrUtility.INHERITANCE_MARKER)) { 1177 annotateTranscript( 1178 "- This is the optimal value because it is the inheritance marker, though the weight was otherwise equal to the next-best."); // triple up arrow 1179 } else { 1180 annotateTranscript( 1181 "- This is the optimal value because it comes earlier than '%s' when the text was sorted, though the weight was otherwise equal to the next-best.", 1182 nValue); 1183 } 1184 annotateTranscript("The Next-best (N) value is '%s', with weight %d", nValue, N); 1185 } 1186 1187 /** This will be changed to true if both kinds of vote are present */ 1188 private boolean bothInheritanceAndBaileyHadVotes = false; 1189 1190 /** 1191 * Resolve the votes. Resolution entails counting votes and setting members for this 1192 * VoteResolver, including winningStatus, winningValue, and many others. 1193 */ resolveVotes()1194 private void resolveVotes() { 1195 annotateTranscript("Resolving votes:"); 1196 resolved = true; 1197 // get the votes for each organization 1198 valuesWithSameVotes.clear(); 1199 totals = organizationToValueAndVote.getTotals(conflictedOrganizations); 1200 /* Note: getKeysetSortedByCount actually returns a LinkedHashSet, "with predictable iteration order". */ 1201 final Set<T> sortedValues = totals.getKeysetSortedByCount(false, votesThenUcaCollator); 1202 if (DEBUG) { 1203 System.out.println("sortedValues :" + sortedValues.toString()); 1204 } 1205 // annotateTranscript("all votes by org: %s", sortedValues); 1206 1207 /* 1208 * If there are no (unconflicted) votes, return baseline (trunk) if not null, 1209 * else INHERITANCE_MARKER if baileySet, else NO_WINNING_VALUE. 1210 * Avoid setting winningValue to null. VoteResolver should be fully in charge of vote resolution. 1211 */ 1212 if (sortedValues.size() == 0) { 1213 if (baselineValue != null) { 1214 setWinningValue(baselineValue); 1215 winningStatus = baselineStatus; 1216 annotateTranscript( 1217 "Winning Value: '%s' with status '%s' because there were no unconflicted votes.", 1218 winningValue, winningStatus); 1219 // Declare the winner here, because we're about to return from the function 1220 } else if (organizationToValueAndVote.baileySet) { 1221 setWinningValue((T) CldrUtility.INHERITANCE_MARKER); 1222 winningStatus = Status.missing; 1223 annotateTranscript( 1224 "Winning Value: '%s' with status '%s' because there were no unconflicted votes, and there was a Bailey value set.", 1225 winningValue, winningStatus); 1226 // Declare the winner here, because we're about to return from the function 1227 } else { 1228 /* 1229 * TODO: When can this still happen? See https://unicode.org/cldr/trac/ticket/11299 "Example C". 1230 * Also http://localhost:8080/cldr-apps/v#/en_CA/Gregorian/ 1231 * -- also http://localhost:8080/cldr-apps/v#/aa/Languages_A_D/ 1232 * xpath //ldml/localeDisplayNames/languages/language[@type="zh_Hans"][@alt="long"] 1233 * See also checkDataRowConsistency in DataSection.java. 1234 */ 1235 setWinningValue((T) NO_WINNING_VALUE); 1236 winningStatus = Status.missing; 1237 annotateTranscript( 1238 "No winning value! status '%s' because there were no unconflicted votes", 1239 winningStatus); 1240 // Declare the non-winner here, because we're about to return from the function 1241 } 1242 valuesWithSameVotes.add(winningValue); 1243 return; // sortedValues.size() == 0, no candidates 1244 } 1245 if (values.size() == 0) { 1246 throw new IllegalArgumentException("No values added to resolver"); 1247 } 1248 1249 /* 1250 * Copy what is in the the totals field of this VoteResolver for all the 1251 * values in sortedValues. This local variable voteCount may be used 1252 * subsequently to make adjustments for vote resolution. Those adjustment 1253 * may affect the winners in vote resolution, while still preserving the original 1254 * voting data including the totals field. 1255 */ 1256 HashMap<T, Long> voteCount = makeVoteCountMap(sortedValues); 1257 1258 /* 1259 * Adjust sortedValues and voteCount as needed to combine "soft" votes for inheritance 1260 * with "hard" votes for the Bailey value. Note that sortedValues and voteCount are 1261 * both local variables. 1262 */ 1263 if (!DROP_HARD_INHERITANCE) { 1264 bothInheritanceAndBaileyHadVotes = 1265 combineInheritanceWithBaileyForVoting(sortedValues, voteCount); 1266 } 1267 1268 /* 1269 * Adjust sortedValues and voteCount as needed for annotation keywords. 1270 */ 1271 if (isUsingKeywordAnnotationVoting()) { 1272 adjustAnnotationVoteCounts(sortedValues, voteCount); 1273 } 1274 1275 /* 1276 * Perform the actual resolution. 1277 * This sets winningValue to the top element of 1278 * sortedValues. 1279 */ 1280 long[] weights = setBestNextAndSameVoteValues(sortedValues, voteCount); 1281 1282 oValue = winningValue; 1283 1284 winningStatus = computeStatus(weights[0], weights[1]); 1285 1286 // if we are not as good as the baseline (trunk), use the baseline 1287 // TODO: how could baselineStatus be null here?? 1288 if (baselineStatus != null && winningStatus.compareTo(baselineStatus) < 0) { 1289 setWinningValue(baselineValue); 1290 annotateTranscript( 1291 "The optimal value so far with status '%s' would not be as good as the baseline status. " 1292 + "Therefore, the winning value is '%s' with status '%s'.", 1293 winningStatus, winningValue, baselineStatus); 1294 winningStatus = baselineStatus; 1295 valuesWithSameVotes.clear(); 1296 valuesWithSameVotes.add(winningValue); 1297 } else { 1298 // Declare the final winner 1299 annotateTranscript( 1300 "The winning value is '%s' with status '%s'.", winningValue, winningStatus); 1301 } 1302 } 1303 1304 /** 1305 * Make a hash for the vote count of each value in the given sorted list, using the totals field 1306 * of this VoteResolver. 1307 * 1308 * <p>This enables subsequent local adjustment of the effective votes, without change to the 1309 * totals field. Purposes include inheritance and annotation voting. 1310 * 1311 * @param sortedValues the sorted list of values (really a LinkedHashSet, "with predictable 1312 * iteration order") 1313 * @return the HashMap 1314 */ makeVoteCountMap(Set<T> sortedValues)1315 private HashMap<T, Long> makeVoteCountMap(Set<T> sortedValues) { 1316 HashMap<T, Long> map = new HashMap<>(); 1317 for (T value : sortedValues) { 1318 map.put(value, totals.getCount(value)); 1319 } 1320 return map; 1321 } 1322 1323 /** 1324 * Adjust the given sortedValues and voteCount, if necessary, to combine "hard" and "soft" 1325 * votes. Do nothing unless both hard and soft votes are present. 1326 * 1327 * <p>For voting resolution in which inheritance plays a role, "soft" votes for inheritance are 1328 * distinct from "hard" (explicit) votes for the Bailey value. For resolution, these two kinds 1329 * of votes are treated in combination. If that combination is winning, then the final winner 1330 * will be the hard item or the soft item, whichever has more votes, the soft item winning if 1331 * they're tied. Except for the soft item being favored as a tie-breaker, this function should 1332 * be symmetrical in its handling of hard and soft votes. 1333 * 1334 * <p>Note: now that "↑↑↑" is permitted to participate directly in voting resolution, it becomes 1335 * significant that with Collator.getInstance(ULocale.ENGLISH), "↑↑↑" sorts before "AAA" just as 1336 * "AAA" sorts before "BBB". 1337 * 1338 * @param sortedValues the set of sorted values, possibly to be modified 1339 * @param voteCount the hash giving the vote count for each value, possibly to be modified 1340 * @return true if both "hard" and "soft" votes existed and were combined, else false 1341 */ combineInheritanceWithBaileyForVoting( Set<T> sortedValues, HashMap<T, Long> voteCount)1342 private boolean combineInheritanceWithBaileyForVoting( 1343 Set<T> sortedValues, HashMap<T, Long> voteCount) { 1344 if (organizationToValueAndVote.baileySet == false 1345 || organizationToValueAndVote.baileyValue == null) { 1346 return false; 1347 } 1348 T hardValue = organizationToValueAndVote.baileyValue; 1349 T softValue = (T) CldrUtility.INHERITANCE_MARKER; 1350 /* 1351 * Check containsKey before get, to avoid NullPointerException. 1352 */ 1353 if (!voteCount.containsKey(hardValue) || !voteCount.containsKey(softValue)) { 1354 return false; 1355 } 1356 long hardCount = voteCount.get(hardValue); 1357 long softCount = voteCount.get(softValue); 1358 if (hardCount == 0 || softCount == 0) { 1359 return false; 1360 } 1361 reallyCombineInheritanceWithBailey( 1362 sortedValues, voteCount, hardValue, softValue, hardCount, softCount); 1363 return true; 1364 } 1365 1366 /** 1367 * Given that both "hard" and "soft" votes exist, combine them 1368 * 1369 * @param sortedValues the set of sorted values, to be modified 1370 * @param voteCount the hash giving the vote count for each value, to be modified 1371 * @param hardValue the bailey value 1372 * @param softValue the inheritance marker 1373 * @param hardCount the number of votes for hardValue 1374 * @param softCount the number of votes for softValue 1375 */ reallyCombineInheritanceWithBailey( Set<T> sortedValues, HashMap<T, Long> voteCount, T hardValue, T softValue, long hardCount, long softCount)1376 private void reallyCombineInheritanceWithBailey( 1377 Set<T> sortedValues, 1378 HashMap<T, Long> voteCount, 1379 T hardValue, 1380 T softValue, 1381 long hardCount, 1382 long softCount) { 1383 final T combValue = (hardCount > softCount) ? hardValue : softValue; 1384 final T skipValue = (hardCount > softCount) ? softValue : hardValue; 1385 final long combinedCount = hardCount + softCount; 1386 voteCount.put(combValue, combinedCount); 1387 voteCount.put(skipValue, 0L); 1388 /* 1389 * Sort again 1390 */ 1391 List<T> list = new ArrayList<>(sortedValues); 1392 list.sort( 1393 (v1, v2) -> { 1394 long c1 = voteCount.get(v1); 1395 long c2 = voteCount.get(v2); 1396 if (c1 != c2) { 1397 return (c1 < c2) ? 1 : -1; // decreasing numeric order (most votes wins) 1398 } 1399 return englishCollator.compare(String.valueOf(v1), String.valueOf(v2)); 1400 }); 1401 /* 1402 * Omit skipValue 1403 */ 1404 sortedValues.clear(); 1405 for (T value : list) { 1406 if (!value.equals(skipValue)) { 1407 sortedValues.add(value); 1408 } 1409 } 1410 } 1411 1412 /** 1413 * Adjust the effective votes for bar-joined annotations, and re-sort the array of values to 1414 * reflect the adjusted vote counts. 1415 * 1416 * <p>Note: "Annotations provide names and keywords for Unicode characters, currently focusing 1417 * on emoji." For example, an annotation "happy | joyful" has two components "happy" and 1418 * "joyful". References: http://unicode.org/cldr/charts/32/annotations/index.html 1419 * https://www.unicode.org/reports/tr35/tr35-general.html#Annotations 1420 * 1421 * <p>http://unicode.org/repos/cldr/tags/latest/common/annotations/ 1422 * 1423 * @param sortedValues the set of sorted values 1424 * @param voteCount the hash giving the vote count for each value in sortedValues 1425 * <p>public for unit testing, see TestAnnotationVotes.java 1426 */ adjustAnnotationVoteCounts(Set<T> sortedValues, HashMap<T, Long> voteCount)1427 public void adjustAnnotationVoteCounts(Set<T> sortedValues, HashMap<T, Long> voteCount) { 1428 if (voteCount == null || sortedValues == null) { 1429 return; 1430 } 1431 annotateTranscript("Vote weights are being adjusted due to annotation keywords."); 1432 1433 // Make compMap map individual components to cumulative vote counts. 1434 HashMap<T, Long> compMap = makeAnnotationComponentMap(sortedValues, voteCount); 1435 1436 // Save a copy of the "raw" vote count before adjustment, since it's needed by 1437 // promoteSuperiorAnnotationSuperset. 1438 HashMap<T, Long> rawVoteCount = new HashMap<>(voteCount); 1439 1440 // Calculate new counts for original values, based on components. 1441 calculateNewCountsBasedOnAnnotationComponents(sortedValues, voteCount, compMap); 1442 1443 // Re-sort sortedValues based on voteCount. 1444 resortValuesBasedOnAdjustedVoteCounts(sortedValues, voteCount); 1445 1446 // If the set that so far is winning has supersets with superior raw vote count, promote the 1447 // supersets. 1448 promoteSuperiorAnnotationSuperset(sortedValues, voteCount, rawVoteCount); 1449 } 1450 1451 /** 1452 * Make a hash that maps individual annotation components to cumulative vote counts. 1453 * 1454 * <p>For example, 3 votes for "a|b" and 2 votes for "a|c" makes 5 votes for "a", 3 for "b", and 1455 * 2 for "c". 1456 * 1457 * @param sortedValues the set of sorted values 1458 * @param voteCount the hash giving the vote count for each value in sortedValues 1459 */ makeAnnotationComponentMap( Set<T> sortedValues, HashMap<T, Long> voteCount)1460 private HashMap<T, Long> makeAnnotationComponentMap( 1461 Set<T> sortedValues, HashMap<T, Long> voteCount) { 1462 HashMap<T, Long> compMap = new HashMap<>(); 1463 annotateTranscript("- First, components are split up and total votes calculated"); 1464 for (T value : sortedValues) { 1465 Long count = voteCount.get(value); 1466 List<T> comps = splitAnnotationIntoComponentsList(value); 1467 for (T comp : comps) { 1468 if (compMap.containsKey(comp)) { 1469 compMap.replace(comp, compMap.get(comp) + count); 1470 } else { 1471 compMap.put(comp, count); 1472 } 1473 } 1474 } 1475 if (transcript != null && !DEBUG) { 1476 for (Entry<T, Long> comp : compMap.entrySet()) { 1477 // TODO: could sort here, or not. 1478 annotateTranscript( 1479 "-- component '%s' has weight %d", 1480 comp.getKey().toString(), comp.getValue()); 1481 } 1482 } 1483 return compMap; 1484 } 1485 1486 /** 1487 * Calculate new counts for original values, based on annotation components. 1488 * 1489 * <p>Find the total votes for each component (e.g., "b" in "b|c"). As the "modified" vote for 1490 * the set, use the geometric mean of the components in the set. 1491 * 1492 * <p>Order the sets by that mean value, then by the smallest number of items in the set, then 1493 * the fallback we always use (alphabetical). 1494 * 1495 * @param sortedValues the set of sorted values 1496 * @param voteCount the hash giving the vote count for each value in sortedValues 1497 * @param compMap the hash that maps individual components to cumulative vote counts 1498 * <p>See http://unicode.org/cldr/trac/ticket/10973 1499 */ calculateNewCountsBasedOnAnnotationComponents( Set<T> sortedValues, HashMap<T, Long> voteCount, HashMap<T, Long> compMap)1500 private void calculateNewCountsBasedOnAnnotationComponents( 1501 Set<T> sortedValues, HashMap<T, Long> voteCount, HashMap<T, Long> compMap) { 1502 voteCount.clear(); 1503 annotateTranscript( 1504 "- Next, the original values get new counts, each based on the geometric mean of the products of all components."); 1505 for (T value : sortedValues) { 1506 List<T> comps = splitAnnotationIntoComponentsList(value); 1507 double product = 1.0; 1508 for (T comp : comps) { 1509 product *= compMap.get(comp); 1510 } 1511 /* Rounding to long integer here loses precision. We tried multiplying by ten before rounding, 1512 * to reduce problems with different doubles getting rounded to identical longs, but that had 1513 * unfortunate side-effects involving thresholds (see getRequiredVotes). An eventual improvement 1514 * may be to use doubles or floats for all vote counts. 1515 */ 1516 Long newCount = Math.round(Math.pow(product, 1.0 / comps.size())); // geometric mean 1517 voteCount.put(value, newCount); 1518 // Don't annotate these here, annotate them once sorted 1519 } 1520 } 1521 1522 /** 1523 * Split an annotation into a list of components. 1524 * 1525 * <p>For example, split "happy | joyful" into ["happy", "joyful"]. 1526 * 1527 * @param value the value like "happy | joyful" 1528 * @return the list like ["happy", "joyful"] 1529 * <p>Called by makeAnnotationComponentMap and 1530 * calculateNewCountsBasedOnAnnotationComponents. Short, but needs encapsulation, should be 1531 * consistent with similar code in DisplayAndInputProcessor.java. 1532 */ splitAnnotationIntoComponentsList(T value)1533 private List<T> splitAnnotationIntoComponentsList(T value) { 1534 return (List<T>) DisplayAndInputProcessor.SPLIT_BAR.splitToList((CharSequence) value); 1535 } 1536 1537 /** 1538 * Re-sort the set of values to match the adjusted vote counts based on annotation components. 1539 * 1540 * <p>Resolve ties using ULocale.ENGLISH collation for consistency with votesThenUcaCollator. 1541 * 1542 * @param sortedValues the set of sorted values, maybe no longer sorted the way we want 1543 * @param voteCount the hash giving the adjusted vote count for each value in sortedValues 1544 */ resortValuesBasedOnAdjustedVoteCounts( Set<T> sortedValues, HashMap<T, Long> voteCount)1545 private void resortValuesBasedOnAdjustedVoteCounts( 1546 Set<T> sortedValues, HashMap<T, Long> voteCount) { 1547 List<T> list = new ArrayList<>(sortedValues); 1548 list.sort( 1549 (v1, v2) -> { 1550 long c1 = voteCount.get(v1), c2 = voteCount.get(v2); 1551 if (c1 != c2) { 1552 return (c1 < c2) ? 1 : -1; // decreasing numeric order (most votes wins) 1553 } 1554 int size1 = splitAnnotationIntoComponentsList(v1).size(); 1555 int size2 = splitAnnotationIntoComponentsList(v2).size(); 1556 if (size1 != size2) { 1557 return (size1 < size2) 1558 ? -1 1559 : 1; // increasing order of size (smallest set wins) 1560 } 1561 return englishCollator.compare(String.valueOf(v1), String.valueOf(v2)); 1562 }); 1563 sortedValues.clear(); 1564 sortedValues.addAll(list); 1565 } 1566 1567 /** 1568 * For annotation votes, if the set that so far is winning has one or more supersets with 1569 * "superior" (see below) raw vote count, promote those supersets to become the new winner, and 1570 * also the new second place if there are two or more superior supersets. 1571 * 1572 * <p>That is, after finding the set X with the largest geometric mean, check whether there are 1573 * any supersets with "superior" raw votes, and that don't exceed the width limit. If so, 1574 * promote Y, the one of those supersets with the most raw votes (using the normal tie breaker), 1575 * to be the winning set. 1576 * 1577 * <p>"Superior" here means that rawVote(Y) ≥ rawVote(X) + 2, where the value 2 (see 1578 * requiredGap) is for the purpose of requiring at least one non-guest vote. 1579 * 1580 * <p>If any other "superior" supersets exist, promote to second place the one with the next 1581 * most raw votes. 1582 * 1583 * <p>Accomplish promotion by increasing vote counts in the voteCount hash. 1584 * 1585 * @param sortedValues the set of sorted values 1586 * @param voteCount the vote count for each value in sortedValues AFTER 1587 * calculateNewCountsBasedOnAnnotationComponents; it gets modified if superior subsets exist 1588 * @param rawVoteCount the vote count for each value in sortedValues BEFORE 1589 * calculateNewCountsBasedOnAnnotationComponents; rawVoteCount is not changed by this 1590 * function 1591 * <p>Reference: https://unicode.org/cldr/trac/ticket/10973 1592 */ promoteSuperiorAnnotationSuperset( Set<T> sortedValues, HashMap<T, Long> voteCount, HashMap<T, Long> rawVoteCount)1593 private void promoteSuperiorAnnotationSuperset( 1594 Set<T> sortedValues, HashMap<T, Long> voteCount, HashMap<T, Long> rawVoteCount) { 1595 final long requiredGap = 2; 1596 T oldWinner = null; 1597 long oldWinnerRawCount = 0; 1598 LinkedHashSet<T> oldWinnerComps = null; 1599 LinkedHashSet<T> superiorSupersets = null; 1600 for (T value : sortedValues) { 1601 // Annotate the means here 1602 final long rawCount = rawVoteCount.get(value); 1603 final long newCount = voteCount.get(value); 1604 if (rawCount != newCount) { 1605 annotateTranscript("-- Value '%s' has updated value '%d'", value, newCount); 1606 } 1607 if (oldWinner == null) { 1608 oldWinner = value; 1609 oldWinnerRawCount = rawVoteCount.get(value); 1610 oldWinnerComps = new LinkedHashSet<>(splitAnnotationIntoComponentsList(value)); 1611 } else { 1612 Set<T> comps = new LinkedHashSet<>(splitAnnotationIntoComponentsList(value)); 1613 if (comps.size() <= CheckWidths.MAX_COMPONENTS_PER_ANNOTATION 1614 && comps.containsAll(oldWinnerComps) 1615 && rawVoteCount.get(value) >= oldWinnerRawCount + requiredGap) { 1616 if (superiorSupersets == null) { 1617 superiorSupersets = new LinkedHashSet<>(); 1618 } 1619 superiorSupersets.add(value); 1620 } 1621 } 1622 } 1623 if (superiorSupersets != null) { 1624 // Sort the supersets by raw vote count, then make their adjusted vote counts higher 1625 // than the old winner's. 1626 resortValuesBasedOnAdjustedVoteCounts(superiorSupersets, rawVoteCount); 1627 T newWinner = null, newSecond; // only adjust votes for first and second place 1628 for (T value : superiorSupersets) { 1629 if (newWinner == null) { 1630 newWinner = value; 1631 long newWinnerCount = voteCount.get(oldWinner) + 2; 1632 annotateTranscript( 1633 "- Optimal value (O) '%s' was promoted to value '%d' due to having a superior raw vote count", 1634 newWinner, newWinnerCount); 1635 voteCount.put(newWinner, newWinnerCount); // more than oldWinner and newSecond 1636 } else { 1637 newSecond = value; 1638 long newSecondCount = voteCount.get(oldWinner) + 1; 1639 annotateTranscript( 1640 "- Next value (N) '%s' was promoted to value '%d' due to having a superior raw vote count", 1641 newSecond, newSecondCount); 1642 voteCount.put( 1643 newSecond, newSecondCount); // more than oldWinner, less than newWinner 1644 break; 1645 } 1646 } 1647 resortValuesBasedOnAdjustedVoteCounts(sortedValues, voteCount); 1648 } 1649 } 1650 1651 /** 1652 * Given a nonempty list of sorted values, and a hash with their vote counts, set these members 1653 * of this VoteResolver: winningValue, nValue, valuesWithSameVotes (which is empty when this 1654 * function is called). 1655 * 1656 * @param sortedValues the set of sorted values 1657 * @param voteCount the hash giving the vote count for each value 1658 * @return an array of two longs, the weights for the best and next-best values. 1659 */ setBestNextAndSameVoteValues(Set<T> sortedValues, HashMap<T, Long> voteCount)1660 private long[] setBestNextAndSameVoteValues(Set<T> sortedValues, HashMap<T, Long> voteCount) { 1661 1662 long[] weightArray = new long[2]; 1663 nValue = null; 1664 1665 /* 1666 * Loop through the sorted values, at least the first (best) for winningValue, 1667 * and the second (if any) for nValue (else nValue stays null), 1668 * and subsequent values that have as many votes as the first, 1669 * to add to valuesWithSameVotes. 1670 */ 1671 int i = -1; 1672 Iterator<T> iterator = sortedValues.iterator(); 1673 for (T value : sortedValues) { 1674 ++i; 1675 long valueWeight = voteCount.get(value); 1676 if (i == 0) { 1677 setWinningValue(value); 1678 weightArray[0] = valueWeight; 1679 valuesWithSameVotes.add(value); 1680 annotateTranscript( 1681 "The optimal value (O) is '%s', with a weight of %d", 1682 winningValue, valueWeight); 1683 if (sortedValues.size() == 1) { 1684 annotateTranscript("- No other values received votes."); // uncontested 1685 } 1686 } else { 1687 if (i == 1) { 1688 // get the next item if there is one 1689 if (iterator.hasNext()) { 1690 nValue = value; 1691 weightArray[1] = valueWeight; 1692 annotateNextBestValue(weightArray[0], weightArray[1], winningValue, nValue); 1693 } 1694 } 1695 if (valueWeight == weightArray[0]) { 1696 valuesWithSameVotes.add(value); 1697 } else { 1698 break; 1699 } 1700 } 1701 } 1702 return weightArray; 1703 } 1704 1705 /** 1706 * Compute the status for the winning value. See: https://cldr.unicode.org/index/process 1707 * 1708 * @param O the weight (vote count) for the best value 1709 * @param N the weight (vote count) for the next-best value 1710 * @return the Status 1711 */ computeStatus(long O, long N)1712 private Status computeStatus(long O, long N) { 1713 if (O > N) { 1714 final int requiredVotes = getRequiredVotes(); 1715 if (O >= requiredVotes) { 1716 final Status computedStatus = Status.approved; 1717 annotateTranscript("O>N, and O>%d: %s", requiredVotes, computedStatus); 1718 return computedStatus; 1719 } 1720 if (O >= 4 && Status.contributed.compareTo(baselineStatus) > 0) { 1721 final Status computedStatus = Status.contributed; 1722 annotateTranscript( 1723 "O>=4, and oldstatus (%s)<contributed: %s", baselineStatus, computedStatus); 1724 return computedStatus; 1725 } 1726 if (O >= 2) { 1727 final int G = organizationToValueAndVote.getOrgCount(winningValue); 1728 if (G >= 2) { 1729 final Status computedStatus = Status.contributed; 1730 annotateTranscript("O>=2, and G (%d)>=2: %s", G, computedStatus); 1731 return computedStatus; 1732 } 1733 } 1734 } 1735 if (O >= N) { 1736 if (O >= 2) { 1737 final Status computedStatus = Status.provisional; 1738 annotateTranscript("O>=N and O>=2: %s", computedStatus); 1739 return computedStatus; 1740 } 1741 } 1742 1743 // otherwise: unconfirmed 1744 final Status computedStatus = Status.unconfirmed; 1745 annotateTranscript("O was not high enough: %s", computedStatus); 1746 return computedStatus; 1747 } 1748 getPossibleWinningStatus()1749 private Status getPossibleWinningStatus() { 1750 if (!resolved) { 1751 resolveVotes(); 1752 } 1753 Status possibleStatus = computeStatus(organizationToValueAndVote.getBestPossibleVote(), 0); 1754 return possibleStatus.compareTo(winningStatus) > 0 ? possibleStatus : winningStatus; 1755 } 1756 1757 /** 1758 * If the winning item is not approved, and if all the people who voted had voted for the 1759 * winning item, would it have made contributed or approved? 1760 * 1761 * @return 1762 */ isDisputed()1763 public boolean isDisputed() { 1764 if (!resolved) { 1765 resolveVotes(); 1766 } 1767 if (winningStatus.compareTo(VoteResolver.Status.contributed) >= 0) { 1768 return false; 1769 } 1770 VoteResolver.Status possibleStatus = getPossibleWinningStatus(); 1771 return possibleStatus.compareTo(Status.contributed) >= 0; 1772 } 1773 getWinningStatus()1774 public Status getWinningStatus() { 1775 if (!resolved) { 1776 resolveVotes(); 1777 } 1778 return winningStatus; 1779 } 1780 1781 /** 1782 * Returns O Value as described in http://cldr.unicode.org/index/process#TOC-Voting-Process. Not 1783 * always the same as the Winning Value. 1784 * 1785 * @return 1786 */ getOValue()1787 private T getOValue() { 1788 if (!resolved) { 1789 resolveVotes(); 1790 } 1791 return oValue; 1792 } 1793 1794 /** 1795 * Returns N Value as described in http://cldr.unicode.org/index/process#TOC-Voting-Process. Not 1796 * always the same as the Winning Value. 1797 * 1798 * @return 1799 */ getNValue()1800 private T getNValue() { 1801 if (!resolved) { 1802 resolveVotes(); 1803 } 1804 return nValue; 1805 } 1806 1807 /** 1808 * Returns Winning Value as described in 1809 * http://cldr.unicode.org/index/process#TOC-Voting-Process. Not always the same as the O Value. 1810 * 1811 * @return 1812 */ getWinningValue()1813 public T getWinningValue() { 1814 if (!resolved) { 1815 resolveVotes(); 1816 } 1817 return winningValue; 1818 } 1819 1820 /** 1821 * Set the Winning Value; if the given value matches Bailey, change it to INHERITANCE_MARKER 1822 * 1823 * @param value the value to set (prior to changeBaileyToInheritance) 1824 */ setWinningValue(T value)1825 private void setWinningValue(T value) { 1826 if (DROP_HARD_INHERITANCE) { 1827 winningValue = changeBaileyToInheritance(value); 1828 } else { 1829 winningValue = value; 1830 } 1831 } 1832 getValuesWithSameVotes()1833 public List<T> getValuesWithSameVotes() { 1834 if (!resolved) { 1835 resolveVotes(); 1836 } 1837 return new ArrayList<>(valuesWithSameVotes); 1838 } 1839 getConflictedOrganizations()1840 public EnumSet<Organization> getConflictedOrganizations() { 1841 if (!resolved) { 1842 resolveVotes(); 1843 } 1844 return conflictedOrganizations; 1845 } 1846 1847 /** 1848 * What value did this organization vote for? 1849 * 1850 * @param org 1851 * @return 1852 */ getOrgVote(Organization org)1853 public T getOrgVote(Organization org) { 1854 return organizationToValueAndVote.getOrgVote(org); 1855 } 1856 getOrgToVotes(Organization org)1857 public Map<T, Long> getOrgToVotes(Organization org) { 1858 return organizationToValueAndVote.getOrgToVotes(org); 1859 } 1860 getNameTime()1861 public Map<String, Long> getNameTime() { 1862 return organizationToValueAndVote.getNameTime(); 1863 } 1864 1865 /** 1866 * Get a String representation of this VoteResolver. This is sent to the client as 1867 * "voteResolver.raw" and is used only for debugging. 1868 * 1869 * <p>Compare SurveyAjax.JSONWriter.wrap(VoteResolver<String>) which creates the data actually 1870 * used by the client. 1871 */ 1872 @Override toString()1873 public String toString() { 1874 return "{" 1875 + "bailey: " 1876 + (organizationToValueAndVote.baileySet 1877 ? ("“" + organizationToValueAndVote.baileyValue + "” ") 1878 : "none ") 1879 + "baseline: {" 1880 + baselineValue 1881 + ", " 1882 + baselineStatus 1883 + "}, " 1884 + organizationToValueAndVote 1885 + ", sameVotes: " 1886 + valuesWithSameVotes 1887 + ", O: " 1888 + getOValue() 1889 + ", N: " 1890 + getNValue() 1891 + ", totals: " 1892 + totals 1893 + ", winning: {" 1894 + getWinningValue() 1895 + ", " 1896 + getWinningStatus() 1897 + "}" 1898 + "}"; 1899 } 1900 getIdToPath(String fileName)1901 public static Map<Integer, String> getIdToPath(String fileName) { 1902 XPathTableHandler myHandler = new XPathTableHandler(); 1903 XMLFileReader xfr = new XMLFileReader().setHandler(myHandler); 1904 xfr.read(fileName, XMLFileReader.CONTENT_HANDLER | XMLFileReader.ERROR_HANDLER, false); 1905 return myHandler.pathIdToPath; 1906 } 1907 1908 static class XPathTableHandler extends XMLFileReader.SimpleHandler { 1909 Matcher matcher = Pattern.compile("id=\"([0-9]+)\"").matcher(""); 1910 Map<Integer, String> pathIdToPath = new HashMap<>(); 1911 1912 @Override handlePathValue(String path, String value)1913 public void handlePathValue(String path, String value) { 1914 // <xpathTable host="tintin.local" date="Tue Apr 29 14:34:32 PDT 2008" count="18266" > 1915 // <xpath 1916 // id="1">//ldml/dates/calendars/calendar[@type="gregorian"]/dateFormats/dateFormatLength[@type="short"]/dateFormat[@type="standard"]/pattern[@type="standard"]</xpath> 1917 if (!matcher.reset(path).find()) { 1918 throw new IllegalArgumentException("Unknown path " + path); 1919 } 1920 pathIdToPath.put(Integer.parseInt(matcher.group(1)), value); 1921 } 1922 } 1923 getBaseToAlternateToInfo( String fileName, VoterInfoList vil)1924 public static Map<Integer, Map<Integer, CandidateInfo>> getBaseToAlternateToInfo( 1925 String fileName, VoterInfoList vil) { 1926 try { 1927 VotesHandler myHandler = new VotesHandler(vil); 1928 XMLFileReader xfr = new XMLFileReader().setHandler(myHandler); 1929 xfr.read(fileName, XMLFileReader.CONTENT_HANDLER | XMLFileReader.ERROR_HANDLER, false); 1930 return myHandler.basepathToInfo; 1931 } catch (Exception e) { 1932 throw new IllegalArgumentException("Can't handle file: " + fileName, e); 1933 } 1934 } 1935 1936 public enum Type { 1937 proposal, 1938 optimal 1939 } 1940 1941 public static class CandidateInfo { 1942 public Status oldStatus; 1943 public Type surveyType; 1944 public Status surveyStatus; 1945 public Set<Integer> voters = new TreeSet<>(); 1946 private final VoterInfoList voterInfoList; 1947 CandidateInfo(VoterInfoList vil)1948 CandidateInfo(VoterInfoList vil) { 1949 this.voterInfoList = vil; 1950 } 1951 1952 @Override toString()1953 public String toString() { 1954 StringBuilder voterString = new StringBuilder("{"); 1955 for (int voter : voters) { 1956 VoterInfo voterInfo = voterInfoList.get(voter); 1957 if (voterString.length() > 1) { 1958 voterString.append(" "); 1959 } 1960 voterString.append(voter); 1961 if (voterInfo != null) { 1962 voterString.append(" ").append(voterInfo); 1963 } 1964 } 1965 voterString.append("}"); 1966 return "{oldStatus: " 1967 + oldStatus 1968 + ", surveyType: " 1969 + surveyType 1970 + ", surveyStatus: " 1971 + surveyStatus 1972 + ", voters: " 1973 + voterString 1974 + "};"; 1975 } 1976 } 1977 1978 /* 1979 * <locale-votes host="tintin.local" date="Tue Apr 29 14:34:32 PDT 2008" 1980 * oldVersion="1.5.1" currentVersion="1.6" resolved="false" locale="zu"> 1981 * <row baseXpath="1"> 1982 * <item xpath="2855" type="proposal" id="1" status="unconfirmed"> 1983 * <old status="unconfirmed"/> 1984 * </item> 1985 * <item xpath="1" type="optimal" id="56810" status="confirmed"> 1986 * <vote user="210"/> 1987 * </item> 1988 * </row> 1989 * ... 1990 * A base path has a set of candidates. Each candidate has various items of information. 1991 */ 1992 static class VotesHandler extends XMLFileReader.SimpleHandler { 1993 private final VoterInfoList voterInfoList; 1994 VotesHandler(VoterInfoList vil)1995 VotesHandler(VoterInfoList vil) { 1996 this.voterInfoList = vil; 1997 } 1998 1999 Map<Integer, Map<Integer, CandidateInfo>> basepathToInfo = new TreeMap<>(); 2000 2001 @Override handlePathValue(String path, String value)2002 public void handlePathValue(String path, String value) { 2003 try { 2004 XPathParts parts = XPathParts.getFrozenInstance(path); 2005 if (parts.size() < 2) { 2006 // empty data 2007 return; 2008 } 2009 int baseId = Integer.parseInt(parts.getAttributeValue(1, "baseXpath")); 2010 Map<Integer, CandidateInfo> info = 2011 basepathToInfo.computeIfAbsent(baseId, k -> new TreeMap<>()); 2012 int itemId = Integer.parseInt(parts.getAttributeValue(2, "xpath")); 2013 CandidateInfo candidateInfo = info.get(itemId); 2014 if (candidateInfo == null) { 2015 info.put(itemId, candidateInfo = new CandidateInfo(voterInfoList)); 2016 candidateInfo.surveyType = Type.valueOf(parts.getAttributeValue(2, "type")); 2017 candidateInfo.surveyStatus = 2018 Status.valueOf( 2019 fixBogusDraftStatusValues( 2020 parts.getAttributeValue(2, "status"))); 2021 // ignore id 2022 } 2023 if (parts.size() < 4) { 2024 return; 2025 } 2026 final String lastElement = parts.getElement(3); 2027 if (lastElement.equals("old")) { 2028 candidateInfo.oldStatus = 2029 Status.valueOf( 2030 fixBogusDraftStatusValues( 2031 parts.getAttributeValue(3, "status"))); 2032 } else if (lastElement.equals("vote")) { 2033 candidateInfo.voters.add(Integer.parseInt(parts.getAttributeValue(3, "user"))); 2034 } else { 2035 throw new IllegalArgumentException("unknown option: " + path); 2036 } 2037 } catch (Exception e) { 2038 throw new IllegalArgumentException("Can't handle path: " + path, e); 2039 } 2040 } 2041 } 2042 2043 public static class UnknownVoterException extends RuntimeException { 2044 private static final long serialVersionUID = 3430877787936678609L; 2045 int voter; 2046 UnknownVoterException(int voter)2047 public UnknownVoterException(int voter) { 2048 this.voter = voter; 2049 } 2050 2051 @Override toString()2052 public String toString() { 2053 return "Unknown voter: " + voter; 2054 } 2055 } 2056 fixBogusDraftStatusValues(String attributeValue)2057 private static String fixBogusDraftStatusValues(String attributeValue) { 2058 if (attributeValue == null) return "approved"; 2059 if ("confirmed".equals(attributeValue)) return "approved"; 2060 if ("true".equals(attributeValue)) return "unconfirmed"; 2061 if ("unknown".equals(attributeValue)) return "unconfirmed"; 2062 return attributeValue; 2063 } 2064 2065 /* 2066 * TODO: either delete this or explain why it's needed 2067 */ size()2068 public int size() { 2069 return values.size(); 2070 } 2071 2072 /** 2073 * Returns a map from value to resolved vote count, in descending order. If the winning item is 2074 * not there, insert at the front. If the baseline (trunk) item is not there, insert at the end. 2075 * 2076 * <p>This map includes intra-org disputes. 2077 * 2078 * @return the map 2079 */ getResolvedVoteCountsIncludingIntraOrgDisputes()2080 public Map<T, Long> getResolvedVoteCountsIncludingIntraOrgDisputes() { 2081 if (!resolved) { 2082 resolveVotes(); 2083 } 2084 Map<T, Long> result = new LinkedHashMap<>(); 2085 if (winningValue != null && !totals.containsKey(winningValue)) { 2086 result.put(winningValue, 0L); 2087 } 2088 for (T value : totals.getKeysetSortedByCount(false, votesThenUcaCollator)) { 2089 result.put(value, totals.get(value)); 2090 } 2091 if (baselineValue != null && !totals.containsKey(baselineValue)) { 2092 result.put(baselineValue, 0L); 2093 } 2094 for (T value : 2095 organizationToValueAndVote.allVotesIncludingIntraOrgDispute.getMap().keySet()) { 2096 if (!result.containsKey(value)) { 2097 result.put(value, 0L); 2098 } 2099 } 2100 if (DEBUG) { 2101 System.out.println("getResolvedVoteCountsIncludingIntraOrgDisputes :" + result); 2102 } 2103 return result; 2104 } 2105 getStatusForOrganization(Organization orgOfUser)2106 public VoteStatus getStatusForOrganization(Organization orgOfUser) { 2107 if (!resolved) { 2108 resolveVotes(); 2109 } 2110 if (Status.provisional.compareTo(winningStatus) >= 0) { 2111 // If the value is provisional, it needs more votes. 2112 return VoteStatus.provisionalOrWorse; 2113 } 2114 T orgVote = organizationToValueAndVote.getOrgVoteRaw(orgOfUser); 2115 if (!equalsOrgVote(winningValue, orgVote)) { 2116 // We voted and lost 2117 return VoteStatus.losing; 2118 } 2119 final int itemsWithVotes = 2120 DROP_HARD_INHERITANCE ? totals.size() : countDistinctValuesWithVotes(); 2121 if (itemsWithVotes > 1) { 2122 // If there are votes for two "distinct" items, we should look at them. 2123 return VoteStatus.disputed; 2124 } 2125 final T singleVotedItem = getSingleVotedItem(); 2126 if (!equalsOrgVote(winningValue, singleVotedItem)) { 2127 // If someone voted but didn't win 2128 return VoteStatus.disputed; 2129 } 2130 if (itemsWithVotes == 0) { 2131 // The value is ok, but we capture that there are no votes, for revealing items like 2132 // unsync'ed 2133 return VoteStatus.ok_novotes; 2134 } else { 2135 // We voted, we won, value is approved, no disputes, have votes 2136 return VoteStatus.ok; 2137 } 2138 } 2139 2140 /** 2141 * Returns value of voted item, in case there is exactly 1. 2142 * 2143 * @return 2144 */ getSingleVotedItem()2145 private T getSingleVotedItem() { 2146 return totals.size() != 1 ? null : totals.iterator().next(); 2147 } 2148 2149 /** 2150 * Should these two values be treated as equivalent for getStatusForOrganization? 2151 * 2152 * @param value 2153 * @param orgVote 2154 * @return true if they are equivalent, false if they are distinct 2155 */ equalsOrgVote(T value, T orgVote)2156 private boolean equalsOrgVote(T value, T orgVote) { 2157 return orgVote == null 2158 || orgVote.equals(value) 2159 || (CldrUtility.INHERITANCE_MARKER.equals(value) 2160 && orgVote.equals(organizationToValueAndVote.baileyValue)) 2161 || (CldrUtility.INHERITANCE_MARKER.equals(orgVote) 2162 && value.equals(organizationToValueAndVote.baileyValue)); 2163 } 2164 2165 /** 2166 * Count the distinct values that have votes. 2167 * 2168 * <p>For this purpose, if there are both votes for inheritance and votes for the specific value 2169 * matching the inherited (bailey) value, they are not "distinct": count them as a single value. 2170 * 2171 * @return the number of distinct values 2172 */ countDistinctValuesWithVotes()2173 private int countDistinctValuesWithVotes() { 2174 if (!resolved) { // must be resolved for bothInheritanceAndBaileyHadVotes 2175 throw new RuntimeException("countDistinctValuesWithVotes !resolved"); 2176 } 2177 int count = organizationToValueAndVote.allVotesIncludingIntraOrgDispute.size(); 2178 if (count > 1 && bothInheritanceAndBaileyHadVotes) { 2179 return count - 1; // prevent showing as "disputed" in dashboard 2180 } 2181 return count; 2182 } 2183 2184 /** 2185 * Should this VoteResolver use keyword annotation voting? 2186 * 2187 * <p>Apply special voting method adjustAnnotationVoteCounts only to certain keyword annotations 2188 * that can have bar-separated values like "happy | joyful". 2189 * 2190 * <p>The paths for keyword annotations start with "//ldml/annotations/annotation" and do NOT 2191 * include Emoji.TYPE_TTS. Both name paths (cf. namePath, getNamePaths) and keyword paths (cf. 2192 * keywordPath, getKeywordPaths) have "//ldml/annotations/annotation". Name paths include 2193 * Emoji.TYPE_TTS, and keyword paths don't. Special voting is only for keyword paths, not for 2194 * name paths. Compare path dependencies in DisplayAndInputProcessor.java. See also 2195 * VoteResolver.splitAnnotationIntoComponentsList. 2196 * 2197 * @return true or false 2198 */ isUsingKeywordAnnotationVoting()2199 private boolean isUsingKeywordAnnotationVoting() { 2200 if (pathHeader == null) { 2201 return false; // this happens in some tests 2202 } 2203 final String path = pathHeader.getOriginalPath(); 2204 return AnnotationUtil.pathIsAnnotation(path) && !path.contains(Emoji.TYPE_TTS); 2205 } 2206 2207 /** 2208 * Is the value locked for this locale+path? 2209 * 2210 * @return true or false 2211 */ isValueLocked()2212 public boolean isValueLocked() { 2213 return valueIsLocked; 2214 } 2215 2216 /** 2217 * Can a user who makes a losing vote flag the locale+path? I.e., is the locale+path locked 2218 * and/or does it require HIGH_BAR votes? 2219 * 2220 * @return true or false 2221 */ canFlagOnLosing()2222 public boolean canFlagOnLosing() { 2223 return valueIsLocked || (getRequiredVotes() == HIGH_BAR); 2224 } 2225 2226 /** 2227 * Calculate VoteResolver.Status 2228 * 2229 * @param baselineFile the 'baseline' file to use 2230 * @param path path the xpath 2231 * @return the Status 2232 */ calculateStatus(CLDRFile baselineFile, String path)2233 public static Status calculateStatus(CLDRFile baselineFile, String path) { 2234 String fullXPath = baselineFile.getFullXPath(path); 2235 if (fullXPath == null) { 2236 fullXPath = path; 2237 } 2238 final XPathParts xpp = XPathParts.getFrozenInstance(fullXPath); 2239 final String draft = xpp.getAttributeValue(-1, LDMLConstants.DRAFT); 2240 Status status = draft == null ? Status.approved : VoteResolver.Status.fromString(draft); 2241 2242 /* 2243 * Reset to missing if the value is inherited from root or code-fallback, unless the XML actually 2244 * contains INHERITANCE_MARKER. Pass false for skipInheritanceMarker so that status will not be 2245 * missing for explicit INHERITANCE_MARKER. 2246 */ 2247 final String srcid = 2248 baselineFile.getSourceLocaleIdExtended( 2249 path, null, false /* skipInheritanceMarker */); 2250 if (srcid.equals(XMLSource.CODE_FALLBACK_ID)) { 2251 status = Status.missing; 2252 } else if (srcid.equals("root")) { 2253 if (!srcid.equals(baselineFile.getLocaleID())) { 2254 status = Status.missing; 2255 } 2256 } 2257 return status; 2258 } 2259 /** 2260 * Get the possibly modified value. If value matches the bailey value or inheritance marker, 2261 * possibly change it from bailey value to inheritance marker, or vice-versa, as needed to meet 2262 * these requirements: 1. If the path changes when getting bailey, then we are inheriting 2263 * sideways. We need to use a hard value. 2. If the value is different from the bailey value, 2264 * can't use inheritance; we need a hard value. 3. Otherwise we use inheritance marker. 2265 * 2266 * <p>These requirements are pragmatic, to work around limitations of the current inheritance 2267 * algorithm, which is hyper-sensitive to the distinction between inheritance marker and bailey, 2268 * which, depending on that distinction, unintentionally tends to change lateral inheritance to 2269 * vertical inheritance, or vice-versa. 2270 * 2271 * <p>This method has consequences affecting vote resolution. For example, assume 2272 * DROP_HARD_INHERITANCE is true. If a user votes for what is currently the inherited value, and 2273 * these requirements call for using inheritance marker, then their vote is stored as 2274 * inheritance marker in the db; if the parent value then changes (even during same release 2275 * cycle), the vote is still a vote for inheritance -- that is how soft inheritence has long 2276 * been intended to work. In the cases where this method returns the hard value matching bailey, 2277 * the user's vote is stored in the db as that hard value; if the parent value then changes, the 2278 * user's vote does not change -- this differs from what we'd like ideally (which is for all 2279 * inh. votes to be "soft"). If and when the inheritance algorithm changes to reduce or 2280 * eliminate the problematic aspects of the hard/soft distinction, this method might no longer 2281 * be needed. 2282 * 2283 * <p>Reference: https://unicode-org.atlassian.net/browse/CLDR-16560 2284 * 2285 * @param path the path 2286 * @param value the input value 2287 * @param cldrFile the CLDRFile for determining inheritance 2288 * @return the possibly modified value 2289 */ reviseInheritanceAsNeeded(String path, String value, CLDRFile cldrFile)2290 public static String reviseInheritanceAsNeeded(String path, String value, CLDRFile cldrFile) { 2291 if (!DROP_HARD_INHERITANCE) { 2292 return value; 2293 } 2294 if (!cldrFile.isResolved()) { 2295 throw new InternalCldrException("must be resolved"); 2296 } 2297 Output<String> pathWhereFound = new Output<>(); 2298 Output<String> localeWhereFound = new Output<>(); 2299 String baileyValue = cldrFile.getBaileyValue(path, pathWhereFound, localeWhereFound); 2300 if (baileyValue != null 2301 && (CldrUtility.INHERITANCE_MARKER.equals(value) || baileyValue.equals(value))) { 2302 // TODO: decide whether to continue treating GlossonymConstructor.PSEUDO_PATH 2303 // (constructed values) as lateral inheritance. This method originally did not 2304 // take constructed values into account, so it implicitly treated constructed 2305 // values as laterally inherited, given that pathWhereFound doesn't equal path. 2306 // This original behavior corresponds to CONSTRUCTED_PSEUDO_PATH_NOT_LATERAL = false. 2307 // Reference: https://unicode-org.atlassian.net/browse/CLDR-16372 2308 final boolean CONSTRUCTED_PSEUDO_PATH_NOT_LATERAL = false; 2309 value = 2310 (pathWhereFound.value.equals(path) 2311 || (CONSTRUCTED_PSEUDO_PATH_NOT_LATERAL 2312 && GlossonymConstructor.PSEUDO_PATH.equals( 2313 pathWhereFound.value))) 2314 ? CldrUtility.INHERITANCE_MARKER 2315 : baileyValue; 2316 } 2317 return value; 2318 } 2319 } 2320