• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 package org.unicode.cldr.util;
2 
3 import com.google.common.base.Objects;
4 import com.google.common.collect.ImmutableSet;
5 import com.ibm.icu.text.Collator;
6 import com.ibm.icu.util.Output;
7 import com.ibm.icu.util.ULocale;
8 import java.sql.Timestamp;
9 import java.util.*;
10 import java.util.Map.Entry;
11 import java.util.regex.Matcher;
12 import java.util.regex.Pattern;
13 import org.unicode.cldr.icu.LDMLConstants;
14 import org.unicode.cldr.test.CheckCLDR;
15 import org.unicode.cldr.test.CheckCLDR.Phase;
16 import org.unicode.cldr.test.CheckWidths;
17 import org.unicode.cldr.test.DisplayAndInputProcessor;
18 
19 /**
20  * This class implements the vote resolution process agreed to by the CLDR committee. Here is an
21  * example of usage:
22  *
23  * <pre>
24  * // before doing anything, initialize the voter data (who are the voters at what levels) with setVoterToInfo.
25  * // We assume this doesn't change often
26  * // here is some fake data:
27  * VoteResolver.setVoterToInfo(Utility.asMap(new Object[][] {
28  *     { 666, new VoterInfo(Organization.google, Level.vetter, &quot;J. Smith&quot;) },
29  *     { 555, new VoterInfo(Organization.google, Level.street, &quot;S. Jones&quot;) },
30  *     { 444, new VoterInfo(Organization.google, Level.vetter, &quot;S. Samuels&quot;) },
31  *     { 333, new VoterInfo(Organization.apple, Level.vetter, &quot;A. Mutton&quot;) },
32  *     { 222, new VoterInfo(Organization.adobe, Level.expert, &quot;A. Aldus&quot;) },
33  *     { 111, new VoterInfo(Organization.ibm, Level.street, &quot;J. Henry&quot;) }, }));
34  *
35  * // you can create a resolver and keep it around. It isn't thread-safe, so either have a separate one per thread (they
36  * // are small), or synchronize.
37  * VoteResolver resolver = new VoteResolver();
38  *
39  * // For any particular base path, set the values
40  * // set the 1.5 status (if we're working on 1.6). This &lt;b&gt;must&lt;/b&gt; be done for each new base path
41  * resolver.newPath(oldValue, oldStatus);
42  * [TODO: function newPath doesn't exist, revise this documentation]
43  *
44  * // now add some values, with who voted for them
45  * resolver.add(value1, voter1);
46  * resolver.add(value1, voter2);
47  * resolver.add(value2, voter3);
48  *
49  * // Once you've done that, you can get the results for the base path
50  * winner = resolver.getWinningValue();
51  * status = resolver.getWinningStatus();
52  * conflicts = resolver.getConflictedOrganizations();
53  * </pre>
54  */
55 public class VoteResolver<T> {
56     public static final boolean DROP_HARD_INHERITANCE = true;
57 
58     private final VoterInfoList voterInfoList;
59 
VoteResolver(VoterInfoList vil)60     public VoteResolver(VoterInfoList vil) {
61         voterInfoList = vil;
62     }
63 
64     private static final boolean DEBUG = false;
65 
66     /** This enables a prose discussion of the voting process. */
67     private DeferredTranscript transcript = null;
68 
enableTranscript()69     public void enableTranscript() {
70         if (transcript == null) {
71             transcript = new DeferredTranscript();
72         }
73     }
74 
disableTranscript()75     public void disableTranscript() {
76         transcript = null;
77     }
78 
getTranscript()79     public String getTranscript() {
80         if (transcript == null) {
81             return null;
82         } else {
83             return transcript.get();
84         }
85     }
86 
87     /**
88      * Add an annotation
89      *
90      * @param fmt
91      * @param args
92      */
annotateTranscript(String fmt, Object... args)93     private final void annotateTranscript(String fmt, Object... args) {
94         if (transcript != null) {
95             transcript.add(fmt, args);
96         }
97     }
98 
99     /**
100      * A placeholder for winningValue when it would otherwise be null. It must match
101      * NO_WINNING_VALUE in the client JavaScript code.
102      */
103     private static final String NO_WINNING_VALUE = "no-winning-value";
104 
105     /**
106      * The status levels according to the committee, in ascending order
107      *
108      * <p>Status corresponds to icons as follows: A checkmark means it’s approved and is slated to
109      * be used. A cross means it’s a missing value. Green/orange check: The item has enough votes to
110      * be used in CLDR. Red/orange/black X: The item does not have enough votes to be used in CLDR,
111      * by most implementations (or is completely missing). Reference: <a
112      * href="https://cldr.unicode.org/translation/getting-started/guide">guide</a>
113      *
114      * <p>When the item is inherited, i.e., winningValue is INHERITANCE_MARKER (↑↑↑), then
115      * orange/red X are replaced by orange/red up-arrow. That change is made only on the client.
116      *
117      * <p>Status.approved: green check Status.contributed: orange check Status.provisional: orange X
118      * (or orange up-arrow if inherited) Status.unconfirmed: red X (or red up-arrow if inherited
119      * Status.missing: black X
120      *
121      * <p>Not to be confused with VoteResolver.VoteStatus
122      */
123     public enum Status {
124         missing,
125         unconfirmed,
126         provisional,
127         contributed,
128         approved;
129 
fromString(String source)130         public static Status fromString(String source) {
131             return source == null ? missing : Status.valueOf(source);
132         }
133     }
134 
135     /**
136      * This is the "high bar" level where flagging is required.
137      *
138      * @see #getRequiredVotes()
139      */
140     public static final int HIGH_BAR = Level.tc.votes;
141 
142     public static final int LOWER_BAR = (2 * Level.vetter.votes);
143 
144     /**
145      * This is the level at which a vote counts. Each level also contains the weight.
146      *
147      * <p>Code related to Level.expert removed 2021-05-18 per CLDR-14597
148      */
149     public enum Level {
150         locked(0 /* votes */, 999 /* stlevel */),
151         guest(1 /* votes */, 10 /* stlevel */),
152         anonymous(0 /* votes */, 8 /* stlevel */),
153         vetter(4 /* votes */, 5 /* stlevel */, /* tcorgvotes */ 6), // org dependent- see getVotes()
154         // Manager and below can manage users
155         manager(4 /* votes */, 2 /* stlevel */),
156         tc(50 /* votes */, 1 /* stlevel */),
157         admin(100 /* votes */, 0 /* stlevel */);
158 
159         /**
160          * PERMANENT_VOTES is used by TC voters to "lock" locale+path permanently (including future
161          * versions, until unlocked), in the current VOTE_VALUE table. It is public for
162          * STFactory.java and PermanentVote.java.
163          */
164         public static final int PERMANENT_VOTES = 1000;
165 
166         /**
167          * LOCKING_VOTES is used (nominally by ADMIN voter, but not really by someone logged in as
168          * ADMIN, instead by combination of two PERMANENT_VOTES) to "lock" locale+path permanently
169          * in the LOCKED_XPATHS table. It is public for STFactory.PerLocaleData.loadVoteValues.
170          */
171         public static final int LOCKING_VOTES = 2000;
172 
173         /** The vote count a user of this level normally votes with */
174         private final int votes;
175 
176         /** The vote count a user of this level normally votes with if a tc org */
177         private final int tcorgvotes;
178 
179         /** The level as an integer, where 0 = admin, ..., 999 = locked */
180         private final int stlevel;
181 
Level(int votes, int stlevel, int tcorgvotes)182         Level(int votes, int stlevel, int tcorgvotes) {
183             this.votes = votes;
184             this.stlevel = stlevel;
185             this.tcorgvotes = tcorgvotes;
186         }
187 
Level(int votes, int stlevel)188         Level(int votes, int stlevel) {
189             this(votes, stlevel, votes);
190         }
191 
192         /**
193          * Get the votes for each level and organization
194          *
195          * @param o the given organization
196          */
getVotes(Organization o)197         public int getVotes(Organization o) {
198             if (this == vetter && o.isTCOrg()) {
199                 return tcorgvotes;
200             }
201             return votes;
202         }
203 
204         /** Get the Survey Tool userlevel for each level. (0=admin, 999=locked) */
getSTLevel()205         public int getSTLevel() {
206             return stlevel;
207         }
208 
209         /**
210          * Find the Level, given ST Level
211          *
212          * @param stlevel
213          * @return the Level corresponding to the integer
214          */
fromSTLevel(int stlevel)215         public static Level fromSTLevel(int stlevel) {
216             for (Level l : Level.values()) {
217                 if (l.getSTLevel() == stlevel) {
218                     return l;
219                 }
220             }
221             return null;
222         }
223 
224         /**
225          * Policy: can this user manage the "other" user's settings?
226          *
227          * @param myOrg the current organization
228          * @param otherLevel the other user's level
229          * @param otherOrg the other user's organization
230          * @return
231          */
isManagerFor(Organization myOrg, Level otherLevel, Organization otherOrg)232         public boolean isManagerFor(Organization myOrg, Level otherLevel, Organization otherOrg) {
233             return (this == admin
234                     || (canManageSomeUsers()
235                             && (myOrg == otherOrg)
236                             && atLeastAsPowerfulAs(otherLevel)));
237         }
238 
239         /**
240          * Policy: Can this user manage any users?
241          *
242          * @return
243          */
canManageSomeUsers()244         public boolean canManageSomeUsers() {
245             return atLeastAsPowerfulAs(manager);
246         }
247 
248         /** Internal: uses the ST Level as a measure of 'power' */
morePowerfulThan(Level other)249         boolean morePowerfulThan(Level other) {
250             return getSTLevel() < other.getSTLevel();
251         }
252 
253         /** Internal: uses the ST Level as a measure of 'power' */
atLeastAsPowerfulAs(Level other)254         boolean atLeastAsPowerfulAs(Level other) {
255             return getSTLevel() <= other.getSTLevel();
256         }
257 
258         /**
259          * Policy: can this user create or set a user to the specified level?
260          *
261          * @param otherLevel the desired new level for the other user
262          *     <p>Note: UserRegistry.canSetUserLevel enforces additional limitations depending on
263          *     more than this user's level and the other user's desired new level
264          */
canCreateOrSetLevelTo(Level otherLevel)265         public boolean canCreateOrSetLevelTo(Level otherLevel) {
266             // Must be a manager at all
267             if (!canManageSomeUsers()) return false;
268             // Cannot elevate privilege
269             return !otherLevel.morePowerfulThan(this);
270         }
271 
272         /**
273          * Can a user with this level and organization vote with the given vote count?
274          *
275          * @param org the given organization
276          * @param withVotes the given vote count
277          * @return true if the user can vote with the given vote count, else false
278          */
canVoteWithCount(Organization org, int withVotes)279         public boolean canVoteWithCount(Organization org, int withVotes) {
280             /*
281              * ADMIN is allowed to vote with LOCKING_VOTES, but not directly in the GUI, only
282              * by two TC voting together with PERMANENT_VOTES. Therefore LOCKING_VOTES is omitted
283              * from the GUI menu (voteCountMenu), but included in canVoteWithCount.
284              */
285             if (withVotes == LOCKING_VOTES && this == admin) {
286                 return true;
287             }
288             Set<Integer> menu = getVoteCountMenu(org);
289             return menu == null ? withVotes == getVotes(org) : menu.contains(withVotes);
290         }
291 
292         /**
293          * If not null, an array of different vote counts from which a user of this level is allowed
294          * to choose.
295          */
296         private ImmutableSet<Integer> voteCountMenu = null;
297 
298         /**
299          * Get the ordered immutable set of different vote counts a user of this level can vote with
300          *
301          * @param ignoredOrg the given organization
302          * @return the set, or null if the user has no choice of vote count
303          */
getVoteCountMenu(Organization ignoredOrg)304         public ImmutableSet<Integer> getVoteCountMenu(Organization ignoredOrg) {
305             // Right now, the organization does not affect the menu.
306             // but update the API to future proof.
307             return voteCountMenu;
308         }
309 
310         /*
311          * Set voteCountMenu for admin and tc in this static block, which will be run after
312          * all the constructors have run, rather than in the constructor itself. For example,
313          * vetter.votes needs to be defined before we can set admin.voteCountMenu.
314          */
315         static {
316             admin.voteCountMenu =
317                     ImmutableSet.of(
318                             guest.votes,
319                             vetter.votes,
320                             vetter.tcorgvotes,
321                             tc.votes,
322                             admin.votes,
323                             PERMANENT_VOTES);
324             /* Not LOCKING_VOTES; see canVoteWithCount */
325             tc.voteCountMenu =
326                     ImmutableSet.of(
327                             guest.votes,
328                             vetter.votes,
329                             vetter.tcorgvotes,
330                             tc.votes,
331                             PERMANENT_VOTES);
332         }
333 
334         // The following methods were moved here from UserRegistry
335         // TODO: remove this todo notice
336 
isAdmin()337         public boolean isAdmin() {
338             return stlevel <= admin.stlevel;
339         }
340 
isTC()341         public boolean isTC() {
342             return stlevel <= tc.stlevel;
343         }
344 
isExactlyManager()345         public boolean isExactlyManager() {
346             return stlevel == manager.stlevel;
347         }
348 
isManagerOrStronger()349         public boolean isManagerOrStronger() {
350             return stlevel <= manager.stlevel;
351         }
352 
isVetter()353         public boolean isVetter() {
354             return stlevel <= vetter.stlevel;
355         }
356 
isGuest()357         public boolean isGuest() {
358             return stlevel <= guest.stlevel;
359         }
360 
isLocked()361         public boolean isLocked() {
362             return stlevel == locked.stlevel;
363         }
364 
isExactlyAnonymous()365         public boolean isExactlyAnonymous() {
366             return stlevel == anonymous.stlevel;
367         }
368 
369         /**
370          * Is this user an administrator 'over' this user? Always true if admin, or if TC in same
371          * org.
372          *
373          * @param myOrg
374          */
isAdminForOrg(Organization myOrg, Organization target)375         public boolean isAdminForOrg(Organization myOrg, Organization target) {
376             return isAdmin() || ((isTC() || stlevel == manager.stlevel) && (myOrg == target));
377         }
378 
canImportOldVotes(CheckCLDR.Phase inPhase)379         public boolean canImportOldVotes(CheckCLDR.Phase inPhase) {
380             return isVetter() && (inPhase == Phase.SUBMISSION);
381         }
382 
canListUsers()383         public boolean canListUsers() {
384             return isManagerOrStronger();
385         }
386 
canCreateUsers()387         public boolean canCreateUsers() {
388             return isTC() || isExactlyManager();
389         }
390 
canEmailUsers()391         public boolean canEmailUsers() {
392             return isTC() || isExactlyManager();
393         }
394 
canModifyUsers()395         public boolean canModifyUsers() {
396             return isTC() || isExactlyManager();
397         }
398 
canCreateOtherOrgs()399         public boolean canCreateOtherOrgs() {
400             return isAdmin();
401         }
402 
canUseVettingSummary()403         public boolean canUseVettingSummary() {
404             return isManagerOrStronger();
405         }
406 
canSubmit(CheckCLDR.Phase inPhase)407         public boolean canSubmit(CheckCLDR.Phase inPhase) {
408             if (inPhase == Phase.FINAL_TESTING) {
409                 return false;
410                 // TODO: Note, this will mean not just READONLY, but VETTING_CLOSED will return
411                 // false here.
412                 // This is probably desired!
413             }
414             return isGuest();
415         }
416 
canCreateSummarySnapshot()417         public boolean canCreateSummarySnapshot() {
418             return isAdmin();
419         }
420 
canMonitorForum()421         public boolean canMonitorForum() {
422             return isTC() || isExactlyManager();
423         }
424 
canSetInterestLocales()425         public boolean canSetInterestLocales() {
426             return isManagerOrStronger();
427         }
428 
canGetEmailList()429         public boolean canGetEmailList() {
430             return isManagerOrStronger();
431         }
432 
433         /** If true, can delete users at their user level or lower. */
canDeleteUsers()434         public boolean canDeleteUsers() {
435             return isAdmin();
436         }
437 
canUseVettingParticipation()438         public boolean canUseVettingParticipation() {
439             return isManagerOrStronger();
440         }
441     }
442 
443     /**
444      * See getStatusForOrganization to see how this is computed.
445      *
446      * <p>Not to be confused with VoteResolver.Status
447      */
448     public enum VoteStatus {
449         /**
450          * The value for the path is either contributed or approved, and the user's organization
451          * didn't vote.
452          */
453         ok_novotes,
454 
455         /**
456          * The value for the path is either contributed or approved, and the user's organization
457          * chose the winning value.
458          */
459         ok,
460 
461         /** The winning value is neither contributed nor approved. */
462         provisionalOrWorse,
463 
464         /**
465          * The user's organization's choice is not winning, and the winning value is either
466          * contributed or approved. There may be insufficient votes to overcome a previously
467          * approved value, or other organizations may be voting against it.
468          */
469         losing,
470 
471         /**
472          * There is a dispute, meaning more than one item with votes, or the item with votes didn't
473          * win.
474          */
475         disputed
476     }
477 
478     /** Internal class for voter information. It is public for testing only */
479     public static class VoterInfo {
480         private Organization organization;
481         private Level level;
482         private String name;
483         /**
484          * A set of locales associated with this voter; it is often empty (as when the user has "*"
485          * for their set of locales); it may not serve any purpose in ordinary operation of Survey
486          * Tool; its main (only?) purpose seems to be for computeMaxVotes, whose only purpose seems
487          * to be creation of localeToOrganizationToMaxVote, which is used only by ConsoleCheckCLDR
488          * (for obscure reason), not by Survey Tool itself.
489          */
490         private final Set<CLDRLocale> locales = new TreeSet<>();
491 
getLocales()492         public Iterable<CLDRLocale> getLocales() {
493             return locales;
494         }
495 
VoterInfo(Organization organization, Level level, String name, LocaleSet localeSet)496         public VoterInfo(Organization organization, Level level, String name, LocaleSet localeSet) {
497             this.setOrganization(organization);
498             this.setLevel(level);
499             this.setName(name);
500             if (!localeSet.isAllLocales()) {
501                 this.locales.addAll(localeSet.getSet());
502             }
503         }
504 
VoterInfo(Organization organization, Level level, String name)505         public VoterInfo(Organization organization, Level level, String name) {
506             this.setOrganization(organization);
507             this.setLevel(level);
508             this.setName(name);
509         }
510 
VoterInfo()511         public VoterInfo() {}
512 
513         @Override
toString()514         public String toString() {
515             return "{" + getName() + ", " + getLevel() + ", " + getOrganization() + "}";
516         }
517 
setOrganization(Organization organization)518         public void setOrganization(Organization organization) {
519             this.organization = organization;
520         }
521 
getOrganization()522         public Organization getOrganization() {
523             return organization;
524         }
525 
setLevel(Level level)526         public void setLevel(Level level) {
527             this.level = level;
528         }
529 
getLevel()530         public Level getLevel() {
531             return level;
532         }
533 
setName(String name)534         public void setName(String name) {
535             this.name = name;
536         }
537 
getName()538         public String getName() {
539             return name;
540         }
541 
addLocale(CLDRLocale locale)542         void addLocale(CLDRLocale locale) {
543             this.locales.add(locale);
544         }
545 
546         @Override
equals(Object obj)547         public boolean equals(Object obj) {
548             if (obj == null) {
549                 return false;
550             }
551             VoterInfo other = (VoterInfo) obj;
552             return organization.equals(other.organization)
553                     && level.equals(other.level)
554                     && name.equals(other.name)
555                     && Objects.equal(locales, other.locales);
556         }
557 
558         @Override
hashCode()559         public int hashCode() {
560             return organization.hashCode() ^ level.hashCode() ^ name.hashCode();
561         }
562     }
563 
564     /**
565      * MaxCounter: make sure that we are always only getting the maximum of the values.
566      *
567      * @author markdavis
568      * @param <T>
569      */
570     static class MaxCounter<T> extends Counter<T> {
MaxCounter(boolean b)571         public MaxCounter(boolean b) {
572             super(b);
573         }
574 
575         /** Add, but only to bring up to the maximum value. */
576         @Override
add(T obj, long countValue, long time)577         public MaxCounter<T> add(T obj, long countValue, long time) {
578             long value = getCount(obj);
579             if ((value <= countValue)) {
580                 super.add(obj, countValue - value, time); // only add the difference!
581             }
582             return this;
583         }
584     }
585 
586     /** Internal class for getting from an organization to its vote. */
587     private class OrganizationToValueAndVote<T> {
588         private final Map<Organization, MaxCounter<T>> orgToVotes =
589                 new EnumMap<>(Organization.class);
590         /**
591          * All votes, even those that aren't any org's vote because they lost an intra-org dispute
592          */
593         private final Counter<T> allVotesIncludingIntraOrgDispute = new Counter<>();
594 
595         private final Map<Organization, Integer> orgToMax = new EnumMap<>(Organization.class);
596         /** The result of {@link #getTotals(EnumSet)} */
597         private final Counter<T> totals = new Counter<>(true);
598 
599         private final Map<String, Long> nameTime = new LinkedHashMap<>();
600 
601         /** map an organization to the value it voted for. */
602         private final Map<Organization, T> orgToAdd = new EnumMap<>(Organization.class);
603 
604         private T baileyValue;
605         private boolean baileySet; // was the bailey value set
606 
OrganizationToValueAndVote()607         OrganizationToValueAndVote() {
608             for (Organization org : Organization.values()) {
609                 orgToVotes.put(org, new MaxCounter<>(true));
610             }
611         }
612 
613         /** Call clear before considering each new path */
clear()614         public void clear() {
615             for (Map.Entry<Organization, MaxCounter<T>> entry : orgToVotes.entrySet()) {
616                 entry.getValue().clear();
617             }
618             orgToAdd.clear();
619             orgToMax.clear();
620             allVotesIncludingIntraOrgDispute.clear();
621             baileyValue = null;
622             baileySet = false;
623             if (transcript != null) {
624                 // there was a transcript before, so retain it
625                 transcript = new DeferredTranscript();
626             }
627         }
628 
getNameTime()629         public Map<String, Long> getNameTime() {
630             return nameTime;
631         }
632 
633         /**
634          * Call this to add votes
635          *
636          * @param value
637          * @param voter
638          * @param withVotes optionally, vote at a non-typical voting level. May not exceed voter's
639          *     maximum allowed level. null = use default level.
640          * @param date
641          */
add(T value, int voter, Integer withVotes, Date date)642         public void add(T value, int voter, Integer withVotes, Date date) {
643             final VoterInfo info = voterInfoList.get(voter);
644             if (info == null) {
645                 throw new UnknownVoterException(voter);
646             }
647             Level level = info.getLevel();
648             if (withVotes == null || !level.canVoteWithCount(info.organization, withVotes)) {
649                 withVotes = level.getVotes(info.organization);
650             }
651             addInternal(value, info, withVotes, date); // do the add
652         }
653 
654         /**
655          * Called by add(T,int,Integer) to actually add a value.
656          *
657          * @param value
658          * @param info
659          * @param votes
660          * @param time
661          * @see #add(Object, int, Integer)
662          */
addInternal(T value, final VoterInfo info, final int votes, Date time)663         private void addInternal(T value, final VoterInfo info, final int votes, Date time) {
664             if (DROP_HARD_INHERITANCE) {
665                 value = changeBaileyToInheritance(value);
666             }
667             /* All votes are added here, even if they will later lose an intra-org dispute. */
668             allVotesIncludingIntraOrgDispute.add(value, votes, time.getTime());
669             nameTime.put(info.getName(), time.getTime());
670             if (DEBUG) {
671                 System.out.println(
672                         "allVotesIncludingIntraOrgDispute Info: "
673                                 + allVotesIncludingIntraOrgDispute);
674             }
675             if (DEBUG) {
676                 System.out.println("VoteInfo: " + info.getName() + info.getOrganization());
677             }
678             Organization organization = info.getOrganization();
679             orgToVotes.get(organization).add(value, votes, time.getTime());
680             if (DEBUG) {
681                 System.out.println(
682                         "Adding now Info: "
683                                 + organization.getDisplayName()
684                                 + info.getName()
685                                 + " is adding: "
686                                 + votes
687                                 + value
688                                 + new Timestamp(time.getTime()));
689             }
690 
691             if (DEBUG) {
692                 System.out.println(
693                         "addInternal: "
694                                 + organization.getDisplayName()
695                                 + " : "
696                                 + orgToVotes.get(organization).toString());
697             }
698 
699             // add the new votes to orgToMax, if they are greater that what was there
700             Integer max = orgToMax.get(info.getOrganization());
701             if (max == null || max < votes) {
702                 orgToMax.put(organization, votes);
703             }
704         }
705 
706         /**
707          * Return the overall vote for each organization. It is the max for each value. When the
708          * organization is conflicted (the top two values have the same vote), the organization is
709          * also added to disputed.
710          *
711          * @param conflictedOrganizations if not null, to be filled in with the set of conflicted
712          *     organizations.
713          */
getTotals(EnumSet<Organization> conflictedOrganizations)714         public Counter<T> getTotals(EnumSet<Organization> conflictedOrganizations) {
715             if (conflictedOrganizations != null) {
716                 conflictedOrganizations.clear();
717             }
718             totals.clear();
719 
720             annotateTranscript("- Getting all totals by organization:");
721             for (Map.Entry<Organization, MaxCounter<T>> entry : orgToVotes.entrySet()) {
722                 Counter<T> items = entry.getValue();
723                 if (items.size() == 0) {
724                     continue;
725                 }
726                 Iterator<T> iterator = items.getKeysetSortedByCount(false).iterator();
727                 T value = iterator.next();
728                 long weight = items.getCount(value);
729                 if (weight == 0) {
730                     continue;
731                 }
732                 annotateTranscript(
733                         "-- Considering %s which has %d item(s)",
734                         entry.getKey().getDisplayName(), items.size());
735                 Organization org = entry.getKey();
736                 if (DEBUG) {
737                     System.out.println("sortedKeys?? " + value + " " + org.getDisplayName());
738                 }
739                 // if there is more than one item, check that it is less
740                 if (iterator.hasNext()) {
741                     T value2 = iterator.next();
742                     long weight2 = items.getCount(value2);
743                     // if the votes for #1 are not better than #2, we have a dispute
744                     if (weight == weight2) {
745                         if (conflictedOrganizations != null) {
746                             annotateTranscript(
747                                     "--- There are conflicts due to different values by users of this organization.");
748                             conflictedOrganizations.add(org);
749                         }
750                     }
751                 }
752                 // This is deprecated, but preserve it until the method is removed.
753                 /*
754                  * TODO: explain the above comment, and follow through. What is deprecated (orgToAdd, or getOrgVote)?
755                  * Preserve until which method is removed (getOrgVote)?
756                  */
757                 orgToAdd.put(org, value);
758 
759                 // We add the max vote for each of the organizations choices
760                 long maxCount = 0;
761                 T considerItem = null;
762                 long considerCount = 0;
763                 long maxtime = 0;
764                 long considerTime = 0;
765                 for (T item : items.keySet()) {
766                     if (DEBUG) {
767                         System.out.println(
768                                 "Items in order: "
769                                         + item.toString()
770                                         + new Timestamp(items.getTime(item)));
771                     }
772                     long count = items.getCount(item);
773                     long time = items.getTime(item);
774                     if (count > maxCount) {
775                         maxCount = count;
776                         maxtime = time;
777                         // tell the 'losing' item
778                         if (considerItem != null) {
779                             annotateTranscript(
780                                     "---- Org is not voting for '%s': there is a higher ranked vote",
781                                     considerItem);
782                         }
783                         considerItem = item;
784                         if (DEBUG) {
785                             System.out.println(
786                                     "count>maxCount: "
787                                             + considerItem
788                                             + ":"
789                                             + new Timestamp(considerTime)
790                                             + " COUNT: "
791                                             + considerCount
792                                             + "MAXCOUNT: "
793                                             + maxCount);
794                         }
795                         considerCount = items.getCount(considerItem);
796                         considerTime = items.getTime(considerItem);
797                     } else if ((time > maxtime) && (count == maxCount)) {
798                         maxtime = time;
799                         // tell the 'losing' item
800                         if (considerItem != null) {
801                             annotateTranscript(
802                                     "---- Org is not voting for '%s': there is a later vote",
803                                     considerItem);
804                         }
805                         considerItem = item;
806                         considerCount = items.getCount(considerItem);
807                         considerTime = items.getTime(considerItem);
808                         if (DEBUG) {
809                             System.out.println(
810                                     "time>maxTime: "
811                                             + considerItem
812                                             + ":"
813                                             + new Timestamp(considerTime));
814                         }
815                     }
816                 }
817                 annotateTranscript(
818                         "--- %s vote is for '%s' with strength %d",
819                         org.getDisplayName(), considerItem, considerCount);
820                 orgToAdd.put(org, considerItem);
821                 totals.add(considerItem, considerCount, considerTime);
822 
823                 if (DEBUG) {
824                     System.out.println("Totals: " + totals + " : " + new Timestamp(considerTime));
825                 }
826             }
827 
828             if (DEBUG) {
829                 System.out.println("FINALTotals: " + totals);
830             }
831             return totals;
832         }
833 
getOrgCount(T winningValue)834         public int getOrgCount(T winningValue) {
835             int orgCount = 0;
836             for (Map.Entry<Organization, MaxCounter<T>> entry : orgToVotes.entrySet()) {
837                 Counter<T> counter = entry.getValue();
838                 long count = counter.getCount(winningValue);
839                 if (count > 0) {
840                     orgCount++;
841                 }
842             }
843             return orgCount;
844         }
845 
getBestPossibleVote()846         private int getBestPossibleVote() {
847             int total = 0;
848             for (Map.Entry<Organization, Integer> entry : orgToMax.entrySet()) {
849                 total += entry.getValue();
850             }
851             return total;
852         }
853 
854         @Override
toString()855         public String toString() {
856             String orgToVotesString = "";
857             for (Entry<Organization, MaxCounter<T>> entry : orgToVotes.entrySet()) {
858                 Counter<T> counter = entry.getValue();
859                 if (counter.size() != 0) {
860                     if (orgToVotesString.length() != 0) {
861                         orgToVotesString += ", ";
862                     }
863                     Organization org = entry.getKey();
864                     orgToVotesString += org.toString() + "=" + counter;
865                 }
866             }
867             EnumSet<Organization> conflicted = EnumSet.noneOf(Organization.class);
868             return "{orgToVotes: "
869                     + orgToVotesString
870                     + ", totals: "
871                     + getTotals(conflicted)
872                     + ", conflicted: "
873                     + conflicted
874                     + "}";
875         }
876 
877         /**
878          * This is now deprecated, since the organization may have multiple votes.
879          *
880          * @param org
881          * @return
882          * @deprecated
883          */
884         @Deprecated
getOrgVote(Organization org)885         public T getOrgVote(Organization org) {
886             return orgToAdd.get(org);
887         }
888 
getOrgVoteRaw(Organization orgOfUser)889         public T getOrgVoteRaw(Organization orgOfUser) {
890             return orgToAdd.get(orgOfUser);
891         }
892 
getOrgToVotes(Organization org)893         public Map<T, Long> getOrgToVotes(Organization org) {
894             Map<T, Long> result = new LinkedHashMap<>();
895             MaxCounter<T> counter = orgToVotes.get(org);
896             for (T item : counter) {
897                 result.put(item, counter.getCount(item));
898             }
899             return result;
900         }
901     }
902 
903     /** Data built internally */
904     private T winningValue;
905 
906     private T oValue; // optimal value; winning if better approval status than old
907     private T nValue; // next to optimal value
908     private final List<T> valuesWithSameVotes = new ArrayList<>();
909     private Counter<T> totals = null;
910 
911     private Status winningStatus;
912     private final EnumSet<Organization> conflictedOrganizations =
913             EnumSet.noneOf(Organization.class);
914     private final OrganizationToValueAndVote<T> organizationToValueAndVote =
915             new OrganizationToValueAndVote<>();
916     private T baselineValue;
917     private Status baselineStatus;
918 
919     private boolean resolved;
920     private boolean valueIsLocked;
921     private int requiredVotes = 0;
922     private final SupplementalDataInfo supplementalDataInfo = SupplementalDataInfo.getInstance();
923     private CLDRLocale locale;
924     private PathHeader pathHeader;
925 
926     private static final Collator englishCollator = Collator.getInstance(ULocale.ENGLISH).freeze();
927 
928     /** Used for comparing objects of type T */
929     private final Comparator<T> objectCollator =
930             (o1, o2) -> englishCollator.compare(String.valueOf(o1), String.valueOf(o2));
931 
932     /**
933      * Set the baseline (or "trunk") value and status for this VoteResolver.
934      *
935      * @param baselineValue the baseline value
936      * @param baselineStatus the baseline status
937      */
setBaseline(T baselineValue, Status baselineStatus)938     public void setBaseline(T baselineValue, Status baselineStatus) {
939         this.baselineValue = baselineValue;
940         this.baselineStatus = baselineValue == null ? Status.missing : baselineStatus;
941     }
942 
getBaselineValue()943     public T getBaselineValue() {
944         return baselineValue;
945     }
946 
getBaselineStatus()947     public Status getBaselineStatus() {
948         return baselineStatus;
949     }
950 
951     /**
952      * Set the locale and PathHeader for this VoteResolver
953      *
954      * <p>You must call this whenever you are using a VoteResolver with a new locale or a new
955      * PathHeader
956      *
957      * @param locale the CLDRLocale
958      * @param pathHeader the PathHeader
959      */
setLocale(CLDRLocale locale, PathHeader pathHeader)960     public void setLocale(CLDRLocale locale, PathHeader pathHeader) {
961         this.locale = locale;
962         this.pathHeader = pathHeader;
963     }
964 
965     /**
966      * What are the required votes for this item?
967      *
968      * @return the number of votes (as of this writing: usually 4, 8 for established locales)
969      */
getRequiredVotes()970     public int getRequiredVotes() {
971         if (requiredVotes == 0) {
972             int preliminaryRequiredVotes =
973                     supplementalDataInfo.getRequiredVotes(locale, pathHeader);
974             if (preliminaryRequiredVotes == HIGH_BAR && baselineStatus != Status.approved) {
975                 requiredVotes = LOWER_BAR;
976             } else {
977                 requiredVotes = preliminaryRequiredVotes;
978             }
979         }
980         return requiredVotes;
981     }
982 
983     /**
984      * Call this method first, for a new base path. You'll then call add for each value associated
985      * with that base path.
986      */
clear()987     public void clear() {
988         baselineValue = null;
989         baselineStatus = Status.missing;
990         requiredVotes = 0;
991         locale = null;
992         pathHeader = null;
993         organizationToValueAndVote.clear();
994         resolved = valueIsLocked = false;
995         values.clear();
996 
997         // TODO: clear these out between reuse
998         // Are there other values that should be cleared?
999         oValue = null;
1000         setWinningValue(null);
1001         nValue = null;
1002 
1003         if (transcript != null) {
1004             transcript.clear();
1005         }
1006     }
1007 
1008     /**
1009      * Get the bailey value (what the inherited value would be if there were no explicit value) for
1010      * this VoteResolver.
1011      *
1012      * <p>Throw an exception if !baileySet.
1013      *
1014      * @return the bailey value.
1015      *     <p>Called by STFactory.PerLocaleData.getResolverInternal in the special circumstance
1016      *     where getWinningValue has returned INHERITANCE_MARKER.
1017      */
getBaileyValue()1018     public T getBaileyValue() {
1019         if (!organizationToValueAndVote.baileySet) {
1020             throw new IllegalArgumentException(
1021                     "setBaileyValue must be called before getBaileyValue");
1022         }
1023         return organizationToValueAndVote.baileyValue;
1024     }
1025 
1026     /**
1027      * Set the Bailey value (what the inherited value would be if there were no explicit value).
1028      * This value is used in handling any CldrUtility.INHERITANCE_MARKER. This value must be set
1029      * <i>before</i> adding values. Usually by calling CLDRFile.getBaileyValue().
1030      */
setBaileyValue(T baileyValue)1031     public void setBaileyValue(T baileyValue) {
1032         organizationToValueAndVote.baileySet = true;
1033         organizationToValueAndVote.baileyValue = baileyValue;
1034     }
1035 
1036     /**
1037      * Call once for each voter for a value. If there are no voters for an item, then call
1038      * add(value);
1039      *
1040      * @param value
1041      * @param voter
1042      * @param withVotes override to lower the user's voting permission. May be null for default.
1043      * @param date
1044      *     <p>Called by getResolverInternal in STFactory, and elsewhere
1045      */
add(T value, int voter, Integer withVotes, Date date)1046     public void add(T value, int voter, Integer withVotes, Date date) {
1047         if (DROP_HARD_INHERITANCE) {
1048             value = changeBaileyToInheritance(value);
1049         }
1050         if (resolved) {
1051             throw new IllegalArgumentException(
1052                     "Must be called after clear, and before any getters.");
1053         }
1054         if (withVotes != null && withVotes == Level.LOCKING_VOTES) {
1055             valueIsLocked = true;
1056         }
1057         organizationToValueAndVote.add(value, voter, withVotes, date);
1058         values.add(value);
1059     }
1060 
1061     /**
1062      * Call once for each voter for a value. If there are no voters for an item, then call
1063      * add(value);
1064      *
1065      * @param value
1066      * @param voter
1067      * @param withVotes override to lower the user's voting permission. May be null for default.
1068      *     <p>Called only for TestUtilities, not used in Survey Tool.
1069      */
add(T value, int voter, Integer withVotes)1070     public void add(T value, int voter, Integer withVotes) {
1071         if (DROP_HARD_INHERITANCE) {
1072             value = changeBaileyToInheritance(value);
1073         }
1074         if (resolved) {
1075             throw new IllegalArgumentException(
1076                     "Must be called after clear, and before any getters.");
1077         }
1078         Date date = new Date();
1079         organizationToValueAndVote.add(value, voter, withVotes, date);
1080         values.add(value);
1081     }
1082 
changeBaileyToInheritance(T value)1083     private <T> T changeBaileyToInheritance(T value) {
1084         if (value != null && value.equals(getBaileyValue())) {
1085             return (T) CldrUtility.INHERITANCE_MARKER;
1086         }
1087         return value;
1088     }
1089 
1090     /** Used only in add(value, voter) for making a pseudo-Date */
1091     private int maxcounter = 100;
1092 
1093     /**
1094      * Call once for each voter for a value. If there are no voters for an item, then call
1095      * add(value);
1096      *
1097      * @param value
1098      * @param voter
1099      *     <p>Called by ConsoleCheckCLDR and TestUtilities; not used in SurveyTool.
1100      */
add(T value, int voter)1101     public void add(T value, int voter) {
1102         Date date = new Date(++maxcounter);
1103         add(value, voter, null, date);
1104     }
1105 
1106     /**
1107      * Call if a value has no voters. It is safe to also call this if there is a voter, just
1108      * unnecessary.
1109      *
1110      * @param value
1111      *     <p>Called by getResolverInternal for the baseline (trunk) value; also called for
1112      *     ConsoleCheckCLDR.
1113      */
add(T value)1114     public void add(T value) {
1115         if (resolved) {
1116             throw new IllegalArgumentException(
1117                     "Must be called after clear, and before any getters.");
1118         }
1119         values.add(value);
1120     }
1121 
1122     private final Set<T> values = new TreeSet<>(objectCollator);
1123 
1124     private final Comparator<T> votesThenUcaCollator =
1125             new Comparator<>() {
1126 
1127                 /**
1128                  * Compare candidate items by vote count, highest vote first. In the case of ties,
1129                  * favor (a) the baseline (trunk) value, then (b) votes for inheritance
1130                  * (INHERITANCE_MARKER), then (c) the alphabetical order (as a last resort).
1131                  *
1132                  * <p>Return negative to favor o1, positive to favor o2.
1133                  *
1134                  * @see VoteResolver#setBestNextAndSameVoteValues(Set, HashMap)
1135                  * @see VoteResolver#annotateNextBestValue(long, long, T, T)
1136                  */
1137                 @Override
1138                 public int compare(T o1, T o2) {
1139                     long v1 = organizationToValueAndVote.allVotesIncludingIntraOrgDispute.get(o1);
1140                     long v2 = organizationToValueAndVote.allVotesIncludingIntraOrgDispute.get(o2);
1141                     if (v1 != v2) {
1142                         return v1 < v2 ? 1 : -1; // highest vote first
1143                     }
1144                     if (o1.equals(baselineValue)) {
1145                         return -1;
1146                     } else if (o2.equals(baselineValue)) {
1147                         return 1;
1148                     }
1149                     if (o1.equals(CldrUtility.INHERITANCE_MARKER)) {
1150                         return -1;
1151                     } else if (o2.equals(CldrUtility.INHERITANCE_MARKER)) {
1152                         return 1;
1153                     }
1154                     return englishCollator.compare(String.valueOf(o1), String.valueOf(o2));
1155                 }
1156             };
1157 
1158     /**
1159      * Annotate why the O (winning) value is winning vs the N (next) value. Assumes that the prior
1160      * annotation mentioned the O value.
1161      *
1162      * @param O optimal value
1163      * @param N next-best value
1164      */
annotateNextBestValue(long O, long N, final T oValue, final T nValue)1165     private void annotateNextBestValue(long O, long N, final T oValue, final T nValue) {
1166         // See the Comparator<> defined immediately above.
1167 
1168         // sortedValues.size() >= 2 - explain why O won and N lost.
1169         // We have to perform the function of the votesThenUcaCollator one more time
1170         if (O > N) {
1171             annotateTranscript(
1172                     "- This is the optimal value because it has the highest weight (voting score).");
1173         } else if (winningValue.equals(baselineValue)) {
1174             annotateTranscript(
1175                     "- This is the optimal value because it is the same as the baseline value, though the weight was otherwise equal to the next-best."); // aka blue star
1176         } else if (winningValue.equals(CldrUtility.INHERITANCE_MARKER)) {
1177             annotateTranscript(
1178                     "- This is the optimal value because it is the inheritance marker, though the weight was otherwise equal to the next-best."); // triple up arrow
1179         } else {
1180             annotateTranscript(
1181                     "- This is the optimal value because it comes earlier than '%s' when the text was sorted, though the weight was otherwise equal to the next-best.",
1182                     nValue);
1183         }
1184         annotateTranscript("The Next-best (N) value is '%s', with weight %d", nValue, N);
1185     }
1186 
1187     /** This will be changed to true if both kinds of vote are present */
1188     private boolean bothInheritanceAndBaileyHadVotes = false;
1189 
1190     /**
1191      * Resolve the votes. Resolution entails counting votes and setting members for this
1192      * VoteResolver, including winningStatus, winningValue, and many others.
1193      */
resolveVotes()1194     private void resolveVotes() {
1195         annotateTranscript("Resolving votes:");
1196         resolved = true;
1197         // get the votes for each organization
1198         valuesWithSameVotes.clear();
1199         totals = organizationToValueAndVote.getTotals(conflictedOrganizations);
1200         /* Note: getKeysetSortedByCount actually returns a LinkedHashSet, "with predictable iteration order". */
1201         final Set<T> sortedValues = totals.getKeysetSortedByCount(false, votesThenUcaCollator);
1202         if (DEBUG) {
1203             System.out.println("sortedValues :" + sortedValues.toString());
1204         }
1205         // annotateTranscript("all votes by org: %s", sortedValues);
1206 
1207         /*
1208          * If there are no (unconflicted) votes, return baseline (trunk) if not null,
1209          * else INHERITANCE_MARKER if baileySet, else NO_WINNING_VALUE.
1210          * Avoid setting winningValue to null. VoteResolver should be fully in charge of vote resolution.
1211          */
1212         if (sortedValues.size() == 0) {
1213             if (baselineValue != null) {
1214                 setWinningValue(baselineValue);
1215                 winningStatus = baselineStatus;
1216                 annotateTranscript(
1217                         "Winning Value: '%s' with status '%s' because there were no unconflicted votes.",
1218                         winningValue, winningStatus);
1219                 // Declare the winner here, because we're about to return from the function
1220             } else if (organizationToValueAndVote.baileySet) {
1221                 setWinningValue((T) CldrUtility.INHERITANCE_MARKER);
1222                 winningStatus = Status.missing;
1223                 annotateTranscript(
1224                         "Winning Value: '%s' with status '%s' because there were no unconflicted votes, and there was a Bailey value set.",
1225                         winningValue, winningStatus);
1226                 // Declare the winner here, because we're about to return from the function
1227             } else {
1228                 /*
1229                  * TODO: When can this still happen? See https://unicode.org/cldr/trac/ticket/11299 "Example C".
1230                  * Also http://localhost:8080/cldr-apps/v#/en_CA/Gregorian/
1231                  * -- also http://localhost:8080/cldr-apps/v#/aa/Languages_A_D/
1232                  *    xpath //ldml/localeDisplayNames/languages/language[@type="zh_Hans"][@alt="long"]
1233                  * See also checkDataRowConsistency in DataSection.java.
1234                  */
1235                 setWinningValue((T) NO_WINNING_VALUE);
1236                 winningStatus = Status.missing;
1237                 annotateTranscript(
1238                         "No winning value! status '%s' because there were no unconflicted votes",
1239                         winningStatus);
1240                 // Declare the non-winner here, because we're about to return from the function
1241             }
1242             valuesWithSameVotes.add(winningValue);
1243             return; // sortedValues.size() == 0, no candidates
1244         }
1245         if (values.size() == 0) {
1246             throw new IllegalArgumentException("No values added to resolver");
1247         }
1248 
1249         /*
1250          * Copy what is in the the totals field of this VoteResolver for all the
1251          * values in sortedValues. This local variable voteCount may be used
1252          * subsequently to make adjustments for vote resolution. Those adjustment
1253          * may affect the winners in vote resolution, while still preserving the original
1254          * voting data including the totals field.
1255          */
1256         HashMap<T, Long> voteCount = makeVoteCountMap(sortedValues);
1257 
1258         /*
1259          * Adjust sortedValues and voteCount as needed to combine "soft" votes for inheritance
1260          * with "hard" votes for the Bailey value. Note that sortedValues and voteCount are
1261          * both local variables.
1262          */
1263         if (!DROP_HARD_INHERITANCE) {
1264             bothInheritanceAndBaileyHadVotes =
1265                     combineInheritanceWithBaileyForVoting(sortedValues, voteCount);
1266         }
1267 
1268         /*
1269          * Adjust sortedValues and voteCount as needed for annotation keywords.
1270          */
1271         if (isUsingKeywordAnnotationVoting()) {
1272             adjustAnnotationVoteCounts(sortedValues, voteCount);
1273         }
1274 
1275         /*
1276          * Perform the actual resolution.
1277          * This sets winningValue to the top element of
1278          * sortedValues.
1279          */
1280         long[] weights = setBestNextAndSameVoteValues(sortedValues, voteCount);
1281 
1282         oValue = winningValue;
1283 
1284         winningStatus = computeStatus(weights[0], weights[1]);
1285 
1286         // if we are not as good as the baseline (trunk), use the baseline
1287         // TODO: how could baselineStatus be null here??
1288         if (baselineStatus != null && winningStatus.compareTo(baselineStatus) < 0) {
1289             setWinningValue(baselineValue);
1290             annotateTranscript(
1291                     "The optimal value so far with status '%s' would not be as good as the baseline status. "
1292                             + "Therefore, the winning value is '%s' with status '%s'.",
1293                     winningStatus, winningValue, baselineStatus);
1294             winningStatus = baselineStatus;
1295             valuesWithSameVotes.clear();
1296             valuesWithSameVotes.add(winningValue);
1297         } else {
1298             // Declare the final winner
1299             annotateTranscript(
1300                     "The winning value is '%s' with status '%s'.", winningValue, winningStatus);
1301         }
1302     }
1303 
1304     /**
1305      * Make a hash for the vote count of each value in the given sorted list, using the totals field
1306      * of this VoteResolver.
1307      *
1308      * <p>This enables subsequent local adjustment of the effective votes, without change to the
1309      * totals field. Purposes include inheritance and annotation voting.
1310      *
1311      * @param sortedValues the sorted list of values (really a LinkedHashSet, "with predictable
1312      *     iteration order")
1313      * @return the HashMap
1314      */
makeVoteCountMap(Set<T> sortedValues)1315     private HashMap<T, Long> makeVoteCountMap(Set<T> sortedValues) {
1316         HashMap<T, Long> map = new HashMap<>();
1317         for (T value : sortedValues) {
1318             map.put(value, totals.getCount(value));
1319         }
1320         return map;
1321     }
1322 
1323     /**
1324      * Adjust the given sortedValues and voteCount, if necessary, to combine "hard" and "soft"
1325      * votes. Do nothing unless both hard and soft votes are present.
1326      *
1327      * <p>For voting resolution in which inheritance plays a role, "soft" votes for inheritance are
1328      * distinct from "hard" (explicit) votes for the Bailey value. For resolution, these two kinds
1329      * of votes are treated in combination. If that combination is winning, then the final winner
1330      * will be the hard item or the soft item, whichever has more votes, the soft item winning if
1331      * they're tied. Except for the soft item being favored as a tie-breaker, this function should
1332      * be symmetrical in its handling of hard and soft votes.
1333      *
1334      * <p>Note: now that "↑↑↑" is permitted to participate directly in voting resolution, it becomes
1335      * significant that with Collator.getInstance(ULocale.ENGLISH), "↑↑↑" sorts before "AAA" just as
1336      * "AAA" sorts before "BBB".
1337      *
1338      * @param sortedValues the set of sorted values, possibly to be modified
1339      * @param voteCount the hash giving the vote count for each value, possibly to be modified
1340      * @return true if both "hard" and "soft" votes existed and were combined, else false
1341      */
combineInheritanceWithBaileyForVoting( Set<T> sortedValues, HashMap<T, Long> voteCount)1342     private boolean combineInheritanceWithBaileyForVoting(
1343             Set<T> sortedValues, HashMap<T, Long> voteCount) {
1344         if (organizationToValueAndVote.baileySet == false
1345                 || organizationToValueAndVote.baileyValue == null) {
1346             return false;
1347         }
1348         T hardValue = organizationToValueAndVote.baileyValue;
1349         T softValue = (T) CldrUtility.INHERITANCE_MARKER;
1350         /*
1351          * Check containsKey before get, to avoid NullPointerException.
1352          */
1353         if (!voteCount.containsKey(hardValue) || !voteCount.containsKey(softValue)) {
1354             return false;
1355         }
1356         long hardCount = voteCount.get(hardValue);
1357         long softCount = voteCount.get(softValue);
1358         if (hardCount == 0 || softCount == 0) {
1359             return false;
1360         }
1361         reallyCombineInheritanceWithBailey(
1362                 sortedValues, voteCount, hardValue, softValue, hardCount, softCount);
1363         return true;
1364     }
1365 
1366     /**
1367      * Given that both "hard" and "soft" votes exist, combine them
1368      *
1369      * @param sortedValues the set of sorted values, to be modified
1370      * @param voteCount the hash giving the vote count for each value, to be modified
1371      * @param hardValue the bailey value
1372      * @param softValue the inheritance marker
1373      * @param hardCount the number of votes for hardValue
1374      * @param softCount the number of votes for softValue
1375      */
reallyCombineInheritanceWithBailey( Set<T> sortedValues, HashMap<T, Long> voteCount, T hardValue, T softValue, long hardCount, long softCount)1376     private void reallyCombineInheritanceWithBailey(
1377             Set<T> sortedValues,
1378             HashMap<T, Long> voteCount,
1379             T hardValue,
1380             T softValue,
1381             long hardCount,
1382             long softCount) {
1383         final T combValue = (hardCount > softCount) ? hardValue : softValue;
1384         final T skipValue = (hardCount > softCount) ? softValue : hardValue;
1385         final long combinedCount = hardCount + softCount;
1386         voteCount.put(combValue, combinedCount);
1387         voteCount.put(skipValue, 0L);
1388         /*
1389          * Sort again
1390          */
1391         List<T> list = new ArrayList<>(sortedValues);
1392         list.sort(
1393                 (v1, v2) -> {
1394                     long c1 = voteCount.get(v1);
1395                     long c2 = voteCount.get(v2);
1396                     if (c1 != c2) {
1397                         return (c1 < c2) ? 1 : -1; // decreasing numeric order (most votes wins)
1398                     }
1399                     return englishCollator.compare(String.valueOf(v1), String.valueOf(v2));
1400                 });
1401         /*
1402          * Omit skipValue
1403          */
1404         sortedValues.clear();
1405         for (T value : list) {
1406             if (!value.equals(skipValue)) {
1407                 sortedValues.add(value);
1408             }
1409         }
1410     }
1411 
1412     /**
1413      * Adjust the effective votes for bar-joined annotations, and re-sort the array of values to
1414      * reflect the adjusted vote counts.
1415      *
1416      * <p>Note: "Annotations provide names and keywords for Unicode characters, currently focusing
1417      * on emoji." For example, an annotation "happy | joyful" has two components "happy" and
1418      * "joyful". References: http://unicode.org/cldr/charts/32/annotations/index.html
1419      * https://www.unicode.org/reports/tr35/tr35-general.html#Annotations
1420      *
1421      * <p>http://unicode.org/repos/cldr/tags/latest/common/annotations/
1422      *
1423      * @param sortedValues the set of sorted values
1424      * @param voteCount the hash giving the vote count for each value in sortedValues
1425      *     <p>public for unit testing, see TestAnnotationVotes.java
1426      */
adjustAnnotationVoteCounts(Set<T> sortedValues, HashMap<T, Long> voteCount)1427     public void adjustAnnotationVoteCounts(Set<T> sortedValues, HashMap<T, Long> voteCount) {
1428         if (voteCount == null || sortedValues == null) {
1429             return;
1430         }
1431         annotateTranscript("Vote weights are being adjusted due to annotation keywords.");
1432 
1433         // Make compMap map individual components to cumulative vote counts.
1434         HashMap<T, Long> compMap = makeAnnotationComponentMap(sortedValues, voteCount);
1435 
1436         // Save a copy of the "raw" vote count before adjustment, since it's needed by
1437         // promoteSuperiorAnnotationSuperset.
1438         HashMap<T, Long> rawVoteCount = new HashMap<>(voteCount);
1439 
1440         // Calculate new counts for original values, based on components.
1441         calculateNewCountsBasedOnAnnotationComponents(sortedValues, voteCount, compMap);
1442 
1443         // Re-sort sortedValues based on voteCount.
1444         resortValuesBasedOnAdjustedVoteCounts(sortedValues, voteCount);
1445 
1446         // If the set that so far is winning has supersets with superior raw vote count, promote the
1447         // supersets.
1448         promoteSuperiorAnnotationSuperset(sortedValues, voteCount, rawVoteCount);
1449     }
1450 
1451     /**
1452      * Make a hash that maps individual annotation components to cumulative vote counts.
1453      *
1454      * <p>For example, 3 votes for "a|b" and 2 votes for "a|c" makes 5 votes for "a", 3 for "b", and
1455      * 2 for "c".
1456      *
1457      * @param sortedValues the set of sorted values
1458      * @param voteCount the hash giving the vote count for each value in sortedValues
1459      */
makeAnnotationComponentMap( Set<T> sortedValues, HashMap<T, Long> voteCount)1460     private HashMap<T, Long> makeAnnotationComponentMap(
1461             Set<T> sortedValues, HashMap<T, Long> voteCount) {
1462         HashMap<T, Long> compMap = new HashMap<>();
1463         annotateTranscript("- First, components are split up and total votes calculated");
1464         for (T value : sortedValues) {
1465             Long count = voteCount.get(value);
1466             List<T> comps = splitAnnotationIntoComponentsList(value);
1467             for (T comp : comps) {
1468                 if (compMap.containsKey(comp)) {
1469                     compMap.replace(comp, compMap.get(comp) + count);
1470                 } else {
1471                     compMap.put(comp, count);
1472                 }
1473             }
1474         }
1475         if (transcript != null && !DEBUG) {
1476             for (Entry<T, Long> comp : compMap.entrySet()) {
1477                 // TODO: could sort here, or not.
1478                 annotateTranscript(
1479                         "-- component '%s' has weight %d",
1480                         comp.getKey().toString(), comp.getValue());
1481             }
1482         }
1483         return compMap;
1484     }
1485 
1486     /**
1487      * Calculate new counts for original values, based on annotation components.
1488      *
1489      * <p>Find the total votes for each component (e.g., "b" in "b|c"). As the "modified" vote for
1490      * the set, use the geometric mean of the components in the set.
1491      *
1492      * <p>Order the sets by that mean value, then by the smallest number of items in the set, then
1493      * the fallback we always use (alphabetical).
1494      *
1495      * @param sortedValues the set of sorted values
1496      * @param voteCount the hash giving the vote count for each value in sortedValues
1497      * @param compMap the hash that maps individual components to cumulative vote counts
1498      *     <p>See http://unicode.org/cldr/trac/ticket/10973
1499      */
calculateNewCountsBasedOnAnnotationComponents( Set<T> sortedValues, HashMap<T, Long> voteCount, HashMap<T, Long> compMap)1500     private void calculateNewCountsBasedOnAnnotationComponents(
1501             Set<T> sortedValues, HashMap<T, Long> voteCount, HashMap<T, Long> compMap) {
1502         voteCount.clear();
1503         annotateTranscript(
1504                 "- Next, the original values get new counts, each based on the geometric mean of the products of all components.");
1505         for (T value : sortedValues) {
1506             List<T> comps = splitAnnotationIntoComponentsList(value);
1507             double product = 1.0;
1508             for (T comp : comps) {
1509                 product *= compMap.get(comp);
1510             }
1511             /* Rounding to long integer here loses precision. We tried multiplying by ten before rounding,
1512              * to reduce problems with different doubles getting rounded to identical longs, but that had
1513              * unfortunate side-effects involving thresholds (see getRequiredVotes). An eventual improvement
1514              * may be to use doubles or floats for all vote counts.
1515              */
1516             Long newCount = Math.round(Math.pow(product, 1.0 / comps.size())); // geometric mean
1517             voteCount.put(value, newCount);
1518             // Don't annotate these here, annotate them once sorted
1519         }
1520     }
1521 
1522     /**
1523      * Split an annotation into a list of components.
1524      *
1525      * <p>For example, split "happy | joyful" into ["happy", "joyful"].
1526      *
1527      * @param value the value like "happy | joyful"
1528      * @return the list like ["happy", "joyful"]
1529      *     <p>Called by makeAnnotationComponentMap and
1530      *     calculateNewCountsBasedOnAnnotationComponents. Short, but needs encapsulation, should be
1531      *     consistent with similar code in DisplayAndInputProcessor.java.
1532      */
splitAnnotationIntoComponentsList(T value)1533     private List<T> splitAnnotationIntoComponentsList(T value) {
1534         return (List<T>) DisplayAndInputProcessor.SPLIT_BAR.splitToList((CharSequence) value);
1535     }
1536 
1537     /**
1538      * Re-sort the set of values to match the adjusted vote counts based on annotation components.
1539      *
1540      * <p>Resolve ties using ULocale.ENGLISH collation for consistency with votesThenUcaCollator.
1541      *
1542      * @param sortedValues the set of sorted values, maybe no longer sorted the way we want
1543      * @param voteCount the hash giving the adjusted vote count for each value in sortedValues
1544      */
resortValuesBasedOnAdjustedVoteCounts( Set<T> sortedValues, HashMap<T, Long> voteCount)1545     private void resortValuesBasedOnAdjustedVoteCounts(
1546             Set<T> sortedValues, HashMap<T, Long> voteCount) {
1547         List<T> list = new ArrayList<>(sortedValues);
1548         list.sort(
1549                 (v1, v2) -> {
1550                     long c1 = voteCount.get(v1), c2 = voteCount.get(v2);
1551                     if (c1 != c2) {
1552                         return (c1 < c2) ? 1 : -1; // decreasing numeric order (most votes wins)
1553                     }
1554                     int size1 = splitAnnotationIntoComponentsList(v1).size();
1555                     int size2 = splitAnnotationIntoComponentsList(v2).size();
1556                     if (size1 != size2) {
1557                         return (size1 < size2)
1558                                 ? -1
1559                                 : 1; // increasing order of size (smallest set wins)
1560                     }
1561                     return englishCollator.compare(String.valueOf(v1), String.valueOf(v2));
1562                 });
1563         sortedValues.clear();
1564         sortedValues.addAll(list);
1565     }
1566 
1567     /**
1568      * For annotation votes, if the set that so far is winning has one or more supersets with
1569      * "superior" (see below) raw vote count, promote those supersets to become the new winner, and
1570      * also the new second place if there are two or more superior supersets.
1571      *
1572      * <p>That is, after finding the set X with the largest geometric mean, check whether there are
1573      * any supersets with "superior" raw votes, and that don't exceed the width limit. If so,
1574      * promote Y, the one of those supersets with the most raw votes (using the normal tie breaker),
1575      * to be the winning set.
1576      *
1577      * <p>"Superior" here means that rawVote(Y) ≥ rawVote(X) + 2, where the value 2 (see
1578      * requiredGap) is for the purpose of requiring at least one non-guest vote.
1579      *
1580      * <p>If any other "superior" supersets exist, promote to second place the one with the next
1581      * most raw votes.
1582      *
1583      * <p>Accomplish promotion by increasing vote counts in the voteCount hash.
1584      *
1585      * @param sortedValues the set of sorted values
1586      * @param voteCount the vote count for each value in sortedValues AFTER
1587      *     calculateNewCountsBasedOnAnnotationComponents; it gets modified if superior subsets exist
1588      * @param rawVoteCount the vote count for each value in sortedValues BEFORE
1589      *     calculateNewCountsBasedOnAnnotationComponents; rawVoteCount is not changed by this
1590      *     function
1591      *     <p>Reference: https://unicode.org/cldr/trac/ticket/10973
1592      */
promoteSuperiorAnnotationSuperset( Set<T> sortedValues, HashMap<T, Long> voteCount, HashMap<T, Long> rawVoteCount)1593     private void promoteSuperiorAnnotationSuperset(
1594             Set<T> sortedValues, HashMap<T, Long> voteCount, HashMap<T, Long> rawVoteCount) {
1595         final long requiredGap = 2;
1596         T oldWinner = null;
1597         long oldWinnerRawCount = 0;
1598         LinkedHashSet<T> oldWinnerComps = null;
1599         LinkedHashSet<T> superiorSupersets = null;
1600         for (T value : sortedValues) {
1601             // Annotate the means here
1602             final long rawCount = rawVoteCount.get(value);
1603             final long newCount = voteCount.get(value);
1604             if (rawCount != newCount) {
1605                 annotateTranscript("-- Value '%s' has updated value '%d'", value, newCount);
1606             }
1607             if (oldWinner == null) {
1608                 oldWinner = value;
1609                 oldWinnerRawCount = rawVoteCount.get(value);
1610                 oldWinnerComps = new LinkedHashSet<>(splitAnnotationIntoComponentsList(value));
1611             } else {
1612                 Set<T> comps = new LinkedHashSet<>(splitAnnotationIntoComponentsList(value));
1613                 if (comps.size() <= CheckWidths.MAX_COMPONENTS_PER_ANNOTATION
1614                         && comps.containsAll(oldWinnerComps)
1615                         && rawVoteCount.get(value) >= oldWinnerRawCount + requiredGap) {
1616                     if (superiorSupersets == null) {
1617                         superiorSupersets = new LinkedHashSet<>();
1618                     }
1619                     superiorSupersets.add(value);
1620                 }
1621             }
1622         }
1623         if (superiorSupersets != null) {
1624             // Sort the supersets by raw vote count, then make their adjusted vote counts higher
1625             // than the old winner's.
1626             resortValuesBasedOnAdjustedVoteCounts(superiorSupersets, rawVoteCount);
1627             T newWinner = null, newSecond; // only adjust votes for first and second place
1628             for (T value : superiorSupersets) {
1629                 if (newWinner == null) {
1630                     newWinner = value;
1631                     long newWinnerCount = voteCount.get(oldWinner) + 2;
1632                     annotateTranscript(
1633                             "- Optimal value (O) '%s' was promoted to value '%d' due to having a superior raw vote count",
1634                             newWinner, newWinnerCount);
1635                     voteCount.put(newWinner, newWinnerCount); // more than oldWinner and newSecond
1636                 } else {
1637                     newSecond = value;
1638                     long newSecondCount = voteCount.get(oldWinner) + 1;
1639                     annotateTranscript(
1640                             "- Next value (N) '%s' was promoted to value '%d' due to having a superior raw vote count",
1641                             newSecond, newSecondCount);
1642                     voteCount.put(
1643                             newSecond, newSecondCount); // more than oldWinner, less than newWinner
1644                     break;
1645                 }
1646             }
1647             resortValuesBasedOnAdjustedVoteCounts(sortedValues, voteCount);
1648         }
1649     }
1650 
1651     /**
1652      * Given a nonempty list of sorted values, and a hash with their vote counts, set these members
1653      * of this VoteResolver: winningValue, nValue, valuesWithSameVotes (which is empty when this
1654      * function is called).
1655      *
1656      * @param sortedValues the set of sorted values
1657      * @param voteCount the hash giving the vote count for each value
1658      * @return an array of two longs, the weights for the best and next-best values.
1659      */
setBestNextAndSameVoteValues(Set<T> sortedValues, HashMap<T, Long> voteCount)1660     private long[] setBestNextAndSameVoteValues(Set<T> sortedValues, HashMap<T, Long> voteCount) {
1661 
1662         long[] weightArray = new long[2];
1663         nValue = null;
1664 
1665         /*
1666          * Loop through the sorted values, at least the first (best) for winningValue,
1667          * and the second (if any) for nValue (else nValue stays null),
1668          * and subsequent values that have as many votes as the first,
1669          * to add to valuesWithSameVotes.
1670          */
1671         int i = -1;
1672         Iterator<T> iterator = sortedValues.iterator();
1673         for (T value : sortedValues) {
1674             ++i;
1675             long valueWeight = voteCount.get(value);
1676             if (i == 0) {
1677                 setWinningValue(value);
1678                 weightArray[0] = valueWeight;
1679                 valuesWithSameVotes.add(value);
1680                 annotateTranscript(
1681                         "The optimal value (O) is '%s', with a weight of %d",
1682                         winningValue, valueWeight);
1683                 if (sortedValues.size() == 1) {
1684                     annotateTranscript("- No other values received votes."); // uncontested
1685                 }
1686             } else {
1687                 if (i == 1) {
1688                     // get the next item if there is one
1689                     if (iterator.hasNext()) {
1690                         nValue = value;
1691                         weightArray[1] = valueWeight;
1692                         annotateNextBestValue(weightArray[0], weightArray[1], winningValue, nValue);
1693                     }
1694                 }
1695                 if (valueWeight == weightArray[0]) {
1696                     valuesWithSameVotes.add(value);
1697                 } else {
1698                     break;
1699                 }
1700             }
1701         }
1702         return weightArray;
1703     }
1704 
1705     /**
1706      * Compute the status for the winning value. See: https://cldr.unicode.org/index/process
1707      *
1708      * @param O the weight (vote count) for the best value
1709      * @param N the weight (vote count) for the next-best value
1710      * @return the Status
1711      */
computeStatus(long O, long N)1712     private Status computeStatus(long O, long N) {
1713         if (O > N) {
1714             final int requiredVotes = getRequiredVotes();
1715             if (O >= requiredVotes) {
1716                 final Status computedStatus = Status.approved;
1717                 annotateTranscript("O>N, and O>%d: %s", requiredVotes, computedStatus);
1718                 return computedStatus;
1719             }
1720             if (O >= 4 && Status.contributed.compareTo(baselineStatus) > 0) {
1721                 final Status computedStatus = Status.contributed;
1722                 annotateTranscript(
1723                         "O>=4, and oldstatus (%s)<contributed: %s", baselineStatus, computedStatus);
1724                 return computedStatus;
1725             }
1726             if (O >= 2) {
1727                 final int G = organizationToValueAndVote.getOrgCount(winningValue);
1728                 if (G >= 2) {
1729                     final Status computedStatus = Status.contributed;
1730                     annotateTranscript("O>=2, and G (%d)>=2: %s", G, computedStatus);
1731                     return computedStatus;
1732                 }
1733             }
1734         }
1735         if (O >= N) {
1736             if (O >= 2) {
1737                 final Status computedStatus = Status.provisional;
1738                 annotateTranscript("O>=N and O>=2: %s", computedStatus);
1739                 return computedStatus;
1740             }
1741         }
1742 
1743         // otherwise: unconfirmed
1744         final Status computedStatus = Status.unconfirmed;
1745         annotateTranscript("O was not high enough: %s", computedStatus);
1746         return computedStatus;
1747     }
1748 
getPossibleWinningStatus()1749     private Status getPossibleWinningStatus() {
1750         if (!resolved) {
1751             resolveVotes();
1752         }
1753         Status possibleStatus = computeStatus(organizationToValueAndVote.getBestPossibleVote(), 0);
1754         return possibleStatus.compareTo(winningStatus) > 0 ? possibleStatus : winningStatus;
1755     }
1756 
1757     /**
1758      * If the winning item is not approved, and if all the people who voted had voted for the
1759      * winning item, would it have made contributed or approved?
1760      *
1761      * @return
1762      */
isDisputed()1763     public boolean isDisputed() {
1764         if (!resolved) {
1765             resolveVotes();
1766         }
1767         if (winningStatus.compareTo(VoteResolver.Status.contributed) >= 0) {
1768             return false;
1769         }
1770         VoteResolver.Status possibleStatus = getPossibleWinningStatus();
1771         return possibleStatus.compareTo(Status.contributed) >= 0;
1772     }
1773 
getWinningStatus()1774     public Status getWinningStatus() {
1775         if (!resolved) {
1776             resolveVotes();
1777         }
1778         return winningStatus;
1779     }
1780 
1781     /**
1782      * Returns O Value as described in http://cldr.unicode.org/index/process#TOC-Voting-Process. Not
1783      * always the same as the Winning Value.
1784      *
1785      * @return
1786      */
getOValue()1787     private T getOValue() {
1788         if (!resolved) {
1789             resolveVotes();
1790         }
1791         return oValue;
1792     }
1793 
1794     /**
1795      * Returns N Value as described in http://cldr.unicode.org/index/process#TOC-Voting-Process. Not
1796      * always the same as the Winning Value.
1797      *
1798      * @return
1799      */
getNValue()1800     private T getNValue() {
1801         if (!resolved) {
1802             resolveVotes();
1803         }
1804         return nValue;
1805     }
1806 
1807     /**
1808      * Returns Winning Value as described in
1809      * http://cldr.unicode.org/index/process#TOC-Voting-Process. Not always the same as the O Value.
1810      *
1811      * @return
1812      */
getWinningValue()1813     public T getWinningValue() {
1814         if (!resolved) {
1815             resolveVotes();
1816         }
1817         return winningValue;
1818     }
1819 
1820     /**
1821      * Set the Winning Value; if the given value matches Bailey, change it to INHERITANCE_MARKER
1822      *
1823      * @param value the value to set (prior to changeBaileyToInheritance)
1824      */
setWinningValue(T value)1825     private void setWinningValue(T value) {
1826         if (DROP_HARD_INHERITANCE) {
1827             winningValue = changeBaileyToInheritance(value);
1828         } else {
1829             winningValue = value;
1830         }
1831     }
1832 
getValuesWithSameVotes()1833     public List<T> getValuesWithSameVotes() {
1834         if (!resolved) {
1835             resolveVotes();
1836         }
1837         return new ArrayList<>(valuesWithSameVotes);
1838     }
1839 
getConflictedOrganizations()1840     public EnumSet<Organization> getConflictedOrganizations() {
1841         if (!resolved) {
1842             resolveVotes();
1843         }
1844         return conflictedOrganizations;
1845     }
1846 
1847     /**
1848      * What value did this organization vote for?
1849      *
1850      * @param org
1851      * @return
1852      */
getOrgVote(Organization org)1853     public T getOrgVote(Organization org) {
1854         return organizationToValueAndVote.getOrgVote(org);
1855     }
1856 
getOrgToVotes(Organization org)1857     public Map<T, Long> getOrgToVotes(Organization org) {
1858         return organizationToValueAndVote.getOrgToVotes(org);
1859     }
1860 
getNameTime()1861     public Map<String, Long> getNameTime() {
1862         return organizationToValueAndVote.getNameTime();
1863     }
1864 
1865     /**
1866      * Get a String representation of this VoteResolver. This is sent to the client as
1867      * "voteResolver.raw" and is used only for debugging.
1868      *
1869      * <p>Compare SurveyAjax.JSONWriter.wrap(VoteResolver<String>) which creates the data actually
1870      * used by the client.
1871      */
1872     @Override
toString()1873     public String toString() {
1874         return "{"
1875                 + "bailey: "
1876                 + (organizationToValueAndVote.baileySet
1877                         ? ("“" + organizationToValueAndVote.baileyValue + "” ")
1878                         : "none ")
1879                 + "baseline: {"
1880                 + baselineValue
1881                 + ", "
1882                 + baselineStatus
1883                 + "}, "
1884                 + organizationToValueAndVote
1885                 + ", sameVotes: "
1886                 + valuesWithSameVotes
1887                 + ", O: "
1888                 + getOValue()
1889                 + ", N: "
1890                 + getNValue()
1891                 + ", totals: "
1892                 + totals
1893                 + ", winning: {"
1894                 + getWinningValue()
1895                 + ", "
1896                 + getWinningStatus()
1897                 + "}"
1898                 + "}";
1899     }
1900 
getIdToPath(String fileName)1901     public static Map<Integer, String> getIdToPath(String fileName) {
1902         XPathTableHandler myHandler = new XPathTableHandler();
1903         XMLFileReader xfr = new XMLFileReader().setHandler(myHandler);
1904         xfr.read(fileName, XMLFileReader.CONTENT_HANDLER | XMLFileReader.ERROR_HANDLER, false);
1905         return myHandler.pathIdToPath;
1906     }
1907 
1908     static class XPathTableHandler extends XMLFileReader.SimpleHandler {
1909         Matcher matcher = Pattern.compile("id=\"([0-9]+)\"").matcher("");
1910         Map<Integer, String> pathIdToPath = new HashMap<>();
1911 
1912         @Override
handlePathValue(String path, String value)1913         public void handlePathValue(String path, String value) {
1914             // <xpathTable host="tintin.local" date="Tue Apr 29 14:34:32 PDT 2008" count="18266" >
1915             // <xpath
1916             // id="1">//ldml/dates/calendars/calendar[@type="gregorian"]/dateFormats/dateFormatLength[@type="short"]/dateFormat[@type="standard"]/pattern[@type="standard"]</xpath>
1917             if (!matcher.reset(path).find()) {
1918                 throw new IllegalArgumentException("Unknown path " + path);
1919             }
1920             pathIdToPath.put(Integer.parseInt(matcher.group(1)), value);
1921         }
1922     }
1923 
getBaseToAlternateToInfo( String fileName, VoterInfoList vil)1924     public static Map<Integer, Map<Integer, CandidateInfo>> getBaseToAlternateToInfo(
1925             String fileName, VoterInfoList vil) {
1926         try {
1927             VotesHandler myHandler = new VotesHandler(vil);
1928             XMLFileReader xfr = new XMLFileReader().setHandler(myHandler);
1929             xfr.read(fileName, XMLFileReader.CONTENT_HANDLER | XMLFileReader.ERROR_HANDLER, false);
1930             return myHandler.basepathToInfo;
1931         } catch (Exception e) {
1932             throw new IllegalArgumentException("Can't handle file: " + fileName, e);
1933         }
1934     }
1935 
1936     public enum Type {
1937         proposal,
1938         optimal
1939     }
1940 
1941     public static class CandidateInfo {
1942         public Status oldStatus;
1943         public Type surveyType;
1944         public Status surveyStatus;
1945         public Set<Integer> voters = new TreeSet<>();
1946         private final VoterInfoList voterInfoList;
1947 
CandidateInfo(VoterInfoList vil)1948         CandidateInfo(VoterInfoList vil) {
1949             this.voterInfoList = vil;
1950         }
1951 
1952         @Override
toString()1953         public String toString() {
1954             StringBuilder voterString = new StringBuilder("{");
1955             for (int voter : voters) {
1956                 VoterInfo voterInfo = voterInfoList.get(voter);
1957                 if (voterString.length() > 1) {
1958                     voterString.append(" ");
1959                 }
1960                 voterString.append(voter);
1961                 if (voterInfo != null) {
1962                     voterString.append(" ").append(voterInfo);
1963                 }
1964             }
1965             voterString.append("}");
1966             return "{oldStatus: "
1967                     + oldStatus
1968                     + ", surveyType: "
1969                     + surveyType
1970                     + ", surveyStatus: "
1971                     + surveyStatus
1972                     + ", voters: "
1973                     + voterString
1974                     + "};";
1975         }
1976     }
1977 
1978     /*
1979      * <locale-votes host="tintin.local" date="Tue Apr 29 14:34:32 PDT 2008"
1980      * oldVersion="1.5.1" currentVersion="1.6" resolved="false" locale="zu">
1981      * <row baseXpath="1">
1982      * <item xpath="2855" type="proposal" id="1" status="unconfirmed">
1983      * <old status="unconfirmed"/>
1984      * </item>
1985      * <item xpath="1" type="optimal" id="56810" status="confirmed">
1986      * <vote user="210"/>
1987      * </item>
1988      * </row>
1989      * ...
1990      * A base path has a set of candidates. Each candidate has various items of information.
1991      */
1992     static class VotesHandler extends XMLFileReader.SimpleHandler {
1993         private final VoterInfoList voterInfoList;
1994 
VotesHandler(VoterInfoList vil)1995         VotesHandler(VoterInfoList vil) {
1996             this.voterInfoList = vil;
1997         }
1998 
1999         Map<Integer, Map<Integer, CandidateInfo>> basepathToInfo = new TreeMap<>();
2000 
2001         @Override
handlePathValue(String path, String value)2002         public void handlePathValue(String path, String value) {
2003             try {
2004                 XPathParts parts = XPathParts.getFrozenInstance(path);
2005                 if (parts.size() < 2) {
2006                     // empty data
2007                     return;
2008                 }
2009                 int baseId = Integer.parseInt(parts.getAttributeValue(1, "baseXpath"));
2010                 Map<Integer, CandidateInfo> info =
2011                         basepathToInfo.computeIfAbsent(baseId, k -> new TreeMap<>());
2012                 int itemId = Integer.parseInt(parts.getAttributeValue(2, "xpath"));
2013                 CandidateInfo candidateInfo = info.get(itemId);
2014                 if (candidateInfo == null) {
2015                     info.put(itemId, candidateInfo = new CandidateInfo(voterInfoList));
2016                     candidateInfo.surveyType = Type.valueOf(parts.getAttributeValue(2, "type"));
2017                     candidateInfo.surveyStatus =
2018                             Status.valueOf(
2019                                     fixBogusDraftStatusValues(
2020                                             parts.getAttributeValue(2, "status")));
2021                     // ignore id
2022                 }
2023                 if (parts.size() < 4) {
2024                     return;
2025                 }
2026                 final String lastElement = parts.getElement(3);
2027                 if (lastElement.equals("old")) {
2028                     candidateInfo.oldStatus =
2029                             Status.valueOf(
2030                                     fixBogusDraftStatusValues(
2031                                             parts.getAttributeValue(3, "status")));
2032                 } else if (lastElement.equals("vote")) {
2033                     candidateInfo.voters.add(Integer.parseInt(parts.getAttributeValue(3, "user")));
2034                 } else {
2035                     throw new IllegalArgumentException("unknown option: " + path);
2036                 }
2037             } catch (Exception e) {
2038                 throw new IllegalArgumentException("Can't handle path: " + path, e);
2039             }
2040         }
2041     }
2042 
2043     public static class UnknownVoterException extends RuntimeException {
2044         private static final long serialVersionUID = 3430877787936678609L;
2045         int voter;
2046 
UnknownVoterException(int voter)2047         public UnknownVoterException(int voter) {
2048             this.voter = voter;
2049         }
2050 
2051         @Override
toString()2052         public String toString() {
2053             return "Unknown voter: " + voter;
2054         }
2055     }
2056 
fixBogusDraftStatusValues(String attributeValue)2057     private static String fixBogusDraftStatusValues(String attributeValue) {
2058         if (attributeValue == null) return "approved";
2059         if ("confirmed".equals(attributeValue)) return "approved";
2060         if ("true".equals(attributeValue)) return "unconfirmed";
2061         if ("unknown".equals(attributeValue)) return "unconfirmed";
2062         return attributeValue;
2063     }
2064 
2065     /*
2066      * TODO: either delete this or explain why it's needed
2067      */
size()2068     public int size() {
2069         return values.size();
2070     }
2071 
2072     /**
2073      * Returns a map from value to resolved vote count, in descending order. If the winning item is
2074      * not there, insert at the front. If the baseline (trunk) item is not there, insert at the end.
2075      *
2076      * <p>This map includes intra-org disputes.
2077      *
2078      * @return the map
2079      */
getResolvedVoteCountsIncludingIntraOrgDisputes()2080     public Map<T, Long> getResolvedVoteCountsIncludingIntraOrgDisputes() {
2081         if (!resolved) {
2082             resolveVotes();
2083         }
2084         Map<T, Long> result = new LinkedHashMap<>();
2085         if (winningValue != null && !totals.containsKey(winningValue)) {
2086             result.put(winningValue, 0L);
2087         }
2088         for (T value : totals.getKeysetSortedByCount(false, votesThenUcaCollator)) {
2089             result.put(value, totals.get(value));
2090         }
2091         if (baselineValue != null && !totals.containsKey(baselineValue)) {
2092             result.put(baselineValue, 0L);
2093         }
2094         for (T value :
2095                 organizationToValueAndVote.allVotesIncludingIntraOrgDispute.getMap().keySet()) {
2096             if (!result.containsKey(value)) {
2097                 result.put(value, 0L);
2098             }
2099         }
2100         if (DEBUG) {
2101             System.out.println("getResolvedVoteCountsIncludingIntraOrgDisputes :" + result);
2102         }
2103         return result;
2104     }
2105 
getStatusForOrganization(Organization orgOfUser)2106     public VoteStatus getStatusForOrganization(Organization orgOfUser) {
2107         if (!resolved) {
2108             resolveVotes();
2109         }
2110         if (Status.provisional.compareTo(winningStatus) >= 0) {
2111             // If the value is provisional, it needs more votes.
2112             return VoteStatus.provisionalOrWorse;
2113         }
2114         T orgVote = organizationToValueAndVote.getOrgVoteRaw(orgOfUser);
2115         if (!equalsOrgVote(winningValue, orgVote)) {
2116             // We voted and lost
2117             return VoteStatus.losing;
2118         }
2119         final int itemsWithVotes =
2120                 DROP_HARD_INHERITANCE ? totals.size() : countDistinctValuesWithVotes();
2121         if (itemsWithVotes > 1) {
2122             // If there are votes for two "distinct" items, we should look at them.
2123             return VoteStatus.disputed;
2124         }
2125         final T singleVotedItem = getSingleVotedItem();
2126         if (!equalsOrgVote(winningValue, singleVotedItem)) {
2127             // If someone voted but didn't win
2128             return VoteStatus.disputed;
2129         }
2130         if (itemsWithVotes == 0) {
2131             // The value is ok, but we capture that there are no votes, for revealing items like
2132             // unsync'ed
2133             return VoteStatus.ok_novotes;
2134         } else {
2135             // We voted, we won, value is approved, no disputes, have votes
2136             return VoteStatus.ok;
2137         }
2138     }
2139 
2140     /**
2141      * Returns value of voted item, in case there is exactly 1.
2142      *
2143      * @return
2144      */
getSingleVotedItem()2145     private T getSingleVotedItem() {
2146         return totals.size() != 1 ? null : totals.iterator().next();
2147     }
2148 
2149     /**
2150      * Should these two values be treated as equivalent for getStatusForOrganization?
2151      *
2152      * @param value
2153      * @param orgVote
2154      * @return true if they are equivalent, false if they are distinct
2155      */
equalsOrgVote(T value, T orgVote)2156     private boolean equalsOrgVote(T value, T orgVote) {
2157         return orgVote == null
2158                 || orgVote.equals(value)
2159                 || (CldrUtility.INHERITANCE_MARKER.equals(value)
2160                         && orgVote.equals(organizationToValueAndVote.baileyValue))
2161                 || (CldrUtility.INHERITANCE_MARKER.equals(orgVote)
2162                         && value.equals(organizationToValueAndVote.baileyValue));
2163     }
2164 
2165     /**
2166      * Count the distinct values that have votes.
2167      *
2168      * <p>For this purpose, if there are both votes for inheritance and votes for the specific value
2169      * matching the inherited (bailey) value, they are not "distinct": count them as a single value.
2170      *
2171      * @return the number of distinct values
2172      */
countDistinctValuesWithVotes()2173     private int countDistinctValuesWithVotes() {
2174         if (!resolved) { // must be resolved for bothInheritanceAndBaileyHadVotes
2175             throw new RuntimeException("countDistinctValuesWithVotes !resolved");
2176         }
2177         int count = organizationToValueAndVote.allVotesIncludingIntraOrgDispute.size();
2178         if (count > 1 && bothInheritanceAndBaileyHadVotes) {
2179             return count - 1; // prevent showing as "disputed" in dashboard
2180         }
2181         return count;
2182     }
2183 
2184     /**
2185      * Should this VoteResolver use keyword annotation voting?
2186      *
2187      * <p>Apply special voting method adjustAnnotationVoteCounts only to certain keyword annotations
2188      * that can have bar-separated values like "happy | joyful".
2189      *
2190      * <p>The paths for keyword annotations start with "//ldml/annotations/annotation" and do NOT
2191      * include Emoji.TYPE_TTS. Both name paths (cf. namePath, getNamePaths) and keyword paths (cf.
2192      * keywordPath, getKeywordPaths) have "//ldml/annotations/annotation". Name paths include
2193      * Emoji.TYPE_TTS, and keyword paths don't. Special voting is only for keyword paths, not for
2194      * name paths. Compare path dependencies in DisplayAndInputProcessor.java. See also
2195      * VoteResolver.splitAnnotationIntoComponentsList.
2196      *
2197      * @return true or false
2198      */
isUsingKeywordAnnotationVoting()2199     private boolean isUsingKeywordAnnotationVoting() {
2200         if (pathHeader == null) {
2201             return false; // this happens in some tests
2202         }
2203         final String path = pathHeader.getOriginalPath();
2204         return AnnotationUtil.pathIsAnnotation(path) && !path.contains(Emoji.TYPE_TTS);
2205     }
2206 
2207     /**
2208      * Is the value locked for this locale+path?
2209      *
2210      * @return true or false
2211      */
isValueLocked()2212     public boolean isValueLocked() {
2213         return valueIsLocked;
2214     }
2215 
2216     /**
2217      * Can a user who makes a losing vote flag the locale+path? I.e., is the locale+path locked
2218      * and/or does it require HIGH_BAR votes?
2219      *
2220      * @return true or false
2221      */
canFlagOnLosing()2222     public boolean canFlagOnLosing() {
2223         return valueIsLocked || (getRequiredVotes() == HIGH_BAR);
2224     }
2225 
2226     /**
2227      * Calculate VoteResolver.Status
2228      *
2229      * @param baselineFile the 'baseline' file to use
2230      * @param path path the xpath
2231      * @return the Status
2232      */
calculateStatus(CLDRFile baselineFile, String path)2233     public static Status calculateStatus(CLDRFile baselineFile, String path) {
2234         String fullXPath = baselineFile.getFullXPath(path);
2235         if (fullXPath == null) {
2236             fullXPath = path;
2237         }
2238         final XPathParts xpp = XPathParts.getFrozenInstance(fullXPath);
2239         final String draft = xpp.getAttributeValue(-1, LDMLConstants.DRAFT);
2240         Status status = draft == null ? Status.approved : VoteResolver.Status.fromString(draft);
2241 
2242         /*
2243          * Reset to missing if the value is inherited from root or code-fallback, unless the XML actually
2244          * contains INHERITANCE_MARKER. Pass false for skipInheritanceMarker so that status will not be
2245          * missing for explicit INHERITANCE_MARKER.
2246          */
2247         final String srcid =
2248                 baselineFile.getSourceLocaleIdExtended(
2249                         path, null, false /* skipInheritanceMarker */);
2250         if (srcid.equals(XMLSource.CODE_FALLBACK_ID)) {
2251             status = Status.missing;
2252         } else if (srcid.equals("root")) {
2253             if (!srcid.equals(baselineFile.getLocaleID())) {
2254                 status = Status.missing;
2255             }
2256         }
2257         return status;
2258     }
2259     /**
2260      * Get the possibly modified value. If value matches the bailey value or inheritance marker,
2261      * possibly change it from bailey value to inheritance marker, or vice-versa, as needed to meet
2262      * these requirements: 1. If the path changes when getting bailey, then we are inheriting
2263      * sideways. We need to use a hard value. 2. If the value is different from the bailey value,
2264      * can't use inheritance; we need a hard value. 3. Otherwise we use inheritance marker.
2265      *
2266      * <p>These requirements are pragmatic, to work around limitations of the current inheritance
2267      * algorithm, which is hyper-sensitive to the distinction between inheritance marker and bailey,
2268      * which, depending on that distinction, unintentionally tends to change lateral inheritance to
2269      * vertical inheritance, or vice-versa.
2270      *
2271      * <p>This method has consequences affecting vote resolution. For example, assume
2272      * DROP_HARD_INHERITANCE is true. If a user votes for what is currently the inherited value, and
2273      * these requirements call for using inheritance marker, then their vote is stored as
2274      * inheritance marker in the db; if the parent value then changes (even during same release
2275      * cycle), the vote is still a vote for inheritance -- that is how soft inheritence has long
2276      * been intended to work. In the cases where this method returns the hard value matching bailey,
2277      * the user's vote is stored in the db as that hard value; if the parent value then changes, the
2278      * user's vote does not change -- this differs from what we'd like ideally (which is for all
2279      * inh. votes to be "soft"). If and when the inheritance algorithm changes to reduce or
2280      * eliminate the problematic aspects of the hard/soft distinction, this method might no longer
2281      * be needed.
2282      *
2283      * <p>Reference: https://unicode-org.atlassian.net/browse/CLDR-16560
2284      *
2285      * @param path the path
2286      * @param value the input value
2287      * @param cldrFile the CLDRFile for determining inheritance
2288      * @return the possibly modified value
2289      */
reviseInheritanceAsNeeded(String path, String value, CLDRFile cldrFile)2290     public static String reviseInheritanceAsNeeded(String path, String value, CLDRFile cldrFile) {
2291         if (!DROP_HARD_INHERITANCE) {
2292             return value;
2293         }
2294         if (!cldrFile.isResolved()) {
2295             throw new InternalCldrException("must be resolved");
2296         }
2297         Output<String> pathWhereFound = new Output<>();
2298         Output<String> localeWhereFound = new Output<>();
2299         String baileyValue = cldrFile.getBaileyValue(path, pathWhereFound, localeWhereFound);
2300         if (baileyValue != null
2301                 && (CldrUtility.INHERITANCE_MARKER.equals(value) || baileyValue.equals(value))) {
2302             // TODO: decide whether to continue treating GlossonymConstructor.PSEUDO_PATH
2303             // (constructed values) as lateral inheritance. This method originally did not
2304             // take constructed values into account, so it implicitly treated constructed
2305             // values as laterally inherited, given that pathWhereFound doesn't equal path.
2306             // This original behavior corresponds to CONSTRUCTED_PSEUDO_PATH_NOT_LATERAL = false.
2307             // Reference: https://unicode-org.atlassian.net/browse/CLDR-16372
2308             final boolean CONSTRUCTED_PSEUDO_PATH_NOT_LATERAL = false;
2309             value =
2310                     (pathWhereFound.value.equals(path)
2311                                     || (CONSTRUCTED_PSEUDO_PATH_NOT_LATERAL
2312                                             && GlossonymConstructor.PSEUDO_PATH.equals(
2313                                                     pathWhereFound.value)))
2314                             ? CldrUtility.INHERITANCE_MARKER
2315                             : baileyValue;
2316         }
2317         return value;
2318     }
2319 }
2320