1 package org.unicode.cldr.unittest; 2 3 import com.google.common.collect.ImmutableMap; 4 import com.google.common.collect.Ordering; 5 import com.ibm.icu.impl.Relation; 6 import com.ibm.icu.impl.UnicodeMap; 7 import com.ibm.icu.impl.Utility; 8 import com.ibm.icu.lang.UCharacter; 9 import com.ibm.icu.lang.UProperty; 10 import com.ibm.icu.text.Collator; 11 import com.ibm.icu.text.UnicodeSet; 12 import com.ibm.icu.util.ULocale; 13 import java.io.StringWriter; 14 import java.text.NumberFormat; 15 import java.util.ArrayList; 16 import java.util.Arrays; 17 import java.util.Collections; 18 import java.util.Comparator; 19 import java.util.HashMap; 20 import java.util.HashSet; 21 import java.util.LinkedHashSet; 22 import java.util.List; 23 import java.util.Map; 24 import java.util.Map.Entry; 25 import java.util.Random; 26 import java.util.Set; 27 import java.util.TreeMap; 28 import java.util.TreeSet; 29 import java.util.regex.Matcher; 30 import org.unicode.cldr.test.SubmissionLocales; 31 import org.unicode.cldr.tool.ConvertLanguageData.InverseComparator; 32 import org.unicode.cldr.util.*; 33 import org.unicode.cldr.util.PathHeader.PageId; 34 import org.unicode.cldr.util.SupplementalDataInfo.PluralInfo.Count; 35 import org.unicode.cldr.util.VettingViewer.MissingStatus; 36 import org.unicode.cldr.util.VoteResolver.Level; 37 import org.unicode.cldr.util.VoteResolver.Status; 38 import org.unicode.cldr.util.VoteResolver.VoteStatus; 39 import org.unicode.cldr.util.VoteResolver.VoterInfo; 40 import org.unicode.cldr.util.props.ICUPropertyFactory; 41 42 public class TestHelper extends TestFmwkPlus { 43 public static boolean DEBUG = true; 44 45 private static final UnicodeSet DIGITS = new UnicodeSet("[0-9]"); 46 static CLDRConfig testInfo = CLDRConfig.getInstance(); 47 private static final SupplementalDataInfo SUPPLEMENTAL_DATA_INFO = 48 testInfo.getSupplementalDataInfo(); 49 private static final int STRING_ID_TEST_COUNT = 1024 * 16; 50 51 final int ONE_VETTER_BAR = Level.vetter.getVotes(Organization.unaffiliated); 52 final int TWO_VETTER_BAR = VoteResolver.LOWER_BAR; 53 main(String[] args)54 public static void main(String[] args) { 55 new TestHelper().run(args); 56 } 57 TestPluralSamples()58 public void TestPluralSamples() { 59 checkPluralSamples("en"); 60 checkPluralSamples("cs"); 61 checkPluralSamples("ar"); 62 } 63 checkPluralSamples(String locale)64 private void checkPluralSamples(String locale) { 65 PluralSamples pluralSamples = PluralSamples.getInstance(locale); 66 Set<Count> counts = SUPPLEMENTAL_DATA_INFO.getPlurals(locale).getCounts(); 67 for (int i = 1; i < 5; ++i) { 68 Map<Count, Double> samplesForDigits = pluralSamples.getSamples(i); 69 if (!counts.containsAll(samplesForDigits.keySet())) { 70 errln( 71 locale 72 + ": mismatch in samples, expected " 73 + counts 74 + ", got: " 75 + samplesForDigits); 76 } else if (samplesForDigits.size() == 0) { 77 errln(locale + ": no sample for digit " + i); 78 } else { 79 logln(locale + " plural samples: " + samplesForDigits); 80 } 81 } 82 } 83 84 public static class StringIdException extends RuntimeException { 85 private static final long serialVersionUID = 1L; 86 } 87 88 public class StringIdThread extends Thread { 89 private final Random r = new Random(); 90 private final int id; 91 StringIdThread(int i)92 StringIdThread(int i) { 93 super("Demo Thread"); 94 id = i; 95 } 96 97 @Override run()98 public void run() { 99 logln("Starting thread: " + this); 100 for (int i = 0; i < STRING_ID_TEST_COUNT; ++i) { 101 String s = String.valueOf(r.nextInt()); 102 long l = StringId.getId(s); 103 String s2 = StringId.getStringFromId(l); 104 if (!s.equals(s2)) { 105 throw new StringIdException(); 106 } 107 } 108 logln("Ending thread: " + this); 109 } 110 111 @Override toString()112 public String toString() { 113 return "StringIdThread " + id; 114 } 115 } 116 TestStringId()117 public void TestStringId() { 118 ArrayList<StringIdThread> threads = new ArrayList<>(); 119 120 for (int i = 0; i < 8; i++) { 121 StringIdThread thread = new StringIdThread(i); 122 threads.add(thread); 123 thread.start(); 124 } 125 for (StringIdThread thread : threads) { 126 try { 127 thread.join(); 128 } catch (InterruptedException e) { 129 errln(e.toString()); 130 } 131 } 132 } 133 TestUrlEscape()134 public void TestUrlEscape() { 135 Matcher byte1 = PatternCache.get("%[A-Za-z0-9]{2}").matcher(""); 136 Matcher byte2 = PatternCache.get("%[A-Za-z0-9]{2}%[A-Za-z0-9]{2}").matcher(""); 137 Matcher byte3 = 138 PatternCache.get("%[A-Za-z0-9]{2}%[A-Za-z0-9]{2}%[A-Za-z0-9]{2}").matcher(""); 139 Matcher byte4 = 140 PatternCache.get("%[A-Za-z0-9]{2}%[A-Za-z0-9]{2}%[A-Za-z0-9]{2}%[A-Za-z0-9]{2}") 141 .matcher(""); 142 for (int i = 1; i <= 0x10FFFF; i = i * 3 / 2 + 1) { 143 String escaped = 144 EscapingUtilities.urlEscape(new StringBuilder().appendCodePoint(i).toString()); 145 logln(Integer.toHexString(i) + " => " + escaped); 146 if (EscapingUtilities.OK_TO_NOT_QUOTE.contains(i)) { 147 assertTrue("Should be unquoted", escaped.length() == 1); 148 } else if (i < 0x80) { 149 assertTrue("Should be %xx", byte1.reset(escaped).matches()); 150 } else if (i < 0x800) { 151 assertTrue("Should be %xx%xx", byte2.reset(escaped).matches()); 152 } else if (i < 0x10000) { 153 assertTrue("Should be %xx%xx%xx", byte3.reset(escaped).matches()); 154 } else { 155 assertTrue("Should be %xx%xx%xx%xx", byte4.reset(escaped).matches()); 156 } 157 } 158 } 159 TestDelegatingIterator()160 public void TestDelegatingIterator() { 161 Set<String> s = new TreeSet<>(Arrays.asList(new String[] {"a", "b", "c"})); 162 Set<String> t = new LinkedHashSet<>(Arrays.asList(new String[] {"f", "d", "e"})); 163 StringBuilder result = new StringBuilder(); 164 165 for (String u : DelegatingIterator.iterable(s, t)) { 166 result.append(u); 167 } 168 assertEquals("Iterator", "abcfde", result.toString()); 169 170 result.setLength(0); 171 for (String u : DelegatingIterator.array("s", "t", "u")) { 172 result.append(u); 173 } 174 assertEquals("Iterator", "stu", result.toString()); 175 176 int count = 0; 177 result.setLength(0); 178 for (int u : DelegatingIterator.array(1, 3, 5)) { 179 count += u; 180 } 181 assertEquals("Iterator", 9, count); 182 183 result.setLength(0); 184 for (Object u : DelegatingIterator.array(1, "t", "u", new UnicodeSet("[a-z]"))) { 185 result.append(u); 186 } 187 assertEquals("Iterator", "1tu[a-z]", result.toString()); 188 } 189 TestUntimedCounter()190 public void TestUntimedCounter() { 191 // simulates how Counter is used in VettingViewer 192 Counter<NotificationCategory> problemCounter = new Counter<>(); 193 problemCounter.increment(NotificationCategory.error); 194 problemCounter.increment(NotificationCategory.error); 195 problemCounter.increment(NotificationCategory.warning); 196 197 assertEquals("problemCounter error", 2, problemCounter.get(NotificationCategory.error)); 198 assertEquals("problemCounter warning", 1, problemCounter.get(NotificationCategory.warning)); 199 assertEquals("problemCounter weLost", 0, problemCounter.get(NotificationCategory.weLost)); 200 201 Counter<NotificationCategory> otherCounter = new Counter<>(); 202 otherCounter.addAll(problemCounter); 203 otherCounter.increment(NotificationCategory.error); 204 205 assertEquals("otherCounter error", 3, otherCounter.get(NotificationCategory.error)); 206 assertEquals("otherCounter warning", 1, otherCounter.get(NotificationCategory.warning)); 207 assertEquals("otherCounter weLost", 0, otherCounter.get(NotificationCategory.weLost)); 208 } 209 TestCounter()210 public void TestCounter() { 211 Counter<String> counter = new Counter<>(true); 212 Comparator<String> uca = 213 new Comparator<>() { 214 Collator col = Collator.getInstance(ULocale.ENGLISH); 215 216 @Override 217 public int compare(String o1, String o2) { 218 return col.compare(o1, o2); 219 } 220 }; 221 InverseComparator ucaDown = new InverseComparator(uca); 222 223 counter.add("c", 95); 224 counter.add("b", 50); 225 counter.add("b", 101); 226 counter.add("a", 100); 227 counter.add("a", -5); 228 counter.add("d", -3); 229 assertEquals("getCount(b)", counter.getCount("b"), 151); 230 assertEquals("getCount(a)", counter.getCount("a"), 95); 231 assertEquals("getCount(a)", counter.getTotal(), 338); 232 assertEquals("getItemCount", counter.getItemCount(), 4); 233 234 assertEquals("getMap", "{a=95, b=151, c=95, d=-3}", counter.toString()); 235 236 assertEquals( 237 "getKeysetSortedByKey", 238 Arrays.asList("a", "b", "c", "d"), 239 new ArrayList<>(counter.getKeysetSortedByKey())); 240 241 assertEquals( 242 "getKeysetSortedByCount(true, ucaDown)", 243 Arrays.asList("d", "c", "a", "b"), 244 new ArrayList<String>(counter.getKeysetSortedByCount(true, ucaDown))); 245 246 assertEquals( 247 "getKeysetSortedByCount(true, null), value", 248 Arrays.asList("d", "a", "c", "b"), 249 new ArrayList<>(counter.getKeysetSortedByCount(true, uca))); 250 251 assertEquals( 252 "getKeysetSortedByCount(false, ucaDown), descending", 253 Arrays.asList("b", "c", "a", "d"), 254 new ArrayList<String>(counter.getKeysetSortedByCount(false, ucaDown))); 255 256 assertEquals( 257 "getKeysetSortedByCount(false, null), descending, value", 258 Arrays.asList("b", "a", "c", "d"), 259 new ArrayList<>(counter.getKeysetSortedByCount(false, uca))); 260 } 261 TestOrganizationOrder()262 public void TestOrganizationOrder() { 263 Map<String, Organization> stringToOrg = new TreeMap<>(); 264 for (Organization org : Organization.values()) { 265 stringToOrg.put(org.toString(), org); 266 } 267 List<Organization> reordered = new ArrayList<>(stringToOrg.values()); 268 List<Organization> plain = Arrays.asList(Organization.values()); 269 for (int i = 0; i < reordered.size(); ++i) { 270 assertEquals("Items not in alphabetical order", reordered.get(i), plain.get(i)); 271 } 272 } 273 TestOrganizationNames()274 public void TestOrganizationNames() { 275 UnicodeSet uppercase = new UnicodeSet("[:uppercase:]"); 276 for (Organization org : Organization.values()) { 277 if (!uppercase.contains(org.getDisplayName().codePointAt(0))) { 278 errln("Organization name isn't titlecased: " + org + ", " + org.getDisplayName()); 279 } 280 assertEquals( 281 "Organization from enum name", org, Organization.fromString(org.toString())); 282 assertEquals( 283 "Organization from display name", 284 org, 285 Organization.fromString(org.getDisplayName())); 286 } 287 } 288 289 static final boolean SHOW_DETAILS = CldrUtility.getProperty("showdetails", false); 290 private static final CharSequence DEBUG_COMMENT = 291 "set up a case of conflict within organization"; 292 293 static class PathValueInfo { 294 private static Map<Integer, String> voteInfo; 295 private CLDRFile file; 296 PathValueInfo(Factory factory, String locale)297 public PathValueInfo(Factory factory, String locale) { 298 this.file = factory.make(locale, false); 299 } 300 getRealValue(int id)301 public String getRealValue(int id) { 302 return file.getStringValue(getRealPath(id)); 303 } 304 getRealPath(int id)305 public String getRealPath(int id) { 306 return voteInfo.get(id); 307 } 308 } 309 310 /** Test user data. Restructured to be easier to read, more typesafe */ 311 public enum TestUser { 312 unaffiliatedS(801, Organization.unaffiliated, Level.guest), 313 gnomeS(701, Organization.gnome, Level.guest), 314 gnomeV(702, Organization.gnome, Level.vetter), 315 googleV(404, Organization.google, Level.vetter), 316 googleS(411, Organization.google, Level.guest), 317 googleV2(424, Organization.google, Level.vetter), 318 appleV(304, Organization.apple, Level.vetter), 319 adobeE(204, Organization.adobe, Level.manager), 320 adobeV(209, Organization.adobe, Level.vetter), 321 ibmS(101, Organization.ibm, Level.guest), 322 microsoftV(134, Organization.microsoft, Level.vetter), 323 ibmE(114, Organization.ibm, Level.manager), 324 ibmT(129, Organization.ibm, Level.tc), 325 unaffiliatedS2(802, Organization.unaffiliated, Level.guest); 326 327 public static final Map<Integer, VoterInfo> TEST_USERS; 328 public final Integer voterId; 329 public final VoterInfo voterInfo; 330 TestUser(int intVoterId, Organization organization, Level level)331 TestUser(int intVoterId, Organization organization, Level level) { 332 voterId = intVoterId; 333 voterInfo = new VoterInfo(organization, level, name()); 334 } 335 336 static { 337 ImmutableMap.Builder<Integer, VoterInfo> temp = ImmutableMap.builder(); 338 for (TestUser testUser : values()) { temp.put(testUser.voterId, testUser.voterInfo)339 temp.put(testUser.voterId, testUser.voterInfo); 340 } 341 TEST_USERS = temp.build(); 342 } 343 } 344 345 public static final Map<Integer, VoterInfo> testdata = TestUser.TEST_USERS; 346 toVoterId(String s)347 private int toVoterId(String s) { 348 return TestUser.valueOf(s).voterId; 349 } 350 351 /** Public to use from other tests */ getTestVoterInfoList()352 public static VoterInfoList getTestVoterInfoList() { 353 return new VoterInfoList().setVoterToInfo(testdata); 354 } 355 TestTrunkStatus()356 public void TestTrunkStatus() { 357 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 358 resolver.setLocale(CLDRLocale.getInstance("de"), null); 359 resolver.setBaileyValue("bailey"); 360 resolver.setBaseline("new-item", Status.approved); 361 assertEquals("", "new-item", resolver.getWinningValue()); 362 363 /* 364 * Formerly last-release would win over trunk in a 2nd scenario here, due to 365 * the difference in status. Now last-release plays no role, that test is obsolete. 366 * Reference: https://unicode.org/cldr/trac/ticket/11916 367 */ 368 } 369 TestVoteResolverNgombaTrunkStatus()370 public void TestVoteResolverNgombaTrunkStatus() { 371 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 372 resolver.setBaileyValue("bailey"); 373 resolver.setLocale(CLDRLocale.getInstance("jgo"), null); 374 final String jgo22trunk = 375 "\uA78C"; // "[a á â ǎ b c d ɛ {ɛ́} {ɛ̂} {ɛ̌} {ɛ̀} {ɛ̄} f ɡ h i í î ǐ j k l m ḿ {m̀} 376 // {m̄} n ń ǹ {n̄} ŋ {ŋ́} {ŋ̀} {ŋ̄} ɔ {ɔ́} {ɔ̂} {ɔ̌} p {pf} s {sh} t {ts} 377 // u ú û ǔ ʉ {ʉ́} {ʉ̂} {ʉ̌} {ʉ̈} v w ẅ y z ꞌ]"; 378 resolver.setBaseline(jgo22trunk, Status.approved); // seed/jgo.xml from 22 379 // trunk 380 logln("SVN: " + jgo22trunk); 381 logln(resolver.toString()); 382 assertEquals("Winning Value", jgo22trunk, resolver.getWinningValue()); 383 } 384 TestVoteStatus()385 public void TestVoteStatus() { 386 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 387 388 resolver.setLocale(CLDRLocale.getInstance("de"), null); 389 resolver.setBaileyValue("bailey"); 390 resolver.setBaseline("foo", Status.approved); 391 resolver.add("fii", toVoterId("adobeE")); 392 resolver.add("fii", toVoterId("appleV")); 393 VoteStatus voteStatus; 394 voteStatus = resolver.getStatusForOrganization(Organization.google); 395 assertEquals("", VoteResolver.VoteStatus.ok, voteStatus); 396 voteStatus = resolver.getStatusForOrganization(Organization.apple); 397 assertEquals("", VoteResolver.VoteStatus.ok, voteStatus); 398 399 // make non-equal foo 400 String s1 = "foo"; 401 String s2 = new StringBuilder("fo").append("o").toString(); 402 if (s1 == s2) { 403 errln("Test problem"); 404 } 405 resolver.clear(); 406 resolver.setBaileyValue("bailey"); 407 resolver.setBaseline(s1, Status.approved); 408 resolver.add(s2, toVoterId("appleV")); 409 voteStatus = resolver.getStatusForOrganization(Organization.apple); 410 assertEquals("", VoteResolver.VoteStatus.ok, voteStatus); 411 } 412 TestLosingStatus()413 public void TestLosingStatus() { 414 // af 415 // losing? {baseline: {BQ, missing}, trunk: {null, null}, 416 // {orgToVotes: , totals: {}, conflicted: []}, 417 // sameVotes: [BQ], O: null, N: null, totals: {}, winning: {BQ, 418 // missing}} 419 // XPath: //ldml/localeDisplayNames/territories/territory[@type="BQ"] 420 // gcvs.openoffice_org.example.com 421 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 422 423 resolver.setLocale(CLDRLocale.getInstance("af"), null); 424 resolver.setBaseline("BQ", Status.missing); 425 resolver.setBaileyValue("bailey"); 426 VoteStatus status = resolver.getStatusForOrganization(Organization.openoffice_org); 427 assertEquals("", VoteResolver.VoteStatus.provisionalOrWorse, status); 428 429 // {lastRelease: {{0}: {1}, missing}, trunk: {null, null}, {orgToVotes: 430 // pakistan={{0}: {1}=8}, totals: {{0}: 431 // {1}=8}, conflicted: []}, sameVotes: [{0}: {1}], O: {0}: {1}, N: null, 432 // totals: {{0}: {1}=8}, winning: {{0}: 433 // {1}, approved}} 434 resolver.clear(); 435 resolver.setBaileyValue("bailey"); 436 // resolver.setLastRelease("{0}: {1}", Status.missing); 437 resolver.add("{0}: {1}", toVoterId("adobeE")); 438 status = resolver.getStatusForOrganization(Organization.openoffice_org); 439 assertEquals("", VoteResolver.VoteStatus.ok, status); 440 441 // {lastRelease: {Arabisch, approved}, trunk: {Arabisch, approved}, 442 // {orgToVotes: , totals: {}, conflicted: []}, 443 // sameVotes: [Arabisch], O: null, N: null, totals: {}, winning: 444 // {Arabisch, approved}} 445 resolver.clear(); 446 resolver.setBaileyValue("bailey"); 447 // resolver.setLastRelease("Arabisch", Status.approved); 448 resolver.setBaseline("Arabisch", Status.approved); 449 status = resolver.getStatusForOrganization(Organization.openoffice_org); 450 assertEquals("", VoteResolver.VoteStatus.ok_novotes, status); 451 } 452 TestTotalVotesStatus()453 public void TestTotalVotesStatus() { 454 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 455 456 Status oldStatus = Status.unconfirmed; 457 458 resolver.setBaileyValue("bailey"); 459 resolver.setLocale(CLDRLocale.getInstance("de"), null); 460 resolver.setBaseline("foo", oldStatus); 461 resolver.add("zebra", toVoterId("googleV")); 462 resolver.add("apple", toVoterId("appleV")); 463 464 // check that alphabetical wins when votes are equal 465 String winner = resolver.getWinningValue(); 466 Status winningStatus = resolver.getWinningStatus(); 467 assertEquals("", "apple", winner); 468 assertEquals("", Status.provisional, winningStatus); 469 470 resolver.clear(); 471 resolver.setBaileyValue("bailey"); 472 resolver.setLocale(CLDRLocale.getInstance("de"), null); 473 resolver.setBaseline("foo", oldStatus); 474 resolver.add("zebra", toVoterId("googleV")); 475 resolver.add("zebra", toVoterId("googleS")); 476 resolver.add("apple", toVoterId("appleV")); 477 478 // check that total votes over alphabetical 479 winner = resolver.getWinningValue(); 480 winningStatus = resolver.getWinningStatus(); 481 assertEquals("", "zebra", winner); 482 assertEquals("", Status.provisional, winningStatus); 483 } 484 TestVoteDowngrade()485 public void TestVoteDowngrade() { 486 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 487 488 Status oldStatus = Status.unconfirmed; 489 490 resolver.setBaileyValue("bailey"); 491 resolver.setLocale(CLDRLocale.getInstance("mt"), null); 492 resolver.setBaseline("foo", oldStatus); 493 resolver.add("aardvark", toVoterId("adobeE")); 494 resolver.add("zebra", toVoterId("ibmT")); 495 assertEquals("", "zebra", resolver.getWinningValue()); // TC vote of 20 496 // beats 497 // expert's 8 498 assertEquals("", Status.approved, resolver.getWinningStatus()); 499 500 resolver.clear(); 501 resolver.setBaileyValue("bailey"); 502 resolver.setLocale(CLDRLocale.getInstance("mt"), null); 503 resolver.setBaseline("foo", oldStatus); 504 resolver.add("aardvark", toVoterId("adobeE")); 505 resolver.add("zebra", toVoterId("ibmT")); 506 resolver.add("aardvark", toVoterId("ibmE")); 507 assertEquals("", "zebra", resolver.getWinningValue()); // TC vote of 20 508 // beats 509 // manager's 4 510 // and its own 511 // manager's 4 512 assertEquals("", Status.approved, resolver.getWinningStatus()); 513 514 resolver.clear(); 515 resolver.setBaileyValue("bailey"); 516 resolver.setLocale(CLDRLocale.getInstance("mt"), null); 517 resolver.setBaseline("foo", oldStatus); 518 resolver.add("aardvark", toVoterId("adobeE")); 519 resolver.add("zebra", toVoterId("ibmT"), Level.vetter.getVotes(Organization.ibm)); // NOTE: 520 // reduced 521 // votes: 522 // as 523 // vetter. 524 resolver.add("aardvark", toVoterId("ibmE")); 525 assertEquals("", "aardvark", resolver.getWinningValue()); // Now 526 // aardvark 527 // wins - 528 // managers 529 // win out as provisional 530 assertEquals("", Status.provisional, resolver.getWinningStatus()); 531 532 resolver.clear(); 533 resolver.setBaileyValue("bailey"); 534 resolver.setLocale(CLDRLocale.getInstance("mt"), null); 535 resolver.setBaseline("foo", oldStatus); 536 resolver.add("aardvark", toVoterId("adobeE")); 537 resolver.add("zebra", toVoterId("ibmT"), Level.vetter.getVotes(Organization.ibm)); // NOTE: 538 // reduced 539 // votes: 540 // as 541 // vetter. 542 assertEquals("", "aardvark", resolver.getWinningValue()); // Now 543 // aardvark 544 // wins - 545 // managers 546 // win out. 547 assertEquals("", Status.provisional, resolver.getWinningStatus()); 548 } 549 TestResolvedVoteCounts()550 public void TestResolvedVoteCounts() { 551 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 552 553 Status oldStatus = Status.unconfirmed; 554 555 resolver.setBaileyValue("bailey"); 556 resolver.setLocale(CLDRLocale.getInstance("de"), null); 557 resolver.setBaseline("foo", oldStatus); 558 resolver.add("zebra", toVoterId("googleV")); 559 resolver.add("apple", toVoterId("appleV")); 560 561 // check that alphabetical wins when votes are equal 562 Map<String, Long> counts = resolver.getResolvedVoteCountsIncludingIntraOrgDisputes(); 563 logln(counts.toString()); 564 assertEquals("", "foo", new ArrayList<>(counts.keySet()).get(2)); 565 566 resolver.clear(); 567 resolver.setBaileyValue("bailey"); 568 resolver.setLocale(CLDRLocale.getInstance("de"), null); 569 resolver.setBaseline("foo", Status.approved); 570 resolver.add("zebra", toVoterId("googleV")); 571 resolver.add("apple", toVoterId("appleV")); 572 counts = resolver.getResolvedVoteCountsIncludingIntraOrgDisputes(); 573 logln(counts.toString()); 574 assertEquals("", "foo", new ArrayList<>(counts.keySet()).get(0)); 575 576 resolver.clear(); 577 resolver.setBaileyValue("bailey"); 578 resolver.setLocale(CLDRLocale.getInstance("de"), null); 579 resolver.setBaseline("foo", Status.approved); 580 resolver.add("zebra", toVoterId("googleS")); 581 counts = resolver.getResolvedVoteCountsIncludingIntraOrgDisputes(); 582 logln(counts.toString()); 583 assertEquals("", "foo", new ArrayList<>(counts.keySet()).get(0)); 584 } 585 verifyRequiredVotes( VoteResolver<String> resolver, String locale, String xpath, Status baselineStatus, int required)586 private void verifyRequiredVotes( 587 VoteResolver<String> resolver, 588 String locale, 589 String xpath, 590 Status baselineStatus, 591 int required) { 592 StringBuilder sb = new StringBuilder(); 593 sb.append("Locale: " + locale); 594 resolver.clear(); 595 resolver.setBaileyValue("bailey"); 596 resolver.setBaseline("foo", baselineStatus); 597 PathHeader ph = null; 598 if (xpath != null) { 599 sb.append(" XPath: " + xpath); 600 ph = PathHeader.getFactory(testInfo.getEnglish()).fromPath(xpath); 601 } 602 resolver.setLocale(CLDRLocale.getInstance(locale), ph); 603 if (!assertEquals( 604 locale + " verifyRequiredVotes: " + ph.toString(), 605 required, 606 resolver.getRequiredVotes())) { 607 int debug = 0; 608 } 609 } 610 TestRequiredVotes()611 public void TestRequiredVotes() { 612 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 613 verifyRequiredVotes( 614 resolver, 615 "mt", 616 "//ldml/localeDisplayNames/languages/language[@type=\"fr_CA\"]", 617 Status.missing, 618 ONE_VETTER_BAR); 619 verifyRequiredVotes( 620 resolver, 621 "fr", 622 "//ldml/localeDisplayNames/languages/language[@type=\"fr_CA\"]", 623 Status.provisional, 624 TWO_VETTER_BAR); 625 verifyRequiredVotes( 626 resolver, 627 "es", 628 "//ldml/numbers/symbols[@numberSystem=\"latn\"]/group", 629 Status.approved, 630 VoteResolver.HIGH_BAR); 631 verifyRequiredVotes( 632 resolver, 633 "es", 634 "//ldml/numbers/symbols[@numberSystem=\"latn\"]/decimal", 635 Status.approved, 636 VoteResolver.HIGH_BAR); 637 verifyRequiredVotes( 638 resolver, 639 "hi", 640 "//ldml/numbers/symbols[@numberSystem=\"deva\"]/decimal", 641 Status.approved, 642 VoteResolver.HIGH_BAR); 643 verifyRequiredVotes( 644 resolver, 645 "hi", 646 "//ldml/numbers/symbols[@numberSystem=\"deva\"]/group", 647 Status.approved, 648 VoteResolver.HIGH_BAR); 649 verifyRequiredVotes( 650 resolver, 651 "ast", 652 "//ldml/numbers/symbols[@numberSystem=\"latn\"]/decimal", 653 Status.approved, 654 ONE_VETTER_BAR); 655 verifyRequiredVotes( 656 resolver, 657 "mt", 658 "//ldml/characters/exemplarCharacters", 659 Status.approved, 660 VoteResolver.HIGH_BAR); 661 verifyRequiredVotes( 662 resolver, 663 "mt", 664 "//ldml/characters/exemplarCharacters", 665 Status.approved, 666 VoteResolver.HIGH_BAR); 667 verifyRequiredVotes( 668 resolver, 669 "mt", 670 "//ldml/characters/exemplarCharacters[@type=\"auxiliary\"]", 671 Status.approved, 672 VoteResolver.HIGH_BAR); 673 verifyRequiredVotes( 674 resolver, 675 "mt", 676 "//ldml/characters/exemplarCharacters[@type=\"numbers\"]", 677 Status.approved, 678 VoteResolver.HIGH_BAR); 679 verifyRequiredVotes( 680 resolver, 681 "mt", 682 "//ldml/characters/exemplarCharacters[@type=\"punctuation\"]", 683 Status.approved, 684 VoteResolver.HIGH_BAR); 685 verifyRequiredVotes( 686 resolver, 687 "mt", 688 "//ldml/characters/exemplarCharacters[@type=\"index\"]", 689 Status.approved, 690 VoteResolver.HIGH_BAR); 691 verifyRequiredVotes( 692 resolver, 693 "es", 694 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/days/dayContext[@type=\"format\"]/dayWidth[@type=\"wide\"]/day[@type=\"sun\"]", 695 Status.approved, 696 VoteResolver.HIGH_BAR); 697 verifyRequiredVotes( 698 resolver, 699 "ast", 700 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/days/dayContext[@type=\"format\"]/dayWidth[@type=\"wide\"]/day[@type=\"sun\"]", 701 Status.approved, 702 ONE_VETTER_BAR); 703 verifyRequiredVotes( 704 resolver, 705 "es", 706 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/months/monthContext[@type=\"format\"]/monthWidth[@type=\"wide\"]/month[@type=\"1\"]", 707 Status.provisional, 708 VoteResolver.LOWER_BAR); 709 verifyRequiredVotes( 710 resolver, 711 "ast", 712 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/months/monthContext[@type=\"format\"]/monthWidth[@type=\"wide\"]/month[@type=\"1\"]", 713 Status.approved, 714 ONE_VETTER_BAR); 715 } 716 717 /** 718 * In sublocales, for a typical path, the required votes should be 4, except for a few specified 719 * locales. 720 */ TestSublocaleRequiredVotes()721 public void TestSublocaleRequiredVotes() { 722 final Set<String> eightVoteSublocales = 723 new HashSet<>( 724 Arrays.asList( 725 "pt_PT", 726 "zh_Hant", 727 "zh_Hant_HK", 728 "en_AU", 729 "en_GB", 730 "es_MX", 731 "fr_CA", 732 "es_419")); 733 final VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 734 final String path = "//ldml/annotations/annotation[@cp=\"��\"][@type=\"tts\"]"; 735 for (String locale : SubmissionLocales.CLDR_OR_HIGH_LEVEL_LOCALES) { 736 if (locale.contains("_")) { 737 int expectedRequiredVotes = 738 eightVoteSublocales.contains(locale) ? TWO_VETTER_BAR : ONE_VETTER_BAR; 739 verifyRequiredVotes(resolver, locale, path, Status.approved, expectedRequiredVotes); 740 } 741 } 742 } 743 TestVoteResolver()744 public void TestVoteResolver() { 745 // to make it easier to debug failures, the first digit is an org, 746 // second is the individual in that org, and 747 // third is the voting weight. 748 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 749 String[] tests = { 750 "bailey=BAILEY", 751 "comment=regression case from John Emmons", 752 "locale=wae", 753 "oldValue=2802", 754 "oldStatus=approved", 755 "304=208027", // Apple vetter 756 // expected values 757 "value=208027", 758 "status=approved", 759 "sameVotes=208027", 760 "conflicts=[]", 761 "check", 762 // first test 763 "oldValue=old-value", 764 "oldStatus=provisional", 765 "comment=Check that identical values get the top overall vote, and that org is maxed (eg vetter + guest = vetter)", 766 "404=next", 767 "411=next", 768 "304=best", 769 // expected values 770 "value=next", 771 "sameVotes=next,best", 772 "conflicts=[]", 773 "status=provisional", 774 "check", 775 "comment=now give next a slight edge (5 to 4) with a different organization", 776 "404=next", 777 "304=best", 778 "801=next", 779 // expected values 780 "value=next", 781 "sameVotes=next", 782 "status=approved", 783 "check", 784 "comment=set up a case of conflict within organization", 785 "404=next", 786 "424=best", 787 // expected values 788 "value=best", // alphabetical 789 "sameVotes=best", 790 "conflicts=[google]", 791 "status=approved", 792 "check", 793 "comment=now cross-organizational conflict, also check for max value in same organization (4, 1) => 4 not 5", 794 "404=next", 795 "424=best", 796 "411=best", 797 "304=best", 798 // expected values 799 "conflicts=[google]", 800 "value=best", 801 "sameVotes=best", 802 "status=approved", 803 "check", 804 "comment=now clear winner 8 over 4", 805 "404=next", 806 // "424=best", 807 "411=best", 808 "304=best", 809 "204=primo", 810 "114=primo", 811 // expected values 812 "conflicts=[]", 813 "value=primo", 814 "sameVotes=primo", 815 "status=approved", 816 "check", 817 "comment=now not so clear, throw in a guest value. So it is 8 to 5. (used to be provisional)", 818 "404=next", 819 // "424=best", 820 "411=best", 821 "304=best", 822 "204=primo", 823 "114=primo", 824 "101=best", 825 // expected values 826 "conflicts=[]", 827 "value=primo", 828 "status=approved", 829 "check", 830 "comment=set up vote of 4 in established locale, with old provisional value", 831 "locale=fr", 832 "404=best", 833 "oldStatus=provisional", 834 // expected values 835 "value=best", 836 "sameVotes=best", 837 "status=contributed", 838 "conflicts=[]", 839 "check", 840 "comment=now set up vote of 4 in established locale, but with old contributed value", 841 "oldStatus=contributed", 842 // expected values 843 "value=old-value", 844 "sameVotes=old-value", 845 "status=contributed", 846 "conflicts=[]", 847 "check", 848 "comment=now set up vote of 1 + 1 in established locale, and with old contributed value", 849 "411=best", 850 "101=best", 851 "oldStatus=contributed", 852 // expected values 853 "value=best", 854 "sameVotes=best", 855 "status=contributed", 856 "conflicts=[]", 857 "check", 858 }; 859 String expectedValue = null; 860 String expectedConflicts = null; 861 Status expectedStatus = null; 862 String oldValue = null; 863 Status oldStatus = null; 864 String baileyValue = null; 865 List<String> sameVotes = null; 866 String locale = null; 867 Map<Integer, String> values = new TreeMap<>(); 868 int counter = -1; 869 870 for (String test : tests) { 871 String[] item = test.split("="); 872 String name = item[0]; 873 String value = item.length < 2 ? null : item[1]; 874 if (name.equalsIgnoreCase("comment")) { 875 logln("#\t" + value); 876 // System.out.println("#\t" + value); 877 if (DEBUG_COMMENT != null && value.contains(DEBUG_COMMENT)) { 878 int x = 0; 879 } 880 } else if (name.equalsIgnoreCase("locale")) { 881 locale = value; 882 } else if (name.equalsIgnoreCase("bailey")) { 883 baileyValue = value; 884 } else if (name.equalsIgnoreCase("oldValue")) { 885 oldValue = value; 886 } else if (name.equalsIgnoreCase("oldStatus")) { 887 oldStatus = Status.valueOf(value); 888 } else if (name.equalsIgnoreCase("value")) { 889 expectedValue = value; 890 } else if (name.equalsIgnoreCase("sameVotes")) { 891 sameVotes = 892 value == null ? new ArrayList<>(0) : Arrays.asList(value.split(",\\s*")); 893 } else if (name.equalsIgnoreCase("status")) { 894 expectedStatus = Status.valueOf(value); 895 } else if (name.equalsIgnoreCase("conflicts")) { 896 expectedConflicts = value; 897 } else if (DIGITS.containsAll(name)) { 898 final int voter = Integer.parseInt(name); 899 if (value == null || value.equals("null")) { 900 values.remove(voter); 901 } else { 902 values.put(voter, value); 903 } 904 } else if (name.equalsIgnoreCase("check")) { 905 counter++; 906 // load the resolver 907 resolver.setBaileyValue(baileyValue); 908 resolver.setLocale(CLDRLocale.getInstance(locale), null); 909 resolver.setBaseline(oldValue, oldStatus); 910 for (int voter : values.keySet()) { 911 resolver.add(values.get(voter), voter); 912 } 913 // print the contents 914 logln(counter + "\t" + values); 915 logln(resolver.toString()); 916 // now print the values 917 assertEquals(counter + " value", expectedValue, resolver.getWinningValue()); 918 assertEquals( 919 counter + " sameVotes", 920 sameVotes.toString(), 921 resolver.getValuesWithSameVotes().toString()); 922 assertEquals(counter + " status", expectedStatus, resolver.getWinningStatus()); 923 assertEquals( 924 counter + " conflicts", 925 expectedConflicts, 926 resolver.getConflictedOrganizations().toString()); 927 resolver.clear(); 928 resolver.setBaileyValue("bailey"); 929 values.clear(); 930 } else { 931 errln("unknown command:\t" + test); 932 } 933 } 934 } 935 936 void assertSpecialLocale(String loc, SpecialLocales.Type type) { 937 assertEquals( 938 "SpecialLocales type for " + loc, 939 type, 940 SpecialLocales.getType(CLDRLocale.getInstance(loc))); 941 } 942 943 public void TestSpecialLocales() { 944 assertSpecialLocale("sr", null); 945 assertSpecialLocale("ha_NE", SpecialLocales.Type.algorithmic); 946 assertSpecialLocale("sr_Latn", SpecialLocales.Type.algorithmic); 947 assertSpecialLocale("sr_Latn_BA", SpecialLocales.Type.algorithmic); 948 assertSpecialLocale("yue_Hans", null); // not readonly, because it is not policy DISCARD 949 assertSpecialLocale("en", SpecialLocales.Type.readonly); 950 assertSpecialLocale("en_ZZ_PROGRAMMERESE", null); // not defined 951 assertSpecialLocale(LocaleNames.UND, null); 952 assertSpecialLocale(LocaleNames.MUL, SpecialLocales.Type.scratch); 953 assertSpecialLocale("mul_ZZ", SpecialLocales.Type.scratch); 954 assertSpecialLocale("und_001", null); // not defined 955 956 CLDRLocale sr_Latn = CLDRLocale.getInstance("sr_Latn"); 957 CLDRLocale sr_Latn_BA = CLDRLocale.getInstance("sr_Latn_BA"); 958 logln("sr_Latn raw comment = " + SpecialLocales.getCommentRaw(sr_Latn)); 959 assertTrue( 960 "sr_Latn raw contains @ sign", SpecialLocales.getCommentRaw(sr_Latn).contains("@")); 961 962 logln("sr_Latn comment = " + SpecialLocales.getComment(sr_Latn)); 963 assertTrue( 964 "sr_Latn comment does NOT contain @ sign", 965 !SpecialLocales.getComment(sr_Latn).contains("@")); 966 logln("sr_Latn_BA raw comment = " + SpecialLocales.getCommentRaw(sr_Latn_BA)); 967 assertTrue( 968 "sr_Latn_BA raw contains '@sr_Latn_BA'", 969 SpecialLocales.getCommentRaw(sr_Latn_BA).contains("@sr_Latn_BA")); 970 } 971 972 public void TestCLDRURLS() { 973 final String KOREAN_LANGUAGE = "//ldml/localeDisplayNames/languages/language[@type=\"ko\"]"; 974 final String KOREAN_LANGUAGE_STRID = "821c2a2fc5c206d"; 975 final CLDRLocale maltese = CLDRLocale.getInstance("mt"); 976 assertEquals( 977 "base", "https://st.unicode.org/cldr-apps", CLDRConfig.getInstance().urls().base()); 978 assertEquals( 979 "locales list", 980 "https://st.unicode.org/cldr-apps/v#locales///", 981 CLDRConfig.getInstance().urls().forSpecial(CLDRURLS.Special.Locales)); 982 assertEquals( 983 "maltese", 984 "https://st.unicode.org/cldr-apps/v#/mt//", 985 CLDRConfig.getInstance().urls().forLocale(maltese)); 986 assertEquals( 987 "korean in maltese", 988 "https://st.unicode.org/cldr-apps/v#/mt//" + KOREAN_LANGUAGE_STRID, 989 CLDRConfig.getInstance().urls().forXpath(maltese, KOREAN_LANGUAGE)); 990 assertEquals( 991 "korean in maltese via stringid", 992 "https://st.unicode.org/cldr-apps/v#/mt//" + KOREAN_LANGUAGE_STRID, 993 CLDRConfig.getInstance().urls().forXpathHexId(maltese, KOREAN_LANGUAGE_STRID)); 994 assertEquals( 995 "south east asia in maltese", 996 "https://st.unicode.org/cldr-apps/v#/mt/C_SEAsia/", 997 CLDRConfig.getInstance().urls().forPage(maltese, PageId.C_SEAsia)); 998 try { 999 String ret = CLDRConfig.getInstance().urls().forXpathHexId(maltese, KOREAN_LANGUAGE); 1000 errln("Error- expected forXpathHexId to choke on an xpath but got " + ret); 1001 } catch (IllegalArgumentException iae) { 1002 logln("GOOD: forXpathHexId Caught expected " + iae); 1003 } 1004 try { 1005 String ret = CLDRConfig.getInstance().urls().forXpath(maltese, KOREAN_LANGUAGE_STRID); 1006 errln("Error- expected forXpath to choke on a hexid but got " + ret); 1007 } catch (IllegalArgumentException iae) { 1008 logln("GOOD: forXpath Caught expected " + iae); 1009 } 1010 1011 assertEquals( 1012 "korean in maltese - absoluteUrl", 1013 "https://st.unicode.org/cldr-apps/v#/mt//" + KOREAN_LANGUAGE_STRID, 1014 CLDRConfig.getInstance().absoluteUrls().forXpath(maltese, KOREAN_LANGUAGE)); 1015 } 1016 1017 static final UnicodeMap<String> SCRIPTS = 1018 ICUPropertyFactory.make().getProperty("script").getUnicodeMap_internal(); 1019 static final UnicodeMap<String> GC = 1020 ICUPropertyFactory.make().getProperty("general_category").getUnicodeMap_internal(); 1021 1022 public void TestUnicodeMapCompose() { 1023 logln("Getting Scripts"); 1024 1025 UnicodeMap.Composer<String> composer = 1026 new UnicodeMap.Composer<>() { 1027 @Override 1028 public String compose(int codepoint, String string, String a, String b) { 1029 return a.toString() + "_" + b.toString(); 1030 } 1031 }; 1032 1033 logln("Trying Compose"); 1034 1035 UnicodeMap<String> composed = 1036 ((UnicodeMap) SCRIPTS.cloneAsThawed()).composeWith(GC, composer); 1037 String last = ""; 1038 for (int i = 0; i < 0x10FFFF; ++i) { 1039 String comp = composed.getValue(i); 1040 String gc = GC.getValue(i); 1041 String sc = SCRIPTS.getValue(i); 1042 if (!comp.equals(composer.compose(i, null, sc, gc))) { 1043 errln("Failed compose at: " + i); 1044 break; 1045 } 1046 if (!last.equals(comp)) { 1047 logln(Utility.hex(i) + "\t" + comp); 1048 last = comp; 1049 } 1050 } 1051 } 1052 1053 private static final int SET_LIMIT = 0x10FFFF; 1054 private static final int CHECK_LIMIT = 0xFFFF; 1055 private static final NumberFormat pf = NumberFormat.getPercentInstance(); 1056 private static final NumberFormat nf = NumberFormat.getInstance(); 1057 1058 public void TestUnicodeMapTime() { 1059 boolean shortTest = getInclusion() < 10; 1060 double hashTime, umTime, icuTime, treeTime; 1061 int warmup = shortTest ? 1 : 20; 1062 umTime = checkUnicodeMapSetTime(warmup, 0); 1063 hashTime = checkUnicodeMapSetTime(warmup, 1); 1064 logln("Percentage: " + pf.format(hashTime / umTime)); 1065 treeTime = checkUnicodeMapSetTime(warmup, 3); 1066 logln("Percentage: " + pf.format(treeTime / umTime)); 1067 1068 if (shortTest) { 1069 return; 1070 } 1071 1072 umTime = checkUnicodeMapGetTime(1000, 0); 1073 hashTime = checkUnicodeMapGetTime(1000, 1); 1074 logln("Percentage: " + pf.format(hashTime / umTime)); 1075 icuTime = checkUnicodeMapGetTime(1000, 2); 1076 logln("Percentage: " + pf.format(icuTime / umTime)); 1077 treeTime = checkUnicodeMapGetTime(1000, 3); 1078 logln("Percentage: " + pf.format(treeTime / umTime)); 1079 } 1080 1081 private static final int propEnum = UProperty.GENERAL_CATEGORY; 1082 1083 private double checkUnicodeMapSetTime(int iterations, int type) { 1084 _checkUnicodeMapSetTime(1, type); 1085 double result = _checkUnicodeMapSetTime(iterations, type); 1086 logln( 1087 (type == 0 ? "UnicodeMap" : type == 1 ? "HashMap" : type == 2 ? "ICU" : "TreeMap") 1088 + "\t" 1089 + nf.format(result)); 1090 return result; 1091 } 1092 1093 private double _checkUnicodeMapSetTime(int iterations, int type) { 1094 UnicodeMap<String> map1 = SCRIPTS; 1095 Map<Integer, String> map2 = map1.putAllCodepointsInto(new HashMap<Integer, String>()); 1096 Map<Integer, String> map3 = new TreeMap<>(map2); 1097 System.gc(); 1098 double start = System.currentTimeMillis(); 1099 for (int j = 0; j < iterations; ++j) 1100 for (int cp = 0; cp <= SET_LIMIT; ++cp) { 1101 int enumValue = UCharacter.getIntPropertyValue(cp, propEnum); 1102 if (enumValue <= 0) continue; // for smaller set 1103 String value = 1104 UCharacter.getPropertyValueName( 1105 propEnum, enumValue, UProperty.NameChoice.LONG); 1106 switch (type) { 1107 case 0: 1108 map1.put(cp, value); 1109 break; 1110 case 1: 1111 map2.put(cp, value); 1112 break; 1113 case 3: 1114 map3.put(cp, value); 1115 break; 1116 } 1117 } 1118 double end = System.currentTimeMillis(); 1119 return (end - start) / 1000 / iterations; 1120 } 1121 1122 private double checkUnicodeMapGetTime(int iterations, int type) { 1123 UnicodeMap<String> map1 = new UnicodeMap<>(); 1124 Map<Integer, String> map2 = map1.putAllCodepointsInto(new HashMap<Integer, String>()); 1125 Map<Integer, String> map3 = new TreeMap<>(); 1126 _checkUnicodeMapGetTime(map1, map2, map3, 1, type); // warmup 1127 double result = _checkUnicodeMapGetTime(map1, map2, map3, iterations, type); 1128 logln( 1129 (type == 0 ? "UnicodeMap" : type == 1 ? "HashMap" : type == 2 ? "ICU" : "TreeMap") 1130 + "\t" 1131 + nf.format(result)); 1132 return result; 1133 } 1134 1135 private double _checkUnicodeMapGetTime( 1136 UnicodeMap<String> map1, 1137 Map<Integer, String> map2, 1138 Map<Integer, String> map3, 1139 int iterations, 1140 int type) { 1141 System.gc(); 1142 double start = System.currentTimeMillis(); 1143 for (int j = 0; j < iterations; ++j) 1144 for (int cp = 0; cp < CHECK_LIMIT; ++cp) { 1145 switch (type) { 1146 case 0: 1147 map1.getValue(cp); 1148 break; 1149 case 1: 1150 map2.get(cp); 1151 break; 1152 case 2: 1153 int enumValue = UCharacter.getIntPropertyValue(cp, propEnum); 1154 UCharacter.getPropertyValueName( 1155 propEnum, enumValue, UProperty.NameChoice.LONG); 1156 break; 1157 case 3: 1158 map3.get(cp); 1159 break; 1160 } 1161 } 1162 double end = System.currentTimeMillis(); 1163 return (end - start) / 1000 / iterations; 1164 } 1165 1166 public void TestStevenTest() { 1167 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 1168 1169 String tests[] = { 1170 "bailey=BAILEY", 1171 "comment=Steven Loomis test case tweaked by Parthinator", 1172 "locale=wae", 1173 "oldValue=_", 1174 "oldStatus=approved", 1175 "304=test", // Apple vetter 1176 // expected values 1177 "value=test", 1178 "status=approved", 1179 "sameVotes=test", 1180 "conflicts=[]", 1181 "check", 1182 1183 // test1 1184 "comment=timestamp case1", 1185 "locale=de", 1186 "oldValue=old-value", 1187 "oldStatus=provisional", 1188 "404=Foo", 1189 "424=Bar", 1190 // expected 1191 "value=Bar", 1192 "status=provisional", 1193 "sameVotes=Bar, test", 1194 "conflicts=[google]", 1195 "check", 1196 1197 // test2 1198 "comment=timestamp case2", 1199 "locale=de", 1200 "oldValue=Bar", 1201 "oldStatus=provisional", 1202 "424=Foo", 1203 "404=Bar", 1204 // expected values 1205 "value=Bar", 1206 "status=provisional", 1207 "sameVotes=Bar, test", 1208 "conflicts=[google]", 1209 "check", 1210 1211 // test 3 1212 "comment=timestamp unaffiliated case", 1213 "locale=de", 1214 "oldValue=_", 1215 "oldStatus=unconfirmed", 1216 // # // G vetter A 1217 // timestamp=1 1218 "801=Foo", 1219 // timestamp=2 1220 "802=Bar", 1221 // expected values 1222 "value=Bar", 1223 "status=contributed", 1224 "sameVotes=Bar", 1225 "conflicts=[google, unaffiliated]", 1226 "check", 1227 }; 1228 1229 String expectedValue = null; 1230 String expectedConflicts = null; 1231 Status expectedStatus = null; 1232 String oldValue = null; 1233 Status oldStatus = null; 1234 String baileyValue = null; 1235 List<String> sameVotes = null; 1236 String locale = null; 1237 int voteEntries = 0; 1238 Map<Integer, String> values = new TreeMap<>(); 1239 Map<Integer, VoteEntries> valuesMap = new TreeMap<>(); 1240 1241 int counter = -1; 1242 1243 for (String test : tests) { 1244 String[] item = test.split("="); 1245 String name = item[0]; 1246 String value = item.length < 2 ? null : item[1]; 1247 if (name.equalsIgnoreCase("comment")) { 1248 logln("#\t" + value); 1249 // System.out.println("#\t" + value); 1250 if (DEBUG_COMMENT != null && value.contains(DEBUG_COMMENT)) { 1251 int x = 0; 1252 } 1253 } else if (name.equalsIgnoreCase("locale")) { 1254 locale = value; 1255 } else if (name.equalsIgnoreCase("oldValue")) { 1256 oldValue = value; 1257 } else if (name.equalsIgnoreCase("oldStatus")) { 1258 oldStatus = Status.valueOf(value); 1259 } else if (name.equalsIgnoreCase("value")) { 1260 expectedValue = value; 1261 } else if (name.equalsIgnoreCase("bailey")) { 1262 baileyValue = value; 1263 } else if (name.equalsIgnoreCase("sameVotes")) { 1264 sameVotes = 1265 value == null ? new ArrayList<>(0) : Arrays.asList(value.split(",\\s*")); 1266 } else if (name.equalsIgnoreCase("status")) { 1267 expectedStatus = Status.valueOf(value); 1268 } else if (name.equalsIgnoreCase("conflicts")) { 1269 expectedConflicts = value; 1270 } else if (DIGITS.containsAll(name)) { 1271 final int voter = Integer.parseInt(name); 1272 if (value == null || value.equals("null")) { 1273 values.remove(voter); 1274 for (Map.Entry<Integer, VoteEntries> entry : valuesMap.entrySet()) { 1275 if (entry.getValue().getVoter() == voter) { 1276 valuesMap.remove(entry.getKey()); 1277 } 1278 } 1279 } else { 1280 values.put(voter, value); 1281 valuesMap.put(++voteEntries, new VoteEntries(voter, value)); 1282 } 1283 } else if (name.equalsIgnoreCase("check")) { 1284 counter++; 1285 // load the resolver 1286 resolver.setBaileyValue(baileyValue); 1287 resolver.setLocale(CLDRLocale.getInstance(locale), null); 1288 resolver.setBaseline(oldValue, oldStatus); 1289 for (int voteEntry : valuesMap.keySet()) { 1290 1291 resolver.add( 1292 valuesMap.get(voteEntry).getValue(), 1293 valuesMap.get(voteEntry).getVoter()); 1294 } 1295 // print the contents 1296 logln(counter + "\t" + values); 1297 logln(resolver.toString()); 1298 // now print the values 1299 assertEquals(counter + " value", expectedValue, resolver.getWinningValue()); 1300 assertEquals( 1301 counter + " sameVotes", 1302 sameVotes.toString(), 1303 resolver.getValuesWithSameVotes().toString()); 1304 assertEquals(counter + " status", expectedStatus, resolver.getWinningStatus()); 1305 assertEquals( 1306 counter + " conflicts", 1307 expectedConflicts, 1308 resolver.getConflictedOrganizations().toString()); 1309 resolver.clear(); 1310 values.clear(); 1311 } else { 1312 errln("unknown command:\t" + test); 1313 } 1314 } 1315 } 1316 1317 public void testBaileyVotes() { 1318 VoterInfoList vil = new VoterInfoList().setVoterToInfo(TestUser.TEST_USERS); 1319 VoteResolver<String> resolver = new VoteResolver<>(vil); 1320 CLDRLocale locale = CLDRLocale.getInstance("de"); 1321 PathHeader path = null; 1322 1323 /* 1324 * Simple case, all = bailey -- should get INHERITANCE_MARKER now that we have "dropped hard inheritance" 1325 */ 1326 resolver.setLocale(locale, path); 1327 resolver.setBaileyValue("bailey"); 1328 resolver.setBaseline("foo", Status.approved); 1329 1330 resolver.add("bailey", TestUser.appleV.voterId); 1331 resolver.add("bailey", TestUser.microsoftV.voterId); 1332 resolver.add("bailey", TestUser.googleV.voterId); 1333 if (VoteResolver.DROP_HARD_INHERITANCE) { 1334 assertEquals( 1335 "Simple case, all = bailey", 1336 CldrUtility.INHERITANCE_MARKER, 1337 resolver.getWinningValue()); 1338 } else { 1339 assertEquals("Simple case, all = bailey", "bailey", resolver.getWinningValue()); 1340 } 1341 1342 /* 1343 * Another simple case, all = INHERITANCE_MARKER 1344 * Added per https://unicode.org/cldr/trac/ticket/11299 1345 */ 1346 resolver.clear(); 1347 resolver.setLocale(locale, path); 1348 resolver.setBaileyValue("bailey"); 1349 resolver.setBaseline("foo", Status.approved); 1350 1351 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.appleV.voterId); 1352 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.microsoftV.voterId); 1353 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.googleV.voterId); 1354 assertEquals( 1355 "Another simple case, all = INHERITANCE_MARKER", 1356 CldrUtility.INHERITANCE_MARKER, 1357 resolver.getWinningValue()); 1358 1359 /* 1360 * INHERITANCE_MARKER should win here, having more votes than bailey. 1361 * Changed per https://unicode.org/cldr/trac/ticket/11299 1362 */ 1363 resolver.clear(); 1364 resolver.setLocale(locale, path); 1365 resolver.setBaileyValue("bailey"); 1366 resolver.setBaseline("foo", Status.approved); 1367 1368 resolver.add("bailey", TestUser.appleV.voterId); 1369 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.microsoftV.voterId); 1370 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.googleV.voterId); 1371 assertEquals( 1372 "The bailey value and explicit value combine to win", 1373 CldrUtility.INHERITANCE_MARKER, 1374 resolver.getWinningValue()); 1375 1376 /* 1377 * INHERITANCE_MARKER should win here, having equal number of votes with bailey; 1378 * first they combine to win over other-vote. 1379 * Changed per https://unicode.org/cldr/trac/ticket/11299 1380 */ 1381 resolver.clear(); 1382 resolver.setLocale(locale, path); 1383 resolver.setBaileyValue("bailey"); 1384 resolver.setBaseline("foo", Status.approved); 1385 1386 resolver.add("bailey", TestUser.appleV.voterId); 1387 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.microsoftV.voterId); 1388 resolver.add("other-vote", TestUser.googleV.voterId); 1389 assertEquals( 1390 "The bailey value and explicit value combine to win again", 1391 CldrUtility.INHERITANCE_MARKER, 1392 resolver.getWinningValue()); 1393 1394 /* 1395 * Split vote, no action 1396 */ 1397 resolver.clear(); 1398 resolver.setLocale(locale, path); 1399 resolver.setBaileyValue("bailey"); 1400 resolver.setBaseline("foo", Status.approved); 1401 1402 resolver.add("bailey", TestUser.appleV.voterId); 1403 resolver.add("not-bailey", TestUser.microsoftV.voterId); 1404 resolver.add("other-vote", TestUser.googleV.voterId); 1405 assertEquals("Split vote, no action", "foo", resolver.getWinningValue()); 1406 1407 /* 1408 * Bailey should lose even if it has MORE votes than INHERITANCE_MARKER, now that we 1409 * have "dropped hard inheritance" 1410 */ 1411 resolver.clear(); 1412 resolver.setLocale(locale, path); 1413 resolver.setBaileyValue("bailey"); 1414 resolver.setBaseline("foo", Status.approved); 1415 1416 resolver.add("bailey", TestUser.googleV.voterId); 1417 resolver.add("bailey", TestUser.appleV.voterId); 1418 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.microsoftV.voterId); 1419 resolver.add("other-vote", TestUser.adobeV.voterId); 1420 resolver.add("other-vote", TestUser.gnomeV.voterId); 1421 if (VoteResolver.DROP_HARD_INHERITANCE) { 1422 assertEquals( 1423 "Bailey never beats INHERITANCE_MARKER", 1424 CldrUtility.INHERITANCE_MARKER, 1425 resolver.getWinningValue()); 1426 } else { 1427 assertEquals( 1428 "Bailey can beat INHERITANCE_MARKER if not dropped", 1429 "bailey", 1430 resolver.getWinningValue()); 1431 } 1432 } 1433 1434 /** Test XMLUploader.writeBulkInfoHtml */ 1435 public void TestBulkUploadHtml() { 1436 StringWriter out = new StringWriter(); 1437 final String bulkStage = "submit"; 1438 try { 1439 XMLUploader.writeBulkInfoHtml(bulkStage, out); 1440 } catch (Exception e) { 1441 errln("Exception for writeBulkInfoHtml in TestBulkUploadHtml: " + e); 1442 } 1443 final String expected = 1444 "<div class='bulkNextInfo'>\n<ul>\n<li class='header'>Bulk Upload:</li>\n" 1445 + "<li class='inactive'>\n<h1>1. upload</h1>\n<h2>Upload XML file</h2>\n</li>\n" 1446 + "<li class='inactive'>\n<h1>2. check</h1>\n<h2>Verify valid XML</h2>\n</li>\n" 1447 + "<li class='inactive'>\n<h1>3. test</h1>\n<h2>Test for CLDR errors</h2>\n</li>\n" 1448 + "<li class='active'>\n<h1>4. submit</h1>\n<h2>Data submitted into SurveyTool</h2>\n</li>\n" 1449 + "</ul>\n</div>\n"; 1450 assertEquals("writeBulkInfoHtml", expected, out.toString()); 1451 } 1452 1453 /** 1454 * Verify that VettingViewer.getMissingStatus returns MissingStatus.PRESENT for a typical path 1455 * in a well-populated locale 1456 * 1457 * <p>Ideally we should also test for MissingStatus.DISPUTED, etc.; that's more difficult 1458 */ 1459 public void TestMissingStatus() { 1460 final String path = 1461 "//ldml/units/unitLength[@type=\"short\"]/unit[@type=\"volume-cup\"]/displayName"; 1462 final String locale = "fr"; 1463 final CLDRFile cldrFile = testInfo.getCLDRFile(locale, true); 1464 final MissingStatus expected = MissingStatus.PRESENT; 1465 // Note: VettingViewer.getMissingStatus reports PRESENT for items with ↑↑↑ and absent if the 1466 // item 1467 // is removed to inherit from root, even though the value obtained is the same in either 1468 // case; 1469 // so for path pick an item that does not have ↑↑↑, otherwise when that item is stripped for 1470 // production data the test will fail. 1471 final MissingStatus status = 1472 VettingViewer.getMissingStatus(cldrFile, path, true /* latin */); 1473 if (status != expected) { 1474 errln( 1475 "Got getMissingStatus = " 1476 + status.toString() 1477 + "; expected " 1478 + expected.toString()); 1479 } 1480 } 1481 1482 /** Check that expected paths are Aliased, and have debugging code */ 1483 public void TestMissingGrammar() { 1484 // https://cldr-smoke.unicode.org/cldr-apps/v#/hu/Length/a4915bf505ffb49 1485 final String path = 1486 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"length-meter\"]/unitPattern[@count=\"one\"][@case=\"accusative\"]"; 1487 checkGrammarCoverage( 1488 "hr", 1489 path, 1490 MissingStatus.PRESENT, 1491 DEBUG, 1492 1, 1493 0, 1494 0, 1495 0, 1496 0); // this isn't a very good test, since we have to adjust each time. Should create 1497 // fake cldr data instead 1498 checkGrammarCoverage("kw", path, MissingStatus.ABSENT, false, 0, 0, 1, 1, 0); 1499 checkGrammarCoverage("en_NZ", path, MissingStatus.ALIASED, DEBUG, 1, 0, 0, 0, 0); 1500 } 1501 1502 /** 1503 * Check the getMissingStatus and getStatus. Note that the values may need to be adjusted in 1504 * successive versions. The sizes are expected sizes. 1505 * 1506 * @param locale 1507 * @param path 1508 * @param statusExpected 1509 * @param debug TODO 1510 */ 1511 public void checkGrammarCoverage( 1512 final String locale, 1513 final String path, 1514 MissingStatus statusExpected, 1515 boolean debug, 1516 int... sizes) { 1517 final CLDRFile cldrFile = testInfo.getCLDRFile(locale, true); 1518 final MissingStatus expected = statusExpected; 1519 final MissingStatus status = 1520 VettingViewer.getMissingStatus(cldrFile, path, true /* latin */); 1521 if (status != expected) { 1522 errln( 1523 locale 1524 + " got getMissingStatus = " 1525 + status.toString() 1526 + "; expected " 1527 + expected.toString()); 1528 } 1529 Iterable<String> pathsToTest = Collections.singleton(path); 1530 Counter<org.unicode.cldr.util.Level> foundCounter = new Counter<>(); 1531 Counter<org.unicode.cldr.util.Level> unconfirmedCounter = new Counter<>(); 1532 Counter<org.unicode.cldr.util.Level> missingCounter = new Counter<>(); 1533 Relation<MissingStatus, String> missingPaths = 1534 new Relation( 1535 new TreeMap<MissingStatus, String>(), TreeSet.class, Ordering.natural()); 1536 Set<String> unconfirmedPaths = new TreeSet<>(); 1537 VettingViewer.getStatus( 1538 pathsToTest, 1539 cldrFile, 1540 PathHeader.getFactory(), 1541 foundCounter, 1542 unconfirmedCounter, 1543 missingCounter, 1544 missingPaths, 1545 unconfirmedPaths); 1546 assertEquals(locale + " foundCounter (0)", sizes[0], foundCounter.getTotal()); 1547 assertEquals(locale + " unconfirmedCounter (1)", sizes[1], unconfirmedCounter.getTotal()); 1548 assertEquals(locale + " missingCounter (2)", sizes[2], missingCounter.getTotal()); 1549 assertEquals(locale + " missingPaths (3)", sizes[3], missingPaths.size()); 1550 assertEquals(locale + " unconfirmedPaths (4)", sizes[4], unconfirmedPaths.size()); 1551 showStatusResults( 1552 locale, 1553 foundCounter, 1554 unconfirmedCounter, 1555 missingCounter, 1556 missingPaths, 1557 unconfirmedPaths); 1558 if (debug) { 1559 foundCounter.clear(); 1560 unconfirmedCounter.clear(); 1561 missingCounter.clear(); 1562 missingPaths.clear(); 1563 unconfirmedPaths.clear(); 1564 pathsToTest = cldrFile.fullIterable(); 1565 VettingViewer.getStatus( 1566 pathsToTest, 1567 cldrFile, 1568 PathHeader.getFactory(), 1569 foundCounter, 1570 unconfirmedCounter, 1571 missingCounter, 1572 missingPaths, 1573 unconfirmedPaths); 1574 showStatusResults( 1575 locale, 1576 foundCounter, 1577 unconfirmedCounter, 1578 missingCounter, 1579 missingPaths, 1580 unconfirmedPaths); 1581 } 1582 } 1583 1584 public void showStatusResults( 1585 final String locale, 1586 Counter<org.unicode.cldr.util.Level> foundCounter, 1587 Counter<org.unicode.cldr.util.Level> unconfirmedCounter, 1588 Counter<org.unicode.cldr.util.Level> missingCounter, 1589 Relation<MissingStatus, String> missingPaths, 1590 Set<String> unconfirmedPaths) { 1591 warnln( 1592 "\n" 1593 + locale 1594 + " foundCounter:\t" 1595 + foundCounter 1596 + "\n" 1597 + locale 1598 + " unconfirmedCounter:\t" 1599 + unconfirmedCounter 1600 + "\n" 1601 + locale 1602 + " missingCounter:\t" 1603 + missingCounter 1604 + "\n" 1605 + locale 1606 + " unconfirmedPaths:\t" 1607 + unconfirmedPaths 1608 + "\n" 1609 + locale 1610 + " missing paths (modern):"); 1611 int count = 0; 1612 for (Entry<MissingStatus, String> entry : missingPaths.entrySet()) { 1613 final MissingStatus missingStatus = entry.getKey(); 1614 final String missingPath = entry.getValue(); 1615 warnln( 1616 ++count 1617 + "\t" 1618 + locale 1619 + "\t" 1620 + missingStatus 1621 + "\t" 1622 + missingPath 1623 + "\t" 1624 + SUPPLEMENTAL_DATA_INFO.getCoverageLevel(missingPath, locale)); 1625 } 1626 } 1627 1628 /** 1629 * Test the function VoteResolver.Level.canCreateOrSetLevelTo() 1630 * 1631 * <p>Compare org.unicode.cldr.unittest.web.TestUserRegistry.TestCanSetUserLevel() 1632 */ 1633 public void TestCanCreateOrSetLevelTo() { 1634 if (Level.vetter.canCreateOrSetLevelTo(Level.guest) 1635 || Level.anonymous.canCreateOrSetLevelTo(Level.guest) 1636 || Level.guest.canCreateOrSetLevelTo(Level.locked) 1637 || Level.locked.canCreateOrSetLevelTo(Level.locked)) { 1638 errln("Only managers and above can change levels at all"); 1639 } 1640 if (Level.manager.canCreateOrSetLevelTo(Level.tc) 1641 || Level.manager.canCreateOrSetLevelTo(Level.admin) 1642 || Level.tc.canCreateOrSetLevelTo(Level.admin)) { 1643 errln("Can’t change anyone to a more privileged level than you"); 1644 } 1645 } 1646 } 1647