1 package org.unicode.cldr.util; 2 3 import java.io.File; 4 import java.io.IOException; 5 import java.io.PrintWriter; 6 import java.util.Arrays; 7 import java.util.HashSet; 8 import java.util.LinkedHashSet; 9 import java.util.Map; 10 import java.util.Set; 11 import java.util.TreeMap; 12 import java.util.TreeSet; 13 import java.util.regex.Pattern; 14 15 import org.unicode.cldr.draft.FileUtilities; 16 import org.unicode.cldr.test.BuildIcuCompactDecimalFormat; 17 import org.unicode.cldr.test.BuildIcuCompactDecimalFormat.CurrencyStyle; 18 import org.unicode.cldr.tool.ChartDelta; 19 import org.unicode.cldr.tool.FormattedFileWriter; 20 import org.unicode.cldr.tool.Option; 21 import org.unicode.cldr.tool.Option.Options; 22 import org.unicode.cldr.tool.ShowData; 23 import org.unicode.cldr.tool.ShowPlurals; 24 import org.unicode.cldr.tool.TablePrinter; 25 import org.unicode.cldr.util.CLDRFile.DraftStatus; 26 import org.unicode.cldr.util.PathHeader.PageId; 27 import org.unicode.cldr.util.SupplementalDataInfo.PluralInfo; 28 29 import com.ibm.icu.text.CompactDecimalFormat; 30 import com.ibm.icu.text.CompactDecimalFormat.CompactStyle; 31 import com.ibm.icu.text.NumberFormat; 32 import com.ibm.icu.util.Currency; 33 import com.ibm.icu.util.ICUUncheckedIOException; 34 import com.ibm.icu.util.ULocale; 35 36 public class VerifyCompactNumbers { 37 38 private static final CLDRConfig CLDR_CONFIG = CLDRConfig.getInstance(); 39 private static final String DIR = CLDRPaths.VERIFY_DIR + "numbers/"; 40 41 final static Options myOptions = new Options(); 42 43 enum MyOptions { 44 organization(".*", "CLDR", "organization"), filter(".*", ".*", "locale filter (regex)"), currency(".*", "EUR", "show currency"),; 45 // boilerplate 46 final Option option; 47 MyOptions(String argumentPattern, String defaultArgument, String helpText)48 MyOptions(String argumentPattern, String defaultArgument, String helpText) { 49 option = myOptions.add(this, argumentPattern, defaultArgument, helpText); 50 } 51 } 52 53 // later, look at DateTimeFormats to set up as an HTML table 54 55 public static final Set<String> USES_GROUPS_OF_4 = new HashSet<>(Arrays.asList("ko", "ja", "zh", "zh_Hant")); 56 57 /** 58 * Produce a set of static tables from the vxml data. Only a stopgap until the above is integrated into ST. 59 * 60 * @param args 61 * @throws IOException 62 */ main(String[] args)63 public static void main(String[] args) throws IOException { 64 myOptions.parse(MyOptions.organization, args, true); 65 new File(DIR).mkdirs(); 66 FileCopier.copy(ShowData.class, "verify-index.html", CLDRPaths.VERIFY_DIR, "index.html"); 67 FileCopier.copy(ChartDelta.class, "index.css", CLDRPaths.VERIFY_DIR, "index.css"); 68 FormattedFileWriter.copyIncludeHtmls(CLDRPaths.VERIFY_DIR); 69 70 String organization = MyOptions.organization.option.getValue(); 71 String filter = MyOptions.filter.option.getValue(); 72 boolean showCurrency = true; // MyOptions.currency.option.doesOccur(); 73 String currencyCode = MyOptions.currency.option.getValue(); 74 75 Factory factory2 = Factory.make(CLDRPaths.MAIN_DIRECTORY, filter); 76 CLDRFile englishCldrFile = factory2.make("en", true); 77 78 SupplementalDataInfo sdi = CLDR_CONFIG.getSupplementalDataInfo(); 79 Set<String> defaultContentLocales = sdi.getDefaultContentLocales(); 80 NumberFormat enf = NumberFormat.getIntegerInstance(ULocale.ENGLISH); 81 enf.setGroupingUsed(false); 82 83 Set<String> availableLanguages = new TreeSet<>(factory2.getAvailableLanguages()); 84 if (Pattern.matches(filter, "pt_PT")) { 85 availableLanguages.add("pt_PT"); 86 } 87 88 PrintWriter plainText = FileUtilities.openUTF8Writer(DIR, "compactTestFile.txt"); 89 DateTimeFormats.writeCss(DIR); 90 final CLDRFile english = CLDR_CONFIG.getEnglish(); 91 92 Map<String, String> indexMap = new TreeMap<>(CLDR_CONFIG.getCollator()); 93 94 for (String locale : availableLanguages) { 95 if (defaultContentLocales.contains(locale)) { 96 continue; 97 } 98 Level level = StandardCodes.make().getLocaleCoverageLevel(organization, locale); 99 if (Level.MODERN.compareTo(level) > 0) { 100 continue; 101 } 102 // TODO: fix to ignore locales with no data. 103 if (locale.equals("ne") || locale.equals("cy")) { 104 continue; 105 } 106 107 PrintWriter out = FileUtilities.openUTF8Writer(DIR, locale + ".html"); 108 String title = "Verify Number Formats: " + englishCldrFile.getName(locale); 109 out.println("<!doctype HTML PUBLIC '-//W3C//DTD HTML 4.0 Transitional//EN'><html><head>\n" + 110 "<meta http-equiv='Content-Type' content='text/html; charset=utf-8'>\n" + 111 "<title>" + title + "</title>\n" + 112 "<link rel='stylesheet' type='text/css' href='index.css'>\n" + 113 "</head><body><h1>" + title + "</h1>\n" 114 + "<p><a href='index.html'>Index</a></p>\n"); 115 116 CLDRFile cldrFile = factory2.make(locale, true, DraftStatus.contributed); 117 118 showNumbers(cldrFile, showCurrency, currencyCode, out, factory2); 119 120 out.println("</body></html>"); 121 out.close(); 122 indexMap.put(english.getName(locale), locale + ".html"); 123 124 } 125 try (PrintWriter index = DateTimeFormats.openIndex(DIR, "Numbers")) { 126 DateTimeFormats.writeIndexMap(indexMap, index); 127 } 128 129 plainText.close(); 130 131 } 132 showNumbers(CLDRFile cldrFile, boolean showCurrency, String currencyCode, Appendable out, Factory factory)133 public static void showNumbers(CLDRFile cldrFile, boolean showCurrency, 134 String currencyCode, Appendable out, Factory factory) { 135 try { 136 Set<String> debugCreationErrors = new LinkedHashSet<>(); 137 Set<String> errors = new LinkedHashSet<>(); 138 String locale = cldrFile.getLocaleID(); 139 140 TablePrinter tablePrinter1 = new TablePrinter() 141 // .setCaption("Timezone Formats") 142 .setTableAttributes("class='dtf-table'") 143 .addColumn("Numeric Format").setHeaderCell(true).setHeaderAttributes("class='dtf-th'") 144 .setCellAttributes("class='dtf-s'") 145 .addColumn("Compact-Short").setHeaderAttributes("class='dtf-th'").setCellAttributes("class='dtf-s'") 146 .addColumn("Compact-Long").setHeaderAttributes("class='dtf-th'").setCellAttributes("class='dtf-s'"); 147 if (showCurrency) { 148 tablePrinter1 149 .addColumn("Compact-Short<br>+Currency") 150 .setHeaderAttributes("class='dtf-th'") 151 .setCellAttributes("class='dtf-s'") 152 // .addColumn("Compact-Short<br>+Unit") 153 // .setHeaderAttributes("class='dtf-th'") 154 // .setCellAttributes("class='dtf-s'") 155 // .addColumn("Compact-Long<br>+Currency") 156 // .addColumn("Compact-Long<br>+Currency-Long") 157 // .addColumn("Numeric Format").setHeaderCell(true).setHeaderAttributes("class='dtf-th'") 158 // .setCellAttributes("class='dtf-s'") 159 ; 160 } 161 // tablePrinter1.addColumn("View").setHeaderCell(true).setHeaderAttributes("class='dtf-th'").setCellAttributes("class='dtf-s'"); 162 163 164 165 ULocale locale2 = new ULocale(locale); 166 ICUServiceBuilder builder = new ICUServiceBuilder().setCldrFile(cldrFile); 167 NumberFormat nf = builder.getNumberFormat(1); 168 169 // nf.setMaximumFractionDigits(0); 170 SupplementalDataInfo sdi = CLDR_CONFIG.getSupplementalDataInfo(); 171 String[] debugOriginals = null; 172 CompactDecimalFormat cdf = BuildIcuCompactDecimalFormat.build(cldrFile, debugCreationErrors, 173 debugOriginals, CompactStyle.SHORT, locale2, CurrencyStyle.PLAIN, currencyCode); 174 captureErrors(debugCreationErrors, errors, locale, "short"); 175 CompactDecimalFormat cdfs = BuildIcuCompactDecimalFormat.build(cldrFile, debugCreationErrors, 176 debugOriginals, CompactStyle.LONG, locale2, CurrencyStyle.PLAIN, currencyCode); 177 captureErrors(debugCreationErrors, errors, locale, "long"); 178 179 CompactDecimalFormat cdfCurr = BuildIcuCompactDecimalFormat.build(cldrFile, debugCreationErrors, 180 debugOriginals, CompactStyle.SHORT, locale2, CurrencyStyle.CURRENCY, currencyCode); 181 captureErrors(debugCreationErrors, errors, locale, "short-curr"); 182 // CompactDecimalFormat cdfU = BuildIcuCompactDecimalFormat.build(cldrFile, debugCreationErrors, 183 // debugOriginals, CompactStyle.SHORT, locale2, CurrencyStyle.UNIT, "EUR"); 184 // captureErrors(debugCreationErrors, errors, locale, "short-kg"); 185 // CompactDecimalFormat cdfsCurr = BuildIcuCompactDecimalFormat.build(cldrFile, debugCreationErrors, 186 // debugOriginals, CompactStyle.SHORT, locale2, CurrencyStyle.CURRENCY, currencyCode); 187 // CompactDecimalFormat cdfsCurrISO = BuildIcuCompactDecimalFormat.build(cldrFile, debugCreationErrors, 188 // debugOriginals, CompactStyle.LONG, locale2, CurrencyStyle.ISO_CURRENCY, "EUR"); 189 190 Set<Double> allSamples = collectSamplesAndSetFormats(currencyCode, locale, sdi, cdf, cdfs, cdfCurr); 191 192 try { 193 for (double source : allSamples) { 194 if (false && source == 22000000 && locale.equals("cs")) { 195 System.out.println("**"); 196 } 197 198 String formattedNumber = nf.format(source); 199 String compactFormattedNumber = cdf == null ? "n/a" : cdf.format(source); 200 String compactLongFormattedNumber = cdfs == null ? "n/a" : cdfs.format(source); 201 String compactCurrFormattedNumber = !showCurrency || cdfs == null ? "n/a" : cdfCurr.format(source); 202 // plainText.println(locale 203 // + "\t__" + source 204 // + "\t__" + compactFormattedNumber 205 // + "\t__" + compactLongFormattedNumber 206 // ); 207 tablePrinter1.addRow() 208 .addCell(formattedNumber) 209 .addCell(compactFormattedNumber) 210 .addCell(compactLongFormattedNumber); 211 if (showCurrency) { 212 tablePrinter1 213 .addCell(compactCurrFormattedNumber) 214 // .addCell(cdfU.format(source)) 215 // .addCell(cdfsCurr.format(source)) 216 // .addCell(cdfsCurrLong.format(source)) 217 // .addCell(cdfsCurrLong.format(source)) 218 //.addCell(formattedNumber) 219 ; 220 } 221 // String view = PathHeader.getLinkedView(surveyUrl, cldrFile, METAZONE_PREFIX + metazone + METAZONE_SUFFIX); 222 // tablePrinter1.addCell(view == null 223 // ? "" 224 // : view); 225 tablePrinter1 226 .finishRow(); 227 } 228 } catch (Exception e) { 229 e.printStackTrace(); 230 } 231 out.append("<p>To correct problems in compact numbers below, please go to " 232 + PathHeader.SECTION_LINK 233 + CLDR_CONFIG.urls().forPage(cldrFile.getLocaleID(), PageId.Compact_Decimal_Formatting) 234 + "'><em>" + PageId.Compact_Decimal_Formatting 235 + "</em></a>.</p>"); 236 out.append(tablePrinter1.toString() + "\n"); 237 out.append("<h3>Plural Rules</h3>"); 238 out.append("<p>Look over the Minimal Pairs to make sure they are ok. " 239 + "Then review the examples in the cell to the left. " 240 + "All of those you should be able to substitute for the numbers in the Minimal Pairs, " 241 + "with an acceptable result. " 242 + "If any would be incorrect, please " 243 + "<a target='ticket' href='" + CLDRURLS.CLDR_NEWTICKET_URL + "'>file a ticket</a>.</p>" 244 + "<p>For more details, see " + 245 "<a target='CLDR-ST-DOCS' href='http://cldr.unicode.org/index/cldr-spec/plural-rules'>Plural Rules</a>.</p>"); 246 ShowPlurals showPlurals = new ShowPlurals(CLDR_CONFIG.getSupplementalDataInfo()); 247 showPlurals.printPluralTable(cldrFile, locale, out, factory); 248 ShowPlurals.appendBlanksForScrolling(out); 249 showErrors(errors, out); 250 showErrors(debugCreationErrors, out); 251 } catch (IOException e) { 252 throw new ICUUncheckedIOException(e); 253 } 254 } 255 collectSamplesAndSetFormats(String currencyCode, String locale, SupplementalDataInfo sdi, CompactDecimalFormat cdf, CompactDecimalFormat cdfs, CompactDecimalFormat cdfCurr)256 public static Set<Double> collectSamplesAndSetFormats(String currencyCode, String locale, SupplementalDataInfo sdi, CompactDecimalFormat cdf, 257 CompactDecimalFormat cdfs, CompactDecimalFormat cdfCurr) { 258 // Collect samples for display 259 // one path for group-3, one for group-4 260 // TODO, fix for indic. 261 int factor = USES_GROUPS_OF_4.contains(locale) ? 10000 : 1000; 262 263 // we want to collect a sample of at least one sample for each plural category for each 264 // power of ten 265 PluralInfo pluralInfo = sdi.getPlurals(locale); 266 Set<Double> samples = new TreeSet<>(); 267 samples.add(1.1d); 268 samples.add(1.5d); 269 samples.add(1100d); 270 collectItems(pluralInfo, 1, 10, samples); 271 collectItems(pluralInfo, 10, 100, samples); 272 collectItems(pluralInfo, 100, 1000, samples); 273 int sigDigits = 3; 274 if (factor > 1000) { 275 collectItems(pluralInfo, 1000, 10000, samples); 276 sigDigits = 4; 277 } 278 if (cdf != null) { 279 cdf.setMaximumSignificantDigits(sigDigits); 280 } 281 if (cdfs != null) { 282 cdfs.setMaximumSignificantDigits(sigDigits); 283 } 284 if (cdfCurr != null) { 285 cdfCurr.setCurrency(Currency.getInstance(currencyCode)); 286 cdfCurr.setMaximumSignificantDigits(sigDigits); 287 } 288 // cdfU.setMaximumSignificantDigits(sigDigits); 289 290 // for (Entry<Count, List<Double>> entry : pluralInfo.getCountToExamplesMap().entrySet()) { 291 // samples.add(entry.getValue().get(0)); 292 // } 293 // 294 // Set<Double> samples2 = new TreeSet<Double>(); 295 // for (int i = 10; i < factor; i *= 10) { 296 // for (Double sample : samples) { 297 // samples2.add(sample*i); 298 // } 299 // } 300 // samples.addAll(samples2); 301 302 Set<Double> allSamples = new TreeSet<>(); 303 for (long i = 1; i <= 100000000000000L; i *= factor) { 304 for (Double sample : samples) { 305 double source = i * sample; 306 allSamples.add(source); 307 } 308 } 309 return allSamples; 310 } 311 312 private static String surveyUrl = CLDR_CONFIG.getProperty("CLDR_SURVEY_URL", 313 "http://st.unicode.org/cldr-apps/survey"); 314 showErrors(Set<String> errors, Appendable out)315 private static void showErrors(Set<String> errors, Appendable out) throws IOException { 316 if (errors.size() != 0) { 317 out.append("<h2>" + "Errors" + "</h2>\n"); 318 for (String s : errors) { 319 out.append("<p>" + s + "</p>\n"); 320 } 321 errors.clear(); 322 } 323 } 324 collectItems(PluralInfo pluralInfo, double start, double limit, Set<Double> samples)325 private static Set<Double> collectItems(PluralInfo pluralInfo, double start, double limit, 326 Set<Double> samples) { 327 // TODO optimize once we have all the keywords 328 Map<String, Double> ones = new TreeMap<>(); 329 for (double i = start; i < limit; ++i) { 330 String cat = pluralInfo.getPluralRules().select(i); 331 if (ones.containsKey(cat)) { 332 continue; 333 } 334 ones.put(cat, i); 335 } 336 samples.addAll(ones.values()); 337 return samples; 338 } 339 captureErrors(Set<String> debugCreationErrors, Set<String> errors, String locale, String length)340 private static void captureErrors(Set<String> debugCreationErrors, Set<String> errors, String locale, String length) { 341 if (debugCreationErrors.size() != 0) { 342 for (String s : debugCreationErrors) { 343 errors.add(locale + "\t" + length + "\t" + s); 344 } 345 debugCreationErrors.clear(); 346 } 347 } 348 } 349