1 package org.unicode.cldr.test; 2 3 import java.io.File; 4 import java.io.IOException; 5 import java.io.InputStream; 6 import java.util.Arrays; 7 import java.util.Date; 8 import java.util.EnumSet; 9 import java.util.HashSet; 10 import java.util.Iterator; 11 import java.util.Objects; 12 import java.util.Set; 13 import java.util.TreeMap; 14 import java.util.TreeSet; 15 import java.util.regex.Matcher; 16 17 import org.unicode.cldr.tool.ToolConfig; 18 import org.unicode.cldr.util.CLDRConfig; 19 import org.unicode.cldr.util.CLDRFile; 20 import org.unicode.cldr.util.CLDRPaths; 21 import org.unicode.cldr.util.CldrUtility; 22 import org.unicode.cldr.util.DateTimeFormats; 23 import org.unicode.cldr.util.DtdType; 24 import org.unicode.cldr.util.Factory; 25 import org.unicode.cldr.util.InputStreamFactory; 26 import org.unicode.cldr.util.LanguageTagParser; 27 import org.unicode.cldr.util.Level; 28 import org.unicode.cldr.util.Organization; 29 import org.unicode.cldr.util.PatternCache; 30 import org.unicode.cldr.util.PrettyPath; 31 import org.unicode.cldr.util.StandardCodes; 32 import org.unicode.cldr.util.XMLFileReader; 33 import org.unicode.cldr.util.XPathParts; 34 import org.xml.sax.ErrorHandler; 35 import org.xml.sax.InputSource; 36 import org.xml.sax.SAXException; 37 import org.xml.sax.SAXParseException; 38 import org.xml.sax.XMLReader; 39 40 import com.ibm.icu.impl.Relation; 41 import com.ibm.icu.text.DateFormatSymbols; 42 import com.ibm.icu.text.SimpleDateFormat; 43 import com.ibm.icu.util.ULocale; 44 45 /** 46 * Simple test that loads each file in the cldr directory, thus verifying that 47 * the DTD works, and also checks that the PrettyPaths work. 48 * 49 * @author markdavis 50 */ 51 public class QuickCheck { 52 private static final Set<String> skipAttributes = new HashSet<String>(Arrays.asList(new String[] { 53 "alt", "draft", "references" })); 54 55 private static String localeRegex; 56 57 private static boolean showInfo = false; 58 59 private static String commonDirectory; 60 private static String mainDirectory; 61 62 private static boolean resolved; 63 64 private static Exception[] internalException = new Exception[1]; 65 66 private static boolean verbose; 67 main(String[] args)68 public static void main(String[] args) throws IOException { 69 CLDRConfig testInfo = ToolConfig.getToolInstance(); 70 Factory factory = testInfo.getCldrFactory(); 71 checkStock(factory); 72 if (true) return; 73 verbose = CldrUtility.getProperty("verbose", "false", "true").matches("(?i)T|TRUE"); 74 localeRegex = CldrUtility.getProperty("locale", ".*"); 75 76 showInfo = CldrUtility.getProperty("showinfo", "false", "true").matches("(?i)T|TRUE"); 77 78 commonDirectory = CLDRPaths.COMMON_DIRECTORY; // Utility.getProperty("common", Utility.COMMON_DIRECTORY); 79 // if (commonDirectory == null) commonDirectory = Utility.COMMON_DIRECTORY 80 // System.out.println("Main Source Directory: " + commonDirectory + 81 // "\t\t(to change, use -DSOURCE=xxx, eg -DSOURCE=C:/cvsdata/unicode/cldr/incoming/proposed/main)"); 82 83 mainDirectory = CldrUtility.getProperty("main", CLDRPaths.COMMON_DIRECTORY + "/main"); 84 // System.out.println("Main Source Directory: " + commonDirectory + 85 // "\t\t(to change, use -DSOURCE=xxx, eg -DSOURCE=C:/cvsdata/unicode/cldr/incoming/proposed/main)"); 86 87 resolved = CldrUtility.getProperty("resolved", "false", "true").matches("(?i)T|TRUE"); 88 89 boolean paths = CldrUtility.getProperty("paths", "true").matches("(?i)T|TRUE"); 90 91 pretty = CldrUtility.getProperty("pretty", "true").matches("(?i)T|TRUE"); 92 93 double startTime = System.currentTimeMillis(); 94 checkDtds(); 95 double deltaTime = System.currentTimeMillis() - startTime; 96 System.out.println("Elapsed: " + deltaTime / 1000.0 + " seconds"); 97 98 if (paths) { 99 System.out.println("Checking paths"); 100 checkPaths(); 101 deltaTime = System.currentTimeMillis() - startTime; 102 System.out.println("Elapsed: " + deltaTime / 1000.0 + " seconds"); 103 System.out.println("Basic Test Passes"); 104 } 105 } 106 checkDtds()107 private static void checkDtds() throws IOException { 108 checkDtds(commonDirectory + "supplemental"); 109 checkDtds(commonDirectory + "collation"); 110 checkDtds(commonDirectory + "main"); 111 checkDtds(commonDirectory + "rbnf"); 112 checkDtds(commonDirectory + "segments"); 113 checkDtds(commonDirectory + "../test"); 114 checkDtds(commonDirectory + "transforms"); 115 } 116 checkDtds(String directory)117 private static void checkDtds(String directory) throws IOException { 118 File directoryFile = new File(directory); 119 File[] listFiles = directoryFile.listFiles(); 120 String canonicalPath = directoryFile.getCanonicalPath(); 121 if (listFiles == null) { 122 throw new IllegalArgumentException("Empty directory: " + canonicalPath); 123 } 124 System.out.println("Checking files for DTD errors in: " + canonicalPath); 125 for (File fileName : listFiles) { 126 if (!fileName.toString().endsWith(".xml")) { 127 continue; 128 } 129 check(fileName); 130 } 131 } 132 133 static class MyErrorHandler implements ErrorHandler { error(SAXParseException exception)134 public void error(SAXParseException exception) throws SAXException { 135 System.out.println("\nerror: " + XMLFileReader.showSAX(exception)); 136 throw exception; 137 } 138 fatalError(SAXParseException exception)139 public void fatalError(SAXParseException exception) throws SAXException { 140 System.out.println("\nfatalError: " + XMLFileReader.showSAX(exception)); 141 throw exception; 142 } 143 warning(SAXParseException exception)144 public void warning(SAXParseException exception) throws SAXException { 145 System.out.println("\nwarning: " + XMLFileReader.showSAX(exception)); 146 throw exception; 147 } 148 } 149 check(File systemID)150 public static void check(File systemID) { 151 try (InputStream fis = InputStreamFactory.createInputStream(systemID)) { 152 // FileInputStream fis = new FileInputStream(systemID); 153 XMLReader xmlReader = XMLFileReader.createXMLReader(true); 154 xmlReader.setErrorHandler(new MyErrorHandler()); 155 InputSource is = new InputSource(fis); 156 is.setSystemId(systemID.toString()); 157 xmlReader.parse(is); 158 // fis.close(); 159 } catch (SAXException | IOException e) { // SAXParseException is a Subtype of SaxException 160 System.out.println("\t" + "Can't read " + systemID); 161 System.out.println("\t" + e.getClass() + "\t" + e.getMessage()); 162 } 163 // catch (SAXException e) { 164 // System.out.println("\t" + "Can't read " + systemID); 165 // System.out.println("\t" + e.getClass() + "\t" + e.getMessage()); 166 // } catch (IOException e) { 167 // System.out.println("\t" + "Can't read " + systemID); 168 // System.out.println("\t" + e.getClass() + "\t" + e.getMessage()); 169 // } 170 } 171 172 static Matcher skipPaths = PatternCache.get("/identity" + "|/alias" + "|\\[@alt=\"proposed").matcher(""); 173 174 private static boolean pretty; 175 checkPaths()176 private static void checkPaths() { 177 Relation<String, String> distinguishing = Relation.<String, String> of(new TreeMap<String, Set<String>>(), TreeSet.class, null); 178 Relation<String, String> nonDistinguishing = Relation.<String, String> of(new TreeMap<String, Set<String>>(), TreeSet.class, null); 179 XPathParts parts = new XPathParts(); 180 Factory cldrFactory = Factory.make(mainDirectory, localeRegex); 181 CLDRFile english = cldrFactory.make("en", true); 182 183 Relation<String, String> pathToLocale = Relation.of( 184 new TreeMap<String, Set<String>>(CLDRFile.getComparator(DtdType.ldml)), 185 TreeSet.class, null); 186 for (String locale : cldrFactory.getAvailable()) { 187 // if (locale.equals("root") && !localeRegex.equals("root")) 188 // continue; 189 CLDRFile file; 190 try { 191 file = cldrFactory.make(locale, resolved); 192 } catch (Exception e) { 193 System.out.println("\nfatalError: " + e.getMessage()); 194 continue; 195 } 196 if (file.isNonInheriting()) 197 continue; 198 DisplayAndInputProcessor displayAndInputProcessor = new DisplayAndInputProcessor(file, false); 199 200 System.out.println(locale + "\t-\t" + english.getName(locale)); 201 DtdType dtdType = null; 202 203 for (Iterator<String> it = file.iterator(); it.hasNext();) { 204 String path = it.next(); 205 if (path.endsWith("/alias")) { 206 continue; 207 } 208 String value = file.getStringValue(path); 209 if (value == null) { 210 throw new IllegalArgumentException(locale + "\tError: in null value at " + path); 211 } 212 String displayValue = displayAndInputProcessor.processForDisplay(path, value); 213 if (!displayValue.equals(value)) { 214 System.out.println("\t" + locale + "\tdisplayAndInputProcessor changes display value <" + value 215 + ">\t=>\t<" + displayValue + ">\t\t" + path); 216 } 217 String inputValue = displayAndInputProcessor.processInput(path, value, internalException); 218 if (internalException[0] != null) { 219 System.out.println("\t" + locale + "\tdisplayAndInputProcessor internal error <" + value 220 + ">\t=>\t<" + inputValue + ">\t\t" + path); 221 internalException[0].printStackTrace(System.out); 222 } 223 if (verbose && !inputValue.equals(value)) { 224 displayAndInputProcessor.processInput(path, value, internalException); // for debugging 225 System.out.println("\t" + locale + "\tdisplayAndInputProcessor changes input value <" + value 226 + ">\t=>\t<" + inputValue + ">\t\t" + path); 227 } 228 229 pathToLocale.put(path, locale); 230 231 // also check for non-distinguishing attributes 232 if (path.contains("/identity")) continue; 233 234 // make sure we don't have problem alts 235 if (path.contains("proposed")) { 236 String sourceLocale = file.getSourceLocaleID(path, null); 237 if (locale.equals(sourceLocale)) { 238 String nonAltPath = CLDRFile.getNondraftNonaltXPath(path); 239 if (!path.equals(nonAltPath)) { 240 String nonAltLocale = file.getSourceLocaleID(nonAltPath, null); 241 String nonAltValue = file.getStringValue(nonAltPath); 242 if (nonAltValue == null || !locale.equals(nonAltLocale)) { 243 System.out.println("\t" + locale + "\tProblem alt=proposed <" + value + ">\t\t" + path); 244 } 245 } 246 } 247 } 248 249 String fullPath = file.getFullXPath(path); 250 parts.set(fullPath); 251 if (dtdType == null) { 252 dtdType = DtdType.valueOf(parts.getElement(0)); 253 } 254 for (int i = 0; i < parts.size(); ++i) { 255 if (parts.getAttributeCount(i) == 0) continue; 256 String element = parts.getElement(i); 257 for (String attribute : parts.getAttributeKeys(i)) { 258 if (skipAttributes.contains(attribute)) continue; 259 if (CLDRFile.isDistinguishing(dtdType, element, attribute)) { 260 distinguishing.put(element, attribute); 261 } else { 262 nonDistinguishing.put(element, attribute); 263 } 264 } 265 } 266 } 267 } 268 System.out.println(); 269 270 System.out.format("Distinguishing Elements: %s" + CldrUtility.LINE_SEPARATOR, distinguishing); 271 System.out.format("Nondistinguishing Elements: %s" + CldrUtility.LINE_SEPARATOR, nonDistinguishing); 272 System.out.format("Skipped %s" + CldrUtility.LINE_SEPARATOR, skipAttributes); 273 274 if (pretty) { 275 if (showInfo) { 276 System.out.println(CldrUtility.LINE_SEPARATOR + "Showing Path to PrettyPath mapping" 277 + CldrUtility.LINE_SEPARATOR); 278 } 279 PrettyPath prettyPath = new PrettyPath().setShowErrors(true); 280 Set<String> badPaths = new TreeSet<String>(); 281 for (String path : pathToLocale.keySet()) { 282 String prettied = prettyPath.getPrettyPath(path, false); 283 if (showInfo) System.out.println(prettied + "\t\t" + path); 284 if (prettied.contains("%%") && !path.contains("/alias")) { 285 badPaths.add(path); 286 } 287 } 288 // now remove root 289 290 if (showInfo) { 291 System.out.println(CldrUtility.LINE_SEPARATOR + "Showing Paths not in root" 292 + CldrUtility.LINE_SEPARATOR); 293 } 294 295 CLDRFile root = cldrFactory.make("root", true); 296 for (Iterator<String> it = root.iterator(); it.hasNext();) { 297 pathToLocale.removeAll(it.next()); 298 } 299 if (showInfo) for (String path : pathToLocale.keySet()) { 300 if (skipPaths.reset(path).find()) { 301 continue; 302 } 303 System.out.println(path + "\t" + pathToLocale.getAll(path)); 304 } 305 306 if (badPaths.size() != 0) { 307 System.out.println("Error: " + badPaths.size() 308 + " Paths were not prettied: use -DSHOW and look for ones with %% in them."); 309 } 310 } 311 } 312 checkStock(Factory factory)313 static void checkStock(Factory factory) { 314 String[][] items = { 315 { "full", "yMMMMEEEEd", "jmmsszzzz" }, 316 { "long", "yMMMMd", "jmmssz" }, 317 { "medium", "yMMMd", "jmmss" }, 318 { "short", "yMd", "jmm" }, 319 }; 320 String calendarID = "gregorian"; 321 String datetimePathPrefix = "//ldml/dates/calendars/calendar[@type=\"" + calendarID + "\"]/"; 322 323 int total = 0; 324 int mismatch = 0; 325 LanguageTagParser ltp = new LanguageTagParser(); 326 Iterable<String> locales = StandardCodes.make().getLocaleCoverageLocales(Organization.cldr, EnumSet.of(Level.MODERN)); 327 for (String locale : locales) { 328 if (!ltp.set(locale).getRegion().isEmpty()) { 329 continue; 330 } 331 CLDRFile file = factory.make(locale, false); 332 DateTimeFormats dtf = new DateTimeFormats(); 333 dtf.set(file, "gregorian", false); 334 for (String[] stockInfo : items) { 335 String length = stockInfo[0]; 336 //ldml/dates/calendars/calendar[@type="gregorian"]/dateFormats/dateFormatLength[@type="full"]/dateFormat[@type="standard"]/pattern[@type="standard"] 337 String path = datetimePathPrefix + "dateFormats/dateFormatLength[@type=\"" + 338 length + "\"]/dateFormat[@type=\"standard\"]/pattern[@type=\"standard\"]"; 339 String stockDatePattern = file.getStringValue(path); 340 String flexibleDatePattern = dtf.getBestPattern(stockInfo[1]); 341 mismatch += showStatus(++total, locale, "date", length, stockInfo[1], stockDatePattern, flexibleDatePattern); 342 path = datetimePathPrefix + "timeFormats/timeFormatLength[@type=\"" + length + 343 "\"]/timeFormat[@type=\"standard\"]/pattern[@type=\"standard\"]"; 344 String stockTimePattern = file.getStringValue(path); 345 String flexibleTimePattern = dtf.getBestPattern(stockInfo[2]); 346 mismatch += showStatus(++total, locale, "time", length, stockInfo[2], stockTimePattern, flexibleTimePattern); 347 } 348 } 349 System.out.println("Mismatches:\t" + mismatch + "\tTotal:\t" + total); 350 } 351 352 static final Date SAMPLE_DATE = new Date(2013 - 1900, 1 - 1, 29, 13, 59, 59); 353 showStatus(int total, String locale, String type, String length, String skeleton, String stockPattern, String flexiblePattern)354 private static int showStatus(int total, String locale, String type, String length, 355 String skeleton, String stockPattern, String flexiblePattern) { 356 ULocale ulocale = new ULocale(locale); 357 DateFormatSymbols dfs = new DateFormatSymbols(ulocale); // just use ICU for now 358 boolean areSame = Objects.equals(stockPattern, flexiblePattern); 359 System.out.println(total 360 + "\t" + (areSame ? "ok" : "diff") 361 + "\t" + locale 362 + "\t" + type 363 + "\t" + length 364 + "\t" + skeleton 365 + "\t" + stockPattern 366 + "\t" + (areSame ? "" : flexiblePattern) 367 + "\t'" + new SimpleDateFormat(stockPattern, dfs, ulocale).format(SAMPLE_DATE) 368 + "\t'" + (areSame ? "" : new SimpleDateFormat(flexiblePattern, dfs, ulocale).format(SAMPLE_DATE))); 369 return areSame ? 0 : 1; 370 } 371 372 }