1 package org.unicode.cldr.test; 2 3 import java.io.File; 4 import java.io.IOException; 5 import java.io.InputStream; 6 import java.util.Arrays; 7 import java.util.Date; 8 import java.util.EnumSet; 9 import java.util.HashSet; 10 import java.util.Iterator; 11 import java.util.Objects; 12 import java.util.Set; 13 import java.util.TreeMap; 14 import java.util.TreeSet; 15 import java.util.regex.Matcher; 16 17 import org.unicode.cldr.tool.ToolConfig; 18 import org.unicode.cldr.util.CLDRConfig; 19 import org.unicode.cldr.util.CLDRFile; 20 import org.unicode.cldr.util.CLDRPaths; 21 import org.unicode.cldr.util.CldrUtility; 22 import org.unicode.cldr.util.DateTimeFormats; 23 import org.unicode.cldr.util.DtdType; 24 import org.unicode.cldr.util.Factory; 25 import org.unicode.cldr.util.InputStreamFactory; 26 import org.unicode.cldr.util.LanguageTagParser; 27 import org.unicode.cldr.util.Level; 28 import org.unicode.cldr.util.Organization; 29 import org.unicode.cldr.util.PatternCache; 30 import org.unicode.cldr.util.PrettyPath; 31 import org.unicode.cldr.util.StandardCodes; 32 import org.unicode.cldr.util.XMLFileReader; 33 import org.unicode.cldr.util.XPathParts; 34 import org.xml.sax.ErrorHandler; 35 import org.xml.sax.InputSource; 36 import org.xml.sax.SAXException; 37 import org.xml.sax.SAXParseException; 38 import org.xml.sax.XMLReader; 39 40 import com.ibm.icu.impl.Relation; 41 import com.ibm.icu.text.DateFormatSymbols; 42 import com.ibm.icu.text.SimpleDateFormat; 43 import com.ibm.icu.util.ULocale; 44 45 /** 46 * Simple test that loads each file in the cldr directory, thus verifying that 47 * the DTD works, and also checks that the PrettyPaths work. 48 * 49 * @author markdavis 50 */ 51 public class QuickCheck { 52 private static final Set<String> skipAttributes = new HashSet<String>(Arrays.asList(new String[] { 53 "alt", "draft", "references" })); 54 55 private static String localeRegex; 56 57 private static boolean showInfo = false; 58 59 private static String commonDirectory; 60 private static String mainDirectory; 61 62 private static boolean resolved; 63 64 private static Exception[] internalException = new Exception[1]; 65 66 private static boolean verbose; 67 main(String[] args)68 public static void main(String[] args) throws IOException { 69 CLDRConfig testInfo = ToolConfig.getToolInstance(); 70 Factory factory = testInfo.getCldrFactory(); 71 checkStock(factory); 72 if (true) return; 73 verbose = CldrUtility.getProperty("verbose", "false", "true").matches("(?i)T|TRUE"); 74 localeRegex = CldrUtility.getProperty("locale", ".*"); 75 76 showInfo = CldrUtility.getProperty("showinfo", "false", "true").matches("(?i)T|TRUE"); 77 78 commonDirectory = CLDRPaths.COMMON_DIRECTORY; // Utility.getProperty("common", Utility.COMMON_DIRECTORY); 79 // if (commonDirectory == null) commonDirectory = Utility.COMMON_DIRECTORY 80 // System.out.println("Main Source Directory: " + commonDirectory + 81 // "\t\t(to change, use -DSOURCE=xxx, eg -DSOURCE=C:/cvsdata/unicode/cldr/incoming/proposed/main)"); 82 83 mainDirectory = CldrUtility.getProperty("main", CLDRPaths.COMMON_DIRECTORY + "/main"); 84 // System.out.println("Main Source Directory: " + commonDirectory + 85 // "\t\t(to change, use -DSOURCE=xxx, eg -DSOURCE=C:/cvsdata/unicode/cldr/incoming/proposed/main)"); 86 87 resolved = CldrUtility.getProperty("resolved", "false", "true").matches("(?i)T|TRUE"); 88 89 boolean paths = CldrUtility.getProperty("paths", "true").matches("(?i)T|TRUE"); 90 91 pretty = CldrUtility.getProperty("pretty", "true").matches("(?i)T|TRUE"); 92 93 double startTime = System.currentTimeMillis(); 94 checkDtds(); 95 double deltaTime = System.currentTimeMillis() - startTime; 96 System.out.println("Elapsed: " + deltaTime / 1000.0 + " seconds"); 97 98 if (paths) { 99 System.out.println("Checking paths"); 100 checkPaths(); 101 deltaTime = System.currentTimeMillis() - startTime; 102 System.out.println("Elapsed: " + deltaTime / 1000.0 + " seconds"); 103 System.out.println("Basic Test Passes"); 104 } 105 } 106 checkDtds()107 private static void checkDtds() throws IOException { 108 checkDtds(commonDirectory + "supplemental"); 109 checkDtds(commonDirectory + "collation"); 110 checkDtds(commonDirectory + "main"); 111 checkDtds(commonDirectory + "rbnf"); 112 checkDtds(commonDirectory + "segments"); 113 checkDtds(commonDirectory + "../test"); 114 checkDtds(commonDirectory + "transforms"); 115 } 116 checkDtds(String directory)117 private static void checkDtds(String directory) throws IOException { 118 File directoryFile = new File(directory); 119 File[] listFiles = directoryFile.listFiles(); 120 String canonicalPath = directoryFile.getCanonicalPath(); 121 if (listFiles == null) { 122 throw new IllegalArgumentException("Empty directory: " + canonicalPath); 123 } 124 System.out.println("Checking files for DTD errors in: " + canonicalPath); 125 for (File fileName : listFiles) { 126 if (!fileName.toString().endsWith(".xml")) { 127 continue; 128 } 129 check(fileName); 130 } 131 } 132 133 static class MyErrorHandler implements ErrorHandler { error(SAXParseException exception)134 public void error(SAXParseException exception) throws SAXException { 135 System.out.println("\nerror: " + XMLFileReader.showSAX(exception)); 136 throw exception; 137 } 138 fatalError(SAXParseException exception)139 public void fatalError(SAXParseException exception) throws SAXException { 140 System.out.println("\nfatalError: " + XMLFileReader.showSAX(exception)); 141 throw exception; 142 } 143 warning(SAXParseException exception)144 public void warning(SAXParseException exception) throws SAXException { 145 System.out.println("\nwarning: " + XMLFileReader.showSAX(exception)); 146 throw exception; 147 } 148 } 149 check(File systemID)150 public static void check(File systemID) { 151 try (InputStream fis = InputStreamFactory.createInputStream(systemID)) { 152 // FileInputStream fis = new FileInputStream(systemID); 153 XMLReader xmlReader = XMLFileReader.createXMLReader(true); 154 xmlReader.setErrorHandler(new MyErrorHandler()); 155 InputSource is = new InputSource(fis); 156 is.setSystemId(systemID.toString()); 157 xmlReader.parse(is); 158 // fis.close(); 159 } catch (SAXException | IOException e) { // SAXParseException is a Subtype of SaxException 160 System.out.println("\t" + "Can't read " + systemID); 161 System.out.println("\t" + e.getClass() + "\t" + e.getMessage()); 162 } 163 // catch (SAXException e) { 164 // System.out.println("\t" + "Can't read " + systemID); 165 // System.out.println("\t" + e.getClass() + "\t" + e.getMessage()); 166 // } catch (IOException e) { 167 // System.out.println("\t" + "Can't read " + systemID); 168 // System.out.println("\t" + e.getClass() + "\t" + e.getMessage()); 169 // } 170 } 171 172 static Matcher skipPaths = PatternCache.get("/identity" + "|/alias" + "|\\[@alt=\"proposed").matcher(""); 173 174 private static boolean pretty; 175 checkPaths()176 private static void checkPaths() { 177 Relation<String, String> distinguishing = Relation.<String, String> of(new TreeMap<String, Set<String>>(), TreeSet.class, null); 178 Relation<String, String> nonDistinguishing = Relation.<String, String> of(new TreeMap<String, Set<String>>(), TreeSet.class, null); 179 Factory cldrFactory = Factory.make(mainDirectory, localeRegex); 180 CLDRFile english = cldrFactory.make("en", true); 181 182 Relation<String, String> pathToLocale = Relation.of( 183 new TreeMap<String, Set<String>>(CLDRFile.getComparator(DtdType.ldml)), 184 TreeSet.class, null); 185 for (String locale : cldrFactory.getAvailable()) { 186 // if (locale.equals("root") && !localeRegex.equals("root")) 187 // continue; 188 CLDRFile file; 189 try { 190 file = cldrFactory.make(locale, resolved); 191 } catch (Exception e) { 192 System.out.println("\nfatalError: " + e.getMessage()); 193 continue; 194 } 195 if (file.isNonInheriting()) 196 continue; 197 DisplayAndInputProcessor displayAndInputProcessor = new DisplayAndInputProcessor(file, false); 198 199 System.out.println(locale + "\t-\t" + english.getName(locale)); 200 DtdType dtdType = null; 201 202 for (Iterator<String> it = file.iterator(); it.hasNext();) { 203 String path = it.next(); 204 if (path.endsWith("/alias")) { 205 continue; 206 } 207 String value = file.getStringValue(path); 208 if (value == null) { 209 throw new IllegalArgumentException(locale + "\tError: in null value at " + path); 210 } 211 String displayValue = displayAndInputProcessor.processForDisplay(path, value); 212 if (!displayValue.equals(value)) { 213 System.out.println("\t" + locale + "\tdisplayAndInputProcessor changes display value <" + value 214 + ">\t=>\t<" + displayValue + ">\t\t" + path); 215 } 216 String inputValue = displayAndInputProcessor.processInput(path, value, internalException); 217 if (internalException[0] != null) { 218 System.out.println("\t" + locale + "\tdisplayAndInputProcessor internal error <" + value 219 + ">\t=>\t<" + inputValue + ">\t\t" + path); 220 internalException[0].printStackTrace(System.out); 221 } 222 if (verbose && !inputValue.equals(value)) { 223 displayAndInputProcessor.processInput(path, value, internalException); // for debugging 224 System.out.println("\t" + locale + "\tdisplayAndInputProcessor changes input value <" + value 225 + ">\t=>\t<" + inputValue + ">\t\t" + path); 226 } 227 228 pathToLocale.put(path, locale); 229 230 // also check for non-distinguishing attributes 231 if (path.contains("/identity")) continue; 232 233 // make sure we don't have problem alts 234 if (path.contains("proposed")) { 235 String sourceLocale = file.getSourceLocaleID(path, null); 236 if (locale.equals(sourceLocale)) { 237 String nonAltPath = CLDRFile.getNondraftNonaltXPath(path); 238 if (!path.equals(nonAltPath)) { 239 String nonAltLocale = file.getSourceLocaleID(nonAltPath, null); 240 String nonAltValue = file.getStringValue(nonAltPath); 241 if (nonAltValue == null || !locale.equals(nonAltLocale)) { 242 System.out.println("\t" + locale + "\tProblem alt=proposed <" + value + ">\t\t" + path); 243 } 244 } 245 } 246 } 247 248 String fullPath = file.getFullXPath(path); 249 XPathParts parts = XPathParts.getFrozenInstance(fullPath); 250 if (dtdType == null) { 251 dtdType = DtdType.valueOf(parts.getElement(0)); 252 } 253 for (int i = 0; i < parts.size(); ++i) { 254 if (parts.getAttributeCount(i) == 0) continue; 255 String element = parts.getElement(i); 256 for (String attribute : parts.getAttributeKeys(i)) { 257 if (skipAttributes.contains(attribute)) continue; 258 if (CLDRFile.isDistinguishing(dtdType, element, attribute)) { 259 distinguishing.put(element, attribute); 260 } else { 261 nonDistinguishing.put(element, attribute); 262 } 263 } 264 } 265 } 266 } 267 System.out.println(); 268 269 System.out.format("Distinguishing Elements: %s" + CldrUtility.LINE_SEPARATOR, distinguishing); 270 System.out.format("Nondistinguishing Elements: %s" + CldrUtility.LINE_SEPARATOR, nonDistinguishing); 271 System.out.format("Skipped %s" + CldrUtility.LINE_SEPARATOR, skipAttributes); 272 273 if (pretty) { 274 if (showInfo) { 275 System.out.println(CldrUtility.LINE_SEPARATOR + "Showing Path to PrettyPath mapping" 276 + CldrUtility.LINE_SEPARATOR); 277 } 278 PrettyPath prettyPath = new PrettyPath().setShowErrors(true); 279 Set<String> badPaths = new TreeSet<String>(); 280 for (String path : pathToLocale.keySet()) { 281 String prettied = prettyPath.getPrettyPath(path, false); 282 if (showInfo) System.out.println(prettied + "\t\t" + path); 283 if (prettied.contains("%%") && !path.contains("/alias")) { 284 badPaths.add(path); 285 } 286 } 287 // now remove root 288 289 if (showInfo) { 290 System.out.println(CldrUtility.LINE_SEPARATOR + "Showing Paths not in root" 291 + CldrUtility.LINE_SEPARATOR); 292 } 293 294 CLDRFile root = cldrFactory.make("root", true); 295 for (Iterator<String> it = root.iterator(); it.hasNext();) { 296 pathToLocale.removeAll(it.next()); 297 } 298 if (showInfo) for (String path : pathToLocale.keySet()) { 299 if (skipPaths.reset(path).find()) { 300 continue; 301 } 302 System.out.println(path + "\t" + pathToLocale.getAll(path)); 303 } 304 305 if (badPaths.size() != 0) { 306 System.out.println("Error: " + badPaths.size() 307 + " Paths were not prettied: use -DSHOW and look for ones with %% in them."); 308 } 309 } 310 } 311 checkStock(Factory factory)312 static void checkStock(Factory factory) { 313 String[][] items = { 314 { "full", "yMMMMEEEEd", "jmmsszzzz" }, 315 { "long", "yMMMMd", "jmmssz" }, 316 { "medium", "yMMMd", "jmmss" }, 317 { "short", "yMd", "jmm" }, 318 }; 319 String calendarID = "gregorian"; 320 String datetimePathPrefix = "//ldml/dates/calendars/calendar[@type=\"" + calendarID + "\"]/"; 321 322 int total = 0; 323 int mismatch = 0; 324 LanguageTagParser ltp = new LanguageTagParser(); 325 Iterable<String> locales = StandardCodes.make().getLocaleCoverageLocales(Organization.cldr, EnumSet.of(Level.MODERN)); 326 for (String locale : locales) { 327 if (!ltp.set(locale).getRegion().isEmpty()) { 328 continue; 329 } 330 CLDRFile file = factory.make(locale, false); 331 DateTimeFormats dtf = new DateTimeFormats(); 332 dtf.set(file, "gregorian", false); 333 for (String[] stockInfo : items) { 334 String length = stockInfo[0]; 335 //ldml/dates/calendars/calendar[@type="gregorian"]/dateFormats/dateFormatLength[@type="full"]/dateFormat[@type="standard"]/pattern[@type="standard"] 336 String path = datetimePathPrefix + "dateFormats/dateFormatLength[@type=\"" + 337 length + "\"]/dateFormat[@type=\"standard\"]/pattern[@type=\"standard\"]"; 338 String stockDatePattern = file.getStringValue(path); 339 String flexibleDatePattern = dtf.getBestPattern(stockInfo[1]); 340 mismatch += showStatus(++total, locale, "date", length, stockInfo[1], stockDatePattern, flexibleDatePattern); 341 path = datetimePathPrefix + "timeFormats/timeFormatLength[@type=\"" + length + 342 "\"]/timeFormat[@type=\"standard\"]/pattern[@type=\"standard\"]"; 343 String stockTimePattern = file.getStringValue(path); 344 String flexibleTimePattern = dtf.getBestPattern(stockInfo[2]); 345 mismatch += showStatus(++total, locale, "time", length, stockInfo[2], stockTimePattern, flexibleTimePattern); 346 } 347 } 348 System.out.println("Mismatches:\t" + mismatch + "\tTotal:\t" + total); 349 } 350 351 static final Date SAMPLE_DATE = new Date(2013 - 1900, 1 - 1, 29, 13, 59, 59); 352 showStatus(int total, String locale, String type, String length, String skeleton, String stockPattern, String flexiblePattern)353 private static int showStatus(int total, String locale, String type, String length, 354 String skeleton, String stockPattern, String flexiblePattern) { 355 ULocale ulocale = new ULocale(locale); 356 DateFormatSymbols dfs = new DateFormatSymbols(ulocale); // just use ICU for now 357 boolean areSame = Objects.equals(stockPattern, flexiblePattern); 358 System.out.println(total 359 + "\t" + (areSame ? "ok" : "diff") 360 + "\t" + locale 361 + "\t" + type 362 + "\t" + length 363 + "\t" + skeleton 364 + "\t" + stockPattern 365 + "\t" + (areSame ? "" : flexiblePattern) 366 + "\t'" + new SimpleDateFormat(stockPattern, dfs, ulocale).format(SAMPLE_DATE) 367 + "\t'" + (areSame ? "" : new SimpleDateFormat(flexiblePattern, dfs, ulocale).format(SAMPLE_DATE))); 368 return areSame ? 0 : 1; 369 } 370 371 }