1 package org.unicode.cldr.test; 2 3 import java.text.ParseException; 4 import java.util.ArrayList; 5 import java.util.Arrays; 6 import java.util.Calendar; 7 import java.util.Collection; 8 import java.util.Date; 9 import java.util.EnumMap; 10 import java.util.HashSet; 11 import java.util.Iterator; 12 import java.util.LinkedHashSet; 13 import java.util.List; 14 import java.util.Locale; 15 import java.util.Map; 16 import java.util.Random; 17 import java.util.Set; 18 import java.util.TreeSet; 19 import java.util.regex.Matcher; 20 import java.util.regex.Pattern; 21 22 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 23 import org.unicode.cldr.util.ApproximateWidth; 24 import org.unicode.cldr.util.CLDRFile; 25 import org.unicode.cldr.util.CLDRFile.Status; 26 import org.unicode.cldr.util.CLDRLocale; 27 import org.unicode.cldr.util.CldrUtility; 28 import org.unicode.cldr.util.DateTimeCanonicalizer.DateTimePatternType; 29 import org.unicode.cldr.util.DayPeriodInfo; 30 import org.unicode.cldr.util.DayPeriodInfo.DayPeriod; 31 import org.unicode.cldr.util.DayPeriodInfo.Type; 32 import org.unicode.cldr.util.Factory; 33 import org.unicode.cldr.util.ICUServiceBuilder; 34 import org.unicode.cldr.util.Level; 35 import org.unicode.cldr.util.LocaleIDParser; 36 import org.unicode.cldr.util.LogicalGrouping; 37 import org.unicode.cldr.util.PathHeader; 38 import org.unicode.cldr.util.PathStarrer; 39 import org.unicode.cldr.util.PatternCache; 40 import org.unicode.cldr.util.PreferredAndAllowedHour; 41 import org.unicode.cldr.util.RegexUtilities; 42 import org.unicode.cldr.util.SupplementalDataInfo; 43 import org.unicode.cldr.util.XPathParts; 44 import org.unicode.cldr.util.props.UnicodeProperty.PatternMatcher; 45 46 import com.ibm.icu.dev.util.CollectionUtilities; 47 import com.ibm.icu.impl.Relation; 48 import com.ibm.icu.text.BreakIterator; 49 import com.ibm.icu.text.DateTimePatternGenerator; 50 import com.ibm.icu.text.DateTimePatternGenerator.VariableField; 51 import com.ibm.icu.text.MessageFormat; 52 import com.ibm.icu.text.NumberFormat; 53 import com.ibm.icu.text.SimpleDateFormat; 54 import com.ibm.icu.text.UnicodeSet; 55 import com.ibm.icu.util.Output; 56 import com.ibm.icu.util.ULocale; 57 58 public class CheckDates extends FactoryCheckCLDR { 59 static boolean GREGORIAN_ONLY = CldrUtility.getProperty("GREGORIAN", false); 60 61 ICUServiceBuilder icuServiceBuilder = new ICUServiceBuilder(); 62 NumberFormat english = NumberFormat.getNumberInstance(ULocale.ENGLISH); 63 PatternMatcher m; 64 DateTimePatternGenerator.FormatParser formatParser = new DateTimePatternGenerator.FormatParser(); 65 DateTimePatternGenerator dateTimePatternGenerator = DateTimePatternGenerator.getEmptyInstance(); 66 private CoverageLevel2 coverageLevel; 67 private SupplementalDataInfo sdi = SupplementalDataInfo.getInstance(); 68 69 // Use the width of the character "0" as the basic unit for checking widths 70 // It's not perfect, but I'm not sure that anything can be. This helps us 71 // weed out some false positives in width checking, like 10月 vs. 十月 72 // in Chinese, which although technically longer, shouldn't trigger an 73 // error. 74 private static final int REFCHAR = ApproximateWidth.getWidth("0"); 75 76 private Level requiredLevel; 77 private String language; 78 private String territory; 79 80 private DayPeriodInfo dateFormatInfoFormat; 81 82 static String[] samples = { 83 // "AD 1970-01-01T00:00:00Z", 84 // "BC 4004-10-23T07:00:00Z", // try a BC date: creation according to Ussher & Lightfoot. Assuming garden of 85 // eden 2 hours ahead of UTC 86 "2005-12-02 12:15:16", 87 // "AD 2100-07-11T10:15:16Z", 88 }; // keep aligned with following 89 static String SampleList = "{0}" 90 // + Utility.LINE_SEPARATOR + "\t\u200E{1}\u200E" + Utility.LINE_SEPARATOR + "\t\u200E{2}\u200E" + 91 // Utility.LINE_SEPARATOR + "\t\u200E{3}\u200E" 92 ; // keep aligned with previous 93 94 private static final String DECIMAL_XPATH = "//ldml/numbers/symbols[@numberSystem='latn']/decimal"; 95 private static final Pattern HOUR_SYMBOL = PatternCache.get("H{1,2}"); 96 private static final Pattern MINUTE_SYMBOL = PatternCache.get("mm"); 97 private static final Pattern YEAR_FIELDS = PatternCache.get("(y|Y|u|U|r){1,5}"); 98 99 static String[] calTypePathsToCheck = { 100 "//ldml/dates/calendars/calendar[@type=\"buddhist\"]", 101 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]", 102 "//ldml/dates/calendars/calendar[@type=\"hebrew\"]", 103 "//ldml/dates/calendars/calendar[@type=\"islamic\"]", 104 "//ldml/dates/calendars/calendar[@type=\"japanese\"]", 105 "//ldml/dates/calendars/calendar[@type=\"roc\"]", 106 }; 107 static String[] calSymbolPathsWhichNeedDistinctValues = { 108 // === for months, days, quarters - format wide & abbrev sets must have distinct values === 109 "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"abbreviated\"]/month", 110 "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"wide\"]/month", 111 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"abbreviated\"]/day", 112 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"short\"]/day", 113 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"wide\"]/day", 114 "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"abbreviated\"]/quarter", 115 "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"wide\"]/quarter", 116 // === for dayPeriods - all values for a given context/width must be distinct === 117 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"abbreviated\"]/dayPeriod", 118 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod", 119 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod", 120 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"abbreviated\"]/dayPeriod", 121 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod", 122 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod", 123 // === for eras - all values for a given context/width should be distinct (warning) === 124 "/eras/eraNames/era", 125 "/eras/eraAbbr/era", // Hmm, root eraAbbr for japanese has many dups, should we change them or drop this test? 126 "/eras/eraNarrow/era", // We may need to allow dups here too 127 }; 128 129 // The following calendar symbol sets need not have distinct values 130 // "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"narrow\"]/month", 131 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"abbreviated\"]/month", 132 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"narrow\"]/month", 133 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"wide\"]/month", 134 // "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"narrow\"]/day", 135 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"abbreviated\"]/day", 136 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"narrow\"]/day", 137 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"wide\"]/day", 138 // "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"narrow\"]/quarter", 139 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"abbreviated\"]/quarter", 140 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"narrow\"]/quarter", 141 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"wide\"]/quarter", 142 143 // The above are followed by trailing pieces such as 144 // "[@type=\"am\"]", 145 // "[@type=\"sun\"]", 146 // "[@type=\"0\"]", 147 // "[@type=\"1\"]", 148 // "[@type=\"12\"]", 149 150 // Map<String, Set<String>> calPathsToSymbolSets; 151 // Map<String, Map<String, String>> calPathsToSymbolMaps = new HashMap<String, Map<String, String>>(); 152 CheckDates(Factory factory)153 public CheckDates(Factory factory) { 154 super(factory); 155 } 156 157 @Override setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors)158 public CheckCLDR setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options, 159 List<CheckStatus> possibleErrors) { 160 if (cldrFileToCheck == null) return this; 161 super.setCldrFileToCheck(cldrFileToCheck, options, possibleErrors); 162 163 icuServiceBuilder.setCldrFile(getResolvedCldrFileToCheck()); 164 // the following is a hack to work around a bug in ICU4J (the snapshot, not the released version). 165 try { 166 bi = BreakIterator.getCharacterInstance(new ULocale(cldrFileToCheck.getLocaleID())); 167 } catch (RuntimeException e) { 168 bi = BreakIterator.getCharacterInstance(new ULocale("")); 169 } 170 CLDRFile resolved = getResolvedCldrFileToCheck(); 171 flexInfo = new FlexibleDateFromCLDR(); // ought to just clear(), but not available. 172 flexInfo.set(resolved); 173 174 // load decimal path specially 175 String decimal = resolved.getWinningValue(DECIMAL_XPATH); 176 if (decimal != null) { 177 flexInfo.checkFlexibles(DECIMAL_XPATH, decimal, DECIMAL_XPATH); 178 } 179 180 String localeID = cldrFileToCheck.getLocaleID(); 181 LocaleIDParser lp = new LocaleIDParser(); 182 territory = lp.set(localeID).getRegion(); 183 language = lp.getLanguage(); 184 if (territory == null || territory.length() == 0) { 185 if (language.equals("root")) { 186 territory = "001"; 187 } else { 188 CLDRLocale loc = CLDRLocale.getInstance(localeID); 189 CLDRLocale defContent = sdi.getDefaultContentFromBase(loc); 190 if (defContent == null) { 191 territory = "001"; 192 } else { 193 territory = defContent.getCountry(); 194 } 195 // Set territory for 12/24 hour clock to Egypt (12 hr) for ar_001 196 // instead of 24 hour (exception). 197 if (territory.equals("001") && language.equals("ar")) { 198 territory = "EG"; 199 } 200 } 201 } 202 coverageLevel = CoverageLevel2.getInstance(sdi, localeID); 203 requiredLevel = options.getRequiredLevel(localeID); 204 205 // load gregorian appendItems 206 for (Iterator<String> it = resolved.iterator("//ldml/dates/calendars/calendar[@type=\"gregorian\"]"); it.hasNext();) { 207 String path = it.next(); 208 String value = resolved.getWinningValue(path); 209 String fullPath = resolved.getFullXPath(path); 210 try { 211 flexInfo.checkFlexibles(path, value, fullPath); 212 } catch (Exception e) { 213 final String message = e.getMessage(); 214 CheckStatus item = new CheckStatus() 215 .setCause(this) 216 .setMainType(CheckStatus.errorType) 217 .setSubtype( 218 message.contains("Conflicting fields") ? Subtype.dateSymbolCollision : Subtype.internalError) 219 .setMessage(message); 220 possibleErrors.add(item); 221 } 222 // possibleErrors.add(flexInfo.getFailurePath(path)); 223 } 224 redundants.clear(); 225 flexInfo.getRedundants(redundants); 226 // Set baseSkeletons = flexInfo.gen.getBaseSkeletons(new TreeSet()); 227 // Set notCovered = new TreeSet(neededFormats); 228 // if (flexInfo.preferred12Hour()) { 229 // notCovered.addAll(neededHours12); 230 // } else { 231 // notCovered.addAll(neededHours24); 232 // } 233 // notCovered.removeAll(baseSkeletons); 234 // if (notCovered.size() != 0) { 235 // possibleErrors.add(new CheckStatus().setCause(this).setType(CheckCLDR.finalErrorType) 236 // .setCheckOnSubmit(false) 237 // .setMessage("Missing availableFormats: {0}", new Object[]{notCovered.toString()})); 238 // } 239 pathsWithConflictingOrder2sample = DateOrder.getOrderingInfo(cldrFileToCheck, resolved, flexInfo.fp); 240 if (pathsWithConflictingOrder2sample == null) { 241 CheckStatus item = new CheckStatus() 242 .setCause(this) 243 .setMainType(CheckStatus.errorType) 244 .setSubtype(Subtype.internalError) 245 .setMessage("DateOrder.getOrderingInfo fails"); 246 possibleErrors.add(item); 247 } 248 249 // calPathsToSymbolMaps.clear(); 250 // for (String calTypePath: calTypePathsToCheck) { 251 // for (String calSymbolPath: calSymbolPathsWhichNeedDistinctValues) { 252 // calPathsToSymbolMaps.put(calTypePath.concat(calSymbolPath), null); 253 // } 254 // } 255 256 dateFormatInfoFormat = sdi.getDayPeriods(Type.format, cldrFileToCheck.getLocaleID()); 257 return this; 258 } 259 260 Map<String, Map<DateOrder, String>> pathsWithConflictingOrder2sample; 261 262 // Set neededFormats = new TreeSet(Arrays.asList(new String[]{ 263 // "yM", "yMMM", "yMd", "yMMMd", "Md", "MMMd","yQ" 264 // })); 265 // Set neededHours12 = new TreeSet(Arrays.asList(new String[]{ 266 // "hm", "hms" 267 // })); 268 // Set neededHours24 = new TreeSet(Arrays.asList(new String[]{ 269 // "Hm", "Hms" 270 // })); 271 /** 272 * hour+minute, hour+minute+second (12 & 24) 273 * year+month, year+month+day (numeric & string) 274 * month+day (numeric & string) 275 * year+quarter 276 */ 277 BreakIterator bi; 278 FlexibleDateFromCLDR flexInfo; 279 Collection<String> redundants = new HashSet<String>(); 280 Status status = new Status(); 281 PathStarrer pathStarrer = new PathStarrer(); 282 stripPrefix(String s)283 private String stripPrefix(String s) { 284 if (s != null && s.lastIndexOf(" ") < 3) { 285 return s.substring(s.lastIndexOf(" ") + 1); 286 } 287 return s; 288 } 289 handleCheck(String path, String fullPath, String value, Options options, List<CheckStatus> result)290 public CheckCLDR handleCheck(String path, String fullPath, String value, Options options, 291 List<CheckStatus> result) { 292 293 if (fullPath == null) { 294 return this; // skip paths that we don't have 295 } 296 297 if (path.indexOf("/dates") < 0 298 || path.endsWith("/default") 299 || path.endsWith("/alias")) { 300 return this; 301 } 302 303 String sourceLocale = getCldrFileToCheck().getSourceLocaleID(path, status); 304 305 if (!path.equals(status.pathWhereFound) || !sourceLocale.equals(getCldrFileToCheck().getLocaleID())) { 306 return this; 307 } 308 309 if (value == null) { 310 return this; 311 } 312 313 if (pathsWithConflictingOrder2sample != null) { 314 Map<DateOrder, String> problem = pathsWithConflictingOrder2sample.get(path); 315 if (problem != null) { 316 CheckStatus item = new CheckStatus() 317 .setCause(this) 318 .setMainType(CheckStatus.warningType) 319 .setSubtype(Subtype.incorrectDatePattern) 320 .setMessage("The ordering of date fields is inconsistent with others: {0}", 321 getValues(getResolvedCldrFileToCheck(), problem.values())); 322 result.add(item); 323 } 324 } 325 326 try { 327 if (path.indexOf("[@type=\"abbreviated\"]") >= 0) { 328 String pathToWide = path.replace("[@type=\"abbreviated\"]", "[@type=\"wide\"]"); 329 String wideValue = getCldrFileToCheck().getWinningValueWithBailey(pathToWide); 330 if (wideValue != null && isTooMuchWiderThan(value, wideValue)) { 331 CheckStatus item = new CheckStatus() 332 .setCause(this) 333 .setMainType(CheckStatus.errorType) 334 .setSubtype(Subtype.abbreviatedDateFieldTooWide) 335 .setMessage("Abbreviated value \"{0}\" can't be longer than the corresponding wide value \"{1}\"", value, 336 wideValue); 337 result.add(item); 338 } 339 for (String lgPath : LogicalGrouping.getPaths(getCldrFileToCheck(), path)) { 340 String lgPathValue = getCldrFileToCheck().getWinningValueWithBailey(lgPath); 341 String lgPathToWide = lgPath.replace("[@type=\"abbreviated\"]", "[@type=\"wide\"]"); 342 String lgPathWideValue = getCldrFileToCheck().getWinningValueWithBailey(lgPathToWide); 343 // This helps us get around things like "de març" vs. "març" in Catalan 344 String thisValueStripped = stripPrefix(value); 345 String wideValueStripped = stripPrefix(wideValue); 346 String lgPathValueStripped = stripPrefix(lgPathValue); 347 String lgPathWideValueStripped = stripPrefix(lgPathWideValue); 348 boolean thisPathHasPeriod = value.contains("."); 349 boolean lgPathHasPeriod = lgPathValue.contains("."); 350 if (!thisValueStripped.equalsIgnoreCase(wideValueStripped) && !lgPathValueStripped.equalsIgnoreCase(lgPathWideValueStripped) && 351 thisPathHasPeriod != lgPathHasPeriod) { 352 CheckStatus.Type et = CheckStatus.errorType; 353 if (path.contains("dayPeriod")) { 354 et = CheckStatus.warningType; 355 } 356 CheckStatus item = new CheckStatus() 357 .setCause(this) 358 .setMainType(et) 359 .setSubtype(Subtype.inconsistentPeriods) 360 .setMessage("Inconsistent use of periods in abbreviations for this section."); 361 result.add(item); 362 break; 363 } 364 } 365 } else if (path.indexOf("[@type=\"narrow\"]") >= 0) { 366 String pathToAbbr = path.replace("[@type=\"narrow\"]", "[@type=\"abbreviated\"]"); 367 String abbrValue = getCldrFileToCheck().getWinningValueWithBailey(pathToAbbr); 368 if (abbrValue != null && isTooMuchWiderThan(value, abbrValue)) { 369 CheckStatus item = new CheckStatus() 370 .setCause(this) 371 .setMainType(CheckStatus.warningType) // Making this just a warning, because there are some oddball cases. 372 .setSubtype(Subtype.narrowDateFieldTooWide) 373 .setMessage("Narrow value \"{0}\" shouldn't be longer than the corresponding abbreviated value \"{1}\"", value, 374 abbrValue); 375 result.add(item); 376 } 377 } else if (path.indexOf("/eraNarrow") >= 0) { 378 String pathToAbbr = path.replace("/eraNarrow", "/eraAbbr"); 379 String abbrValue = getCldrFileToCheck().getWinningValueWithBailey(pathToAbbr); 380 if (abbrValue != null && isTooMuchWiderThan(value, abbrValue)) { 381 CheckStatus item = new CheckStatus() 382 .setCause(this) 383 .setMainType(CheckStatus.errorType) 384 .setSubtype(Subtype.narrowDateFieldTooWide) 385 .setMessage("Narrow value \"{0}\" can't be longer than the corresponding abbreviated value \"{1}\"", value, 386 abbrValue); 387 result.add(item); 388 } 389 } else if (path.indexOf("/eraAbbr") >= 0) { 390 String pathToWide = path.replace("/eraAbbr", "/eraNames"); 391 String wideValue = getCldrFileToCheck().getWinningValueWithBailey(pathToWide); 392 if (wideValue != null && isTooMuchWiderThan(value, wideValue)) { 393 CheckStatus item = new CheckStatus() 394 .setCause(this) 395 .setMainType(CheckStatus.errorType) 396 .setSubtype(Subtype.abbreviatedDateFieldTooWide) 397 .setMessage("Abbreviated value \"{0}\" can't be longer than the corresponding wide value \"{1}\"", value, 398 wideValue); 399 result.add(item); 400 } 401 402 } 403 404 String failure = flexInfo.checkValueAgainstSkeleton(path, value); 405 if (failure != null) { 406 result.add(new CheckStatus() 407 .setCause(this) 408 .setMainType(CheckStatus.errorType) 409 .setSubtype(Subtype.illegalDatePattern) 410 .setMessage(failure)); 411 } 412 413 final String collisionPrefix = "//ldml/dates/calendars/calendar"; 414 main: if (path.startsWith(collisionPrefix)) { 415 int pos = path.indexOf("\"]"); // end of first type 416 if (pos < 0 || skipPath(path)) { // skip narrow, no-calendar 417 break main; 418 } 419 pos += 2; 420 String myType = getLastType(path); 421 if (myType == null) { 422 break main; 423 } 424 String myMainType = getMainType(path); 425 426 String calendarPrefix = path.substring(0, pos); 427 boolean endsWithDisplayName = path.endsWith("displayName"); // special hack, these shouldn't be in 428 // calendar. 429 430 Set<String> retrievedPaths = new HashSet<String>(); 431 getResolvedCldrFileToCheck().getPathsWithValue(value, calendarPrefix, null, retrievedPaths); 432 if (retrievedPaths.size() < 2) { 433 break main; 434 } 435 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraAbbr/era[@type="0"], 436 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraNames/era[@type="0"], 437 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraNarrow/era[@type="0"]] 438 Type type = null; 439 DayPeriod dayPeriod = null; 440 final boolean isDayPeriod = path.contains("dayPeriod"); 441 if (isDayPeriod) { 442 XPathParts parts = XPathParts.getFrozenInstance(fullPath); 443 type = Type.fromString(parts.getAttributeValue(5, "type")); 444 dayPeriod = DayPeriod.valueOf(parts.getAttributeValue(-1, "type")); 445 } 446 447 // TODO redo above and below in terms of parts instead of searching strings 448 449 Set<String> filteredPaths = new HashSet<String>(); 450 Output<Integer> sampleError = new Output<>(); 451 452 for (String item : retrievedPaths) { 453 if (item.equals(path) 454 || skipPath(item) 455 || endsWithDisplayName != item.endsWith("displayName")) { 456 continue; 457 } 458 String otherType = getLastType(item); 459 if (myType.equals(otherType)) { // we don't care about items with the same type value 460 continue; 461 } 462 String mainType = getMainType(item); 463 if (!myMainType.equals(mainType)) { // we *only* care about items with the same type value 464 continue; 465 } 466 if (isDayPeriod) { 467 //ldml/dates/calendars/calendar[@type="gregorian"]/dayPeriods/dayPeriodContext[@type="format"]/dayPeriodWidth[@type="wide"]/dayPeriod[@type="am"] 468 XPathParts itemParts = XPathParts.getFrozenInstance(item); 469 Type itemType = Type.fromString(itemParts.getAttributeValue(5, "type")); 470 DayPeriod itemDayPeriod = DayPeriod.valueOf(itemParts.getAttributeValue(-1, "type")); 471 472 if (!dateFormatInfoFormat.collisionIsError(type, dayPeriod, itemType, itemDayPeriod, sampleError)) { 473 continue; 474 } 475 } 476 filteredPaths.add(item); 477 } 478 if (filteredPaths.size() == 0) { 479 break main; 480 } 481 Set<String> others = new TreeSet<String>(); 482 for (String path2 : filteredPaths) { 483 PathHeader pathHeader = getPathHeaderFactory().fromPath(path2); 484 others.add(pathHeader.getHeaderCode()); 485 } 486 CheckStatus.Type statusType = getPhase() == Phase.SUBMISSION || getPhase() == Phase.BUILD 487 ? CheckStatus.warningType 488 : CheckStatus.errorType; 489 final CheckStatus checkStatus = new CheckStatus() 490 .setCause(this) 491 .setMainType(statusType) 492 .setSubtype(Subtype.dateSymbolCollision); 493 if (sampleError.value == null) { 494 checkStatus.setMessage("The date value “{0}” is the same as what is used for a different item: {1}", 495 value, others.toString()); 496 } else { 497 checkStatus.setMessage("The date value “{0}” is the same as what is used for a different item: {1}. Sample problem: {2}", 498 value, others.toString(), sampleError.value / DayPeriodInfo.HOUR); 499 } 500 result.add(checkStatus); 501 } 502 503 // result.add(new CheckStatus() 504 // .setCause(this).setMainType(statusType).setSubtype(Subtype.dateSymbolCollision) 505 // .setMessage("Date symbol value {0} duplicates an earlier symbol in the same set, for {1}", value, 506 // typeForPrev)); 507 508 // // Test for duplicate date symbol names (in format wide/abbrev months/days/quarters, or any context/width 509 // dayPeriods/eras) 510 // int truncateAt = path.lastIndexOf("[@type="); // want path without any final [@type="sun"], [@type="12"], 511 // etc. 512 // if ( truncateAt >= 0 ) { 513 // String truncPath = path.substring(0,truncateAt); 514 // if ( calPathsToSymbolMaps.containsKey(truncPath) ) { 515 // // Need to check whether this symbol duplicates another 516 // String type = path.substring(truncateAt); // the final part e.g. [@type="am"] 517 // Map<String, String> mapForThisPath = calPathsToSymbolMaps.get(truncPath); 518 // if ( mapForThisPath == null ) { 519 // mapForThisPath = new HashMap<String, String>(); 520 // mapForThisPath.put(value, type); 521 // calPathsToSymbolMaps.put(truncPath, mapForThisPath); 522 // } else if ( !mapForThisPath.containsKey(value) ) { 523 // mapForThisPath.put(value, type); 524 // calPathsToSymbolMaps.put(truncPath, mapForThisPath); 525 // } else { 526 // // this value duplicates a previous one in the same set. May be only a warning. 527 // String statusType = CheckStatus.errorType; 528 // String typeForPrev = mapForThisPath.get(value); 529 // if (path.contains("/eras/")) { 530 // statusType = CheckStatus.warningType; 531 // } else if (path.contains("/dayPeriods/")) { 532 // // certain duplicates only merit a warning: 533 // // "am" and "morning", "noon" and "midDay", "pm" and "afternoon" 534 // String typeEquiv = dayPeriodsEquivMap.get(type); 535 // if ( typeForPrev.equals(typeEquiv) ) { 536 // statusType = CheckStatus.warningType; 537 // } 538 // } 539 // result.add(new CheckStatus() 540 // .setCause(this).setMainType(statusType).setSubtype(Subtype.dateSymbolCollision) 541 // .setMessage("Date symbol value {0} duplicates an earlier symbol in the same set, for {1}", value, 542 // typeForPrev)); 543 // } 544 // } 545 // } 546 547 DateTimePatternType dateTypePatternType = DateTimePatternType.fromPath(path); 548 if (DateTimePatternType.STOCK_AVAILABLE_INTERVAL_PATTERNS.contains(dateTypePatternType)) { 549 boolean patternBasicallyOk = false; 550 try { 551 if (dateTypePatternType != DateTimePatternType.INTERVAL) { 552 SimpleDateFormat sdf = new SimpleDateFormat(value); 553 } 554 formatParser.set(value); 555 patternBasicallyOk = true; 556 } catch (RuntimeException e) { 557 String message = e.getMessage(); 558 if (message.contains("Illegal datetime field:")) { 559 CheckStatus item = new CheckStatus().setCause(this) 560 .setMainType(CheckStatus.errorType) 561 .setSubtype(Subtype.illegalDatePattern) 562 .setMessage(message); 563 result.add(item); 564 } else { 565 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 566 .setSubtype(Subtype.illegalDatePattern) 567 .setMessage("Illegal date format pattern {0}", new Object[] { e }); 568 result.add(item); 569 } 570 } 571 if (patternBasicallyOk) { 572 checkPattern(dateTypePatternType, path, fullPath, value, result); 573 } 574 } else if (path.contains("hourFormat")) { 575 int semicolonPos = value.indexOf(';'); 576 if (semicolonPos < 0) { 577 CheckStatus item = new CheckStatus() 578 .setCause(this) 579 .setMainType(CheckStatus.errorType) 580 .setSubtype(Subtype.illegalDatePattern) 581 .setMessage( 582 "Value should contain a positive hour format and a negative hour format separated by a semicolon."); 583 result.add(item); 584 } else { 585 String[] formats = value.split(";"); 586 if (formats[0].equals(formats[1])) { 587 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 588 .setSubtype(Subtype.illegalDatePattern) 589 .setMessage("The hour formats should not be the same."); 590 result.add(item); 591 } else { 592 checkHasHourMinuteSymbols(formats[0], result); 593 checkHasHourMinuteSymbols(formats[1], result); 594 } 595 } 596 } 597 } catch (ParseException e) { 598 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 599 .setSubtype(Subtype.illegalDatePattern) 600 .setMessage("ParseException in creating date format {0}", new Object[] { e }); 601 result.add(item); 602 } catch (Exception e) { 603 // e.printStackTrace(); 604 // HACK 605 if (!HACK_CONFLICTING.matcher(e.getMessage()).find()) { 606 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 607 .setSubtype(Subtype.illegalDatePattern) 608 .setMessage("Error in creating date format {0}", new Object[] { e }); 609 result.add(item); 610 } 611 } 612 return this; 613 } 614 isTooMuchWiderThan(String shortString, String longString)615 private boolean isTooMuchWiderThan(String shortString, String longString) { 616 // We all 1/3 the width of the reference character as a "fudge factor" in determining the allowable width 617 return ApproximateWidth.getWidth(shortString) > ApproximateWidth.getWidth(longString) + REFCHAR / 3; 618 } 619 620 /** 621 * Check for the presence of hour and minute symbols. 622 * 623 * @param value 624 * the value to be checked 625 * @param result 626 * the list to add any errors to. 627 */ checkHasHourMinuteSymbols(String value, List<CheckStatus> result)628 private void checkHasHourMinuteSymbols(String value, List<CheckStatus> result) { 629 boolean hasHourSymbol = HOUR_SYMBOL.matcher(value).find(); 630 boolean hasMinuteSymbol = MINUTE_SYMBOL.matcher(value).find(); 631 if (!hasHourSymbol && !hasMinuteSymbol) { 632 result.add(createErrorCheckStatus().setMessage("The hour and minute symbols are missing from {0}.", value)); 633 } else if (!hasHourSymbol) { 634 result.add(createErrorCheckStatus() 635 .setMessage("The hour symbol (H or HH) should be present in {0}.", value)); 636 } else if (!hasMinuteSymbol) { 637 result.add(createErrorCheckStatus().setMessage("The minute symbol (mm) should be present in {0}.", value)); 638 } 639 } 640 641 /** 642 * Convenience method for creating errors. 643 * 644 * @return 645 */ createErrorCheckStatus()646 private CheckStatus createErrorCheckStatus() { 647 return new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 648 .setSubtype(Subtype.illegalDatePattern); 649 } 650 skipPath(String path)651 public boolean skipPath(String path) { 652 return path.contains("arrow") 653 || path.contains("/availableFormats") 654 || path.contains("/interval") 655 || path.contains("/dateTimeFormat") 656 // || path.contains("/dayPeriod[") 657 // && !path.endsWith("=\"pm\"]") 658 // && !path.endsWith("=\"am\"]") 659 ; 660 } 661 getLastType(String path)662 public String getLastType(String path) { 663 int secondType = path.lastIndexOf("[@type=\""); 664 if (secondType < 0) { 665 return null; 666 } 667 secondType += 8; 668 int secondEnd = path.indexOf("\"]", secondType); 669 if (secondEnd < 0) { 670 return null; 671 } 672 return path.substring(secondType, secondEnd); 673 } 674 getMainType(String path)675 public String getMainType(String path) { 676 int secondType = path.indexOf("\"]/"); 677 if (secondType < 0) { 678 return null; 679 } 680 secondType += 3; 681 int secondEnd = path.indexOf("/", secondType); 682 if (secondEnd < 0) { 683 return null; 684 } 685 return path.substring(secondType, secondEnd); 686 } 687 getValues(CLDRFile resolvedCldrFileToCheck, Collection<String> values)688 private String getValues(CLDRFile resolvedCldrFileToCheck, Collection<String> values) { 689 Set<String> results = new TreeSet<String>(); 690 for (String path : values) { 691 final String stringValue = resolvedCldrFileToCheck.getStringValue(path); 692 if (stringValue != null) { 693 results.add(stringValue); 694 } 695 } 696 return "{" + CollectionUtilities.join(results, "},{") + "}"; 697 } 698 699 static final Pattern HACK_CONFLICTING = PatternCache.get("Conflicting fields:\\s+M+,\\s+l"); 700 handleGetExamples(String path, String fullPath, String value, Options options, List<CheckStatus> result)701 public CheckCLDR handleGetExamples(String path, String fullPath, String value, Options options, List<CheckStatus> result) { 702 if (path.indexOf("/dates") < 0 || path.indexOf("gregorian") < 0) return this; 703 try { 704 if (path.indexOf("/pattern") >= 0 && path.indexOf("/dateTimeFormat") < 0 705 || path.indexOf("/dateFormatItem") >= 0) { 706 checkPattern2(path, fullPath, value, result); 707 } 708 } catch (Exception e) { 709 // don't worry about errors 710 } 711 return this; 712 } 713 714 // Calendar myCal = Calendar.getInstance(TimeZone.getTimeZone("America/Denver")); 715 // TimeZone denver = TimeZone.getTimeZone("America/Denver"); 716 static final SimpleDateFormat neutralFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", ULocale.ENGLISH); 717 static { 718 neutralFormat.setTimeZone(ExampleGenerator.ZONE_SAMPLE); 719 } 720 XPathParts pathParts = new XPathParts(null, null); 721 722 // Get Date-Time in milliseconds getDateTimeinMillis(int year, int month, int date, int hourOfDay, int minute, int second)723 private static long getDateTimeinMillis(int year, int month, int date, int hourOfDay, int minute, int second) { 724 Calendar cal = Calendar.getInstance(); 725 cal.set(year, month, date, hourOfDay, minute, second); 726 return cal.getTimeInMillis(); 727 } 728 729 static long date1950 = getDateTimeinMillis(1950, 0, 1, 0, 0, 0); 730 static long date2010 = getDateTimeinMillis(2010, 0, 1, 0, 0, 0); 731 static long date4004BC = getDateTimeinMillis(-4004, 9, 23, 2, 0, 0); 732 static Random random = new Random(0); 733 checkPattern(DateTimePatternType dateTypePatternType, String path, String fullPath, String value, List<CheckStatus> result)734 private void checkPattern(DateTimePatternType dateTypePatternType, String path, String fullPath, String value, List<CheckStatus> result) 735 throws ParseException { 736 String skeleton = dateTimePatternGenerator.getSkeletonAllowingDuplicates(value); 737 String skeletonCanonical = dateTimePatternGenerator.getCanonicalSkeletonAllowingDuplicates(value); 738 739 if (value.contains("MMM.") || value.contains("LLL.") || value.contains("E.") || value.contains("eee.") 740 || value.contains("ccc.") || value.contains("QQQ.") || value.contains("qqq.")) { 741 result 742 .add(new CheckStatus() 743 .setCause(this) 744 .setMainType(CheckStatus.warningType) 745 .setSubtype(Subtype.incorrectDatePattern) 746 .setMessage( 747 "Your pattern ({0}) is probably incorrect; abbreviated month/weekday/quarter names that need a period should include it in the name, rather than adding it to the pattern.", 748 value)); 749 } 750 751 pathParts.set(path); 752 String calendar = pathParts.findAttributeValue("calendar", "type"); 753 String id; 754 switch (dateTypePatternType) { 755 case AVAILABLE: 756 id = pathParts.getAttributeValue(-1, "id"); 757 break; 758 case INTERVAL: 759 id = pathParts.getAttributeValue(-2, "id"); 760 break; 761 case STOCK: 762 id = pathParts.getAttributeValue(-3, "type"); 763 break; 764 default: 765 throw new IllegalArgumentException(); 766 } 767 768 if (dateTypePatternType == DateTimePatternType.AVAILABLE || dateTypePatternType == DateTimePatternType.INTERVAL) { 769 String idCanonical = dateTimePatternGenerator.getCanonicalSkeletonAllowingDuplicates(id); 770 if (skeleton.isEmpty()) { 771 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 772 .setSubtype(Subtype.incorrectDatePattern) 773 // "Internal ID ({0}) doesn't match generated ID ({1}) for pattern ({2}). " + 774 .setMessage("Your pattern ({1}) is incorrect for ID ({0}). " + 775 "You need to supply a pattern according to http://cldr.org/translation/date-time-patterns.", 776 id, value)); 777 } else if (!dateTimePatternGenerator.skeletonsAreSimilar(idCanonical, skeletonCanonical)) { 778 String fixedValue = dateTimePatternGenerator.replaceFieldTypes(value, id); 779 result 780 .add(new CheckStatus() 781 .setCause(this) 782 .setMainType(CheckStatus.errorType) 783 .setSubtype(Subtype.incorrectDatePattern) 784 // "Internal ID ({0}) doesn't match generated ID ({1}) for pattern ({2}). " + 785 .setMessage( 786 "Your pattern ({2}) doesn't correspond to what is asked for. Yours would be right for an ID ({1}) but not for the ID ({0}). " 787 + 788 "Please change your pattern to match what was asked, such as ({3}), with the right punctuation and/or ordering for your language. See http://cldr.org/translation/date-time-patterns.", 789 id, skeletonCanonical, value, fixedValue)); 790 } 791 if (dateTypePatternType == DateTimePatternType.AVAILABLE) { 792 // index y+w+ must correpond to pattern containing only Y+ and w+ 793 if (idCanonical.matches("y+w+") && !(skeleton.matches("Y+w+") || skeleton.matches("w+Y+"))) { 794 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType).setSubtype(Subtype.incorrectDatePattern) 795 .setMessage("For id {0}, the pattern ({1}) must contain fields Y and w, and no others.", id, value)); 796 } 797 // index M+W msut correspond to pattern containing only M+/L+ and W 798 if (idCanonical.matches("M+W") && !(skeletonCanonical.matches("M+W") || skeletonCanonical.matches("WM+"))) { 799 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType).setSubtype(Subtype.incorrectDatePattern) 800 .setMessage("For id {0}, the pattern ({1}) must contain fields M or L, plus W, and no others.", id, value)); 801 } 802 } 803 String failureMessage = (String) flexInfo.getFailurePath(path); 804 if (failureMessage != null) { 805 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 806 .setSubtype(Subtype.illegalDatePattern) 807 .setMessage("{0}", new Object[] { failureMessage })); 808 } 809 810 // if (redundants.contains(value)) { 811 // result.add(new CheckStatus().setCause(this).setType(CheckStatus.errorType) 812 // .setMessage("Redundant with some pattern (or combination)", new Object[]{})); 813 // } 814 } 815 // String calendar = pathParts.findAttributeValue("calendar", "type"); 816 // if (path.indexOf("\"full\"") >= 0) { 817 // // for date, check that era is preserved 818 // // TODO fix naked constants 819 // SimpleDateFormat y = icuServiceBuilder.getDateFormat(calendar, 4, 4); 820 // //String trial = "BC 4004-10-23T2:00:00Z"; 821 // //Date dateSource = neutralFormat.parse(trial); 822 // Date dateSource = new Date(date4004BC); 823 // int year = dateSource.getYear() + 1900; 824 // if (year > 0) { 825 // year = 1-year; 826 // dateSource.setYear(year - 1900); 827 // } 828 // //myCal.setTime(dateSource); 829 // String result2 = y.format(dateSource); 830 // Date backAgain; 831 // try { 832 // 833 // backAgain = y.parse(result2,parsePosition); 834 // } catch (ParseException e) { 835 // // TODO Auto-generated catch block 836 // e.printStackTrace(); 837 // } 838 // //String isoBackAgain = neutralFormat.format(backAgain); 839 // 840 // if (false && path.indexOf("/dateFormat") >= 0 && year != backAgain.getYear()) { 841 // CheckStatus item = new CheckStatus().setCause(this).setType(CheckStatus.errorType) 842 // .setMessage("Need Era (G) in full format.", new Object[]{}); 843 // result.add(item); 844 // } 845 846 // formatParser.set(value); 847 // String newValue = toString(formatParser); 848 // if (!newValue.equals(value)) { 849 // CheckStatus item = new CheckStatus().setType(CheckStatus.warningType) 850 // .setMessage("Canonical form would be {0}", new Object[]{newValue}); 851 // result.add(item); 852 // } 853 // find the variable fields 854 855 if (dateTypePatternType == DateTimePatternType.STOCK) { 856 int style = 0; 857 String len = pathParts.findAttributeValue("timeFormatLength", "type"); 858 DateOrTime dateOrTime = DateOrTime.time; 859 if (len == null) { 860 dateOrTime = DateOrTime.date; 861 style += 4; 862 len = pathParts.findAttributeValue("dateFormatLength", "type"); 863 if (len == null) { 864 len = pathParts.findAttributeValue("dateTimeFormatLength", "type"); 865 dateOrTime = DateOrTime.dateTime; 866 } 867 } 868 869 DateTimeLengths dateTimeLength = DateTimeLengths.valueOf(len.toUpperCase(Locale.ENGLISH)); 870 871 if (calendar.equals("gregorian") && !"root".equals(getCldrFileToCheck().getLocaleID())) { 872 checkValue(dateTimeLength, dateOrTime, value, result); 873 } 874 if (dateOrTime == DateOrTime.dateTime) { 875 return; // We don't need to do the rest for date/time combo patterns. 876 } 877 style += dateTimeLength.ordinal(); 878 // do regex match with skeletonCanonical but report errors using skeleton; they have corresponding field lengths 879 if (!dateTimePatterns[style].matcher(skeletonCanonical).matches() 880 && !calendar.equals("chinese") 881 && !calendar.equals("hebrew")) { 882 int i = RegexUtilities.findMismatch(dateTimePatterns[style], skeletonCanonical); 883 String skeletonPosition = skeleton.substring(0, i) + "☹" + skeleton.substring(i); 884 result.add(new CheckStatus() 885 .setCause(this) 886 .setMainType(CheckStatus.errorType) 887 .setSubtype(Subtype.missingOrExtraDateField) 888 .setMessage("Field is missing, extra, or the wrong length. Expected {0} [Internal: {1} / {2}]", 889 new Object[] { dateTimeMessage[style], skeletonPosition, dateTimePatterns[style].pattern() })); 890 } 891 } else if (dateTypePatternType == DateTimePatternType.INTERVAL) { 892 if (id.contains("y")) { 893 String greatestDifference = pathParts.findAttributeValue("greatestDifference", "id"); 894 int requiredYearFieldCount = 1; 895 if ("y".equals(greatestDifference)) { 896 requiredYearFieldCount = 2; 897 } 898 int yearFieldCount = 0; 899 Matcher yearFieldMatcher = YEAR_FIELDS.matcher(value); 900 while (yearFieldMatcher.find()) { 901 yearFieldCount++; 902 } 903 if (yearFieldCount < requiredYearFieldCount) { 904 result.add(new CheckStatus() 905 .setCause(this) 906 .setMainType(CheckStatus.errorType) 907 .setSubtype(Subtype.missingOrExtraDateField) 908 .setMessage("Not enough year fields in interval pattern. Must have {0} but only found {1}", 909 new Object[] { requiredYearFieldCount, yearFieldCount })); 910 } 911 } 912 } 913 914 if (value.contains("G") && calendar.equals("gregorian")) { 915 GyState actual = GyState.forPattern(value); 916 GyState expected = getExpectedGy(getCldrFileToCheck().getLocaleID()); 917 if (actual != expected) { 918 result.add(new CheckStatus() 919 .setCause(this) 920 .setMainType(CheckStatus.warningType) 921 .setSubtype(Subtype.unexpectedOrderOfEraYear) 922 .setMessage("Unexpected order of era/year. Expected {0}, but got {1} in 〈{2}〉 for {3}/{4}", 923 expected, actual, value, calendar, id)); 924 } 925 } 926 } 927 928 enum DateOrTime { 929 date, time, dateTime 930 } 931 932 static final Map<DateOrTime, Relation<DateTimeLengths, String>> STOCK_PATTERNS = new EnumMap<DateOrTime, Relation<DateTimeLengths, String>>( 933 DateOrTime.class); 934 935 // add(Map<DateOrTime, Relation<DateTimeLengths, String>> stockPatterns, DateOrTime dateOrTime, DateTimeLengths dateTimeLength, String... keys)936 private static void add(Map<DateOrTime, Relation<DateTimeLengths, String>> stockPatterns, 937 DateOrTime dateOrTime, DateTimeLengths dateTimeLength, String... keys) { 938 Relation<DateTimeLengths, String> rel = STOCK_PATTERNS.get(dateOrTime); 939 if (rel == null) { 940 STOCK_PATTERNS.put(dateOrTime, rel = Relation.of(new EnumMap<DateTimeLengths, Set<String>>(DateTimeLengths.class), LinkedHashSet.class)); 941 } 942 rel.putAll(dateTimeLength, Arrays.asList(keys)); 943 } 944 945 /* Ticket #4936 946 value(short time) = value(hm) or value(Hm) 947 value(medium time) = value(hms) or value(Hms) 948 value(long time) = value(medium time+z) 949 value(full time) = value(medium time+zzzz) 950 */ 951 static { add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.SHORT, "hm", "Hm")952 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.SHORT, "hm", "Hm"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.MEDIUM, "hms", "Hms")953 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.MEDIUM, "hms", "Hms"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.LONG, "hms*z", "Hms*z")954 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.LONG, "hms*z", "Hms*z"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.FULL, "hms*zzzz", "Hms*zzzz")955 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.FULL, "hms*zzzz", "Hms*zzzz"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.SHORT, "yMd")956 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.SHORT, "yMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.MEDIUM, "yMMMd")957 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.MEDIUM, "yMMMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.LONG, "yMMMMd", "yMMMd")958 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.LONG, "yMMMMd", "yMMMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.FULL, "yMMMMEd", "yMMMEd")959 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.FULL, "yMMMMEd", "yMMMEd"); 960 } 961 962 static final String AVAILABLE_PREFIX = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\""; 963 static final String AVAILABLE_SUFFIX = "\"]"; 964 static final String APPEND_TIMEZONE = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/appendItems/appendItem[@request=\"Timezone\"]"; 965 checkValue(DateTimeLengths dateTimeLength, DateOrTime dateOrTime, String value, List<CheckStatus> result)966 private void checkValue(DateTimeLengths dateTimeLength, DateOrTime dateOrTime, String value, List<CheckStatus> result) { 967 // Check consistency of the pattern vs. supplemental wrt 12 vs. 24 hour clock. 968 if (dateOrTime == DateOrTime.time) { 969 PreferredAndAllowedHour pref = sdi.getTimeData().get(territory); 970 if (pref == null) { 971 pref = sdi.getTimeData().get("001"); 972 } 973 String checkForHour, clockType; 974 if (pref.preferred.equals(PreferredAndAllowedHour.HourStyle.h)) { 975 checkForHour = "h"; 976 clockType = "12"; 977 } else { 978 checkForHour = "H"; 979 clockType = "24"; 980 } 981 if (!value.contains(checkForHour)) { 982 CheckStatus.Type errType = CheckStatus.errorType; 983 // French/Canada is strange, they use 24 hr clock while en_CA uses 12. 984 if (language.equals("fr") && territory.equals("CA")) { 985 errType = CheckStatus.warningType; 986 } 987 988 result.add(new CheckStatus().setCause(this).setMainType(errType) 989 .setSubtype(Subtype.inconsistentTimePattern) 990 .setMessage("Time format inconsistent with supplemental time data for territory \"" + territory + "\"." 991 + " Use '" + checkForHour + "' for " + clockType + " hour clock.")); 992 } 993 } 994 if (dateOrTime == DateOrTime.dateTime) { 995 boolean inQuotes = false; 996 for (int i = 0; i < value.length(); i++) { 997 char ch = value.charAt(i); 998 if (ch == '\'') { 999 inQuotes = !inQuotes; 1000 } 1001 if (!inQuotes && (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')) { 1002 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 1003 .setSubtype(Subtype.patternContainsInvalidCharacters) 1004 .setMessage("Unquoted letter \"{0}\" in dateTime format.", ch)); 1005 } 1006 } 1007 } else { 1008 Set<String> keys = STOCK_PATTERNS.get(dateOrTime).get(dateTimeLength); 1009 StringBuilder b = new StringBuilder(); 1010 boolean onlyNulls = true; 1011 int countMismatches = 0; 1012 boolean errorOnMissing = false; 1013 String timezonePattern = null; 1014 Set<String> bases = new LinkedHashSet<String>(); 1015 for (String key : keys) { 1016 int star = key.indexOf('*'); 1017 boolean hasStar = star >= 0; 1018 String base = !hasStar ? key : key.substring(0, star); 1019 bases.add(base); 1020 String xpath = AVAILABLE_PREFIX + base + AVAILABLE_SUFFIX; 1021 String value1 = getCldrFileToCheck().getStringValue(xpath); 1022 // String localeFound = getCldrFileToCheck().getSourceLocaleID(xpath, null); && !localeFound.equals("root") && !localeFound.equals("code-fallback") 1023 if (value1 != null) { 1024 onlyNulls = false; 1025 if (hasStar) { 1026 String zone = key.substring(star + 1); 1027 timezonePattern = getResolvedCldrFileToCheck().getStringValue(APPEND_TIMEZONE); 1028 value1 = MessageFormat.format(timezonePattern, value1, zone); 1029 } 1030 if (equalsExceptWidth(value, value1)) { 1031 return; 1032 } 1033 } else { 1034 // Example, if the requiredLevel for the locale is moderate, 1035 // and the level for the path is modern, then we'll skip the error, 1036 // but if the level for the path is basic, then we won't 1037 Level pathLevel = coverageLevel.getLevel(xpath); 1038 if (requiredLevel.compareTo(pathLevel) >= 0) { 1039 errorOnMissing = true; 1040 } 1041 } 1042 add(b, base, value1); 1043 countMismatches++; 1044 } 1045 if (!onlyNulls) { 1046 if (timezonePattern != null) { 1047 b.append(" (with appendZonePattern: “" + timezonePattern + "”)"); 1048 } 1049 String msg = countMismatches != 1 1050 ? "{1}-{0} → “{2}” didn't match any of the corresponding flexible skeletons: [{3}]. This or the flexible patterns needs to be changed." 1051 : "{1}-{0} → “{2}” didn't match the corresponding flexible skeleton: {3}. This or the flexible pattern needs to be changed."; 1052 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.warningType) 1053 .setSubtype(Subtype.inconsistentDatePattern) 1054 .setMessage(msg, 1055 dateTimeLength, dateOrTime, value, b)); 1056 } else { 1057 if (errorOnMissing) { 1058 String msg = countMismatches != 1 1059 ? "{1}-{0} → “{2}” doesn't have at least one value for a corresponding flexible skeleton {3}, which needs to be added." 1060 : "{1}-{0} → “{2}” doesn't have a value for the corresponding flexible skeleton {3}, which needs to be added."; 1061 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.warningType) 1062 .setSubtype(Subtype.missingDatePattern) 1063 .setMessage(msg, 1064 dateTimeLength, dateOrTime, value, CollectionUtilities.join(bases, ", "))); 1065 } 1066 } 1067 } 1068 } 1069 add(StringBuilder b, String key, String value1)1070 private void add(StringBuilder b, String key, String value1) { 1071 if (value1 == null) { 1072 return; 1073 } 1074 if (b.length() != 0) { 1075 b.append(" or "); 1076 } 1077 b.append(key + (value1 == null ? " - missing" : " → “" + value1 + "”")); 1078 } 1079 equalsExceptWidth(String value1, String value2)1080 private boolean equalsExceptWidth(String value1, String value2) { 1081 if (value1.equals(value2)) { 1082 return true; 1083 } else if (value2 == null) { 1084 return false; 1085 } 1086 1087 List<Object> items1 = new ArrayList<Object>(formatParser.set(value1).getItems()); // clone 1088 List<Object> items2 = formatParser.set(value2).getItems(); 1089 if (items1.size() != items2.size()) { 1090 return false; 1091 } 1092 Iterator<Object> it2 = items2.iterator(); 1093 for (Object item1 : items1) { 1094 Object item2 = it2.next(); 1095 if (item1.equals(item2)) { 1096 continue; 1097 } 1098 if (item1 instanceof VariableField && item2 instanceof VariableField) { 1099 // simple test for now, ignore widths 1100 if (item1.toString().charAt(0) == item2.toString().charAt(0)) { 1101 continue; 1102 } 1103 } 1104 return false; 1105 } 1106 return true; 1107 } 1108 1109 static final Set<String> YgLanguages = new HashSet<String>(Arrays.asList( 1110 "ar", "cs", "da", "de", "en", "es", "fa", "fi", "fr", "he", "hr", "id", "it", "nb", "nl", "pt", "ru", "sv", "tr")); 1111 getExpectedGy(String localeID)1112 private GyState getExpectedGy(String localeID) { 1113 // hack for now 1114 int firstBar = localeID.indexOf('_'); 1115 String lang = firstBar < 0 ? localeID : localeID.substring(0, firstBar); 1116 return YgLanguages.contains(lang) ? GyState.YEAR_ERA : GyState.ERA_YEAR; 1117 } 1118 1119 enum GyState { 1120 YEAR_ERA, ERA_YEAR, OTHER; 1121 static DateTimePatternGenerator.FormatParser formatParser = new DateTimePatternGenerator.FormatParser(); 1122 1123 static synchronized GyState forPattern(String value) { 1124 formatParser.set(value); 1125 int last = -1; 1126 for (Object x : formatParser.getItems()) { 1127 if (x instanceof VariableField) { 1128 int type = ((VariableField) x).getType(); 1129 if (type == DateTimePatternGenerator.ERA && last == DateTimePatternGenerator.YEAR) { 1130 return GyState.YEAR_ERA; 1131 } else if (type == DateTimePatternGenerator.YEAR && last == DateTimePatternGenerator.ERA) { 1132 return GyState.ERA_YEAR; 1133 } 1134 last = type; 1135 } 1136 } 1137 return GyState.OTHER; 1138 } 1139 } 1140 1141 enum DateTimeLengths { 1142 SHORT, MEDIUM, LONG, FULL 1143 }; 1144 1145 // The patterns below should only use the *canonical* characters for each field type: 1146 // y (not Y, u, U) 1147 // Q (not q) 1148 // M (not L) 1149 // E (not e, c) 1150 // a (not b, B) 1151 // H or h (not k or K) 1152 // v (not z, Z, V) 1153 static final Pattern[] dateTimePatterns = { 1154 PatternCache.get("a*(h|hh|H|HH)(m|mm)"), // time-short 1155 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)"), // time-medium 1156 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)(v+)"), // time-long 1157 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)(v+)"), // time-full 1158 PatternCache.get("G*y{1,4}M{1,2}(d|dd)"), // date-short; allow yyy for Minguo/ROC calendar 1159 PatternCache.get("G*y(yyy)?M{1,3}(d|dd)"), // date-medium 1160 PatternCache.get("G*y(yyy)?M{1,4}(d|dd)"), // date-long 1161 PatternCache.get("G*y(yyy)?M{1,4}E*(d|dd)"), // date-full 1162 PatternCache.get(".*"), // datetime-short 1163 PatternCache.get(".*"), // datetime-medium 1164 PatternCache.get(".*"), // datetime-long 1165 PatternCache.get(".*"), // datetime-full 1166 }; 1167 1168 static final String[] dateTimeMessage = { 1169 "hours (H, HH, h, or hh), and minutes (m or mm)", // time-short 1170 "hours (H, HH, h, or hh), minutes (m or mm), and seconds (s or ss)", // time-medium 1171 "hours (H, HH, h, or hh), minutes (m or mm), and seconds (s or ss); optionally timezone (z, zzzz, v, vvvv)", // time-long 1172 "hours (H, HH, h, or hh), minutes (m or mm), seconds (s or ss), and timezone (z, zzzz, v, vvvv)", // time-full 1173 "year (y, yy, yyyy), month (M or MM), and day (d or dd); optionally era (G)", // date-short 1174 "year (y), month (M, MM, or MMM), and day (d or dd); optionally era (G)", // date-medium 1175 "year (y), month (M, ... MMMM), and day (d or dd); optionally era (G)", // date-long 1176 "year (y), month (M, ... MMMM), and day (d or dd); optionally day of week (EEEE or cccc) or era (G)", // date-full 1177 }; 1178 1179 public String toString(DateTimePatternGenerator.FormatParser formatParser) { 1180 StringBuffer result = new StringBuffer(); 1181 for (Object x : formatParser.getItems()) { 1182 if (x instanceof DateTimePatternGenerator.VariableField) { 1183 result.append(x.toString()); 1184 } else { 1185 result.append(formatParser.quoteLiteral(x.toString())); 1186 } 1187 } 1188 return result.toString(); 1189 } 1190 1191 private void checkPattern2(String path, String fullPath, String value, List<CheckStatus> result) throws ParseException { 1192 pathParts.set(path); 1193 String calendar = pathParts.findAttributeValue("calendar", "type"); 1194 SimpleDateFormat x = icuServiceBuilder.getDateFormat(calendar, value); 1195 x.setTimeZone(ExampleGenerator.ZONE_SAMPLE); 1196 1197 // Object[] arguments = new Object[samples.length]; 1198 // for (int i = 0; i < samples.length; ++i) { 1199 // String source = getRandomDate(date1950, date2010); // samples[i]; 1200 // Date dateSource = neutralFormat.parse(source); 1201 // String formatted = x.format(dateSource); 1202 // String reparsed; 1203 // 1204 // parsePosition.setIndex(0); 1205 // Date parsed = x.parse(formatted, parsePosition); 1206 // if (parsePosition.getIndex() != formatted.length()) { 1207 // reparsed = "Couldn't parse past: " + formatted.substring(0,parsePosition.getIndex()); 1208 // } else { 1209 // reparsed = neutralFormat.format(parsed); 1210 // } 1211 // 1212 // arguments[i] = source + " \u2192 \u201C\u200E" + formatted + "\u200E\u201D \u2192 " + reparsed; 1213 // } 1214 // result.add(new CheckStatus() 1215 // .setCause(this).setType(CheckStatus.exampleType) 1216 // .setMessage(SampleList, arguments)); 1217 result.add(new MyCheckStatus() 1218 .setFormat(x) 1219 .setCause(this).setMainType(CheckStatus.demoType)); 1220 } 1221 1222 static final UnicodeSet XGRAPHEME = new UnicodeSet("[[:mark:][:grapheme_extend:][:punctuation:]]"); 1223 static final UnicodeSet DIGIT = new UnicodeSet("[:decimal_number:]"); 1224 1225 static public class MyCheckStatus extends CheckStatus { 1226 private SimpleDateFormat df; 1227 1228 public MyCheckStatus setFormat(SimpleDateFormat df) { 1229 this.df = df; 1230 return this; 1231 } 1232 1233 public SimpleDemo getDemo() { 1234 return new MyDemo().setFormat(df); 1235 } 1236 } 1237 1238 static class MyDemo extends FormatDemo { 1239 private SimpleDateFormat df; 1240 1241 protected String getPattern() { 1242 return df.toPattern(); 1243 } 1244 1245 protected String getSampleInput() { 1246 return neutralFormat.format(ExampleGenerator.DATE_SAMPLE); 1247 } 1248 1249 public MyDemo setFormat(SimpleDateFormat df) { 1250 this.df = df; 1251 return this; 1252 } 1253 1254 protected void getArguments(Map<String, String> inout) { 1255 currentPattern = currentInput = currentFormatted = currentReparsed = "?"; 1256 Date d; 1257 try { 1258 currentPattern = inout.get("pattern"); 1259 if (currentPattern != null) 1260 df.applyPattern(currentPattern); 1261 else 1262 currentPattern = getPattern(); 1263 } catch (Exception e) { 1264 currentPattern = "Use format like: ##,###.##"; 1265 return; 1266 } 1267 try { 1268 currentInput = (String) inout.get("input"); 1269 if (currentInput == null) { 1270 currentInput = getSampleInput(); 1271 } 1272 d = neutralFormat.parse(currentInput); 1273 } catch (Exception e) { 1274 currentInput = "Use neutral format like: 1993-11-31 13:49:02"; 1275 return; 1276 } 1277 try { 1278 currentFormatted = df.format(d); 1279 } catch (Exception e) { 1280 currentFormatted = "Can't format: " + e.getMessage(); 1281 return; 1282 } 1283 try { 1284 parsePosition.setIndex(0); 1285 Date n = df.parse(currentFormatted, parsePosition); 1286 if (parsePosition.getIndex() != currentFormatted.length()) { 1287 currentReparsed = "Couldn't parse past: " + "\u200E" 1288 + currentFormatted.substring(0, parsePosition.getIndex()) + "\u200E"; 1289 } else { 1290 currentReparsed = neutralFormat.format(n); 1291 } 1292 } catch (Exception e) { 1293 currentReparsed = "Can't parse: " + e.getMessage(); 1294 } 1295 } 1296 1297 } 1298 } 1299