1 package org.unicode.cldr.test; 2 3 import java.text.ParseException; 4 import java.util.ArrayList; 5 import java.util.Arrays; 6 import java.util.Calendar; 7 import java.util.Collection; 8 import java.util.Date; 9 import java.util.EnumMap; 10 import java.util.HashMap; 11 import java.util.HashSet; 12 import java.util.Iterator; 13 import java.util.LinkedHashSet; 14 import java.util.List; 15 import java.util.Locale; 16 import java.util.Map; 17 import java.util.Random; 18 import java.util.Set; 19 import java.util.TreeSet; 20 import java.util.regex.Matcher; 21 import java.util.regex.Pattern; 22 23 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 24 import org.unicode.cldr.util.ApproximateWidth; 25 import org.unicode.cldr.util.CLDRFile; 26 import org.unicode.cldr.util.CLDRFile.Status; 27 import org.unicode.cldr.util.CLDRLocale; 28 import org.unicode.cldr.util.CLDRURLS; 29 import org.unicode.cldr.util.CldrUtility; 30 import org.unicode.cldr.util.DateTimeCanonicalizer.DateTimePatternType; 31 import org.unicode.cldr.util.DayPeriodInfo; 32 import org.unicode.cldr.util.DayPeriodInfo.DayPeriod; 33 import org.unicode.cldr.util.DayPeriodInfo.Type; 34 import org.unicode.cldr.util.Factory; 35 import org.unicode.cldr.util.ICUServiceBuilder; 36 import org.unicode.cldr.util.Level; 37 import org.unicode.cldr.util.LocaleIDParser; 38 import org.unicode.cldr.util.LogicalGrouping; 39 import org.unicode.cldr.util.PathHeader; 40 import org.unicode.cldr.util.PathStarrer; 41 import org.unicode.cldr.util.PatternCache; 42 import org.unicode.cldr.util.PreferredAndAllowedHour; 43 import org.unicode.cldr.util.RegexUtilities; 44 import org.unicode.cldr.util.SupplementalDataInfo; 45 import org.unicode.cldr.util.XPathParts; 46 import org.unicode.cldr.util.props.UnicodeProperty.PatternMatcher; 47 48 import com.google.common.base.Joiner; 49 import com.ibm.icu.impl.Relation; 50 import com.ibm.icu.text.BreakIterator; 51 import com.ibm.icu.text.DateTimePatternGenerator; 52 import com.ibm.icu.text.DateTimePatternGenerator.VariableField; 53 import com.ibm.icu.text.MessageFormat; 54 import com.ibm.icu.text.NumberFormat; 55 import com.ibm.icu.text.SimpleDateFormat; 56 import com.ibm.icu.text.UnicodeSet; 57 import com.ibm.icu.util.Output; 58 import com.ibm.icu.util.ULocale; 59 60 public class CheckDates extends FactoryCheckCLDR { 61 static boolean GREGORIAN_ONLY = CldrUtility.getProperty("GREGORIAN", false); 62 63 ICUServiceBuilder icuServiceBuilder = new ICUServiceBuilder(); 64 NumberFormat english = NumberFormat.getNumberInstance(ULocale.ENGLISH); 65 PatternMatcher m; 66 DateTimePatternGenerator.FormatParser formatParser = new DateTimePatternGenerator.FormatParser(); 67 DateTimePatternGenerator dateTimePatternGenerator = DateTimePatternGenerator.getEmptyInstance(); 68 private CoverageLevel2 coverageLevel; 69 private SupplementalDataInfo sdi = SupplementalDataInfo.getInstance(); 70 // Ordered list of this CLDRFile and parent CLDRFiles up to root 71 List<CLDRFile> parentCLDRFiles = new ArrayList<>(); 72 // Map from calendar type (i.e. "gregorian", "generic", "chinese") to DateTimePatternGenerator instance for that type 73 Map<String, DateTimePatternGenerator> dtpgForType = new HashMap<>(); 74 75 // Use the width of the character "0" as the basic unit for checking widths 76 // It's not perfect, but I'm not sure that anything can be. This helps us 77 // weed out some false positives in width checking, like 10月 vs. 十月 78 // in Chinese, which although technically longer, shouldn't trigger an 79 // error. 80 private static final int REFCHAR = ApproximateWidth.getWidth("0"); 81 82 private Level requiredLevel; 83 private String language; 84 private String territory; 85 86 private DayPeriodInfo dateFormatInfoFormat; 87 88 static String[] samples = { 89 // "AD 1970-01-01T00:00:00Z", 90 // "BC 4004-10-23T07:00:00Z", // try a BC date: creation according to Ussher & Lightfoot. Assuming garden of 91 // eden 2 hours ahead of UTC 92 "2005-12-02 12:15:16", 93 // "AD 2100-07-11T10:15:16Z", 94 }; // keep aligned with following 95 static String SampleList = "{0}" 96 // + Utility.LINE_SEPARATOR + "\t\u200E{1}\u200E" + Utility.LINE_SEPARATOR + "\t\u200E{2}\u200E" + 97 // Utility.LINE_SEPARATOR + "\t\u200E{3}\u200E" 98 ; // keep aligned with previous 99 100 private static final String DECIMAL_XPATH = "//ldml/numbers/symbols[@numberSystem='latn']/decimal"; 101 private static final Pattern HOUR_SYMBOL = PatternCache.get("H{1,2}"); 102 private static final Pattern MINUTE_SYMBOL = PatternCache.get("mm"); 103 private static final Pattern YEAR_FIELDS = PatternCache.get("(y|Y|u|U|r){1,5}"); 104 105 private static String CALENDAR_ID_PREFIX = "/calendar[@type=\""; 106 107 static String[] calTypePathsToCheck = { 108 "//ldml/dates/calendars/calendar[@type=\"buddhist\"]", 109 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]", 110 "//ldml/dates/calendars/calendar[@type=\"hebrew\"]", 111 "//ldml/dates/calendars/calendar[@type=\"islamic\"]", 112 "//ldml/dates/calendars/calendar[@type=\"japanese\"]", 113 "//ldml/dates/calendars/calendar[@type=\"roc\"]", 114 }; 115 static String[] calSymbolPathsWhichNeedDistinctValues = { 116 // === for months, days, quarters - format wide & abbrev sets must have distinct values === 117 "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"abbreviated\"]/month", 118 "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"wide\"]/month", 119 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"abbreviated\"]/day", 120 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"short\"]/day", 121 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"wide\"]/day", 122 "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"abbreviated\"]/quarter", 123 "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"wide\"]/quarter", 124 // === for dayPeriods - all values for a given context/width must be distinct === 125 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"abbreviated\"]/dayPeriod", 126 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod", 127 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod", 128 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"abbreviated\"]/dayPeriod", 129 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod", 130 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod", 131 // === for eras - all values for a given context/width should be distinct (warning) === 132 "/eras/eraNames/era", 133 "/eras/eraAbbr/era", // Hmm, root eraAbbr for japanese has many dups, should we change them or drop this test? 134 "/eras/eraNarrow/era", // We may need to allow dups here too 135 }; 136 137 // The following calendar symbol sets need not have distinct values 138 // "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"narrow\"]/month", 139 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"abbreviated\"]/month", 140 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"narrow\"]/month", 141 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"wide\"]/month", 142 // "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"narrow\"]/day", 143 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"abbreviated\"]/day", 144 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"narrow\"]/day", 145 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"wide\"]/day", 146 // "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"narrow\"]/quarter", 147 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"abbreviated\"]/quarter", 148 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"narrow\"]/quarter", 149 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"wide\"]/quarter", 150 151 // The above are followed by trailing pieces such as 152 // "[@type=\"am\"]", 153 // "[@type=\"sun\"]", 154 // "[@type=\"0\"]", 155 // "[@type=\"1\"]", 156 // "[@type=\"12\"]", 157 158 // Map<String, Set<String>> calPathsToSymbolSets; 159 // Map<String, Map<String, String>> calPathsToSymbolMaps = new HashMap<String, Map<String, String>>(); 160 CheckDates(Factory factory)161 public CheckDates(Factory factory) { 162 super(factory); 163 } 164 165 @Override setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors)166 public CheckCLDR setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options, 167 List<CheckStatus> possibleErrors) { 168 if (cldrFileToCheck == null) return this; 169 super.setCldrFileToCheck(cldrFileToCheck, options, possibleErrors); 170 171 icuServiceBuilder.setCldrFile(getResolvedCldrFileToCheck()); 172 // the following is a hack to work around a bug in ICU4J (the snapshot, not the released version). 173 try { 174 bi = BreakIterator.getCharacterInstance(new ULocale(cldrFileToCheck.getLocaleID())); 175 } catch (RuntimeException e) { 176 bi = BreakIterator.getCharacterInstance(new ULocale("")); 177 } 178 CLDRFile resolved = getResolvedCldrFileToCheck(); 179 flexInfo = new FlexibleDateFromCLDR(); // ought to just clear(), but not available. 180 flexInfo.set(resolved); 181 182 // load decimal path specially 183 String decimal = resolved.getWinningValue(DECIMAL_XPATH); 184 if (decimal != null) { 185 flexInfo.checkFlexibles(DECIMAL_XPATH, decimal, DECIMAL_XPATH); 186 } 187 188 String localeID = cldrFileToCheck.getLocaleID(); 189 LocaleIDParser lp = new LocaleIDParser(); 190 territory = lp.set(localeID).getRegion(); 191 language = lp.getLanguage(); 192 if (territory == null || territory.length() == 0) { 193 if (language.equals("root")) { 194 territory = "001"; 195 } else { 196 CLDRLocale loc = CLDRLocale.getInstance(localeID); 197 CLDRLocale defContent = sdi.getDefaultContentFromBase(loc); 198 if (defContent == null) { 199 territory = "001"; 200 } else { 201 territory = defContent.getCountry(); 202 } 203 // Set territory for 12/24 hour clock to Egypt (12 hr) for ar_001 204 // instead of 24 hour (exception). 205 if (territory.equals("001") && language.equals("ar")) { 206 territory = "EG"; 207 } 208 } 209 } 210 coverageLevel = CoverageLevel2.getInstance(sdi, localeID); 211 requiredLevel = options.getRequiredLevel(localeID); 212 213 // load gregorian appendItems 214 for (Iterator<String> it = resolved.iterator("//ldml/dates/calendars/calendar[@type=\"gregorian\"]"); it.hasNext();) { 215 String path = it.next(); 216 String value = resolved.getWinningValue(path); 217 String fullPath = resolved.getFullXPath(path); 218 try { 219 flexInfo.checkFlexibles(path, value, fullPath); 220 } catch (Exception e) { 221 final String message = e.getMessage(); 222 CheckStatus item = new CheckStatus() 223 .setCause(this) 224 .setMainType(CheckStatus.errorType) 225 .setSubtype( 226 message.contains("Conflicting fields") ? Subtype.dateSymbolCollision : Subtype.internalError) 227 .setMessage(message); 228 possibleErrors.add(item); 229 } 230 // possibleErrors.add(flexInfo.getFailurePath(path)); 231 } 232 redundants.clear(); 233 /* 234 * TODO: NullPointerException may be thrown in ICU here during cldr-unittest TestAll 235 */ 236 flexInfo.getRedundants(redundants); 237 // Set baseSkeletons = flexInfo.gen.getBaseSkeletons(new TreeSet()); 238 // Set notCovered = new TreeSet(neededFormats); 239 // if (flexInfo.preferred12Hour()) { 240 // notCovered.addAll(neededHours12); 241 // } else { 242 // notCovered.addAll(neededHours24); 243 // } 244 // notCovered.removeAll(baseSkeletons); 245 // if (notCovered.size() != 0) { 246 // possibleErrors.add(new CheckStatus().setCause(this).setType(CheckCLDR.finalErrorType) 247 // .setCheckOnSubmit(false) 248 // .setMessage("Missing availableFormats: {0}", new Object[]{notCovered.toString()})); 249 // } 250 pathsWithConflictingOrder2sample = DateOrder.getOrderingInfo(cldrFileToCheck, resolved, flexInfo.fp); 251 if (pathsWithConflictingOrder2sample == null) { 252 CheckStatus item = new CheckStatus() 253 .setCause(this) 254 .setMainType(CheckStatus.errorType) 255 .setSubtype(Subtype.internalError) 256 .setMessage("DateOrder.getOrderingInfo fails"); 257 possibleErrors.add(item); 258 } 259 260 // calPathsToSymbolMaps.clear(); 261 // for (String calTypePath: calTypePathsToCheck) { 262 // for (String calSymbolPath: calSymbolPathsWhichNeedDistinctValues) { 263 // calPathsToSymbolMaps.put(calTypePath.concat(calSymbolPath), null); 264 // } 265 // } 266 267 dateFormatInfoFormat = sdi.getDayPeriods(Type.format, cldrFileToCheck.getLocaleID()); 268 269 // Make new list of parent CLDRFiles 270 parentCLDRFiles.clear(); 271 parentCLDRFiles.add(cldrFileToCheck); 272 while ((localeID = LocaleIDParser.getParent(localeID)) != null) { 273 CLDRFile resolvedParentCLDRFile = getFactory().make(localeID, true); 274 parentCLDRFiles.add(resolvedParentCLDRFile); 275 } 276 // Clear out map of DateTimePatternGenerators for calendarType 277 dtpgForType.clear(); 278 279 return this; 280 } 281 282 Map<String, Map<DateOrder, String>> pathsWithConflictingOrder2sample; 283 284 // Set neededFormats = new TreeSet(Arrays.asList(new String[]{ 285 // "yM", "yMMM", "yMd", "yMMMd", "Md", "MMMd","yQ" 286 // })); 287 // Set neededHours12 = new TreeSet(Arrays.asList(new String[]{ 288 // "hm", "hms" 289 // })); 290 // Set neededHours24 = new TreeSet(Arrays.asList(new String[]{ 291 // "Hm", "Hms" 292 // })); 293 /** 294 * hour+minute, hour+minute+second (12 & 24) 295 * year+month, year+month+day (numeric & string) 296 * month+day (numeric & string) 297 * year+quarter 298 */ 299 BreakIterator bi; 300 FlexibleDateFromCLDR flexInfo; 301 Collection<String> redundants = new HashSet<>(); 302 Status status = new Status(); 303 PathStarrer pathStarrer = new PathStarrer(); 304 stripPrefix(String s)305 private String stripPrefix(String s) { 306 if (s != null) { 307 int prefEnd = s.lastIndexOf(" "); 308 if (prefEnd < 0 || prefEnd >= 3) { 309 prefEnd = s.lastIndexOf("\u2019"); // as in d’ 310 } 311 if (prefEnd >= 0 && prefEnd < 3) { 312 return s.substring(prefEnd + 1); 313 } 314 } 315 return s; 316 } 317 318 @Override handleCheck(String path, String fullPath, String value, Options options, List<CheckStatus> result)319 public CheckCLDR handleCheck(String path, String fullPath, String value, Options options, 320 List<CheckStatus> result) { 321 322 if (fullPath == null) { 323 return this; // skip paths that we don't have 324 } 325 326 if (path.indexOf("/dates") < 0 327 || path.endsWith("/default") 328 || path.endsWith("/alias")) { 329 return this; 330 } 331 332 String sourceLocale = getCldrFileToCheck().getSourceLocaleID(path, status); 333 334 if (!path.equals(status.pathWhereFound) || !sourceLocale.equals(getCldrFileToCheck().getLocaleID())) { 335 return this; 336 } 337 338 if (value == null) { 339 return this; 340 } 341 342 if (pathsWithConflictingOrder2sample != null) { 343 Map<DateOrder, String> problem = pathsWithConflictingOrder2sample.get(path); 344 if (problem != null) { 345 CheckStatus item = new CheckStatus() 346 .setCause(this) 347 .setMainType(CheckStatus.warningType) 348 .setSubtype(Subtype.incorrectDatePattern) 349 .setMessage("The ordering of date fields is inconsistent with others: {0}", 350 getValues(getResolvedCldrFileToCheck(), problem.values())); 351 result.add(item); 352 } 353 } 354 355 try { 356 if (path.indexOf("[@type=\"abbreviated\"]") >= 0) { 357 String pathToWide = path.replace("[@type=\"abbreviated\"]", "[@type=\"wide\"]"); 358 String wideValue = getCldrFileToCheck().getWinningValueWithBailey(pathToWide); 359 if (wideValue != null && isTooMuchWiderThan(value, wideValue)) { 360 CheckStatus item = new CheckStatus() 361 .setCause(this) 362 .setMainType(CheckStatus.errorType) 363 .setSubtype(Subtype.abbreviatedDateFieldTooWide) 364 .setMessage("Abbreviated value \"{0}\" can't be longer than the corresponding wide value \"{1}\"", value, 365 wideValue); 366 result.add(item); 367 } 368 Set<String> grouping = LogicalGrouping.getPaths(getCldrFileToCheck(), path); 369 if (grouping != null) { 370 for (String lgPath : grouping) { 371 String lgPathValue = getCldrFileToCheck().getWinningValueWithBailey(lgPath); 372 if (lgPathValue == null) { 373 continue; 374 } 375 String lgPathToWide = lgPath.replace("[@type=\"abbreviated\"]", "[@type=\"wide\"]"); 376 String lgPathWideValue = getCldrFileToCheck().getWinningValueWithBailey(lgPathToWide); 377 // This helps us get around things like "de març" vs. "març" in Catalan 378 String thisValueStripped = stripPrefix(value); 379 String wideValueStripped = stripPrefix(wideValue); 380 String lgPathValueStripped = stripPrefix(lgPathValue); 381 String lgPathWideValueStripped = stripPrefix(lgPathWideValue); 382 boolean thisPathHasPeriod = value.contains("."); 383 boolean lgPathHasPeriod = lgPathValue.contains("."); 384 if (!thisValueStripped.equalsIgnoreCase(wideValueStripped) && !lgPathValueStripped.equalsIgnoreCase(lgPathWideValueStripped) && 385 thisPathHasPeriod != lgPathHasPeriod) { 386 CheckStatus.Type et = CheckStatus.errorType; 387 if (path.contains("dayPeriod")) { 388 et = CheckStatus.warningType; 389 } 390 CheckStatus item = new CheckStatus() 391 .setCause(this) 392 .setMainType(et) 393 .setSubtype(Subtype.inconsistentPeriods) 394 .setMessage("Inconsistent use of periods in abbreviations for this section."); 395 result.add(item); 396 break; 397 } 398 } 399 } 400 } else if (path.indexOf("[@type=\"narrow\"]") >= 0) { 401 String pathToAbbr = path.replace("[@type=\"narrow\"]", "[@type=\"abbreviated\"]"); 402 String abbrValue = getCldrFileToCheck().getWinningValueWithBailey(pathToAbbr); 403 if (abbrValue != null && isTooMuchWiderThan(value, abbrValue)) { 404 CheckStatus item = new CheckStatus() 405 .setCause(this) 406 .setMainType(CheckStatus.warningType) // Making this just a warning, because there are some oddball cases. 407 .setSubtype(Subtype.narrowDateFieldTooWide) 408 .setMessage("Narrow value \"{0}\" shouldn't be longer than the corresponding abbreviated value \"{1}\"", value, 409 abbrValue); 410 result.add(item); 411 } 412 } else if (path.indexOf("/eraNarrow") >= 0) { 413 String pathToAbbr = path.replace("/eraNarrow", "/eraAbbr"); 414 String abbrValue = getCldrFileToCheck().getWinningValueWithBailey(pathToAbbr); 415 if (abbrValue != null && isTooMuchWiderThan(value, abbrValue)) { 416 CheckStatus item = new CheckStatus() 417 .setCause(this) 418 .setMainType(CheckStatus.errorType) 419 .setSubtype(Subtype.narrowDateFieldTooWide) 420 .setMessage("Narrow value \"{0}\" can't be longer than the corresponding abbreviated value \"{1}\"", value, 421 abbrValue); 422 result.add(item); 423 } 424 } else if (path.indexOf("/eraAbbr") >= 0) { 425 String pathToWide = path.replace("/eraAbbr", "/eraNames"); 426 String wideValue = getCldrFileToCheck().getWinningValueWithBailey(pathToWide); 427 if (wideValue != null && isTooMuchWiderThan(value, wideValue)) { 428 CheckStatus item = new CheckStatus() 429 .setCause(this) 430 .setMainType(CheckStatus.errorType) 431 .setSubtype(Subtype.abbreviatedDateFieldTooWide) 432 .setMessage("Abbreviated value \"{0}\" can't be longer than the corresponding wide value \"{1}\"", value, 433 wideValue); 434 result.add(item); 435 } 436 437 } 438 439 String failure = flexInfo.checkValueAgainstSkeleton(path, value); 440 if (failure != null) { 441 result.add(new CheckStatus() 442 .setCause(this) 443 .setMainType(CheckStatus.errorType) 444 .setSubtype(Subtype.illegalDatePattern) 445 .setMessage(failure)); 446 } 447 448 final String collisionPrefix = "//ldml/dates/calendars/calendar"; 449 main: if (path.startsWith(collisionPrefix)) { 450 int pos = path.indexOf("\"]"); // end of first type 451 if (pos < 0 || skipPath(path)) { // skip narrow, no-calendar 452 break main; 453 } 454 pos += 2; 455 String myType = getLastType(path); 456 if (myType == null) { 457 break main; 458 } 459 String myMainType = getMainType(path); 460 461 String calendarPrefix = path.substring(0, pos); 462 boolean endsWithDisplayName = path.endsWith("displayName"); // special hack, these shouldn't be in 463 // calendar. 464 465 Set<String> retrievedPaths = new HashSet<>(); 466 getResolvedCldrFileToCheck().getPathsWithValue(value, calendarPrefix, null, retrievedPaths); 467 if (retrievedPaths.size() < 2) { 468 break main; 469 } 470 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraAbbr/era[@type="0"], 471 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraNames/era[@type="0"], 472 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraNarrow/era[@type="0"]] 473 Type type = null; 474 DayPeriod dayPeriod = null; 475 final boolean isDayPeriod = path.contains("dayPeriod"); 476 if (isDayPeriod) { 477 XPathParts parts = XPathParts.getFrozenInstance(fullPath); 478 type = Type.fromString(parts.getAttributeValue(5, "type")); 479 dayPeriod = DayPeriod.valueOf(parts.getAttributeValue(-1, "type")); 480 } 481 482 // TODO redo above and below in terms of parts instead of searching strings 483 484 Set<String> filteredPaths = new HashSet<>(); 485 Output<Integer> sampleError = new Output<>(); 486 487 for (String item : retrievedPaths) { 488 XPathParts itemParts = XPathParts.getFrozenInstance(item); 489 if (item.equals(path) 490 || skipPath(item) 491 || endsWithDisplayName != item.endsWith("displayName") 492 || itemParts.containsElement("alias")) { 493 continue; 494 } 495 String otherType = getLastType(item); 496 if (myType.equals(otherType)) { // we don't care about items with the same type value 497 continue; 498 } 499 String mainType = getMainType(item); 500 if (!myMainType.equals(mainType)) { // we *only* care about items with the same type value 501 continue; 502 } 503 if (isDayPeriod) { 504 //ldml/dates/calendars/calendar[@type="gregorian"]/dayPeriods/dayPeriodContext[@type="format"]/dayPeriodWidth[@type="wide"]/dayPeriod[@type="am"] 505 Type itemType = Type.fromString(itemParts.getAttributeValue(5, "type")); 506 DayPeriod itemDayPeriod = DayPeriod.valueOf(itemParts.getAttributeValue(-1, "type")); 507 508 if (!dateFormatInfoFormat.collisionIsError(type, dayPeriod, itemType, itemDayPeriod, sampleError)) { 509 continue; 510 } 511 } 512 filteredPaths.add(item); 513 } 514 if (filteredPaths.size() == 0) { 515 break main; 516 } 517 Set<String> others = new TreeSet<>(); 518 for (String path2 : filteredPaths) { 519 PathHeader pathHeader = getPathHeaderFactory().fromPath(path2); 520 others.add(pathHeader.getHeaderCode()); 521 } 522 CheckStatus.Type statusType = getPhase() == Phase.SUBMISSION || getPhase() == Phase.BUILD 523 ? CheckStatus.warningType 524 : CheckStatus.errorType; 525 final CheckStatus checkStatus = new CheckStatus() 526 .setCause(this) 527 .setMainType(statusType) 528 .setSubtype(Subtype.dateSymbolCollision); 529 if (sampleError.value == null) { 530 checkStatus.setMessage("The date value “{0}” is the same as what is used for a different item: {1}", 531 value, others.toString()); 532 } else { 533 checkStatus.setMessage("The date value “{0}” is the same as what is used for a different item: {1}. Sample problem: {2}", 534 value, others.toString(), sampleError.value / DayPeriodInfo.HOUR); 535 } 536 result.add(checkStatus); 537 } 538 DateTimePatternType dateTypePatternType = DateTimePatternType.fromPath(path); 539 if (DateTimePatternType.STOCK_AVAILABLE_INTERVAL_PATTERNS.contains(dateTypePatternType)) { 540 boolean patternBasicallyOk = false; 541 try { 542 formatParser.set(value); 543 patternBasicallyOk = true; 544 } catch (RuntimeException e) { 545 String message = e.getMessage(); 546 if (message.contains("Illegal datetime field:")) { 547 CheckStatus item = new CheckStatus().setCause(this) 548 .setMainType(CheckStatus.errorType) 549 .setSubtype(Subtype.illegalDatePattern) 550 .setMessage(message); 551 result.add(item); 552 } else { 553 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 554 .setSubtype(Subtype.illegalDatePattern) 555 .setMessage("Illegal date format pattern {0}", new Object[] { e }); 556 result.add(item); 557 } 558 } 559 if (patternBasicallyOk) { 560 checkPattern(dateTypePatternType, path, fullPath, value, result); 561 } 562 } else if (path.contains("datetimeSkeleton") && !path.contains("[@alt=")) { // cannot test any alt skeletons 563 // Get calendar type from //ldml/dates/calendars/calendar[@type="..."]/ 564 int startIndex = path.indexOf(CALENDAR_ID_PREFIX); 565 if (startIndex > 0) { 566 startIndex += CALENDAR_ID_PREFIX.length(); 567 int endIndex = path.indexOf("\"]", startIndex); 568 String calendarType = path.substring(startIndex,endIndex); 569 // Get pattern generated from datetimeSkeleton 570 DateTimePatternGenerator dtpg = getDTPGForCalendarType(calendarType); 571 String patternFromSkeleton = dtpg.getBestPattern(value); 572 // Get actual stock pattern 573 String patternPath = path.replace("/datetimeSkeleton", "/pattern[@type=\"standard\"]"); 574 String patternStock = getCldrFileToCheck().getWinningValue(patternPath); 575 // Compare and flag error if mismatch 576 if (!patternFromSkeleton.equals(patternStock)) { 577 CheckStatus item = new CheckStatus() 578 .setCause(this) 579 .setMainType(CheckStatus.warningType) 580 .setSubtype(Subtype.inconsistentDatePattern) 581 .setMessage( 582 "Pattern \"{0}\" from datetimeSkeleton should match corresponding standard pattern \"{1}\", adjust availableFormats to fix.", 583 patternFromSkeleton, patternStock); 584 result.add(item); 585 } 586 } 587 } else if (path.contains("hourFormat")) { 588 int semicolonPos = value.indexOf(';'); 589 if (semicolonPos < 0) { 590 CheckStatus item = new CheckStatus() 591 .setCause(this) 592 .setMainType(CheckStatus.errorType) 593 .setSubtype(Subtype.illegalDatePattern) 594 .setMessage( 595 "Value should contain a positive hour format and a negative hour format separated by a semicolon."); 596 result.add(item); 597 } else { 598 String[] formats = value.split(";"); 599 if (formats[0].equals(formats[1])) { 600 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 601 .setSubtype(Subtype.illegalDatePattern) 602 .setMessage("The hour formats should not be the same."); 603 result.add(item); 604 } else { 605 checkHasHourMinuteSymbols(formats[0], result); 606 checkHasHourMinuteSymbols(formats[1], result); 607 } 608 } 609 } 610 } catch (ParseException e) { 611 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 612 .setSubtype(Subtype.illegalDatePattern) 613 .setMessage("ParseException in creating date format {0}", new Object[] { e }); 614 result.add(item); 615 } catch (Exception e) { 616 // e.printStackTrace(); 617 // HACK 618 String msg = e.getMessage(); 619 if (msg == null || !HACK_CONFLICTING.matcher(msg).find()) { 620 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 621 .setSubtype(Subtype.illegalDatePattern) 622 .setMessage("Error in creating date format {0}", new Object[] { e }); 623 result.add(item); 624 } 625 } 626 return this; 627 } 628 isTooMuchWiderThan(String shortString, String longString)629 private boolean isTooMuchWiderThan(String shortString, String longString) { 630 // We all 1/3 the width of the reference character as a "fudge factor" in determining the allowable width 631 return ApproximateWidth.getWidth(shortString) > ApproximateWidth.getWidth(longString) + REFCHAR / 3; 632 } 633 634 /** 635 * Check for the presence of hour and minute symbols. 636 * 637 * @param value 638 * the value to be checked 639 * @param result 640 * the list to add any errors to. 641 */ checkHasHourMinuteSymbols(String value, List<CheckStatus> result)642 private void checkHasHourMinuteSymbols(String value, List<CheckStatus> result) { 643 boolean hasHourSymbol = HOUR_SYMBOL.matcher(value).find(); 644 boolean hasMinuteSymbol = MINUTE_SYMBOL.matcher(value).find(); 645 if (!hasHourSymbol && !hasMinuteSymbol) { 646 result.add(createErrorCheckStatus().setMessage("The hour and minute symbols are missing from {0}.", value)); 647 } else if (!hasHourSymbol) { 648 result.add(createErrorCheckStatus() 649 .setMessage("The hour symbol (H or HH) should be present in {0}.", value)); 650 } else if (!hasMinuteSymbol) { 651 result.add(createErrorCheckStatus().setMessage("The minute symbol (mm) should be present in {0}.", value)); 652 } 653 } 654 655 /** 656 * Convenience method for creating errors. 657 * 658 * @return 659 */ createErrorCheckStatus()660 private CheckStatus createErrorCheckStatus() { 661 return new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 662 .setSubtype(Subtype.illegalDatePattern); 663 } 664 skipPath(String path)665 public boolean skipPath(String path) { 666 return path.contains("arrow") 667 || path.contains("/availableFormats") 668 || path.contains("/interval") 669 || path.contains("/dateTimeFormat") 670 // || path.contains("/dayPeriod[") 671 // && !path.endsWith("=\"pm\"]") 672 // && !path.endsWith("=\"am\"]") 673 ; 674 } 675 getLastType(String path)676 public String getLastType(String path) { 677 int secondType = path.lastIndexOf("[@type=\""); 678 if (secondType < 0) { 679 return null; 680 } 681 secondType += 8; 682 int secondEnd = path.indexOf("\"]", secondType); 683 if (secondEnd < 0) { 684 return null; 685 } 686 return path.substring(secondType, secondEnd); 687 } 688 getMainType(String path)689 public String getMainType(String path) { 690 int secondType = path.indexOf("\"]/"); 691 if (secondType < 0) { 692 return null; 693 } 694 secondType += 3; 695 int secondEnd = path.indexOf("/", secondType); 696 if (secondEnd < 0) { 697 return null; 698 } 699 return path.substring(secondType, secondEnd); 700 } 701 getValues(CLDRFile resolvedCldrFileToCheck, Collection<String> values)702 private String getValues(CLDRFile resolvedCldrFileToCheck, Collection<String> values) { 703 Set<String> results = new TreeSet<>(); 704 for (String path : values) { 705 final String stringValue = resolvedCldrFileToCheck.getStringValue(path); 706 if (stringValue != null) { 707 results.add(stringValue); 708 } 709 } 710 return "{" + Joiner.on("},{").join(results) + "}"; 711 } 712 713 static final Pattern HACK_CONFLICTING = PatternCache.get("Conflicting fields:\\s+M+,\\s+l"); 714 715 @Override handleGetExamples(String path, String fullPath, String value, Options options, List<CheckStatus> result)716 public CheckCLDR handleGetExamples(String path, String fullPath, String value, Options options, List<CheckStatus> result) { 717 if (path.indexOf("/dates") < 0 || path.indexOf("gregorian") < 0) return this; 718 try { 719 if (path.indexOf("/pattern") >= 0 && path.indexOf("/dateTimeFormat") < 0 720 || path.indexOf("/dateFormatItem") >= 0) { 721 checkPattern2(path, value, result); 722 } 723 } catch (Exception e) { 724 // don't worry about errors 725 } 726 return this; 727 } 728 729 static final SimpleDateFormat neutralFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", ULocale.ENGLISH); 730 static { 731 neutralFormat.setTimeZone(ExampleGenerator.ZONE_SAMPLE); 732 } 733 734 // Get Date-Time in milliseconds getDateTimeinMillis(int year, int month, int date, int hourOfDay, int minute, int second)735 private static long getDateTimeinMillis(int year, int month, int date, int hourOfDay, int minute, int second) { 736 Calendar cal = Calendar.getInstance(); 737 cal.set(year, month, date, hourOfDay, minute, second); 738 return cal.getTimeInMillis(); 739 } 740 741 static long date1950 = getDateTimeinMillis(1950, 0, 1, 0, 0, 0); 742 static long date2010 = getDateTimeinMillis(2010, 0, 1, 0, 0, 0); 743 static long date4004BC = getDateTimeinMillis(-4004, 9, 23, 2, 0, 0); 744 static Random random = new Random(0); 745 checkPattern(DateTimePatternType dateTypePatternType, String path, String fullPath, String value, List<CheckStatus> result)746 private void checkPattern(DateTimePatternType dateTypePatternType, String path, String fullPath, String value, List<CheckStatus> result) 747 throws ParseException { 748 String skeleton = dateTimePatternGenerator.getSkeletonAllowingDuplicates(value); 749 String skeletonCanonical = dateTimePatternGenerator.getCanonicalSkeletonAllowingDuplicates(value); 750 751 if (value.contains("MMM.") || value.contains("LLL.") || value.contains("E.") || value.contains("eee.") 752 || value.contains("ccc.") || value.contains("QQQ.") || value.contains("qqq.")) { 753 result 754 .add(new CheckStatus() 755 .setCause(this) 756 .setMainType(CheckStatus.warningType) 757 .setSubtype(Subtype.incorrectDatePattern) 758 .setMessage( 759 "Your pattern ({0}) is probably incorrect; abbreviated month/weekday/quarter names that need a period should include it in the name, rather than adding it to the pattern.", 760 value)); 761 } 762 XPathParts pathParts = XPathParts.getFrozenInstance(path); 763 String calendar = pathParts.findAttributeValue("calendar", "type"); 764 String id; 765 switch (dateTypePatternType) { 766 case AVAILABLE: 767 id = pathParts.getAttributeValue(-1, "id"); 768 break; 769 case INTERVAL: 770 id = pathParts.getAttributeValue(-2, "id"); 771 break; 772 case STOCK: 773 id = pathParts.getAttributeValue(-3, "type"); 774 break; 775 default: 776 throw new IllegalArgumentException(); 777 } 778 779 if (dateTypePatternType == DateTimePatternType.AVAILABLE || dateTypePatternType == DateTimePatternType.INTERVAL) { 780 String idCanonical = dateTimePatternGenerator.getCanonicalSkeletonAllowingDuplicates(id); 781 if (skeleton.isEmpty()) { 782 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 783 .setSubtype(Subtype.incorrectDatePattern) 784 // "Internal ID ({0}) doesn't match generated ID ({1}) for pattern ({2}). " + 785 .setMessage("Your pattern ({1}) is incorrect for ID ({0}). " + 786 "You need to supply a pattern according to " + CLDRURLS.DATE_TIME_PATTERNS_URL + ".", 787 id, value)); 788 } else if (!dateTimePatternGenerator.skeletonsAreSimilar(idCanonical, skeletonCanonical)) { 789 String fixedValue = dateTimePatternGenerator.replaceFieldTypes(value, id); 790 result 791 .add(new CheckStatus() 792 .setCause(this) 793 .setMainType(CheckStatus.errorType) 794 .setSubtype(Subtype.incorrectDatePattern) 795 // "Internal ID ({0}) doesn't match generated ID ({1}) for pattern ({2}). " + 796 .setMessage( 797 "Your pattern ({2}) doesn't correspond to what is asked for. Yours would be right for an ID ({1}) but not for the ID ({0}). " 798 + 799 "Please change your pattern to match what was asked, such as ({3}), with the right punctuation and/or ordering for your language. See " 800 + CLDRURLS.DATE_TIME_PATTERNS_URL + ".", 801 id, skeletonCanonical, value, fixedValue)); 802 } 803 if (dateTypePatternType == DateTimePatternType.AVAILABLE) { 804 // index y+w+ must correpond to pattern containing only Y+ and w+ 805 if (idCanonical.matches("y+w+") && !(skeleton.matches("Y+w+") || skeleton.matches("w+Y+"))) { 806 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType).setSubtype(Subtype.incorrectDatePattern) 807 .setMessage("For id {0}, the pattern ({1}) must contain fields Y and w, and no others.", id, value)); 808 } 809 // index M+W msut correspond to pattern containing only M+/L+ and W 810 if (idCanonical.matches("M+W") && !(skeletonCanonical.matches("M+W") || skeletonCanonical.matches("WM+"))) { 811 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType).setSubtype(Subtype.incorrectDatePattern) 812 .setMessage("For id {0}, the pattern ({1}) must contain fields M or L, plus W, and no others.", id, value)); 813 } 814 } 815 String failureMessage = (String) flexInfo.getFailurePath(path); 816 if (failureMessage != null) { 817 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 818 .setSubtype(Subtype.illegalDatePattern) 819 .setMessage("{0}", new Object[] { failureMessage })); 820 } 821 } 822 if (dateTypePatternType == DateTimePatternType.STOCK) { 823 int style = 0; 824 String len = pathParts.findAttributeValue("timeFormatLength", "type"); 825 DateOrTime dateOrTime = DateOrTime.time; 826 if (len == null) { 827 dateOrTime = DateOrTime.date; 828 style += 4; 829 len = pathParts.findAttributeValue("dateFormatLength", "type"); 830 if (len == null) { 831 len = pathParts.findAttributeValue("dateTimeFormatLength", "type"); 832 dateOrTime = DateOrTime.dateTime; 833 } 834 } 835 836 DateTimeLengths dateTimeLength = DateTimeLengths.valueOf(len.toUpperCase(Locale.ENGLISH)); 837 838 if (calendar.equals("gregorian") && !"root".equals(getCldrFileToCheck().getLocaleID())) { 839 checkValue(dateTimeLength, dateOrTime, value, result); 840 } 841 if (dateOrTime == DateOrTime.dateTime) { 842 return; // We don't need to do the rest for date/time combo patterns. 843 } 844 style += dateTimeLength.ordinal(); 845 // do regex match with skeletonCanonical but report errors using skeleton; they have corresponding field lengths 846 if (!dateTimePatterns[style].matcher(skeletonCanonical).matches() 847 && !calendar.equals("chinese") 848 && !calendar.equals("hebrew")) { 849 int i = RegexUtilities.findMismatch(dateTimePatterns[style], skeletonCanonical); 850 String skeletonPosition = skeleton.substring(0, i) + "☹" + skeleton.substring(i); 851 result.add(new CheckStatus() 852 .setCause(this) 853 .setMainType(CheckStatus.errorType) 854 .setSubtype(Subtype.missingOrExtraDateField) 855 .setMessage("Field is missing, extra, or the wrong length. Expected {0} [Internal: {1} / {2}]", 856 new Object[] { dateTimeMessage[style], skeletonPosition, dateTimePatterns[style].pattern() })); 857 } 858 } else if (dateTypePatternType == DateTimePatternType.INTERVAL) { 859 if (id.contains("y")) { 860 String greatestDifference = pathParts.findAttributeValue("greatestDifference", "id"); 861 int requiredYearFieldCount = 1; 862 if ("y".equals(greatestDifference)) { 863 requiredYearFieldCount = 2; 864 } 865 int yearFieldCount = 0; 866 Matcher yearFieldMatcher = YEAR_FIELDS.matcher(value); 867 while (yearFieldMatcher.find()) { 868 yearFieldCount++; 869 } 870 if (yearFieldCount < requiredYearFieldCount) { 871 result.add(new CheckStatus() 872 .setCause(this) 873 .setMainType(CheckStatus.errorType) 874 .setSubtype(Subtype.missingOrExtraDateField) 875 .setMessage("Not enough year fields in interval pattern. Must have {0} but only found {1}", 876 new Object[] { requiredYearFieldCount, yearFieldCount })); 877 } 878 } 879 } 880 881 if (value.contains("G") && calendar.equals("gregorian")) { 882 GyState actual = GyState.forPattern(value); 883 GyState expected = getExpectedGy(getCldrFileToCheck().getLocaleID()); 884 if (actual != expected) { 885 result.add(new CheckStatus() 886 .setCause(this) 887 .setMainType(CheckStatus.warningType) 888 .setSubtype(Subtype.unexpectedOrderOfEraYear) 889 .setMessage("Unexpected order of era/year. Expected {0}, but got {1} in 〈{2}〉 for {3}/{4}", 890 expected, actual, value, calendar, id)); 891 } 892 } 893 } 894 895 enum DateOrTime { 896 date, time, dateTime 897 } 898 899 static final Map<DateOrTime, Relation<DateTimeLengths, String>> STOCK_PATTERNS = new EnumMap<>( 900 DateOrTime.class); 901 902 // add(Map<DateOrTime, Relation<DateTimeLengths, String>> stockPatterns, DateOrTime dateOrTime, DateTimeLengths dateTimeLength, String... keys)903 private static void add(Map<DateOrTime, Relation<DateTimeLengths, String>> stockPatterns, 904 DateOrTime dateOrTime, DateTimeLengths dateTimeLength, String... keys) { 905 Relation<DateTimeLengths, String> rel = STOCK_PATTERNS.get(dateOrTime); 906 if (rel == null) { 907 STOCK_PATTERNS.put(dateOrTime, rel = Relation.of(new EnumMap<DateTimeLengths, Set<String>>(DateTimeLengths.class), LinkedHashSet.class)); 908 } 909 rel.putAll(dateTimeLength, Arrays.asList(keys)); 910 } 911 912 /* Ticket #4936 913 value(short time) = value(hm) or value(Hm) 914 value(medium time) = value(hms) or value(Hms) 915 value(long time) = value(medium time+z) 916 value(full time) = value(medium time+zzzz) 917 */ 918 static { add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.SHORT, "hm", "Hm")919 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.SHORT, "hm", "Hm"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.MEDIUM, "hms", "Hms")920 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.MEDIUM, "hms", "Hms"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.LONG, "hms*z", "Hms*z")921 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.LONG, "hms*z", "Hms*z"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.FULL, "hms*zzzz", "Hms*zzzz")922 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.FULL, "hms*zzzz", "Hms*zzzz"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.SHORT, "yMd")923 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.SHORT, "yMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.MEDIUM, "yMMMd")924 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.MEDIUM, "yMMMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.LONG, "yMMMMd", "yMMMd")925 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.LONG, "yMMMMd", "yMMMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.FULL, "yMMMMEd", "yMMMEd")926 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.FULL, "yMMMMEd", "yMMMEd"); 927 } 928 929 static final String AVAILABLE_PREFIX = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\""; 930 static final String AVAILABLE_SUFFIX = "\"]"; 931 static final String APPEND_TIMEZONE = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/appendItems/appendItem[@request=\"Timezone\"]"; 932 checkValue(DateTimeLengths dateTimeLength, DateOrTime dateOrTime, String value, List<CheckStatus> result)933 private void checkValue(DateTimeLengths dateTimeLength, DateOrTime dateOrTime, String value, List<CheckStatus> result) { 934 // Check consistency of the pattern vs. supplemental wrt 12 vs. 24 hour clock. 935 if (dateOrTime == DateOrTime.time) { 936 PreferredAndAllowedHour pref = sdi.getTimeData().get(territory); 937 if (pref == null) { 938 pref = sdi.getTimeData().get("001"); 939 } 940 String checkForHour, clockType; 941 if (pref.preferred.equals(PreferredAndAllowedHour.HourStyle.h)) { 942 checkForHour = "h"; 943 clockType = "12"; 944 } else { 945 checkForHour = "H"; 946 clockType = "24"; 947 } 948 if (!value.contains(checkForHour)) { 949 CheckStatus.Type errType = CheckStatus.errorType; 950 // French/Canada is strange, they use 24 hr clock while en_CA uses 12. 951 if (language.equals("fr") && territory.equals("CA")) { 952 errType = CheckStatus.warningType; 953 } 954 955 result.add(new CheckStatus().setCause(this).setMainType(errType) 956 .setSubtype(Subtype.inconsistentTimePattern) 957 .setMessage("Time format inconsistent with supplemental time data for territory \"" + territory + "\"." 958 + " Use '" + checkForHour + "' for " + clockType + " hour clock.")); 959 } 960 } 961 if (dateOrTime == DateOrTime.dateTime) { 962 boolean inQuotes = false; 963 for (int i = 0; i < value.length(); i++) { 964 char ch = value.charAt(i); 965 if (ch == '\'') { 966 inQuotes = !inQuotes; 967 } 968 if (!inQuotes && (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')) { 969 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 970 .setSubtype(Subtype.patternContainsInvalidCharacters) 971 .setMessage("Unquoted letter \"{0}\" in dateTime format.", ch)); 972 } 973 } 974 } else { 975 Set<String> keys = STOCK_PATTERNS.get(dateOrTime).get(dateTimeLength); 976 StringBuilder b = new StringBuilder(); 977 boolean onlyNulls = true; 978 int countMismatches = 0; 979 boolean errorOnMissing = false; 980 String timezonePattern = null; 981 Set<String> bases = new LinkedHashSet<>(); 982 for (String key : keys) { 983 int star = key.indexOf('*'); 984 boolean hasStar = star >= 0; 985 String base = !hasStar ? key : key.substring(0, star); 986 bases.add(base); 987 String xpath = AVAILABLE_PREFIX + base + AVAILABLE_SUFFIX; 988 String value1 = getCldrFileToCheck().getStringValue(xpath); 989 // String localeFound = getCldrFileToCheck().getSourceLocaleID(xpath, null); && !localeFound.equals("root") && !localeFound.equals("code-fallback") 990 if (value1 != null) { 991 onlyNulls = false; 992 if (hasStar) { 993 String zone = key.substring(star + 1); 994 timezonePattern = getResolvedCldrFileToCheck().getStringValue(APPEND_TIMEZONE); 995 value1 = MessageFormat.format(timezonePattern, value1, zone); 996 } 997 if (equalsExceptWidth(value, value1)) { 998 return; 999 } 1000 } else { 1001 // Example, if the requiredLevel for the locale is moderate, 1002 // and the level for the path is modern, then we'll skip the error, 1003 // but if the level for the path is basic, then we won't 1004 Level pathLevel = coverageLevel.getLevel(xpath); 1005 if (requiredLevel.compareTo(pathLevel) >= 0) { 1006 errorOnMissing = true; 1007 } 1008 } 1009 add(b, base, value1); 1010 countMismatches++; 1011 } 1012 if (!onlyNulls) { 1013 if (timezonePattern != null) { 1014 b.append(" (with appendZonePattern: “" + timezonePattern + "”)"); 1015 } 1016 String msg = countMismatches != 1 1017 ? "{1}-{0} → “{2}” didn't match any of the corresponding flexible skeletons: [{3}]. This or the flexible patterns needs to be changed." 1018 : "{1}-{0} → “{2}” didn't match the corresponding flexible skeleton: {3}. This or the flexible pattern needs to be changed."; 1019 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.warningType) 1020 .setSubtype(Subtype.inconsistentDatePattern) 1021 .setMessage(msg, 1022 dateTimeLength, dateOrTime, value, b)); 1023 } else { 1024 if (errorOnMissing) { 1025 String msg = countMismatches != 1 1026 ? "{1}-{0} → “{2}” doesn't have at least one value for a corresponding flexible skeleton {3}, which needs to be added." 1027 : "{1}-{0} → “{2}” doesn't have a value for the corresponding flexible skeleton {3}, which needs to be added."; 1028 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.warningType) 1029 .setSubtype(Subtype.missingDatePattern) 1030 .setMessage(msg, 1031 dateTimeLength, dateOrTime, value, Joiner.on(", ").join(bases))); 1032 } 1033 } 1034 } 1035 } 1036 add(StringBuilder b, String key, String value1)1037 private void add(StringBuilder b, String key, String value1) { 1038 if (value1 == null) { 1039 return; 1040 } 1041 if (b.length() != 0) { 1042 b.append(" or "); 1043 } 1044 b.append(key + (value1 == null ? " - missing" : " → “" + value1 + "”")); 1045 } 1046 equalsExceptWidth(String value1, String value2)1047 private boolean equalsExceptWidth(String value1, String value2) { 1048 if (value1.equals(value2)) { 1049 return true; 1050 } else if (value2 == null) { 1051 return false; 1052 } 1053 1054 List<Object> items1 = new ArrayList<>(formatParser.set(value1).getItems()); // clone 1055 List<Object> items2 = formatParser.set(value2).getItems(); 1056 if (items1.size() != items2.size()) { 1057 return false; 1058 } 1059 Iterator<Object> it2 = items2.iterator(); 1060 for (Object item1 : items1) { 1061 Object item2 = it2.next(); 1062 if (item1.equals(item2)) { 1063 continue; 1064 } 1065 if (item1 instanceof VariableField && item2 instanceof VariableField) { 1066 // simple test for now, ignore widths 1067 if (item1.toString().charAt(0) == item2.toString().charAt(0)) { 1068 continue; 1069 } 1070 } 1071 return false; 1072 } 1073 return true; 1074 } 1075 1076 static final Set<String> YgLanguages = new HashSet<>(Arrays.asList( 1077 "ar", "cs", "da", "de", "en", "es", "fa", "fi", "fr", "he", "hr", "id", "it", "nl", "no", "pt", "ru", "sv", "tr")); 1078 getExpectedGy(String localeID)1079 private GyState getExpectedGy(String localeID) { 1080 // hack for now 1081 int firstBar = localeID.indexOf('_'); 1082 String lang = firstBar < 0 ? localeID : localeID.substring(0, firstBar); 1083 return YgLanguages.contains(lang) ? GyState.YEAR_ERA : GyState.ERA_YEAR; 1084 } 1085 1086 enum GyState { 1087 YEAR_ERA, ERA_YEAR, OTHER; 1088 static DateTimePatternGenerator.FormatParser formatParser = new DateTimePatternGenerator.FormatParser(); 1089 1090 static synchronized GyState forPattern(String value) { 1091 formatParser.set(value); 1092 int last = -1; 1093 for (Object x : formatParser.getItems()) { 1094 if (x instanceof VariableField) { 1095 int type = ((VariableField) x).getType(); 1096 if (type == DateTimePatternGenerator.ERA && last == DateTimePatternGenerator.YEAR) { 1097 return GyState.YEAR_ERA; 1098 } else if (type == DateTimePatternGenerator.YEAR && last == DateTimePatternGenerator.ERA) { 1099 return GyState.ERA_YEAR; 1100 } 1101 last = type; 1102 } 1103 } 1104 return GyState.OTHER; 1105 } 1106 } 1107 1108 enum DateTimeLengths { 1109 SHORT, MEDIUM, LONG, FULL 1110 } 1111 1112 // The patterns below should only use the *canonical* characters for each field type: 1113 // y (not Y, u, U) 1114 // Q (not q) 1115 // M (not L) 1116 // E (not e, c) 1117 // a (not b, B) 1118 // H or h (not k or K) 1119 // v (not z, Z, V) 1120 static final Pattern[] dateTimePatterns = { 1121 PatternCache.get("a*(h|hh|H|HH)(m|mm)"), // time-short 1122 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)"), // time-medium 1123 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)(v+)"), // time-long 1124 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)(v+)"), // time-full 1125 PatternCache.get("G*y{1,4}M{1,2}(d|dd)"), // date-short; allow yyy for Minguo/ROC calendar 1126 PatternCache.get("G*y(yyy)?M{1,3}(d|dd)"), // date-medium 1127 PatternCache.get("G*y(yyy)?M{1,4}(d|dd)"), // date-long 1128 PatternCache.get("G*y(yyy)?M{1,4}E*(d|dd)"), // date-full 1129 PatternCache.get(".*"), // datetime-short 1130 PatternCache.get(".*"), // datetime-medium 1131 PatternCache.get(".*"), // datetime-long 1132 PatternCache.get(".*"), // datetime-full 1133 }; 1134 1135 static final String[] dateTimeMessage = { 1136 "hours (H, HH, h, or hh), and minutes (m or mm)", // time-short 1137 "hours (H, HH, h, or hh), minutes (m or mm), and seconds (s or ss)", // time-medium 1138 "hours (H, HH, h, or hh), minutes (m or mm), and seconds (s or ss); optionally timezone (z, zzzz, v, vvvv)", // time-long 1139 "hours (H, HH, h, or hh), minutes (m or mm), seconds (s or ss), and timezone (z, zzzz, v, vvvv)", // time-full 1140 "year (y, yy, yyyy), month (M or MM), and day (d or dd); optionally era (G)", // date-short 1141 "year (y), month (M, MM, or MMM), and day (d or dd); optionally era (G)", // date-medium 1142 "year (y), month (M, ... MMMM), and day (d or dd); optionally era (G)", // date-long 1143 "year (y), month (M, ... MMMM), and day (d or dd); optionally day of week (EEEE or cccc) or era (G)", // date-full 1144 }; 1145 1146 public String toString(DateTimePatternGenerator.FormatParser formatParser) { 1147 StringBuffer result = new StringBuffer(); 1148 for (Object x : formatParser.getItems()) { 1149 if (x instanceof DateTimePatternGenerator.VariableField) { 1150 result.append(x.toString()); 1151 } else { 1152 result.append(formatParser.quoteLiteral(x.toString())); 1153 } 1154 } 1155 return result.toString(); 1156 } 1157 1158 private void checkPattern2(String path, String value, List<CheckStatus> result) throws ParseException { 1159 XPathParts pathParts = XPathParts.getFrozenInstance(path); 1160 String calendar = pathParts.findAttributeValue("calendar", "type"); 1161 SimpleDateFormat x = icuServiceBuilder.getDateFormat(calendar, value); 1162 x.setTimeZone(ExampleGenerator.ZONE_SAMPLE); 1163 result.add(new MyCheckStatus() 1164 .setFormat(x) 1165 .setCause(this).setMainType(CheckStatus.demoType)); 1166 } 1167 1168 private DateTimePatternGenerator getDTPGForCalendarType(String calendarType) { 1169 DateTimePatternGenerator dtpg = dtpgForType.get(calendarType); 1170 if (dtpg == null) { 1171 dtpg = flexInfo.getDTPGForCalendarType(calendarType, parentCLDRFiles); 1172 dtpgForType.put(calendarType, dtpg); 1173 } 1174 return dtpg; 1175 } 1176 1177 static final UnicodeSet XGRAPHEME = new UnicodeSet("[[:mark:][:grapheme_extend:][:punctuation:]]"); 1178 static final UnicodeSet DIGIT = new UnicodeSet("[:decimal_number:]"); 1179 1180 static public class MyCheckStatus extends CheckStatus { 1181 private SimpleDateFormat df; 1182 1183 public MyCheckStatus setFormat(SimpleDateFormat df) { 1184 this.df = df; 1185 return this; 1186 } 1187 1188 @Override 1189 public SimpleDemo getDemo() { 1190 return new MyDemo().setFormat(df); 1191 } 1192 } 1193 1194 static class MyDemo extends FormatDemo { 1195 private SimpleDateFormat df; 1196 1197 @Override 1198 protected String getPattern() { 1199 return df.toPattern(); 1200 } 1201 1202 @Override 1203 protected String getSampleInput() { 1204 return neutralFormat.format(ExampleGenerator.DATE_SAMPLE); 1205 } 1206 1207 public MyDemo setFormat(SimpleDateFormat df) { 1208 this.df = df; 1209 return this; 1210 } 1211 1212 @Override 1213 protected void getArguments(Map<String, String> inout) { 1214 currentPattern = currentInput = currentFormatted = currentReparsed = "?"; 1215 Date d; 1216 try { 1217 currentPattern = inout.get("pattern"); 1218 if (currentPattern != null) 1219 df.applyPattern(currentPattern); 1220 else 1221 currentPattern = getPattern(); 1222 } catch (Exception e) { 1223 currentPattern = "Use format like: ##,###.##"; 1224 return; 1225 } 1226 try { 1227 currentInput = inout.get("input"); 1228 if (currentInput == null) { 1229 currentInput = getSampleInput(); 1230 } 1231 d = neutralFormat.parse(currentInput); 1232 } catch (Exception e) { 1233 currentInput = "Use neutral format like: 1993-11-31 13:49:02"; 1234 return; 1235 } 1236 try { 1237 currentFormatted = df.format(d); 1238 } catch (Exception e) { 1239 currentFormatted = "Can't format: " + e.getMessage(); 1240 return; 1241 } 1242 try { 1243 parsePosition.setIndex(0); 1244 Date n = df.parse(currentFormatted, parsePosition); 1245 if (parsePosition.getIndex() != currentFormatted.length()) { 1246 currentReparsed = "Couldn't parse past: " + "\u200E" 1247 + currentFormatted.substring(0, parsePosition.getIndex()) + "\u200E"; 1248 } else { 1249 currentReparsed = neutralFormat.format(n); 1250 } 1251 } catch (Exception e) { 1252 currentReparsed = "Can't parse: " + e.getMessage(); 1253 } 1254 } 1255 1256 } 1257 } 1258