1 package org.unicode.cldr.test; 2 3 import java.text.ParseException; 4 import java.util.ArrayList; 5 import java.util.Arrays; 6 import java.util.Calendar; 7 import java.util.Collection; 8 import java.util.Date; 9 import java.util.EnumMap; 10 import java.util.HashMap; 11 import java.util.HashSet; 12 import java.util.Iterator; 13 import java.util.LinkedHashSet; 14 import java.util.List; 15 import java.util.Locale; 16 import java.util.Map; 17 import java.util.Random; 18 import java.util.Set; 19 import java.util.TreeSet; 20 import java.util.regex.Matcher; 21 import java.util.regex.Pattern; 22 23 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 24 import org.unicode.cldr.util.ApproximateWidth; 25 import org.unicode.cldr.util.CLDRFile; 26 import org.unicode.cldr.util.CLDRFile.Status; 27 import org.unicode.cldr.util.CLDRLocale; 28 import org.unicode.cldr.util.CldrUtility; 29 import org.unicode.cldr.util.DateTimeCanonicalizer.DateTimePatternType; 30 import org.unicode.cldr.util.DayPeriodInfo; 31 import org.unicode.cldr.util.DayPeriodInfo.DayPeriod; 32 import org.unicode.cldr.util.DayPeriodInfo.Type; 33 import org.unicode.cldr.util.Factory; 34 import org.unicode.cldr.util.ICUServiceBuilder; 35 import org.unicode.cldr.util.Level; 36 import org.unicode.cldr.util.LocaleIDParser; 37 import org.unicode.cldr.util.LogicalGrouping; 38 import org.unicode.cldr.util.PathHeader; 39 import org.unicode.cldr.util.PathStarrer; 40 import org.unicode.cldr.util.PatternCache; 41 import org.unicode.cldr.util.PreferredAndAllowedHour; 42 import org.unicode.cldr.util.RegexUtilities; 43 import org.unicode.cldr.util.SupplementalDataInfo; 44 import org.unicode.cldr.util.XPathParts; 45 import org.unicode.cldr.util.props.UnicodeProperty.PatternMatcher; 46 47 import com.google.common.base.Joiner; 48 import com.ibm.icu.impl.Relation; 49 import com.ibm.icu.text.BreakIterator; 50 import com.ibm.icu.text.DateTimePatternGenerator; 51 import com.ibm.icu.text.DateTimePatternGenerator.VariableField; 52 import com.ibm.icu.text.MessageFormat; 53 import com.ibm.icu.text.NumberFormat; 54 import com.ibm.icu.text.SimpleDateFormat; 55 import com.ibm.icu.text.UnicodeSet; 56 import com.ibm.icu.util.Output; 57 import com.ibm.icu.util.ULocale; 58 59 public class CheckDates extends FactoryCheckCLDR { 60 static boolean GREGORIAN_ONLY = CldrUtility.getProperty("GREGORIAN", false); 61 62 ICUServiceBuilder icuServiceBuilder = new ICUServiceBuilder(); 63 NumberFormat english = NumberFormat.getNumberInstance(ULocale.ENGLISH); 64 PatternMatcher m; 65 DateTimePatternGenerator.FormatParser formatParser = new DateTimePatternGenerator.FormatParser(); 66 DateTimePatternGenerator dateTimePatternGenerator = DateTimePatternGenerator.getEmptyInstance(); 67 private CoverageLevel2 coverageLevel; 68 private SupplementalDataInfo sdi = SupplementalDataInfo.getInstance(); 69 // Ordered list of this CLDRFile and parent CLDRFiles up to root 70 List<CLDRFile> parentCLDRFiles = new ArrayList<CLDRFile>(); 71 // Map from calendar type (i.e. "gregorian", "generic", "chinese") to DateTimePatternGenerator instance for that type 72 Map<String, DateTimePatternGenerator> dtpgForType = new HashMap<String, DateTimePatternGenerator>(); 73 74 // Use the width of the character "0" as the basic unit for checking widths 75 // It's not perfect, but I'm not sure that anything can be. This helps us 76 // weed out some false positives in width checking, like 10月 vs. 十月 77 // in Chinese, which although technically longer, shouldn't trigger an 78 // error. 79 private static final int REFCHAR = ApproximateWidth.getWidth("0"); 80 81 private Level requiredLevel; 82 private String language; 83 private String territory; 84 85 private DayPeriodInfo dateFormatInfoFormat; 86 87 static String[] samples = { 88 // "AD 1970-01-01T00:00:00Z", 89 // "BC 4004-10-23T07:00:00Z", // try a BC date: creation according to Ussher & Lightfoot. Assuming garden of 90 // eden 2 hours ahead of UTC 91 "2005-12-02 12:15:16", 92 // "AD 2100-07-11T10:15:16Z", 93 }; // keep aligned with following 94 static String SampleList = "{0}" 95 // + Utility.LINE_SEPARATOR + "\t\u200E{1}\u200E" + Utility.LINE_SEPARATOR + "\t\u200E{2}\u200E" + 96 // Utility.LINE_SEPARATOR + "\t\u200E{3}\u200E" 97 ; // keep aligned with previous 98 99 private static final String DECIMAL_XPATH = "//ldml/numbers/symbols[@numberSystem='latn']/decimal"; 100 private static final Pattern HOUR_SYMBOL = PatternCache.get("H{1,2}"); 101 private static final Pattern MINUTE_SYMBOL = PatternCache.get("mm"); 102 private static final Pattern YEAR_FIELDS = PatternCache.get("(y|Y|u|U|r){1,5}"); 103 104 private static String CALENDAR_ID_PREFIX = "/calendar[@type=\""; 105 106 static String[] calTypePathsToCheck = { 107 "//ldml/dates/calendars/calendar[@type=\"buddhist\"]", 108 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]", 109 "//ldml/dates/calendars/calendar[@type=\"hebrew\"]", 110 "//ldml/dates/calendars/calendar[@type=\"islamic\"]", 111 "//ldml/dates/calendars/calendar[@type=\"japanese\"]", 112 "//ldml/dates/calendars/calendar[@type=\"roc\"]", 113 }; 114 static String[] calSymbolPathsWhichNeedDistinctValues = { 115 // === for months, days, quarters - format wide & abbrev sets must have distinct values === 116 "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"abbreviated\"]/month", 117 "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"wide\"]/month", 118 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"abbreviated\"]/day", 119 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"short\"]/day", 120 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"wide\"]/day", 121 "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"abbreviated\"]/quarter", 122 "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"wide\"]/quarter", 123 // === for dayPeriods - all values for a given context/width must be distinct === 124 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"abbreviated\"]/dayPeriod", 125 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod", 126 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod", 127 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"abbreviated\"]/dayPeriod", 128 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod", 129 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod", 130 // === for eras - all values for a given context/width should be distinct (warning) === 131 "/eras/eraNames/era", 132 "/eras/eraAbbr/era", // Hmm, root eraAbbr for japanese has many dups, should we change them or drop this test? 133 "/eras/eraNarrow/era", // We may need to allow dups here too 134 }; 135 136 // The following calendar symbol sets need not have distinct values 137 // "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"narrow\"]/month", 138 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"abbreviated\"]/month", 139 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"narrow\"]/month", 140 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"wide\"]/month", 141 // "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"narrow\"]/day", 142 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"abbreviated\"]/day", 143 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"narrow\"]/day", 144 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"wide\"]/day", 145 // "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"narrow\"]/quarter", 146 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"abbreviated\"]/quarter", 147 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"narrow\"]/quarter", 148 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"wide\"]/quarter", 149 150 // The above are followed by trailing pieces such as 151 // "[@type=\"am\"]", 152 // "[@type=\"sun\"]", 153 // "[@type=\"0\"]", 154 // "[@type=\"1\"]", 155 // "[@type=\"12\"]", 156 157 // Map<String, Set<String>> calPathsToSymbolSets; 158 // Map<String, Map<String, String>> calPathsToSymbolMaps = new HashMap<String, Map<String, String>>(); 159 CheckDates(Factory factory)160 public CheckDates(Factory factory) { 161 super(factory); 162 } 163 164 @Override setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors)165 public CheckCLDR setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options, 166 List<CheckStatus> possibleErrors) { 167 if (cldrFileToCheck == null) return this; 168 super.setCldrFileToCheck(cldrFileToCheck, options, possibleErrors); 169 170 icuServiceBuilder.setCldrFile(getResolvedCldrFileToCheck()); 171 // the following is a hack to work around a bug in ICU4J (the snapshot, not the released version). 172 try { 173 bi = BreakIterator.getCharacterInstance(new ULocale(cldrFileToCheck.getLocaleID())); 174 } catch (RuntimeException e) { 175 bi = BreakIterator.getCharacterInstance(new ULocale("")); 176 } 177 CLDRFile resolved = getResolvedCldrFileToCheck(); 178 flexInfo = new FlexibleDateFromCLDR(); // ought to just clear(), but not available. 179 flexInfo.set(resolved); 180 181 // load decimal path specially 182 String decimal = resolved.getWinningValue(DECIMAL_XPATH); 183 if (decimal != null) { 184 flexInfo.checkFlexibles(DECIMAL_XPATH, decimal, DECIMAL_XPATH); 185 } 186 187 String localeID = cldrFileToCheck.getLocaleID(); 188 LocaleIDParser lp = new LocaleIDParser(); 189 territory = lp.set(localeID).getRegion(); 190 language = lp.getLanguage(); 191 if (territory == null || territory.length() == 0) { 192 if (language.equals("root")) { 193 territory = "001"; 194 } else { 195 CLDRLocale loc = CLDRLocale.getInstance(localeID); 196 CLDRLocale defContent = sdi.getDefaultContentFromBase(loc); 197 if (defContent == null) { 198 territory = "001"; 199 } else { 200 territory = defContent.getCountry(); 201 } 202 // Set territory for 12/24 hour clock to Egypt (12 hr) for ar_001 203 // instead of 24 hour (exception). 204 if (territory.equals("001") && language.equals("ar")) { 205 territory = "EG"; 206 } 207 } 208 } 209 coverageLevel = CoverageLevel2.getInstance(sdi, localeID); 210 requiredLevel = options.getRequiredLevel(localeID); 211 212 // load gregorian appendItems 213 for (Iterator<String> it = resolved.iterator("//ldml/dates/calendars/calendar[@type=\"gregorian\"]"); it.hasNext();) { 214 String path = it.next(); 215 String value = resolved.getWinningValue(path); 216 String fullPath = resolved.getFullXPath(path); 217 try { 218 flexInfo.checkFlexibles(path, value, fullPath); 219 } catch (Exception e) { 220 final String message = e.getMessage(); 221 CheckStatus item = new CheckStatus() 222 .setCause(this) 223 .setMainType(CheckStatus.errorType) 224 .setSubtype( 225 message.contains("Conflicting fields") ? Subtype.dateSymbolCollision : Subtype.internalError) 226 .setMessage(message); 227 possibleErrors.add(item); 228 } 229 // possibleErrors.add(flexInfo.getFailurePath(path)); 230 } 231 redundants.clear(); 232 /* 233 * TODO: NullPointerException may be thrown in ICU here during cldr-unittest TestAll 234 */ 235 flexInfo.getRedundants(redundants); 236 // Set baseSkeletons = flexInfo.gen.getBaseSkeletons(new TreeSet()); 237 // Set notCovered = new TreeSet(neededFormats); 238 // if (flexInfo.preferred12Hour()) { 239 // notCovered.addAll(neededHours12); 240 // } else { 241 // notCovered.addAll(neededHours24); 242 // } 243 // notCovered.removeAll(baseSkeletons); 244 // if (notCovered.size() != 0) { 245 // possibleErrors.add(new CheckStatus().setCause(this).setType(CheckCLDR.finalErrorType) 246 // .setCheckOnSubmit(false) 247 // .setMessage("Missing availableFormats: {0}", new Object[]{notCovered.toString()})); 248 // } 249 pathsWithConflictingOrder2sample = DateOrder.getOrderingInfo(cldrFileToCheck, resolved, flexInfo.fp); 250 if (pathsWithConflictingOrder2sample == null) { 251 CheckStatus item = new CheckStatus() 252 .setCause(this) 253 .setMainType(CheckStatus.errorType) 254 .setSubtype(Subtype.internalError) 255 .setMessage("DateOrder.getOrderingInfo fails"); 256 possibleErrors.add(item); 257 } 258 259 // calPathsToSymbolMaps.clear(); 260 // for (String calTypePath: calTypePathsToCheck) { 261 // for (String calSymbolPath: calSymbolPathsWhichNeedDistinctValues) { 262 // calPathsToSymbolMaps.put(calTypePath.concat(calSymbolPath), null); 263 // } 264 // } 265 266 dateFormatInfoFormat = sdi.getDayPeriods(Type.format, cldrFileToCheck.getLocaleID()); 267 268 // Make new list of parent CLDRFiles 269 parentCLDRFiles.clear(); 270 parentCLDRFiles.add(cldrFileToCheck); 271 while ((localeID = LocaleIDParser.getParent(localeID)) != null) { 272 CLDRFile resolvedParentCLDRFile = getFactory().make(localeID, true); 273 parentCLDRFiles.add(resolvedParentCLDRFile); 274 } 275 // Clear out map of DateTimePatternGenerators for calendarType 276 dtpgForType.clear(); 277 278 return this; 279 } 280 281 Map<String, Map<DateOrder, String>> pathsWithConflictingOrder2sample; 282 283 // Set neededFormats = new TreeSet(Arrays.asList(new String[]{ 284 // "yM", "yMMM", "yMd", "yMMMd", "Md", "MMMd","yQ" 285 // })); 286 // Set neededHours12 = new TreeSet(Arrays.asList(new String[]{ 287 // "hm", "hms" 288 // })); 289 // Set neededHours24 = new TreeSet(Arrays.asList(new String[]{ 290 // "Hm", "Hms" 291 // })); 292 /** 293 * hour+minute, hour+minute+second (12 & 24) 294 * year+month, year+month+day (numeric & string) 295 * month+day (numeric & string) 296 * year+quarter 297 */ 298 BreakIterator bi; 299 FlexibleDateFromCLDR flexInfo; 300 Collection<String> redundants = new HashSet<>(); 301 Status status = new Status(); 302 PathStarrer pathStarrer = new PathStarrer(); 303 stripPrefix(String s)304 private String stripPrefix(String s) { 305 if (s != null) { 306 int prefEnd = s.lastIndexOf(" "); 307 if (prefEnd < 0 || prefEnd >= 3) { 308 prefEnd = s.lastIndexOf("\u2019"); // as in d’ 309 } 310 if (prefEnd >= 0 && prefEnd < 3) { 311 return s.substring(prefEnd + 1); 312 } 313 } 314 return s; 315 } 316 317 @Override handleCheck(String path, String fullPath, String value, Options options, List<CheckStatus> result)318 public CheckCLDR handleCheck(String path, String fullPath, String value, Options options, 319 List<CheckStatus> result) { 320 321 if (fullPath == null) { 322 return this; // skip paths that we don't have 323 } 324 325 if (path.indexOf("/dates") < 0 326 || path.endsWith("/default") 327 || path.endsWith("/alias")) { 328 return this; 329 } 330 331 String sourceLocale = getCldrFileToCheck().getSourceLocaleID(path, status); 332 333 if (!path.equals(status.pathWhereFound) || !sourceLocale.equals(getCldrFileToCheck().getLocaleID())) { 334 return this; 335 } 336 337 if (value == null) { 338 return this; 339 } 340 341 if (pathsWithConflictingOrder2sample != null) { 342 Map<DateOrder, String> problem = pathsWithConflictingOrder2sample.get(path); 343 if (problem != null) { 344 CheckStatus item = new CheckStatus() 345 .setCause(this) 346 .setMainType(CheckStatus.warningType) 347 .setSubtype(Subtype.incorrectDatePattern) 348 .setMessage("The ordering of date fields is inconsistent with others: {0}", 349 getValues(getResolvedCldrFileToCheck(), problem.values())); 350 result.add(item); 351 } 352 } 353 354 try { 355 if (path.indexOf("[@type=\"abbreviated\"]") >= 0) { 356 String pathToWide = path.replace("[@type=\"abbreviated\"]", "[@type=\"wide\"]"); 357 String wideValue = getCldrFileToCheck().getWinningValueWithBailey(pathToWide); 358 if (wideValue != null && isTooMuchWiderThan(value, wideValue)) { 359 CheckStatus item = new CheckStatus() 360 .setCause(this) 361 .setMainType(CheckStatus.errorType) 362 .setSubtype(Subtype.abbreviatedDateFieldTooWide) 363 .setMessage("Abbreviated value \"{0}\" can't be longer than the corresponding wide value \"{1}\"", value, 364 wideValue); 365 result.add(item); 366 } 367 Set<String> grouping = LogicalGrouping.getPaths(getCldrFileToCheck(), path); 368 if (grouping != null) { 369 for (String lgPath : grouping) { 370 String lgPathValue = getCldrFileToCheck().getWinningValueWithBailey(lgPath); 371 if (lgPathValue == null) { 372 continue; 373 } 374 String lgPathToWide = lgPath.replace("[@type=\"abbreviated\"]", "[@type=\"wide\"]"); 375 String lgPathWideValue = getCldrFileToCheck().getWinningValueWithBailey(lgPathToWide); 376 // This helps us get around things like "de març" vs. "març" in Catalan 377 String thisValueStripped = stripPrefix(value); 378 String wideValueStripped = stripPrefix(wideValue); 379 String lgPathValueStripped = stripPrefix(lgPathValue); 380 String lgPathWideValueStripped = stripPrefix(lgPathWideValue); 381 boolean thisPathHasPeriod = value.contains("."); 382 boolean lgPathHasPeriod = lgPathValue.contains("."); 383 if (!thisValueStripped.equalsIgnoreCase(wideValueStripped) && !lgPathValueStripped.equalsIgnoreCase(lgPathWideValueStripped) && 384 thisPathHasPeriod != lgPathHasPeriod) { 385 CheckStatus.Type et = CheckStatus.errorType; 386 if (path.contains("dayPeriod")) { 387 et = CheckStatus.warningType; 388 } 389 CheckStatus item = new CheckStatus() 390 .setCause(this) 391 .setMainType(et) 392 .setSubtype(Subtype.inconsistentPeriods) 393 .setMessage("Inconsistent use of periods in abbreviations for this section."); 394 result.add(item); 395 break; 396 } 397 } 398 } 399 } else if (path.indexOf("[@type=\"narrow\"]") >= 0) { 400 String pathToAbbr = path.replace("[@type=\"narrow\"]", "[@type=\"abbreviated\"]"); 401 String abbrValue = getCldrFileToCheck().getWinningValueWithBailey(pathToAbbr); 402 if (abbrValue != null && isTooMuchWiderThan(value, abbrValue)) { 403 CheckStatus item = new CheckStatus() 404 .setCause(this) 405 .setMainType(CheckStatus.warningType) // Making this just a warning, because there are some oddball cases. 406 .setSubtype(Subtype.narrowDateFieldTooWide) 407 .setMessage("Narrow value \"{0}\" shouldn't be longer than the corresponding abbreviated value \"{1}\"", value, 408 abbrValue); 409 result.add(item); 410 } 411 } else if (path.indexOf("/eraNarrow") >= 0) { 412 String pathToAbbr = path.replace("/eraNarrow", "/eraAbbr"); 413 String abbrValue = getCldrFileToCheck().getWinningValueWithBailey(pathToAbbr); 414 if (abbrValue != null && isTooMuchWiderThan(value, abbrValue)) { 415 CheckStatus item = new CheckStatus() 416 .setCause(this) 417 .setMainType(CheckStatus.errorType) 418 .setSubtype(Subtype.narrowDateFieldTooWide) 419 .setMessage("Narrow value \"{0}\" can't be longer than the corresponding abbreviated value \"{1}\"", value, 420 abbrValue); 421 result.add(item); 422 } 423 } else if (path.indexOf("/eraAbbr") >= 0) { 424 String pathToWide = path.replace("/eraAbbr", "/eraNames"); 425 String wideValue = getCldrFileToCheck().getWinningValueWithBailey(pathToWide); 426 if (wideValue != null && isTooMuchWiderThan(value, wideValue)) { 427 CheckStatus item = new CheckStatus() 428 .setCause(this) 429 .setMainType(CheckStatus.errorType) 430 .setSubtype(Subtype.abbreviatedDateFieldTooWide) 431 .setMessage("Abbreviated value \"{0}\" can't be longer than the corresponding wide value \"{1}\"", value, 432 wideValue); 433 result.add(item); 434 } 435 436 } 437 438 String failure = flexInfo.checkValueAgainstSkeleton(path, value); 439 if (failure != null) { 440 result.add(new CheckStatus() 441 .setCause(this) 442 .setMainType(CheckStatus.errorType) 443 .setSubtype(Subtype.illegalDatePattern) 444 .setMessage(failure)); 445 } 446 447 final String collisionPrefix = "//ldml/dates/calendars/calendar"; 448 main: if (path.startsWith(collisionPrefix)) { 449 int pos = path.indexOf("\"]"); // end of first type 450 if (pos < 0 || skipPath(path)) { // skip narrow, no-calendar 451 break main; 452 } 453 pos += 2; 454 String myType = getLastType(path); 455 if (myType == null) { 456 break main; 457 } 458 String myMainType = getMainType(path); 459 460 String calendarPrefix = path.substring(0, pos); 461 boolean endsWithDisplayName = path.endsWith("displayName"); // special hack, these shouldn't be in 462 // calendar. 463 464 Set<String> retrievedPaths = new HashSet<>(); 465 getResolvedCldrFileToCheck().getPathsWithValue(value, calendarPrefix, null, retrievedPaths); 466 if (retrievedPaths.size() < 2) { 467 break main; 468 } 469 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraAbbr/era[@type="0"], 470 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraNames/era[@type="0"], 471 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraNarrow/era[@type="0"]] 472 Type type = null; 473 DayPeriod dayPeriod = null; 474 final boolean isDayPeriod = path.contains("dayPeriod"); 475 if (isDayPeriod) { 476 XPathParts parts = XPathParts.getFrozenInstance(fullPath); 477 type = Type.fromString(parts.getAttributeValue(5, "type")); 478 dayPeriod = DayPeriod.valueOf(parts.getAttributeValue(-1, "type")); 479 } 480 481 // TODO redo above and below in terms of parts instead of searching strings 482 483 Set<String> filteredPaths = new HashSet<>(); 484 Output<Integer> sampleError = new Output<>(); 485 486 for (String item : retrievedPaths) { 487 XPathParts itemParts = XPathParts.getFrozenInstance(item); 488 if (item.equals(path) 489 || skipPath(item) 490 || endsWithDisplayName != item.endsWith("displayName") 491 || itemParts.containsElement("alias")) { 492 continue; 493 } 494 String otherType = getLastType(item); 495 if (myType.equals(otherType)) { // we don't care about items with the same type value 496 continue; 497 } 498 String mainType = getMainType(item); 499 if (!myMainType.equals(mainType)) { // we *only* care about items with the same type value 500 continue; 501 } 502 if (isDayPeriod) { 503 //ldml/dates/calendars/calendar[@type="gregorian"]/dayPeriods/dayPeriodContext[@type="format"]/dayPeriodWidth[@type="wide"]/dayPeriod[@type="am"] 504 Type itemType = Type.fromString(itemParts.getAttributeValue(5, "type")); 505 DayPeriod itemDayPeriod = DayPeriod.valueOf(itemParts.getAttributeValue(-1, "type")); 506 507 if (!dateFormatInfoFormat.collisionIsError(type, dayPeriod, itemType, itemDayPeriod, sampleError)) { 508 continue; 509 } 510 } 511 filteredPaths.add(item); 512 } 513 if (filteredPaths.size() == 0) { 514 break main; 515 } 516 Set<String> others = new TreeSet<>(); 517 for (String path2 : filteredPaths) { 518 PathHeader pathHeader = getPathHeaderFactory().fromPath(path2); 519 others.add(pathHeader.getHeaderCode()); 520 } 521 CheckStatus.Type statusType = getPhase() == Phase.SUBMISSION || getPhase() == Phase.BUILD 522 ? CheckStatus.warningType 523 : CheckStatus.errorType; 524 final CheckStatus checkStatus = new CheckStatus() 525 .setCause(this) 526 .setMainType(statusType) 527 .setSubtype(Subtype.dateSymbolCollision); 528 if (sampleError.value == null) { 529 checkStatus.setMessage("The date value “{0}” is the same as what is used for a different item: {1}", 530 value, others.toString()); 531 } else { 532 checkStatus.setMessage("The date value “{0}” is the same as what is used for a different item: {1}. Sample problem: {2}", 533 value, others.toString(), sampleError.value / DayPeriodInfo.HOUR); 534 } 535 result.add(checkStatus); 536 } 537 DateTimePatternType dateTypePatternType = DateTimePatternType.fromPath(path); 538 if (DateTimePatternType.STOCK_AVAILABLE_INTERVAL_PATTERNS.contains(dateTypePatternType)) { 539 boolean patternBasicallyOk = false; 540 try { 541 formatParser.set(value); 542 patternBasicallyOk = true; 543 } catch (RuntimeException e) { 544 String message = e.getMessage(); 545 if (message.contains("Illegal datetime field:")) { 546 CheckStatus item = new CheckStatus().setCause(this) 547 .setMainType(CheckStatus.errorType) 548 .setSubtype(Subtype.illegalDatePattern) 549 .setMessage(message); 550 result.add(item); 551 } else { 552 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 553 .setSubtype(Subtype.illegalDatePattern) 554 .setMessage("Illegal date format pattern {0}", new Object[] { e }); 555 result.add(item); 556 } 557 } 558 if (patternBasicallyOk) { 559 checkPattern(dateTypePatternType, path, fullPath, value, result); 560 } 561 } else if (path.contains("datetimeSkeleton") && !path.contains("[@alt=")) { // cannot test any alt skeletons 562 // Get calendar type from //ldml/dates/calendars/calendar[@type="..."]/ 563 int startIndex = path.indexOf(CALENDAR_ID_PREFIX); 564 if (startIndex > 0) { 565 startIndex += CALENDAR_ID_PREFIX.length(); 566 int endIndex = path.indexOf("\"]", startIndex); 567 String calendarType = path.substring(startIndex,endIndex); 568 // Get pattern generated from datetimeSkeleton 569 DateTimePatternGenerator dtpg = getDTPGForCalendarType(calendarType); 570 String patternFromSkeleton = dtpg.getBestPattern(value); 571 // Get actual stock pattern 572 String patternPath = path.replace("/datetimeSkeleton", "/pattern[@type=\"standard\"]"); 573 String patternStock = getCldrFileToCheck().getWinningValue(patternPath); 574 // Compare and flag error if mismatch 575 if (!patternFromSkeleton.equals(patternStock)) { 576 CheckStatus item = new CheckStatus() 577 .setCause(this) 578 .setMainType(CheckStatus.warningType) 579 .setSubtype(Subtype.inconsistentDatePattern) 580 .setMessage( 581 "Pattern \"{0}\" from datetimeSkeleton should match corresponding standard pattern \"{1}\", adjust availableFormats to fix.", 582 patternFromSkeleton, patternStock); 583 result.add(item); 584 } 585 } 586 } else if (path.contains("hourFormat")) { 587 int semicolonPos = value.indexOf(';'); 588 if (semicolonPos < 0) { 589 CheckStatus item = new CheckStatus() 590 .setCause(this) 591 .setMainType(CheckStatus.errorType) 592 .setSubtype(Subtype.illegalDatePattern) 593 .setMessage( 594 "Value should contain a positive hour format and a negative hour format separated by a semicolon."); 595 result.add(item); 596 } else { 597 String[] formats = value.split(";"); 598 if (formats[0].equals(formats[1])) { 599 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 600 .setSubtype(Subtype.illegalDatePattern) 601 .setMessage("The hour formats should not be the same."); 602 result.add(item); 603 } else { 604 checkHasHourMinuteSymbols(formats[0], result); 605 checkHasHourMinuteSymbols(formats[1], result); 606 } 607 } 608 } 609 } catch (ParseException e) { 610 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 611 .setSubtype(Subtype.illegalDatePattern) 612 .setMessage("ParseException in creating date format {0}", new Object[] { e }); 613 result.add(item); 614 } catch (Exception e) { 615 // e.printStackTrace(); 616 // HACK 617 String msg = e.getMessage(); 618 if (msg == null || !HACK_CONFLICTING.matcher(msg).find()) { 619 CheckStatus item = new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 620 .setSubtype(Subtype.illegalDatePattern) 621 .setMessage("Error in creating date format {0}", new Object[] { e }); 622 result.add(item); 623 } 624 } 625 return this; 626 } 627 isTooMuchWiderThan(String shortString, String longString)628 private boolean isTooMuchWiderThan(String shortString, String longString) { 629 // We all 1/3 the width of the reference character as a "fudge factor" in determining the allowable width 630 return ApproximateWidth.getWidth(shortString) > ApproximateWidth.getWidth(longString) + REFCHAR / 3; 631 } 632 633 /** 634 * Check for the presence of hour and minute symbols. 635 * 636 * @param value 637 * the value to be checked 638 * @param result 639 * the list to add any errors to. 640 */ checkHasHourMinuteSymbols(String value, List<CheckStatus> result)641 private void checkHasHourMinuteSymbols(String value, List<CheckStatus> result) { 642 boolean hasHourSymbol = HOUR_SYMBOL.matcher(value).find(); 643 boolean hasMinuteSymbol = MINUTE_SYMBOL.matcher(value).find(); 644 if (!hasHourSymbol && !hasMinuteSymbol) { 645 result.add(createErrorCheckStatus().setMessage("The hour and minute symbols are missing from {0}.", value)); 646 } else if (!hasHourSymbol) { 647 result.add(createErrorCheckStatus() 648 .setMessage("The hour symbol (H or HH) should be present in {0}.", value)); 649 } else if (!hasMinuteSymbol) { 650 result.add(createErrorCheckStatus().setMessage("The minute symbol (mm) should be present in {0}.", value)); 651 } 652 } 653 654 /** 655 * Convenience method for creating errors. 656 * 657 * @return 658 */ createErrorCheckStatus()659 private CheckStatus createErrorCheckStatus() { 660 return new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 661 .setSubtype(Subtype.illegalDatePattern); 662 } 663 skipPath(String path)664 public boolean skipPath(String path) { 665 return path.contains("arrow") 666 || path.contains("/availableFormats") 667 || path.contains("/interval") 668 || path.contains("/dateTimeFormat") 669 // || path.contains("/dayPeriod[") 670 // && !path.endsWith("=\"pm\"]") 671 // && !path.endsWith("=\"am\"]") 672 ; 673 } 674 getLastType(String path)675 public String getLastType(String path) { 676 int secondType = path.lastIndexOf("[@type=\""); 677 if (secondType < 0) { 678 return null; 679 } 680 secondType += 8; 681 int secondEnd = path.indexOf("\"]", secondType); 682 if (secondEnd < 0) { 683 return null; 684 } 685 return path.substring(secondType, secondEnd); 686 } 687 getMainType(String path)688 public String getMainType(String path) { 689 int secondType = path.indexOf("\"]/"); 690 if (secondType < 0) { 691 return null; 692 } 693 secondType += 3; 694 int secondEnd = path.indexOf("/", secondType); 695 if (secondEnd < 0) { 696 return null; 697 } 698 return path.substring(secondType, secondEnd); 699 } 700 getValues(CLDRFile resolvedCldrFileToCheck, Collection<String> values)701 private String getValues(CLDRFile resolvedCldrFileToCheck, Collection<String> values) { 702 Set<String> results = new TreeSet<>(); 703 for (String path : values) { 704 final String stringValue = resolvedCldrFileToCheck.getStringValue(path); 705 if (stringValue != null) { 706 results.add(stringValue); 707 } 708 } 709 return "{" + Joiner.on("},{").join(results) + "}"; 710 } 711 712 static final Pattern HACK_CONFLICTING = PatternCache.get("Conflicting fields:\\s+M+,\\s+l"); 713 714 @Override handleGetExamples(String path, String fullPath, String value, Options options, List<CheckStatus> result)715 public CheckCLDR handleGetExamples(String path, String fullPath, String value, Options options, List<CheckStatus> result) { 716 if (path.indexOf("/dates") < 0 || path.indexOf("gregorian") < 0) return this; 717 try { 718 if (path.indexOf("/pattern") >= 0 && path.indexOf("/dateTimeFormat") < 0 719 || path.indexOf("/dateFormatItem") >= 0) { 720 checkPattern2(path, value, result); 721 } 722 } catch (Exception e) { 723 // don't worry about errors 724 } 725 return this; 726 } 727 728 static final SimpleDateFormat neutralFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", ULocale.ENGLISH); 729 static { 730 neutralFormat.setTimeZone(ExampleGenerator.ZONE_SAMPLE); 731 } 732 733 // Get Date-Time in milliseconds getDateTimeinMillis(int year, int month, int date, int hourOfDay, int minute, int second)734 private static long getDateTimeinMillis(int year, int month, int date, int hourOfDay, int minute, int second) { 735 Calendar cal = Calendar.getInstance(); 736 cal.set(year, month, date, hourOfDay, minute, second); 737 return cal.getTimeInMillis(); 738 } 739 740 static long date1950 = getDateTimeinMillis(1950, 0, 1, 0, 0, 0); 741 static long date2010 = getDateTimeinMillis(2010, 0, 1, 0, 0, 0); 742 static long date4004BC = getDateTimeinMillis(-4004, 9, 23, 2, 0, 0); 743 static Random random = new Random(0); 744 checkPattern(DateTimePatternType dateTypePatternType, String path, String fullPath, String value, List<CheckStatus> result)745 private void checkPattern(DateTimePatternType dateTypePatternType, String path, String fullPath, String value, List<CheckStatus> result) 746 throws ParseException { 747 String skeleton = dateTimePatternGenerator.getSkeletonAllowingDuplicates(value); 748 String skeletonCanonical = dateTimePatternGenerator.getCanonicalSkeletonAllowingDuplicates(value); 749 750 if (value.contains("MMM.") || value.contains("LLL.") || value.contains("E.") || value.contains("eee.") 751 || value.contains("ccc.") || value.contains("QQQ.") || value.contains("qqq.")) { 752 result 753 .add(new CheckStatus() 754 .setCause(this) 755 .setMainType(CheckStatus.warningType) 756 .setSubtype(Subtype.incorrectDatePattern) 757 .setMessage( 758 "Your pattern ({0}) is probably incorrect; abbreviated month/weekday/quarter names that need a period should include it in the name, rather than adding it to the pattern.", 759 value)); 760 } 761 XPathParts pathParts = XPathParts.getFrozenInstance(path); 762 String calendar = pathParts.findAttributeValue("calendar", "type"); 763 String id; 764 switch (dateTypePatternType) { 765 case AVAILABLE: 766 id = pathParts.getAttributeValue(-1, "id"); 767 break; 768 case INTERVAL: 769 id = pathParts.getAttributeValue(-2, "id"); 770 break; 771 case STOCK: 772 id = pathParts.getAttributeValue(-3, "type"); 773 break; 774 default: 775 throw new IllegalArgumentException(); 776 } 777 778 if (dateTypePatternType == DateTimePatternType.AVAILABLE || dateTypePatternType == DateTimePatternType.INTERVAL) { 779 String idCanonical = dateTimePatternGenerator.getCanonicalSkeletonAllowingDuplicates(id); 780 if (skeleton.isEmpty()) { 781 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 782 .setSubtype(Subtype.incorrectDatePattern) 783 // "Internal ID ({0}) doesn't match generated ID ({1}) for pattern ({2}). " + 784 .setMessage("Your pattern ({1}) is incorrect for ID ({0}). " + 785 "You need to supply a pattern according to http://cldr.org/translation/date-time-patterns.", 786 id, value)); 787 } else if (!dateTimePatternGenerator.skeletonsAreSimilar(idCanonical, skeletonCanonical)) { 788 String fixedValue = dateTimePatternGenerator.replaceFieldTypes(value, id); 789 result 790 .add(new CheckStatus() 791 .setCause(this) 792 .setMainType(CheckStatus.errorType) 793 .setSubtype(Subtype.incorrectDatePattern) 794 // "Internal ID ({0}) doesn't match generated ID ({1}) for pattern ({2}). " + 795 .setMessage( 796 "Your pattern ({2}) doesn't correspond to what is asked for. Yours would be right for an ID ({1}) but not for the ID ({0}). " 797 + 798 "Please change your pattern to match what was asked, such as ({3}), with the right punctuation and/or ordering for your language. See http://cldr.org/translation/date-time-patterns.", 799 id, skeletonCanonical, value, fixedValue)); 800 } 801 if (dateTypePatternType == DateTimePatternType.AVAILABLE) { 802 // index y+w+ must correpond to pattern containing only Y+ and w+ 803 if (idCanonical.matches("y+w+") && !(skeleton.matches("Y+w+") || skeleton.matches("w+Y+"))) { 804 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType).setSubtype(Subtype.incorrectDatePattern) 805 .setMessage("For id {0}, the pattern ({1}) must contain fields Y and w, and no others.", id, value)); 806 } 807 // index M+W msut correspond to pattern containing only M+/L+ and W 808 if (idCanonical.matches("M+W") && !(skeletonCanonical.matches("M+W") || skeletonCanonical.matches("WM+"))) { 809 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType).setSubtype(Subtype.incorrectDatePattern) 810 .setMessage("For id {0}, the pattern ({1}) must contain fields M or L, plus W, and no others.", id, value)); 811 } 812 } 813 String failureMessage = (String) flexInfo.getFailurePath(path); 814 if (failureMessage != null) { 815 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 816 .setSubtype(Subtype.illegalDatePattern) 817 .setMessage("{0}", new Object[] { failureMessage })); 818 } 819 } 820 if (dateTypePatternType == DateTimePatternType.STOCK) { 821 int style = 0; 822 String len = pathParts.findAttributeValue("timeFormatLength", "type"); 823 DateOrTime dateOrTime = DateOrTime.time; 824 if (len == null) { 825 dateOrTime = DateOrTime.date; 826 style += 4; 827 len = pathParts.findAttributeValue("dateFormatLength", "type"); 828 if (len == null) { 829 len = pathParts.findAttributeValue("dateTimeFormatLength", "type"); 830 dateOrTime = DateOrTime.dateTime; 831 } 832 } 833 834 DateTimeLengths dateTimeLength = DateTimeLengths.valueOf(len.toUpperCase(Locale.ENGLISH)); 835 836 if (calendar.equals("gregorian") && !"root".equals(getCldrFileToCheck().getLocaleID())) { 837 checkValue(dateTimeLength, dateOrTime, value, result); 838 } 839 if (dateOrTime == DateOrTime.dateTime) { 840 return; // We don't need to do the rest for date/time combo patterns. 841 } 842 style += dateTimeLength.ordinal(); 843 // do regex match with skeletonCanonical but report errors using skeleton; they have corresponding field lengths 844 if (!dateTimePatterns[style].matcher(skeletonCanonical).matches() 845 && !calendar.equals("chinese") 846 && !calendar.equals("hebrew")) { 847 int i = RegexUtilities.findMismatch(dateTimePatterns[style], skeletonCanonical); 848 String skeletonPosition = skeleton.substring(0, i) + "☹" + skeleton.substring(i); 849 result.add(new CheckStatus() 850 .setCause(this) 851 .setMainType(CheckStatus.errorType) 852 .setSubtype(Subtype.missingOrExtraDateField) 853 .setMessage("Field is missing, extra, or the wrong length. Expected {0} [Internal: {1} / {2}]", 854 new Object[] { dateTimeMessage[style], skeletonPosition, dateTimePatterns[style].pattern() })); 855 } 856 } else if (dateTypePatternType == DateTimePatternType.INTERVAL) { 857 if (id.contains("y")) { 858 String greatestDifference = pathParts.findAttributeValue("greatestDifference", "id"); 859 int requiredYearFieldCount = 1; 860 if ("y".equals(greatestDifference)) { 861 requiredYearFieldCount = 2; 862 } 863 int yearFieldCount = 0; 864 Matcher yearFieldMatcher = YEAR_FIELDS.matcher(value); 865 while (yearFieldMatcher.find()) { 866 yearFieldCount++; 867 } 868 if (yearFieldCount < requiredYearFieldCount) { 869 result.add(new CheckStatus() 870 .setCause(this) 871 .setMainType(CheckStatus.errorType) 872 .setSubtype(Subtype.missingOrExtraDateField) 873 .setMessage("Not enough year fields in interval pattern. Must have {0} but only found {1}", 874 new Object[] { requiredYearFieldCount, yearFieldCount })); 875 } 876 } 877 } 878 879 if (value.contains("G") && calendar.equals("gregorian")) { 880 GyState actual = GyState.forPattern(value); 881 GyState expected = getExpectedGy(getCldrFileToCheck().getLocaleID()); 882 if (actual != expected) { 883 result.add(new CheckStatus() 884 .setCause(this) 885 .setMainType(CheckStatus.warningType) 886 .setSubtype(Subtype.unexpectedOrderOfEraYear) 887 .setMessage("Unexpected order of era/year. Expected {0}, but got {1} in 〈{2}〉 for {3}/{4}", 888 expected, actual, value, calendar, id)); 889 } 890 } 891 } 892 893 enum DateOrTime { 894 date, time, dateTime 895 } 896 897 static final Map<DateOrTime, Relation<DateTimeLengths, String>> STOCK_PATTERNS = new EnumMap<>( 898 DateOrTime.class); 899 900 // add(Map<DateOrTime, Relation<DateTimeLengths, String>> stockPatterns, DateOrTime dateOrTime, DateTimeLengths dateTimeLength, String... keys)901 private static void add(Map<DateOrTime, Relation<DateTimeLengths, String>> stockPatterns, 902 DateOrTime dateOrTime, DateTimeLengths dateTimeLength, String... keys) { 903 Relation<DateTimeLengths, String> rel = STOCK_PATTERNS.get(dateOrTime); 904 if (rel == null) { 905 STOCK_PATTERNS.put(dateOrTime, rel = Relation.of(new EnumMap<DateTimeLengths, Set<String>>(DateTimeLengths.class), LinkedHashSet.class)); 906 } 907 rel.putAll(dateTimeLength, Arrays.asList(keys)); 908 } 909 910 /* Ticket #4936 911 value(short time) = value(hm) or value(Hm) 912 value(medium time) = value(hms) or value(Hms) 913 value(long time) = value(medium time+z) 914 value(full time) = value(medium time+zzzz) 915 */ 916 static { add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.SHORT, "hm", "Hm")917 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.SHORT, "hm", "Hm"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.MEDIUM, "hms", "Hms")918 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.MEDIUM, "hms", "Hms"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.LONG, "hms*z", "Hms*z")919 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.LONG, "hms*z", "Hms*z"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.FULL, "hms*zzzz", "Hms*zzzz")920 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.FULL, "hms*zzzz", "Hms*zzzz"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.SHORT, "yMd")921 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.SHORT, "yMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.MEDIUM, "yMMMd")922 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.MEDIUM, "yMMMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.LONG, "yMMMMd", "yMMMd")923 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.LONG, "yMMMMd", "yMMMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.FULL, "yMMMMEd", "yMMMEd")924 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.FULL, "yMMMMEd", "yMMMEd"); 925 } 926 927 static final String AVAILABLE_PREFIX = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\""; 928 static final String AVAILABLE_SUFFIX = "\"]"; 929 static final String APPEND_TIMEZONE = "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/appendItems/appendItem[@request=\"Timezone\"]"; 930 checkValue(DateTimeLengths dateTimeLength, DateOrTime dateOrTime, String value, List<CheckStatus> result)931 private void checkValue(DateTimeLengths dateTimeLength, DateOrTime dateOrTime, String value, List<CheckStatus> result) { 932 // Check consistency of the pattern vs. supplemental wrt 12 vs. 24 hour clock. 933 if (dateOrTime == DateOrTime.time) { 934 PreferredAndAllowedHour pref = sdi.getTimeData().get(territory); 935 if (pref == null) { 936 pref = sdi.getTimeData().get("001"); 937 } 938 String checkForHour, clockType; 939 if (pref.preferred.equals(PreferredAndAllowedHour.HourStyle.h)) { 940 checkForHour = "h"; 941 clockType = "12"; 942 } else { 943 checkForHour = "H"; 944 clockType = "24"; 945 } 946 if (!value.contains(checkForHour)) { 947 CheckStatus.Type errType = CheckStatus.errorType; 948 // French/Canada is strange, they use 24 hr clock while en_CA uses 12. 949 if (language.equals("fr") && territory.equals("CA")) { 950 errType = CheckStatus.warningType; 951 } 952 953 result.add(new CheckStatus().setCause(this).setMainType(errType) 954 .setSubtype(Subtype.inconsistentTimePattern) 955 .setMessage("Time format inconsistent with supplemental time data for territory \"" + territory + "\"." 956 + " Use '" + checkForHour + "' for " + clockType + " hour clock.")); 957 } 958 } 959 if (dateOrTime == DateOrTime.dateTime) { 960 boolean inQuotes = false; 961 for (int i = 0; i < value.length(); i++) { 962 char ch = value.charAt(i); 963 if (ch == '\'') { 964 inQuotes = !inQuotes; 965 } 966 if (!inQuotes && (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')) { 967 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.errorType) 968 .setSubtype(Subtype.patternContainsInvalidCharacters) 969 .setMessage("Unquoted letter \"{0}\" in dateTime format.", ch)); 970 } 971 } 972 } else { 973 Set<String> keys = STOCK_PATTERNS.get(dateOrTime).get(dateTimeLength); 974 StringBuilder b = new StringBuilder(); 975 boolean onlyNulls = true; 976 int countMismatches = 0; 977 boolean errorOnMissing = false; 978 String timezonePattern = null; 979 Set<String> bases = new LinkedHashSet<>(); 980 for (String key : keys) { 981 int star = key.indexOf('*'); 982 boolean hasStar = star >= 0; 983 String base = !hasStar ? key : key.substring(0, star); 984 bases.add(base); 985 String xpath = AVAILABLE_PREFIX + base + AVAILABLE_SUFFIX; 986 String value1 = getCldrFileToCheck().getStringValue(xpath); 987 // String localeFound = getCldrFileToCheck().getSourceLocaleID(xpath, null); && !localeFound.equals("root") && !localeFound.equals("code-fallback") 988 if (value1 != null) { 989 onlyNulls = false; 990 if (hasStar) { 991 String zone = key.substring(star + 1); 992 timezonePattern = getResolvedCldrFileToCheck().getStringValue(APPEND_TIMEZONE); 993 value1 = MessageFormat.format(timezonePattern, value1, zone); 994 } 995 if (equalsExceptWidth(value, value1)) { 996 return; 997 } 998 } else { 999 // Example, if the requiredLevel for the locale is moderate, 1000 // and the level for the path is modern, then we'll skip the error, 1001 // but if the level for the path is basic, then we won't 1002 Level pathLevel = coverageLevel.getLevel(xpath); 1003 if (requiredLevel.compareTo(pathLevel) >= 0) { 1004 errorOnMissing = true; 1005 } 1006 } 1007 add(b, base, value1); 1008 countMismatches++; 1009 } 1010 if (!onlyNulls) { 1011 if (timezonePattern != null) { 1012 b.append(" (with appendZonePattern: “" + timezonePattern + "”)"); 1013 } 1014 String msg = countMismatches != 1 1015 ? "{1}-{0} → “{2}” didn't match any of the corresponding flexible skeletons: [{3}]. This or the flexible patterns needs to be changed." 1016 : "{1}-{0} → “{2}” didn't match the corresponding flexible skeleton: {3}. This or the flexible pattern needs to be changed."; 1017 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.warningType) 1018 .setSubtype(Subtype.inconsistentDatePattern) 1019 .setMessage(msg, 1020 dateTimeLength, dateOrTime, value, b)); 1021 } else { 1022 if (errorOnMissing) { 1023 String msg = countMismatches != 1 1024 ? "{1}-{0} → “{2}” doesn't have at least one value for a corresponding flexible skeleton {3}, which needs to be added." 1025 : "{1}-{0} → “{2}” doesn't have a value for the corresponding flexible skeleton {3}, which needs to be added."; 1026 result.add(new CheckStatus().setCause(this).setMainType(CheckStatus.warningType) 1027 .setSubtype(Subtype.missingDatePattern) 1028 .setMessage(msg, 1029 dateTimeLength, dateOrTime, value, Joiner.on(", ").join(bases))); 1030 } 1031 } 1032 } 1033 } 1034 add(StringBuilder b, String key, String value1)1035 private void add(StringBuilder b, String key, String value1) { 1036 if (value1 == null) { 1037 return; 1038 } 1039 if (b.length() != 0) { 1040 b.append(" or "); 1041 } 1042 b.append(key + (value1 == null ? " - missing" : " → “" + value1 + "”")); 1043 } 1044 equalsExceptWidth(String value1, String value2)1045 private boolean equalsExceptWidth(String value1, String value2) { 1046 if (value1.equals(value2)) { 1047 return true; 1048 } else if (value2 == null) { 1049 return false; 1050 } 1051 1052 List<Object> items1 = new ArrayList<>(formatParser.set(value1).getItems()); // clone 1053 List<Object> items2 = formatParser.set(value2).getItems(); 1054 if (items1.size() != items2.size()) { 1055 return false; 1056 } 1057 Iterator<Object> it2 = items2.iterator(); 1058 for (Object item1 : items1) { 1059 Object item2 = it2.next(); 1060 if (item1.equals(item2)) { 1061 continue; 1062 } 1063 if (item1 instanceof VariableField && item2 instanceof VariableField) { 1064 // simple test for now, ignore widths 1065 if (item1.toString().charAt(0) == item2.toString().charAt(0)) { 1066 continue; 1067 } 1068 } 1069 return false; 1070 } 1071 return true; 1072 } 1073 1074 static final Set<String> YgLanguages = new HashSet<>(Arrays.asList( 1075 "ar", "cs", "da", "de", "en", "es", "fa", "fi", "fr", "he", "hr", "id", "it", "nl", "no", "pt", "ru", "sv", "tr")); 1076 getExpectedGy(String localeID)1077 private GyState getExpectedGy(String localeID) { 1078 // hack for now 1079 int firstBar = localeID.indexOf('_'); 1080 String lang = firstBar < 0 ? localeID : localeID.substring(0, firstBar); 1081 return YgLanguages.contains(lang) ? GyState.YEAR_ERA : GyState.ERA_YEAR; 1082 } 1083 1084 enum GyState { 1085 YEAR_ERA, ERA_YEAR, OTHER; 1086 static DateTimePatternGenerator.FormatParser formatParser = new DateTimePatternGenerator.FormatParser(); 1087 1088 static synchronized GyState forPattern(String value) { 1089 formatParser.set(value); 1090 int last = -1; 1091 for (Object x : formatParser.getItems()) { 1092 if (x instanceof VariableField) { 1093 int type = ((VariableField) x).getType(); 1094 if (type == DateTimePatternGenerator.ERA && last == DateTimePatternGenerator.YEAR) { 1095 return GyState.YEAR_ERA; 1096 } else if (type == DateTimePatternGenerator.YEAR && last == DateTimePatternGenerator.ERA) { 1097 return GyState.ERA_YEAR; 1098 } 1099 last = type; 1100 } 1101 } 1102 return GyState.OTHER; 1103 } 1104 } 1105 1106 enum DateTimeLengths { 1107 SHORT, MEDIUM, LONG, FULL 1108 } 1109 1110 // The patterns below should only use the *canonical* characters for each field type: 1111 // y (not Y, u, U) 1112 // Q (not q) 1113 // M (not L) 1114 // E (not e, c) 1115 // a (not b, B) 1116 // H or h (not k or K) 1117 // v (not z, Z, V) 1118 static final Pattern[] dateTimePatterns = { 1119 PatternCache.get("a*(h|hh|H|HH)(m|mm)"), // time-short 1120 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)"), // time-medium 1121 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)(v+)"), // time-long 1122 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)(v+)"), // time-full 1123 PatternCache.get("G*y{1,4}M{1,2}(d|dd)"), // date-short; allow yyy for Minguo/ROC calendar 1124 PatternCache.get("G*y(yyy)?M{1,3}(d|dd)"), // date-medium 1125 PatternCache.get("G*y(yyy)?M{1,4}(d|dd)"), // date-long 1126 PatternCache.get("G*y(yyy)?M{1,4}E*(d|dd)"), // date-full 1127 PatternCache.get(".*"), // datetime-short 1128 PatternCache.get(".*"), // datetime-medium 1129 PatternCache.get(".*"), // datetime-long 1130 PatternCache.get(".*"), // datetime-full 1131 }; 1132 1133 static final String[] dateTimeMessage = { 1134 "hours (H, HH, h, or hh), and minutes (m or mm)", // time-short 1135 "hours (H, HH, h, or hh), minutes (m or mm), and seconds (s or ss)", // time-medium 1136 "hours (H, HH, h, or hh), minutes (m or mm), and seconds (s or ss); optionally timezone (z, zzzz, v, vvvv)", // time-long 1137 "hours (H, HH, h, or hh), minutes (m or mm), seconds (s or ss), and timezone (z, zzzz, v, vvvv)", // time-full 1138 "year (y, yy, yyyy), month (M or MM), and day (d or dd); optionally era (G)", // date-short 1139 "year (y), month (M, MM, or MMM), and day (d or dd); optionally era (G)", // date-medium 1140 "year (y), month (M, ... MMMM), and day (d or dd); optionally era (G)", // date-long 1141 "year (y), month (M, ... MMMM), and day (d or dd); optionally day of week (EEEE or cccc) or era (G)", // date-full 1142 }; 1143 1144 public String toString(DateTimePatternGenerator.FormatParser formatParser) { 1145 StringBuffer result = new StringBuffer(); 1146 for (Object x : formatParser.getItems()) { 1147 if (x instanceof DateTimePatternGenerator.VariableField) { 1148 result.append(x.toString()); 1149 } else { 1150 result.append(formatParser.quoteLiteral(x.toString())); 1151 } 1152 } 1153 return result.toString(); 1154 } 1155 1156 private void checkPattern2(String path, String value, List<CheckStatus> result) throws ParseException { 1157 XPathParts pathParts = XPathParts.getFrozenInstance(path); 1158 String calendar = pathParts.findAttributeValue("calendar", "type"); 1159 SimpleDateFormat x = icuServiceBuilder.getDateFormat(calendar, value); 1160 x.setTimeZone(ExampleGenerator.ZONE_SAMPLE); 1161 result.add(new MyCheckStatus() 1162 .setFormat(x) 1163 .setCause(this).setMainType(CheckStatus.demoType)); 1164 } 1165 1166 private DateTimePatternGenerator getDTPGForCalendarType(String calendarType) { 1167 DateTimePatternGenerator dtpg = dtpgForType.get(calendarType); 1168 if (dtpg == null) { 1169 dtpg = flexInfo.getDTPGForCalendarType(calendarType, parentCLDRFiles); 1170 dtpgForType.put(calendarType, dtpg); 1171 } 1172 return dtpg; 1173 } 1174 1175 static final UnicodeSet XGRAPHEME = new UnicodeSet("[[:mark:][:grapheme_extend:][:punctuation:]]"); 1176 static final UnicodeSet DIGIT = new UnicodeSet("[:decimal_number:]"); 1177 1178 static public class MyCheckStatus extends CheckStatus { 1179 private SimpleDateFormat df; 1180 1181 public MyCheckStatus setFormat(SimpleDateFormat df) { 1182 this.df = df; 1183 return this; 1184 } 1185 1186 @Override 1187 public SimpleDemo getDemo() { 1188 return new MyDemo().setFormat(df); 1189 } 1190 } 1191 1192 static class MyDemo extends FormatDemo { 1193 private SimpleDateFormat df; 1194 1195 @Override 1196 protected String getPattern() { 1197 return df.toPattern(); 1198 } 1199 1200 @Override 1201 protected String getSampleInput() { 1202 return neutralFormat.format(ExampleGenerator.DATE_SAMPLE); 1203 } 1204 1205 public MyDemo setFormat(SimpleDateFormat df) { 1206 this.df = df; 1207 return this; 1208 } 1209 1210 @Override 1211 protected void getArguments(Map<String, String> inout) { 1212 currentPattern = currentInput = currentFormatted = currentReparsed = "?"; 1213 Date d; 1214 try { 1215 currentPattern = inout.get("pattern"); 1216 if (currentPattern != null) 1217 df.applyPattern(currentPattern); 1218 else 1219 currentPattern = getPattern(); 1220 } catch (Exception e) { 1221 currentPattern = "Use format like: ##,###.##"; 1222 return; 1223 } 1224 try { 1225 currentInput = inout.get("input"); 1226 if (currentInput == null) { 1227 currentInput = getSampleInput(); 1228 } 1229 d = neutralFormat.parse(currentInput); 1230 } catch (Exception e) { 1231 currentInput = "Use neutral format like: 1993-11-31 13:49:02"; 1232 return; 1233 } 1234 try { 1235 currentFormatted = df.format(d); 1236 } catch (Exception e) { 1237 currentFormatted = "Can't format: " + e.getMessage(); 1238 return; 1239 } 1240 try { 1241 parsePosition.setIndex(0); 1242 Date n = df.parse(currentFormatted, parsePosition); 1243 if (parsePosition.getIndex() != currentFormatted.length()) { 1244 currentReparsed = "Couldn't parse past: " + "\u200E" 1245 + currentFormatted.substring(0, parsePosition.getIndex()) + "\u200E"; 1246 } else { 1247 currentReparsed = neutralFormat.format(n); 1248 } 1249 } catch (Exception e) { 1250 currentReparsed = "Can't parse: " + e.getMessage(); 1251 } 1252 } 1253 1254 } 1255 } 1256