1 package org.unicode.cldr.test; 2 3 import com.google.common.cache.Cache; 4 import com.google.common.cache.CacheBuilder; 5 import com.google.common.cache.CacheLoader; 6 import com.google.common.cache.LoadingCache; 7 import java.util.ArrayList; 8 import java.util.List; 9 import java.util.Map.Entry; 10 import java.util.concurrent.ConcurrentHashMap; 11 import java.util.concurrent.ExecutionException; 12 import java.util.logging.Level; 13 import java.util.logging.Logger; 14 import org.unicode.cldr.test.CheckCLDR.CheckStatus; 15 import org.unicode.cldr.test.CheckCLDR.Options; 16 import org.unicode.cldr.util.CLDRConfig; 17 import org.unicode.cldr.util.CLDRFile; 18 import org.unicode.cldr.util.CLDRLocale; 19 import org.unicode.cldr.util.CLDRLocale.SublocaleProvider; 20 import org.unicode.cldr.util.Factory; 21 import org.unicode.cldr.util.Pair; 22 import org.unicode.cldr.util.XMLSource; 23 24 /** 25 * Caches tests and examples Call XMLSource.addListener() on the instance to notify it of changes to 26 * the XMLSource. 27 * 28 * @author srl 29 * @see XMLSource#addListener(org.unicode.cldr.util.XMLSource.Listener) 30 */ 31 public class TestCache implements XMLSource.Listener { 32 private static final Logger logger = Logger.getLogger(TestCache.class.getSimpleName()); 33 34 public class TestResultBundle { 35 private final CheckCLDR cc = CheckCLDR.getCheckAll(getFactory(), nameMatcher); 36 final CLDRFile file; 37 private final CheckCLDR.Options options; 38 private final ConcurrentHashMap<Pair<String, String>, List<CheckStatus>> pathCache; 39 protected final List<CheckStatus> possibleProblems = new ArrayList<>(); 40 TestResultBundle(CheckCLDR.Options cldrOptions)41 protected TestResultBundle(CheckCLDR.Options cldrOptions) { 42 options = cldrOptions; 43 pathCache = new ConcurrentHashMap<>(); 44 file = getFactory().make(options.getLocale().getBaseName(), true); 45 synchronized (cc) { 46 cc.setCldrFileToCheck(file, options, possibleProblems); 47 } 48 } 49 50 /** 51 * Check the given value for the given path, using this TestResultBundle for options, 52 * pathCache and cc (CheckCLDR). 53 * 54 * @param path the path 55 * @param result the list to which CheckStatus objects may be added; this function clears 56 * any objects that might already be in it 57 * @param value the value to be checked 58 */ check(String path, List<CheckStatus> result, String value)59 public void check(String path, List<CheckStatus> result, String value) { 60 /* 61 * result.clear() is needed to avoid phantom warnings in the Info Panel, if we're called 62 * with non-empty result (leftover from another row) and we get cachedResult != null. 63 * cc.check() also calls result.clear() (at least as of 2018-11-20) so in that case it's 64 * currently redundant here. Clear it here unconditionally to be sure. 65 */ 66 result.clear(); 67 Pair<String, String> key = new Pair<>(path, value); 68 List<CheckStatus> cachedResult = 69 pathCache.computeIfAbsent( 70 key, 71 (Pair<String, String> k) -> { 72 List<CheckStatus> l = new ArrayList<CheckStatus>(); 73 synchronized (cc) { 74 cc.check( 75 k.getFirst(), 76 file.getFullXPath(k.getFirst()), 77 k.getSecond(), 78 options, 79 l); 80 } 81 return l; 82 }); 83 if (cachedResult != null) { 84 result.addAll(cachedResult); 85 } 86 } 87 getExamples(String path, String value, List<CheckStatus> result)88 public void getExamples(String path, String value, List<CheckStatus> result) { 89 synchronized (cc) { 90 cc.getExamples(path, file.getFullXPath(path), value, options, result); 91 } 92 } 93 getPossibleProblems()94 public List<CheckStatus> getPossibleProblems() { 95 return possibleProblems; 96 } 97 } 98 99 private static final boolean DEBUG = false; 100 101 /* 102 * TODO: document whether CLDR_TESTCACHE_SIZE is set on production server, and if so to what, and why; 103 * evaluate why the fallback 12 for CLDR_TESTCACHE_SIZE is appropriate or too small. Consider not 104 * using maximumSize() at all, depending on softValues() instead to garbage collect only when needed. 105 */ 106 private LoadingCache<CheckCLDR.Options, TestResultBundle> testResultCache = 107 CacheBuilder.newBuilder() 108 .maximumSize(CLDRConfig.getInstance().getProperty("CLDR_TESTCACHE_SIZE", 12)) 109 .softValues() 110 .build( 111 new CacheLoader<CheckCLDR.Options, TestResultBundle>() { 112 113 @Override 114 public TestResultBundle load(Options key) throws Exception { 115 return new TestResultBundle(key); 116 } 117 }); 118 119 private final Factory factory; 120 121 private String nameMatcher = ".*"; 122 123 /** Get the bundle for this test */ getBundle(final CheckCLDR.Options options)124 public TestResultBundle getBundle(final CheckCLDR.Options options) { 125 TestResultBundle b; 126 try { 127 b = testResultCache.get(options); 128 } catch (ExecutionException e) { 129 logger.log(Level.SEVERE, e, () -> "Failed to load " + options); 130 throw new RuntimeException(e); 131 } 132 return b; 133 } 134 getFactory()135 protected Factory getFactory() { 136 return factory; 137 } 138 139 /** construct a new TestCache with this factory. Intended for use from within Factory. */ TestCache(Factory f)140 public TestCache(Factory f) { 141 this.factory = f; 142 logger.fine(() -> toString() + " - init(" + f + ")"); 143 } 144 145 /** Change which checks are run. Invalidates all caches. */ setNameMatcher(String nameMatcher)146 public void setNameMatcher(String nameMatcher) { 147 logger.finest(() -> toString() + " - setNameMatcher(" + nameMatcher + ")"); 148 this.nameMatcher = nameMatcher; 149 invalidateAllCached(); 150 } 151 152 /** 153 * Convert this TestCache to a string 154 * 155 * <p>Used only for debugging? 156 */ 157 @Override toString()158 public String toString() { 159 StringBuilder stats = new StringBuilder(); 160 stats.append( 161 "{" 162 + this.getClass().getSimpleName() 163 + super.toString() 164 + " F=" 165 + factory.getClass().getSimpleName() 166 + " Size: " 167 + testResultCache.size() 168 + " ("); 169 int good = 0; 170 int total = 0; 171 for (Entry<Options, TestResultBundle> k : testResultCache.asMap().entrySet()) { 172 Options key = k.getKey(); 173 TestResultBundle bundle = k.getValue(); 174 if (bundle != null) { 175 good++; 176 } 177 if (DEBUG) { 178 stats.append("," + k.getKey() + "=" + key); 179 } 180 total++; 181 } 182 stats.append(" " + good + "/" + total + "}"); 183 return stats.toString(); 184 } 185 186 /** 187 * Update the caches as needed, given that the value has changed for this xpath and source. 188 * 189 * @param xpath the xpath 190 * @param source the XMLSource 191 */ 192 @Override valueChanged(String xpath, XMLSource source)193 public void valueChanged(String xpath, XMLSource source) { 194 CLDRLocale locale = CLDRLocale.getInstance(source.getLocaleID()); 195 valueChangedInvalidateRecursively(xpath, locale); 196 } 197 198 /** 199 * Update the caches as needed, given that the value has changed for this xpath and locale. 200 * 201 * <p>Called by valueChanged(String xpath, XMLSource source), and also calls itself recursively 202 * for sublocales 203 * 204 * @param xpath the xpath 205 * @param locale the CLDRLocale 206 */ valueChangedInvalidateRecursively(String xpath, final CLDRLocale locale)207 private void valueChangedInvalidateRecursively(String xpath, final CLDRLocale locale) { 208 logger.finer(() -> "BundDelLoc " + locale + " @ " + xpath); 209 /* 210 * Call self recursively for all sub-locales 211 */ 212 for (CLDRLocale sub : ((SublocaleProvider) getFactory()).subLocalesOf(locale)) { 213 valueChangedInvalidateRecursively(xpath, sub); 214 } 215 /* 216 * Update caching for TestResultBundle 217 */ 218 updateTestResultCache(xpath, locale); 219 /* 220 * Update caching for ExampleGenerator 221 */ 222 updateExampleGeneratorCache(xpath, locale); 223 } 224 225 /** 226 * Update the cache of TestResultBundle objects, per valueChanged 227 * 228 * @param xpath the xpath whose value has changed 229 * @param locale the CLDRLocale 230 * <p>Called by valueChangedInvalidateRecursively 231 */ updateTestResultCache( @uppressWarnings"unused") String xpath, CLDRLocale locale)232 private void updateTestResultCache( 233 @SuppressWarnings("unused") String xpath, CLDRLocale locale) { 234 if (!testResultCache.asMap().isEmpty()) { 235 // Filter the testResultCache to only remove the items where the locale matches 236 List<Options> toRemove = new ArrayList<>(); 237 for (Options k : testResultCache.asMap().keySet()) { 238 if (k.getLocale().equals(locale)) { 239 toRemove.add(k); 240 } 241 } 242 if (!DEBUG) { 243 // no logging is done, simply invalidate all items 244 testResultCache.invalidateAll(toRemove); 245 } else { 246 // avoid concurrent remove 247 for (CheckCLDR.Options k : toRemove) { 248 testResultCache.invalidate(k); 249 System.err.println("BundDel " + k); 250 } 251 } 252 } 253 } 254 255 /** 256 * Per-locale testResultCache of ExampleGenerator objects 257 * 258 * <p>Re-use the TestCache implementation of XMLSource.Listener for ExampleGenerator objects in 259 * addition to TestResultBundle objects. The actual caches are distinct, only the Listener 260 * interface is shared. 261 * 262 * <p>ExampleGenerator objects are for generating examples, rather than for checking validity, 263 * unlike other TestCache-related objects such as TestResultBundle. Still, ExampleGenerator has 264 * similar dependence on locales, paths, and values, and needs similar treatment for caching and 265 * performance. ExampleGenerator is in the same package ("test") as TestCache. 266 * 267 * <p>There are currently unused (?) files Registerable.java and LocaleChangeRegistry.java that 268 * appear to have been intended for a similar purpose. They are in the web package. 269 * 270 * <p>Reference: https://unicode-org.atlassian.net/browse/CLDR-12020 271 */ 272 private static Cache<String, ExampleGenerator> exampleGeneratorCache = 273 CacheBuilder.newBuilder().softValues().build(); 274 275 /** 276 * Get an ExampleGenerator for the given locale, etc. 277 * 278 * <p>Use a cache for performance. 279 * 280 * @param locale the CLDRLocale 281 * @param ourSrc the CLDRFile for the locale 282 * @param translationHintsFile the CLDRFile for translation hints (English) 283 * @return the ExampleGenerator 284 * <p>Called by DataPage.make for use in SurveyTool. 285 * <p>Note: other objects also have functions named "getExampleGenerator": 286 * org.unicode.cldr.unittest.TestExampleGenerator.getExampleGenerator(String) 287 * org.unicode.cldr.test.ConsoleCheckCLDR.getExampleGenerator() 288 */ getExampleGenerator( CLDRLocale locale, CLDRFile ourSrc, CLDRFile translationHintsFile)289 public static ExampleGenerator getExampleGenerator( 290 CLDRLocale locale, CLDRFile ourSrc, CLDRFile translationHintsFile) { 291 boolean egCacheIsEnabled = true; 292 if (!egCacheIsEnabled) { 293 return new ExampleGenerator(ourSrc, translationHintsFile); 294 } 295 /* 296 * TODO: consider get(locString, Callable) instead of getIfPresent and put. 297 */ 298 String locString = locale.toString(); 299 ExampleGenerator eg = exampleGeneratorCache.getIfPresent(locString); 300 if (eg == null) { 301 synchronized (exampleGeneratorCache) { 302 eg = exampleGeneratorCache.getIfPresent(locString); 303 if (eg == null) { 304 eg = new ExampleGenerator(ourSrc, translationHintsFile); 305 exampleGeneratorCache.put(locString, eg); 306 } 307 } 308 } 309 return eg; 310 } 311 312 /** 313 * Update the cached ExampleGenerator, per valueChanged 314 * 315 * @param xpath the xpath whose value has changed 316 * @param locale the CLDRLocale determining which ExampleGenerator to update 317 * <p>Called by valueChangedInvalidateRecursively 318 */ updateExampleGeneratorCache(String xpath, CLDRLocale locale)319 private static void updateExampleGeneratorCache(String xpath, CLDRLocale locale) { 320 ExampleGenerator eg = exampleGeneratorCache.getIfPresent(locale.toString()); 321 if (eg != null) { 322 /* 323 * Each ExampleGenerator has its own internal cache, which is not the same 324 * as exampleGeneratorCache. 325 * 326 * We could call exampleGeneratorCache.invalidate(locale.toString()) but that would be 327 * too drastic, effectively throwing away the ExampleGenerator for the entire locale. 328 * Ideally eg.updateCache will only clear the minimum set of examples (in its internal 329 * cache) required due to dependence on the given xpath. 330 */ 331 eg.updateCache(xpath); 332 } 333 } 334 335 /** Public for tests. Invalidate cache. */ invalidateAllCached()336 public void invalidateAllCached() { 337 logger.fine(() -> toString() + " - invalidateAllCached()"); 338 testResultCache.invalidateAll(); 339 exampleGeneratorCache.invalidateAll(); 340 } 341 } 342