1 package org.unicode.cldr.tool; 2 3 import com.google.common.collect.Multimap; 4 import com.google.common.collect.TreeMultimap; 5 import java.io.File; 6 import java.io.IOException; 7 import java.io.PrintWriter; 8 import java.util.ArrayList; 9 import java.util.Collections; 10 import java.util.HashSet; 11 import java.util.List; 12 import java.util.Set; 13 import java.util.TreeSet; 14 import org.unicode.cldr.draft.FileUtilities; 15 import org.unicode.cldr.test.ExampleGenerator; 16 import org.unicode.cldr.util.CLDRConfig; 17 import org.unicode.cldr.util.CLDRFile; 18 import org.unicode.cldr.util.CLDRPaths; 19 import org.unicode.cldr.util.CLDRTool; 20 import org.unicode.cldr.util.Factory; 21 import org.unicode.cldr.util.LocaleIDParser; 22 import org.unicode.cldr.util.PathStarrer; 23 import org.unicode.cldr.util.RecordingCLDRFile; 24 import org.unicode.cldr.util.XMLSource; 25 26 @CLDRTool(alias = "generate-example-dependencies", description = "Generate example dependencies") 27 public class GenerateExampleDependencies { 28 private static final String OUTPUT_FILE_NAME = "ExampleDependencies.java"; 29 private static final String DO_NOT_EDIT = 30 "/* DO NOT EDIT THIS FILE, instead regenerate it using " 31 + GenerateExampleDependencies.class.getSimpleName() 32 + ".java */"; 33 private final CLDRFile englishFile; 34 private final Factory factory; 35 private final Set<String> locales; 36 private final String outputDir; 37 private final PathStarrer pathStarrer; 38 main(String[] args)39 public static void main(String[] args) throws IOException { 40 new GenerateExampleDependencies().run(); 41 } 42 43 /** 44 * Find dependencies where changing the value of one path changes example-generation for another 45 * path. 46 * 47 * <p>The goal is to optimize example caching by only regenerating examples when necessary. 48 */ GenerateExampleDependencies()49 public GenerateExampleDependencies() { 50 CLDRConfig info = CLDRConfig.getInstance(); 51 englishFile = info.getEnglish(); 52 factory = info.getCldrFactory(); 53 locales = factory.getAvailable(); 54 outputDir = CLDRPaths.GEN_DIRECTORY + "test" + File.separator; 55 pathStarrer = new PathStarrer().setSubstitutionPattern("*"); 56 } 57 run()58 public void run() throws IOException { 59 int localeCount = locales.size(); 60 System.out.println( 61 "Looping through " + localeCount + " locales ... (this may take an hour)"); 62 final Multimap<String, String> dependencies = TreeMultimap.create(); 63 int i = 0; 64 for (String localeId : locales) { 65 int percent = (++i * 100) / localeCount; 66 System.out.println(localeId + " " + i + "/" + localeCount + " " + percent + "%"); 67 addDependenciesForLocale(dependencies, localeId); 68 } 69 System.out.println("Creating " + outputDir + OUTPUT_FILE_NAME + " ..."); 70 PrintWriter writer = FileUtilities.openUTF8Writer(outputDir, OUTPUT_FILE_NAME); 71 int dependenciesWritten = writeDependenciesToFile(dependencies, writer); 72 System.out.println( 73 "Wrote " 74 + dependenciesWritten 75 + " dependencies to " 76 + outputDir 77 + GenerateExampleDependencies.OUTPUT_FILE_NAME); 78 System.out.println( 79 "If it looks OK, you can move it to the proper location, replacing the old version."); 80 } 81 addDependenciesForLocale(Multimap<String, String> dependencies, String localeId)82 private void addDependenciesForLocale(Multimap<String, String> dependencies, String localeId) { 83 RecordingCLDRFile cldrFile = makeRecordingCldrFile(localeId); 84 cldrFile.disableCaching(); 85 86 Set<String> paths = new TreeSet<>(cldrFile.getComparator()); 87 cldrFile.forEach(paths::add); // time-consuming 88 89 ExampleGenerator egTest = new ExampleGenerator(cldrFile, englishFile); 90 // Caching MUST be disabled for egTest.ICUServiceBuilder to prevent some dependencies from 91 // being missed 92 egTest.setCachingEnabled(false); 93 94 for (String pathB : paths) { 95 if (skipPathForDependencies(pathB)) { 96 continue; 97 } 98 String valueB = cldrFile.getStringValue(pathB); 99 if (valueB == null) { 100 continue; 101 } 102 String starredB = pathStarrer.set(pathB); 103 cldrFile.clearRecordedPaths(); 104 egTest.getExampleHtml(pathB, valueB); 105 HashSet<String> pathsA = cldrFile.getRecordedPaths(); 106 for (String pathA : pathsA) { 107 if (pathA.equals(pathB) || skipPathForDependencies(pathA)) { 108 continue; 109 } 110 String starredA = pathStarrer.set(pathA); 111 dependencies.put(starredA, starredB); 112 } 113 } 114 } 115 makeRecordingCldrFile(String localeId)116 private RecordingCLDRFile makeRecordingCldrFile(String localeId) { 117 XMLSource topSource = factory.makeSource(localeId); 118 List<XMLSource> parents = getParentSources(factory, localeId); 119 XMLSource[] a = new XMLSource[parents.size()]; 120 return new RecordingCLDRFile(topSource, parents.toArray(a)); 121 } 122 123 /** 124 * Get the parent sources for the given localeId 125 * 126 * @param factory the Factory for makeSource 127 * @param localeId the locale ID 128 * @return the List of XMLSource objects 129 */ getParentSources(Factory factory, String localeId)130 private static List<XMLSource> getParentSources(Factory factory, String localeId) { 131 List<XMLSource> parents = new ArrayList<>(); 132 for (String currentLocaleID = LocaleIDParser.getParent(localeId); 133 currentLocaleID != null; 134 currentLocaleID = LocaleIDParser.getParent(currentLocaleID)) { 135 parents.add(factory.makeSource(currentLocaleID)); 136 } 137 return parents; 138 } 139 140 /** 141 * Should the given path be skipped when testing example-generator path dependencies? 142 * 143 * @param path the path in question 144 * @return true to skip, else false 145 */ skipPathForDependencies(String path)146 private static boolean skipPathForDependencies(String path) { 147 return path.endsWith("/alias") || path.startsWith("//ldml/identity"); 148 } 149 150 /** 151 * Write the given map of example-generator path dependencies to a java file. 152 * 153 * @param dependencies the multimap of example-generator path dependencies 154 * @param writer the PrintWriter 155 * @return the number of dependencies written 156 */ writeDependenciesToFile(Multimap<String, String> dependencies, PrintWriter writer)157 private int writeDependenciesToFile(Multimap<String, String> dependencies, PrintWriter writer) { 158 writer.println("package org.unicode.cldr.test;"); 159 writer.println(DO_NOT_EDIT); 160 writer.println("import com.google.common.collect.ImmutableSetMultimap;"); 161 writer.println(""); 162 writer.println("public class ExampleDependencies {"); 163 writer.println(" public static ImmutableSetMultimap<String, String> dependencies ="); 164 writer.println(" new ImmutableSetMultimap.Builder<String, String>()"); 165 int dependenciesWritten = 0; 166 ArrayList<String> listA = new ArrayList<>(dependencies.keySet()); 167 Collections.sort(listA); 168 for (String pathA : listA) { 169 ArrayList<String> listB = new ArrayList<>(dependencies.get(pathA)); 170 Collections.sort(listB); 171 String a = "\"" + pathA.replaceAll("\"", "\\\\\"") + "\""; 172 writer.println(" .putAll(" + a + ","); 173 int remainingCount = listB.size(); 174 for (String pathB : listB) { 175 String b = "\"" + pathB.replaceAll("\"", "\\\\\"") + "\""; 176 String endOfLine = --remainingCount > 0 ? "," : ")"; 177 writer.println(" " + b + endOfLine); 178 ++dependenciesWritten; 179 } 180 } 181 writer.println(" .build();"); 182 writer.println("}"); 183 writer.println(DO_NOT_EDIT); 184 writer.close(); 185 return dependenciesWritten; 186 } 187 } 188