1 package org.unicode.cldr.draft; 2 3 import java.io.File; 4 import java.io.IOException; 5 import java.io.PrintWriter; 6 import java.util.ArrayList; 7 import java.util.Arrays; 8 import java.util.Collection; 9 import java.util.Collections; 10 import java.util.HashSet; 11 import java.util.Iterator; 12 import java.util.LinkedHashMap; 13 import java.util.LinkedHashSet; 14 import java.util.List; 15 import java.util.Map; 16 import java.util.Set; 17 import java.util.TreeMap; 18 import java.util.TreeSet; 19 20 import org.unicode.cldr.util.CLDRFile; 21 import org.unicode.cldr.util.CLDRPaths; 22 import org.unicode.cldr.util.DtdType; 23 import org.unicode.cldr.util.ElementAttributeInfo; 24 import org.unicode.cldr.util.Factory; 25 import org.unicode.cldr.util.XPathParts; 26 27 import com.ibm.icu.impl.Relation; 28 import com.ibm.icu.impl.Row; 29 import com.ibm.icu.impl.Row.R2; 30 import com.ibm.icu.impl.Utility; 31 import com.ibm.icu.util.ICUUncheckedIOException; 32 33 public class JsonConverter { 34 35 private static final String FILES = "el.*"; 36 private static final String MAIN_DIRECTORY = CLDRPaths.MAIN_DIRECTORY;// CldrUtility.SUPPLEMENTAL_DIRECTORY; 37 // //CldrUtility.MAIN_DIRECTORY; 38 private static final String OUT_DIRECTORY = CLDRPaths.GEN_DIRECTORY + "/jason/"; // CldrUtility.MAIN_DIRECTORY; 39 private static boolean COMPACT = false; 40 static final Set<String> REPLACING_BASE = !COMPACT ? Collections.EMPTY_SET : new HashSet<>( 41 Arrays.asList("type id key count".split("\\s"))); 42 static final Set<String> EXTRA_DISTINGUISHING = new HashSet<>( 43 Arrays.asList("locales territory desired supported".split("\\s"))); 44 static final Relation<String, String> mainInfo = ElementAttributeInfo.getInstance(DtdType.ldml) 45 .getElement2Attributes(); 46 static final Relation<String, String> suppInfo = ElementAttributeInfo.getInstance(DtdType.supplementalData) 47 .getElement2Attributes(); 48 main(String[] args)49 public static void main(String[] args) throws IOException { 50 final String subdirectory = new File(MAIN_DIRECTORY).getName(); 51 final Factory cldrFactory = Factory.make(MAIN_DIRECTORY, FILES); 52 final Set<String> locales = new TreeSet<>(cldrFactory.getAvailable()); 53 /* 54 * TODO: "parts" is always empty, so all the code using it is wasted! 55 */ 56 final XPathParts parts = new XPathParts(); 57 for (String locale : locales) { 58 System.out.println("Converting:\t" + locale); 59 final CLDRFile file = cldrFactory.make(locale, false); 60 Relation<String, String> element2Attributes = file.isNonInheriting() ? suppInfo : mainInfo; 61 final Item main = new TableItem(null); 62 DtdType dtdType = null; 63 for (Iterator<String> it = file.iterator("", file.getComparator()); it.hasNext();) { 64 final String xpath = it.next(); 65 final String fullXpath = file.getFullXPath(xpath); 66 String value = file.getStringValue(xpath); 67 XPathParts oldParts = XPathParts.getFrozenInstance(fullXpath).cloneAsThawed(); // not frozen, rewrite can modify 68 if (dtdType == null) { 69 dtdType = DtdType.valueOf(parts.getElement(0)); 70 } 71 rewrite(dtdType, oldParts, value, element2Attributes, parts); 72 System.out.println(parts); 73 Item current = main; 74 int size = parts.size(); 75 76 for (int i = 0; i < size - 1; ++i) { 77 final String element = parts.getElement(i); 78 Map<String, String> actualAttributeKeys = parts.getAttributes(i); 79 Set<String> keySet = actualAttributeKeys.keySet(); 80 if (keySet.size() != 0) { 81 Item temp = current.makeSubItem(element, Item.Type.unorderedItem); 82 for (String attribute : keySet) { 83 temp.put(attribute, actualAttributeKeys.get(attribute)); 84 } 85 } 86 if (i < size - 2) { 87 current = current.makeSubItem(element, 88 actualAttributeKeys.containsKey("_q") ? Item.Type.orderedItem : Item.Type.unorderedItem); 89 } else { 90 current.put(element, parts.getElement(i + 1)); 91 } 92 } 93 } 94 PrintWriter out = FileUtilities.openUTF8Writer(OUT_DIRECTORY + subdirectory, locale + ".json"); 95 main.print(out, 0); 96 out.close(); 97 } 98 } 99 100 static Relation<String, String> extraDistinguishing = Relation.of(new TreeMap<String, Set<String>>(), LinkedHashSet.class); 101 static { putAll(extraDistinguishing, "dayPeriodRule", "earlyMorning", "before", "from")102 putAll(extraDistinguishing, "dayPeriodRule", "earlyMorning", "before", "from"); 103 } 104 putAll(Relation r, K key, V... values)105 static <K, V> void putAll(Relation r, K key, V... values) { 106 r.putAll(key, Arrays.asList(values)); 107 } 108 isDistinguishing(DtdType dtdType, final String element, final String attribute)109 private static boolean isDistinguishing(DtdType dtdType, final String element, final String attribute) { 110 // <mapZone other="Afghanistan" territory="001" type="Asia/Kabul"/> result is the type! 111 // <deprecatedItems elements="variant" attributes="type" values="BOKMAL NYNORSK AALAND POLYTONI"/> 112 // ugly: if there are values, then everything else is distinguishing, ow if there are attibutes, elements are 113 if (element.equals("deprecatedItems")) { 114 115 } 116 Set<String> extras = extraDistinguishing.getAll(element); 117 if (extras != null && extras.contains(attribute)) return true; 118 if (EXTRA_DISTINGUISHING.contains(attribute)) return true; 119 return CLDRFile.isDistinguishing(dtdType, element, attribute); 120 } 121 rewrite(DtdType dtdType, XPathParts parts, String value, Relation<String, String> element2Attributes, XPathParts out)122 private static void rewrite(DtdType dtdType, XPathParts parts, String value, 123 Relation<String, String> element2Attributes, XPathParts out) { 124 out.clear(); 125 int size = parts.size(); 126 for (int i = 1; i < size; ++i) { 127 final String element = parts.getElement(i); 128 out.addElement(element); 129 130 // turn a path into a revised path. All distinguished attributes (including those not currently on the 131 // string) 132 // get turned into extra element/element pairs, starting with _ 133 // all non-distinguishing attributes get turned into separate children 134 // a/b[@non="y"][@dist="x"]/w : z => 135 // a/b/_dist/x/_non=y 136 // a/b/_dist/x/w=z 137 Collection<String> actualAttributeKeys = parts.getAttributeKeys(i); 138 boolean isOrdered = actualAttributeKeys.contains("_q"); 139 Set<String> possibleAttributeKeys = element2Attributes.getAll(element); 140 141 for (final String attribute : actualAttributeKeys) { 142 String attributeValue = parts.getAttributeValue(i, attribute); 143 if (!isDistinguishing(dtdType, element, attribute)) { 144 out.addAttribute(attribute, attributeValue); 145 } 146 } 147 if (possibleAttributeKeys != null) { 148 for (final String attribute : possibleAttributeKeys) { 149 if (isDistinguishing(dtdType, element, attribute)) { 150 if (attribute.equals("alt")) continue; // TODO fix 151 String attributeValue = parts.getAttributeValue(i, attribute); 152 out.addElement("_" + attribute); 153 if (attributeValue == null) { 154 attributeValue = "?"; 155 } 156 out.addElement(attributeValue); 157 } 158 } 159 } 160 if (isOrdered) { 161 Map<String, String> lastAttributes = out.getAttributes(-2); 162 lastAttributes.put("_q", "_q"); 163 } 164 } 165 if (value.length() > 0) { 166 out.addElement(value); 167 } 168 169 if (!COMPACT) { 170 return; 171 } 172 if (parts.getElement(-1).equals("type")) { 173 String key = parts.getAttributeValue(-1, "key"); 174 if (key != null) { 175 parts.setElement(-2, key + "Key"); 176 parts.putAttributeValue(-1, "key", null); 177 } 178 // fall thru 179 } 180 if (parts.getElement(1).equals("localeDisplayNames")) { 181 String element2 = parts.getElement(2); 182 if (!element2.endsWith("Pattern")) { 183 if (element2.endsWith("s")) { 184 element2 = element2.substring(0, element2.length() - 1); 185 } 186 parts.setElement(2, element2 + "Names"); 187 } 188 parts.removeElement(1); 189 } 190 if (parts.getElement(1).equals("dates")) { 191 parts.removeElement(1); 192 String element1 = parts.getElement(1); 193 if (element1.equals("timeZoneNames")) { 194 String main = parts.getElement(2); 195 if (main.equals("zone") || main.equals("metazone")) { 196 parts.setElement(1, main + "Names"); 197 } 198 return; 199 } 200 } 201 if (parts.getElement(1).equals("numbers") && parts.getElement(2).equals("currencies")) { 202 parts.removeElement(1); 203 return; 204 } 205 } 206 207 static class ElementName { 208 String oldBase; 209 String base; 210 boolean replacedBase; 211 StringBuilder suffix = new StringBuilder(); 212 reset(String element)213 public void reset(String element) { 214 suffix.setLength(0); 215 base = oldBase = element; 216 replacedBase = false; 217 } 218 add(String attribute, String attributeValue)219 public void add(String attribute, String attributeValue) { 220 if (REPLACING_BASE.contains(attribute)) { 221 if (replacedBase) { 222 System.out.println("ERROR: Two replacement types on same element!!\t" + oldBase + "," + base + "," 223 + attribute + "," + attributeValue); 224 } else { 225 replacedBase = true; 226 base = attributeValue; 227 return; 228 } 229 } 230 suffix.append('$').append(attribute).append('=').append(attributeValue); 231 } 232 233 @Override toString()234 public String toString() { 235 if (suffix == null) { 236 return base; 237 } 238 return base + suffix; 239 } 240 } 241 242 static abstract class Item { 243 protected Item parent; 244 Item(Item parent)245 public Item(Item parent) { 246 this.parent = parent; 247 } 248 size()249 public abstract int size(); 250 251 enum Type { 252 unorderedItem, orderedItem 253 } 254 print(Appendable result, int i)255 public abstract Appendable print(Appendable result, int i); 256 indent(Appendable result, int i)257 protected Appendable indent(Appendable result, int i) throws IOException { 258 return result.append(getIndent(i)); 259 } 260 getIndent(int i)261 protected String getIndent(int i) { 262 return Utility.repeat(" ", i); 263 } 264 appendString(Appendable result, String string, int indent)265 public Appendable appendString(Appendable result, String string, int indent) throws IOException { 266 result.append('"'); 267 for (int i = 0; i < string.length(); ++i) { 268 // http://www.json.org/ 269 // any-Unicode-character-except-"-or-\-or-control-character 270 // uses UTF16 271 char ch = string.charAt(i); 272 switch (ch) { 273 case '\"': 274 result.append("\\\""); 275 break; 276 case '\\': 277 result.append("\\\\"); 278 break; 279 case '/': 280 result.append("\\/"); 281 break; 282 case '\b': 283 result.append("\\b"); 284 break; 285 case '\f': 286 result.append("\\f"); 287 break; 288 case '\n': 289 if (indent < 0) { 290 result.append("\\n"); 291 } else { 292 result.append('\n').append(getIndent(indent)); 293 } 294 break; 295 case '\r': 296 result.append("\\r"); 297 break; 298 case '\t': 299 result.append("\\t"); 300 break; 301 default: 302 if (ch <= 0x1F || 0x7F <= ch && ch <= 0x9F) { 303 result.append("\\u").append(Utility.hex(ch, 4)); 304 } else { 305 result.append(ch); 306 } 307 break; 308 } 309 } 310 return result.append('"'); 311 } 312 313 @Override toString()314 public String toString() { 315 return print(new StringBuilder(), 0).toString(); 316 } 317 create(Type ordered)318 protected Item create(Type ordered) { 319 switch (ordered) { 320 case unorderedItem: 321 return new TableItem(this); 322 case orderedItem: 323 return new ArrayItem(this); 324 default: 325 throw new UnsupportedOperationException(); 326 } 327 } 328 makeSubItem(String element, Type ordered)329 public abstract Item makeSubItem(String element, Type ordered); 330 put(String element, String value)331 public abstract void put(String element, String value); 332 getRoot()333 public Item getRoot() { 334 if (parent == null) { 335 return this; 336 } else { 337 return parent.getRoot(); 338 } 339 } 340 } 341 342 static class TableItem extends Item { TableItem(Item parent)343 public TableItem(Item parent) { 344 super(parent); 345 } 346 347 private Map<String, Item> map = new LinkedHashMap<>(); 348 get(String element)349 public Item get(String element) { 350 return map.get(element); 351 } 352 353 @Override put(String element, String value)354 public void put(String element, String value) { 355 Item old = map.get(element); 356 if (old != null) { 357 if (old instanceof StringItem) { 358 if (value.equals(((StringItem) old).value)) { 359 return; 360 } 361 } 362 throw new IllegalArgumentException("ERROR: Table already has object: " + element + ", " + old + ", " 363 + value + ", " + getRoot().toString()); 364 } 365 map.put(element, new StringItem(value)); 366 } 367 368 @Override makeSubItem(String element, Type ordered)369 public Item makeSubItem(String element, Type ordered) { 370 Item result = map.get(element); 371 if (result != null) { 372 return result; 373 } 374 result = create(ordered); 375 result.parent = this; 376 377 map.put(element, result); 378 return result; 379 } 380 381 @Override print(Appendable result, int i)382 public Appendable print(Appendable result, int i) { 383 try { 384 if (map.size() == 0) { 385 result.append("{}"); 386 return result; 387 } 388 result.append("{\n"); 389 boolean first = true; 390 for (String key : map.keySet()) { 391 Item value = map.get(key); 392 if (first) { 393 first = false; 394 } else { 395 result.append(",\n"); 396 } 397 indent(result, i + 1); 398 appendString(result, key, -1).append(" : "); 399 value.print(result, i + 1); 400 } 401 result.append("\n"); 402 indent(result, i).append("}"); 403 return result; 404 } catch (IOException e) { 405 throw new ICUUncheckedIOException(e); 406 } 407 } 408 409 @Override size()410 public int size() { 411 return map.size(); 412 } 413 } 414 415 static class ArrayItem extends Item { ArrayItem(Item parent)416 public ArrayItem(Item parent) { 417 super(parent); 418 } 419 420 private List<Row.R2<String, Item>> list = new ArrayList<>(); 421 422 @Override print(Appendable result, int i)423 public Appendable print(Appendable result, int i) { 424 try { 425 if (list.size() == 0) { 426 result.append("[]"); 427 return result; 428 } 429 430 result.append("[\n"); 431 for (int j = 0; j < list.size(); ++j) { 432 if (j != 0) { 433 result.append(",\n"); 434 } 435 indent(result, i + 1); 436 R2<String, Item> row = list.get(j); 437 result.append("{"); 438 appendString(result, row.get0(), i + 1); 439 result.append(" : "); 440 row.get1().print(result, i + 1); 441 result.append("}"); 442 } 443 result.append("\n"); 444 indent(result, i).append("]"); 445 return result; 446 } catch (IOException e) { 447 throw new IllegalArgumentException(e); 448 } 449 } 450 451 @Override makeSubItem(String element, Type ordered)452 public Item makeSubItem(String element, Type ordered) { 453 Item result = create(ordered); 454 list.add(Row.of(element, result)); 455 return result; 456 } 457 458 @Override put(String element, String value)459 public void put(String element, String value) { 460 list.add(Row.of(element, (Item) new StringItem(value))); 461 } 462 463 @Override size()464 public int size() { 465 return list.size(); 466 } 467 } 468 469 static class StringItem extends Item { 470 private String value; 471 StringItem(String value2)472 public StringItem(String value2) { 473 super(null); 474 value = value2; 475 } 476 477 @Override print(Appendable result, int i)478 public Appendable print(Appendable result, int i) { 479 try { 480 return appendString(result, value, i + 1); 481 } catch (IOException e) { 482 throw new IllegalArgumentException(e); 483 } 484 } 485 486 @Override makeSubItem(String element, Type ordered)487 public Item makeSubItem(String element, Type ordered) { 488 throw new UnsupportedOperationException(); 489 } 490 491 @Override put(String element, String value)492 public void put(String element, String value) { 493 throw new UnsupportedOperationException(); 494 } 495 496 @Override size()497 public int size() { 498 throw new UnsupportedOperationException(); 499 } 500 } 501 } 502