• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 package org.unicode.cldr.draft;
2 
3 import java.io.File;
4 import java.io.IOException;
5 import java.io.PrintWriter;
6 import java.util.ArrayList;
7 import java.util.Arrays;
8 import java.util.Collection;
9 import java.util.Collections;
10 import java.util.HashSet;
11 import java.util.Iterator;
12 import java.util.LinkedHashMap;
13 import java.util.LinkedHashSet;
14 import java.util.List;
15 import java.util.Map;
16 import java.util.Set;
17 import java.util.TreeMap;
18 import java.util.TreeSet;
19 
20 import org.unicode.cldr.util.CLDRFile;
21 import org.unicode.cldr.util.CLDRPaths;
22 import org.unicode.cldr.util.DtdType;
23 import org.unicode.cldr.util.ElementAttributeInfo;
24 import org.unicode.cldr.util.Factory;
25 import org.unicode.cldr.util.XPathParts;
26 
27 import com.ibm.icu.impl.Relation;
28 import com.ibm.icu.impl.Row;
29 import com.ibm.icu.impl.Row.R2;
30 import com.ibm.icu.impl.Utility;
31 import com.ibm.icu.util.ICUUncheckedIOException;
32 
33 public class JsonConverter {
34 
35     private static final String FILES = "el.*";
36     private static final String MAIN_DIRECTORY = CLDRPaths.MAIN_DIRECTORY;// CldrUtility.SUPPLEMENTAL_DIRECTORY;
37                                                                           // //CldrUtility.MAIN_DIRECTORY;
38     private static final String OUT_DIRECTORY = CLDRPaths.GEN_DIRECTORY + "/jason/"; // CldrUtility.MAIN_DIRECTORY;
39     private static boolean COMPACT = false;
40     static final Set<String> REPLACING_BASE = !COMPACT ? Collections.EMPTY_SET : new HashSet<>(
41         Arrays.asList("type id key count".split("\\s")));
42     static final Set<String> EXTRA_DISTINGUISHING = new HashSet<>(
43         Arrays.asList("locales territory desired supported".split("\\s")));
44     static final Relation<String, String> mainInfo = ElementAttributeInfo.getInstance(DtdType.ldml)
45         .getElement2Attributes();
46     static final Relation<String, String> suppInfo = ElementAttributeInfo.getInstance(DtdType.supplementalData)
47         .getElement2Attributes();
48 
main(String[] args)49     public static void main(String[] args) throws IOException {
50         final String subdirectory = new File(MAIN_DIRECTORY).getName();
51         final Factory cldrFactory = Factory.make(MAIN_DIRECTORY, FILES);
52         final Set<String> locales = new TreeSet<>(cldrFactory.getAvailable());
53         /*
54          * TODO: "parts" is always empty, so all the code using it is wasted!
55          */
56         final XPathParts parts = new XPathParts();
57         for (String locale : locales) {
58             System.out.println("Converting:\t" + locale);
59             final CLDRFile file = cldrFactory.make(locale, false);
60             Relation<String, String> element2Attributes = file.isNonInheriting() ? suppInfo : mainInfo;
61             final Item main = new TableItem(null);
62             DtdType dtdType = null;
63             for (Iterator<String> it = file.iterator("", file.getComparator()); it.hasNext();) {
64                 final String xpath = it.next();
65                 final String fullXpath = file.getFullXPath(xpath);
66                 String value = file.getStringValue(xpath);
67                 XPathParts oldParts = XPathParts.getFrozenInstance(fullXpath).cloneAsThawed(); // not frozen, rewrite can modify
68                 if (dtdType == null) {
69                     dtdType = DtdType.valueOf(parts.getElement(0));
70                 }
71                 rewrite(dtdType, oldParts, value, element2Attributes, parts);
72                 System.out.println(parts);
73                 Item current = main;
74                 int size = parts.size();
75 
76                 for (int i = 0; i < size - 1; ++i) {
77                     final String element = parts.getElement(i);
78                     Map<String, String> actualAttributeKeys = parts.getAttributes(i);
79                     Set<String> keySet = actualAttributeKeys.keySet();
80                     if (keySet.size() != 0) {
81                         Item temp = current.makeSubItem(element, Item.Type.unorderedItem);
82                         for (String attribute : keySet) {
83                             temp.put(attribute, actualAttributeKeys.get(attribute));
84                         }
85                     }
86                     if (i < size - 2) {
87                         current = current.makeSubItem(element,
88                             actualAttributeKeys.containsKey("_q") ? Item.Type.orderedItem : Item.Type.unorderedItem);
89                     } else {
90                         current.put(element, parts.getElement(i + 1));
91                     }
92                 }
93             }
94             PrintWriter out = FileUtilities.openUTF8Writer(OUT_DIRECTORY + subdirectory, locale + ".json");
95             main.print(out, 0);
96             out.close();
97         }
98     }
99 
100     static Relation<String, String> extraDistinguishing = Relation.of(new TreeMap<String, Set<String>>(), LinkedHashSet.class);
101     static {
putAll(extraDistinguishing, "dayPeriodRule", "earlyMorning", "before", "from")102         putAll(extraDistinguishing, "dayPeriodRule", "earlyMorning", "before", "from");
103     }
104 
putAll(Relation r, K key, V... values)105     static <K, V> void putAll(Relation r, K key, V... values) {
106         r.putAll(key, Arrays.asList(values));
107     }
108 
isDistinguishing(DtdType dtdType, final String element, final String attribute)109     private static boolean isDistinguishing(DtdType dtdType, final String element, final String attribute) {
110         // <mapZone other="Afghanistan" territory="001" type="Asia/Kabul"/> result is the type!
111         // <deprecatedItems elements="variant" attributes="type" values="BOKMAL NYNORSK AALAND POLYTONI"/>
112         // ugly: if there are values, then everything else is distinguishing, ow if there are attibutes, elements are
113         if (element.equals("deprecatedItems")) {
114 
115         }
116         Set<String> extras = extraDistinguishing.getAll(element);
117         if (extras != null && extras.contains(attribute)) return true;
118         if (EXTRA_DISTINGUISHING.contains(attribute)) return true;
119         return CLDRFile.isDistinguishing(dtdType, element, attribute);
120     }
121 
rewrite(DtdType dtdType, XPathParts parts, String value, Relation<String, String> element2Attributes, XPathParts out)122     private static void rewrite(DtdType dtdType, XPathParts parts, String value,
123         Relation<String, String> element2Attributes, XPathParts out) {
124         out.clear();
125         int size = parts.size();
126         for (int i = 1; i < size; ++i) {
127             final String element = parts.getElement(i);
128             out.addElement(element);
129 
130             // turn a path into a revised path. All distinguished attributes (including those not currently on the
131             // string)
132             // get turned into extra element/element pairs, starting with _
133             // all non-distinguishing attributes get turned into separate children
134             // a/b[@non="y"][@dist="x"]/w : z =>
135             // a/b/_dist/x/_non=y
136             // a/b/_dist/x/w=z
137             Collection<String> actualAttributeKeys = parts.getAttributeKeys(i);
138             boolean isOrdered = actualAttributeKeys.contains("_q");
139             Set<String> possibleAttributeKeys = element2Attributes.getAll(element);
140 
141             for (final String attribute : actualAttributeKeys) {
142                 String attributeValue = parts.getAttributeValue(i, attribute);
143                 if (!isDistinguishing(dtdType, element, attribute)) {
144                     out.addAttribute(attribute, attributeValue);
145                 }
146             }
147             if (possibleAttributeKeys != null) {
148                 for (final String attribute : possibleAttributeKeys) {
149                     if (isDistinguishing(dtdType, element, attribute)) {
150                         if (attribute.equals("alt")) continue; // TODO fix
151                         String attributeValue = parts.getAttributeValue(i, attribute);
152                         out.addElement("_" + attribute);
153                         if (attributeValue == null) {
154                             attributeValue = "?";
155                         }
156                         out.addElement(attributeValue);
157                     }
158                 }
159             }
160             if (isOrdered) {
161                 Map<String, String> lastAttributes = out.getAttributes(-2);
162                 lastAttributes.put("_q", "_q");
163             }
164         }
165         if (value.length() > 0) {
166             out.addElement(value);
167         }
168 
169         if (!COMPACT) {
170             return;
171         }
172         if (parts.getElement(-1).equals("type")) {
173             String key = parts.getAttributeValue(-1, "key");
174             if (key != null) {
175                 parts.setElement(-2, key + "Key");
176                 parts.putAttributeValue(-1, "key", null);
177             }
178             // fall thru
179         }
180         if (parts.getElement(1).equals("localeDisplayNames")) {
181             String element2 = parts.getElement(2);
182             if (!element2.endsWith("Pattern")) {
183                 if (element2.endsWith("s")) {
184                     element2 = element2.substring(0, element2.length() - 1);
185                 }
186                 parts.setElement(2, element2 + "Names");
187             }
188             parts.removeElement(1);
189         }
190         if (parts.getElement(1).equals("dates")) {
191             parts.removeElement(1);
192             String element1 = parts.getElement(1);
193             if (element1.equals("timeZoneNames")) {
194                 String main = parts.getElement(2);
195                 if (main.equals("zone") || main.equals("metazone")) {
196                     parts.setElement(1, main + "Names");
197                 }
198                 return;
199             }
200         }
201         if (parts.getElement(1).equals("numbers") && parts.getElement(2).equals("currencies")) {
202             parts.removeElement(1);
203             return;
204         }
205     }
206 
207     static class ElementName {
208         String oldBase;
209         String base;
210         boolean replacedBase;
211         StringBuilder suffix = new StringBuilder();
212 
reset(String element)213         public void reset(String element) {
214             suffix.setLength(0);
215             base = oldBase = element;
216             replacedBase = false;
217         }
218 
add(String attribute, String attributeValue)219         public void add(String attribute, String attributeValue) {
220             if (REPLACING_BASE.contains(attribute)) {
221                 if (replacedBase) {
222                     System.out.println("ERROR: Two replacement types on same element!!\t" + oldBase + "," + base + ","
223                         + attribute + "," + attributeValue);
224                 } else {
225                     replacedBase = true;
226                     base = attributeValue;
227                     return;
228                 }
229             }
230             suffix.append('$').append(attribute).append('=').append(attributeValue);
231         }
232 
233         @Override
toString()234         public String toString() {
235             if (suffix == null) {
236                 return base;
237             }
238             return base + suffix;
239         }
240     }
241 
242     static abstract class Item {
243         protected Item parent;
244 
Item(Item parent)245         public Item(Item parent) {
246             this.parent = parent;
247         }
248 
size()249         public abstract int size();
250 
251         enum Type {
252             unorderedItem, orderedItem
253         }
254 
print(Appendable result, int i)255         public abstract Appendable print(Appendable result, int i);
256 
indent(Appendable result, int i)257         protected Appendable indent(Appendable result, int i) throws IOException {
258             return result.append(getIndent(i));
259         }
260 
getIndent(int i)261         protected String getIndent(int i) {
262             return Utility.repeat("    ", i);
263         }
264 
appendString(Appendable result, String string, int indent)265         public Appendable appendString(Appendable result, String string, int indent) throws IOException {
266             result.append('"');
267             for (int i = 0; i < string.length(); ++i) {
268                 // http://www.json.org/
269                 // any-Unicode-character-except-"-or-\-or-control-character
270                 // uses UTF16
271                 char ch = string.charAt(i);
272                 switch (ch) {
273                 case '\"':
274                     result.append("\\\"");
275                     break;
276                 case '\\':
277                     result.append("\\\\");
278                     break;
279                 case '/':
280                     result.append("\\/");
281                     break;
282                 case '\b':
283                     result.append("\\b");
284                     break;
285                 case '\f':
286                     result.append("\\f");
287                     break;
288                 case '\n':
289                     if (indent < 0) {
290                         result.append("\\n");
291                     } else {
292                         result.append('\n').append(getIndent(indent));
293                     }
294                     break;
295                 case '\r':
296                     result.append("\\r");
297                     break;
298                 case '\t':
299                     result.append("\\t");
300                     break;
301                 default:
302                     if (ch <= 0x1F || 0x7F <= ch && ch <= 0x9F) {
303                         result.append("\\u").append(Utility.hex(ch, 4));
304                     } else {
305                         result.append(ch);
306                     }
307                     break;
308                 }
309             }
310             return result.append('"');
311         }
312 
313         @Override
toString()314         public String toString() {
315             return print(new StringBuilder(), 0).toString();
316         }
317 
create(Type ordered)318         protected Item create(Type ordered) {
319             switch (ordered) {
320             case unorderedItem:
321                 return new TableItem(this);
322             case orderedItem:
323                 return new ArrayItem(this);
324             default:
325                 throw new UnsupportedOperationException();
326             }
327         }
328 
makeSubItem(String element, Type ordered)329         public abstract Item makeSubItem(String element, Type ordered);
330 
put(String element, String value)331         public abstract void put(String element, String value);
332 
getRoot()333         public Item getRoot() {
334             if (parent == null) {
335                 return this;
336             } else {
337                 return parent.getRoot();
338             }
339         }
340     }
341 
342     static class TableItem extends Item {
TableItem(Item parent)343         public TableItem(Item parent) {
344             super(parent);
345         }
346 
347         private Map<String, Item> map = new LinkedHashMap<>();
348 
get(String element)349         public Item get(String element) {
350             return map.get(element);
351         }
352 
353         @Override
put(String element, String value)354         public void put(String element, String value) {
355             Item old = map.get(element);
356             if (old != null) {
357                 if (old instanceof StringItem) {
358                     if (value.equals(((StringItem) old).value)) {
359                         return;
360                     }
361                 }
362                 throw new IllegalArgumentException("ERROR: Table already has object: " + element + ", " + old + ", "
363                     + value + ", " + getRoot().toString());
364             }
365             map.put(element, new StringItem(value));
366         }
367 
368         @Override
makeSubItem(String element, Type ordered)369         public Item makeSubItem(String element, Type ordered) {
370             Item result = map.get(element);
371             if (result != null) {
372                 return result;
373             }
374             result = create(ordered);
375             result.parent = this;
376 
377             map.put(element, result);
378             return result;
379         }
380 
381         @Override
print(Appendable result, int i)382         public Appendable print(Appendable result, int i) {
383             try {
384                 if (map.size() == 0) {
385                     result.append("{}");
386                     return result;
387                 }
388                 result.append("{\n");
389                 boolean first = true;
390                 for (String key : map.keySet()) {
391                     Item value = map.get(key);
392                     if (first) {
393                         first = false;
394                     } else {
395                         result.append(",\n");
396                     }
397                     indent(result, i + 1);
398                     appendString(result, key, -1).append(" : ");
399                     value.print(result, i + 1);
400                 }
401                 result.append("\n");
402                 indent(result, i).append("}");
403                 return result;
404             } catch (IOException e) {
405                 throw new ICUUncheckedIOException(e);
406             }
407         }
408 
409         @Override
size()410         public int size() {
411             return map.size();
412         }
413     }
414 
415     static class ArrayItem extends Item {
ArrayItem(Item parent)416         public ArrayItem(Item parent) {
417             super(parent);
418         }
419 
420         private List<Row.R2<String, Item>> list = new ArrayList<>();
421 
422         @Override
print(Appendable result, int i)423         public Appendable print(Appendable result, int i) {
424             try {
425                 if (list.size() == 0) {
426                     result.append("[]");
427                     return result;
428                 }
429 
430                 result.append("[\n");
431                 for (int j = 0; j < list.size(); ++j) {
432                     if (j != 0) {
433                         result.append(",\n");
434                     }
435                     indent(result, i + 1);
436                     R2<String, Item> row = list.get(j);
437                     result.append("{");
438                     appendString(result, row.get0(), i + 1);
439                     result.append(" : ");
440                     row.get1().print(result, i + 1);
441                     result.append("}");
442                 }
443                 result.append("\n");
444                 indent(result, i).append("]");
445                 return result;
446             } catch (IOException e) {
447                 throw new IllegalArgumentException(e);
448             }
449         }
450 
451         @Override
makeSubItem(String element, Type ordered)452         public Item makeSubItem(String element, Type ordered) {
453             Item result = create(ordered);
454             list.add(Row.of(element, result));
455             return result;
456         }
457 
458         @Override
put(String element, String value)459         public void put(String element, String value) {
460             list.add(Row.of(element, (Item) new StringItem(value)));
461         }
462 
463         @Override
size()464         public int size() {
465             return list.size();
466         }
467     }
468 
469     static class StringItem extends Item {
470         private String value;
471 
StringItem(String value2)472         public StringItem(String value2) {
473             super(null);
474             value = value2;
475         }
476 
477         @Override
print(Appendable result, int i)478         public Appendable print(Appendable result, int i) {
479             try {
480                 return appendString(result, value, i + 1);
481             } catch (IOException e) {
482                 throw new IllegalArgumentException(e);
483             }
484         }
485 
486         @Override
makeSubItem(String element, Type ordered)487         public Item makeSubItem(String element, Type ordered) {
488             throw new UnsupportedOperationException();
489         }
490 
491         @Override
put(String element, String value)492         public void put(String element, String value) {
493             throw new UnsupportedOperationException();
494         }
495 
496         @Override
size()497         public int size() {
498             throw new UnsupportedOperationException();
499         }
500     }
501 }
502