• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 package org.unicode.cldr.draft;
2 
3 import java.io.File;
4 import java.io.IOException;
5 import java.io.PrintWriter;
6 import java.util.ArrayList;
7 import java.util.Arrays;
8 import java.util.Collection;
9 import java.util.Collections;
10 import java.util.HashSet;
11 import java.util.Iterator;
12 import java.util.LinkedHashMap;
13 import java.util.LinkedHashSet;
14 import java.util.List;
15 import java.util.Map;
16 import java.util.Set;
17 import java.util.TreeMap;
18 import java.util.TreeSet;
19 
20 import org.unicode.cldr.util.CLDRFile;
21 import org.unicode.cldr.util.CLDRPaths;
22 import org.unicode.cldr.util.DtdType;
23 import org.unicode.cldr.util.ElementAttributeInfo;
24 import org.unicode.cldr.util.Factory;
25 import org.unicode.cldr.util.XPathParts;
26 
27 import com.ibm.icu.impl.Relation;
28 import com.ibm.icu.impl.Row;
29 import com.ibm.icu.impl.Row.R2;
30 import com.ibm.icu.impl.Utility;
31 import com.ibm.icu.util.ICUUncheckedIOException;
32 
33 public class JsonConverter {
34 
35     private static final String FILES = "el.*";
36     private static final String MAIN_DIRECTORY = CLDRPaths.MAIN_DIRECTORY;// CldrUtility.SUPPLEMENTAL_DIRECTORY;
37                                                                           // //CldrUtility.MAIN_DIRECTORY;
38     private static final String OUT_DIRECTORY = CLDRPaths.GEN_DIRECTORY + "/jason/"; // CldrUtility.MAIN_DIRECTORY;
39     private static boolean COMPACT = false;
40     static final Set<String> REPLACING_BASE = !COMPACT ? Collections.EMPTY_SET : new HashSet<String>(
41         Arrays.asList("type id key count".split("\\s")));
42     static final Set<String> EXTRA_DISTINGUISHING = new HashSet<String>(
43         Arrays.asList("locales territory desired supported".split("\\s")));
44     static final Relation<String, String> mainInfo = ElementAttributeInfo.getInstance(DtdType.ldml)
45         .getElement2Attributes();
46     static final Relation<String, String> suppInfo = ElementAttributeInfo.getInstance(DtdType.supplementalData)
47         .getElement2Attributes();
48 
main(String[] args)49     public static void main(String[] args) throws IOException {
50         final String subdirectory = new File(MAIN_DIRECTORY).getName();
51         final Factory cldrFactory = Factory.make(MAIN_DIRECTORY, FILES);
52         final Set<String> locales = new TreeSet<String>(cldrFactory.getAvailable());
53         final XPathParts oldParts = new XPathParts();
54         final XPathParts parts = new XPathParts();
55         // ElementName elementName = new ElementName();
56         // LinkedHashMap<String, String> nonDistinguishing = new LinkedHashMap<String, String>();
57         for (String locale : locales) {
58             System.out.println("Converting:\t" + locale);
59             final CLDRFile file = cldrFactory.make(locale, false);
60             Relation<String, String> element2Attributes = file.isNonInheriting() ? suppInfo : mainInfo;
61             final Item main = new TableItem(null);
62             DtdType dtdType = null;
63             for (Iterator<String> it = file.iterator("", file.getComparator()); it.hasNext();) {
64                 final String xpath = it.next();
65                 final String fullXpath = file.getFullXPath(xpath);
66                 String value = file.getStringValue(xpath);
67                 oldParts.set(fullXpath);
68                 if (dtdType == null) {
69                     dtdType = DtdType.valueOf(parts.getElement(0));
70                 }
71                 rewrite(dtdType, oldParts, value, element2Attributes, parts);
72                 System.out.println(parts);
73                 Item current = main;
74                 int size = parts.size();
75 
76                 for (int i = 0; i < size - 1; ++i) {
77                     final String element = parts.getElement(i);
78                     Map<String, String> actualAttributeKeys = parts.getAttributes(i);
79                     Set<String> keySet = actualAttributeKeys.keySet();
80                     if (keySet.size() != 0) {
81                         Item temp = current.makeSubItem(element, Item.Type.unorderedItem);
82                         for (String attribute : keySet) {
83                             temp.put(attribute, actualAttributeKeys.get(attribute));
84                         }
85                     }
86                     if (i < size - 2) {
87                         current = current.makeSubItem(element,
88                             actualAttributeKeys.containsKey("_q") ? Item.Type.orderedItem : Item.Type.unorderedItem);
89                     } else {
90                         current.put(element, parts.getElement(i + 1));
91                     }
92                 }
93             }
94             PrintWriter out = FileUtilities.openUTF8Writer(OUT_DIRECTORY + subdirectory, locale + ".json");
95             main.print(out, 0);
96             out.close();
97         }
98     }
99 
100     static Relation<String, String> extraDistinguishing = Relation.of(new TreeMap<String, Set<String>>(), LinkedHashSet.class);
101     static {
putAll(extraDistinguishing, "dayPeriodRule", "earlyMorning", "before", "from")102         putAll(extraDistinguishing, "dayPeriodRule", "earlyMorning", "before", "from");
103     }
104 
putAll(Relation r, K key, V... values)105     static <K, V> void putAll(Relation r, K key, V... values) {
106         r.putAll(key, Arrays.asList(values));
107     }
108 
isDistinguishing(DtdType dtdType, final String element, final String attribute)109     private static boolean isDistinguishing(DtdType dtdType, final String element, final String attribute) {
110         // <mapZone other="Afghanistan" territory="001" type="Asia/Kabul"/> result is the type!
111         // <deprecatedItems elements="variant" attributes="type" values="BOKMAL NYNORSK AALAND POLYTONI"/>
112         // ugly: if there are values, then everything else is distinguishing, ow if there are attibutes, elements are
113         if (element.equals("deprecatedItems")) {
114 
115         }
116         Set<String> extras = extraDistinguishing.getAll(element);
117         if (extras != null && extras.contains(attribute)) return true;
118         if (EXTRA_DISTINGUISHING.contains(attribute)) return true;
119         return CLDRFile.isDistinguishing(dtdType, element, attribute);
120     }
121 
rewrite(DtdType dtdType, XPathParts parts, String value, Relation<String, String> element2Attributes, XPathParts out)122     private static void rewrite(DtdType dtdType, XPathParts parts, String value,
123         Relation<String, String> element2Attributes, XPathParts out) {
124         out.clear();
125         int size = parts.size();
126         for (int i = 1; i < size; ++i) {
127             final String element = parts.getElement(i);
128             out.addElement(element);
129 
130             // turn a path into a revised path. All distinguished attributes (including those not currently on the
131             // string)
132             // get turned into extra element/element pairs, starting with _
133             // all non-distinguishing attributes get turned into separate children
134             // a/b[@non="y"][@dist="x"]/w : z =>
135             // a/b/_dist/x/_non=y
136             // a/b/_dist/x/w=z
137             Collection<String> actualAttributeKeys = parts.getAttributeKeys(i);
138             boolean isOrdered = actualAttributeKeys.contains("_q");
139             Set<String> possibleAttributeKeys = element2Attributes.getAll(element);
140 
141             for (final String attribute : actualAttributeKeys) {
142                 String attributeValue = parts.getAttributeValue(i, attribute);
143                 if (!isDistinguishing(dtdType, element, attribute)) {
144                     out.addAttribute(attribute, attributeValue);
145                 }
146             }
147             if (possibleAttributeKeys != null) {
148                 for (final String attribute : possibleAttributeKeys) {
149                     if (isDistinguishing(dtdType, element, attribute)) {
150                         if (attribute.equals("alt")) continue; // TODO fix
151                         String attributeValue = parts.getAttributeValue(i, attribute);
152                         out.addElement("_" + attribute);
153                         if (attributeValue == null) {
154                             attributeValue = "?";
155                         }
156                         out.addElement(attributeValue);
157                     }
158                 }
159             }
160             if (isOrdered) {
161                 Map<String, String> lastAttributes = out.getAttributes(-2);
162                 lastAttributes.put("_q", "_q");
163             }
164         }
165         if (value.length() > 0) {
166             out.addElement(value);
167         }
168 
169         if (!COMPACT) {
170             return;
171         }
172         if (parts.getElement(-1).equals("type")) {
173             String key = parts.getAttributeValue(-1, "key");
174             if (key != null) {
175                 parts.setElement(-2, key + "Key");
176                 parts.putAttributeValue(-1, "key", null);
177             }
178             // fall thru
179         }
180         if (parts.getElement(1).equals("localeDisplayNames")) {
181             String element2 = parts.getElement(2);
182             if (!element2.endsWith("Pattern")) {
183                 if (element2.endsWith("s")) {
184                     element2 = element2.substring(0, element2.length() - 1);
185                 }
186                 parts.setElement(2, element2 + "Names");
187             }
188             parts.removeElement(1);
189         }
190         if (parts.getElement(1).equals("dates")) {
191             parts.removeElement(1);
192             String element1 = parts.getElement(1);
193             if (element1.equals("timeZoneNames")) {
194                 String main = parts.getElement(2);
195                 if (main.equals("zone") || main.equals("metazone")) {
196                     parts.setElement(1, main + "Names");
197                 }
198                 return;
199             }
200         }
201         if (parts.getElement(1).equals("numbers") && parts.getElement(2).equals("currencies")) {
202             parts.removeElement(1);
203             return;
204         }
205     }
206 
207     static class ElementName {
208         String oldBase;
209         String base;
210         boolean replacedBase;
211         StringBuilder suffix = new StringBuilder();
212 
reset(String element)213         public void reset(String element) {
214             suffix.setLength(0);
215             base = oldBase = element;
216             replacedBase = false;
217         }
218 
add(String attribute, String attributeValue)219         public void add(String attribute, String attributeValue) {
220             if (REPLACING_BASE.contains(attribute)) {
221                 if (replacedBase) {
222                     System.out.println("ERROR: Two replacement types on same element!!\t" + oldBase + "," + base + ","
223                         + attribute + "," + attributeValue);
224                 } else {
225                     replacedBase = true;
226                     base = attributeValue;
227                     return;
228                 }
229             }
230             suffix.append('$').append(attribute).append('=').append(attributeValue);
231         }
232 
toString()233         public String toString() {
234             if (suffix == null) {
235                 return base;
236             }
237             return base + suffix;
238         }
239     }
240 
241     static abstract class Item {
242         protected Item parent;
243 
Item(Item parent)244         public Item(Item parent) {
245             this.parent = parent;
246         }
247 
size()248         public abstract int size();
249 
250         enum Type {
251             unorderedItem, orderedItem
252         }
253 
print(Appendable result, int i)254         public abstract Appendable print(Appendable result, int i);
255 
indent(Appendable result, int i)256         protected Appendable indent(Appendable result, int i) throws IOException {
257             return result.append(getIndent(i));
258         }
259 
getIndent(int i)260         protected String getIndent(int i) {
261             return Utility.repeat("    ", i);
262         }
263 
appendString(Appendable result, String string, int indent)264         public Appendable appendString(Appendable result, String string, int indent) throws IOException {
265             result.append('"');
266             for (int i = 0; i < string.length(); ++i) {
267                 // http://www.json.org/
268                 // any-Unicode-character-except-"-or-\-or-control-character
269                 // uses UTF16
270                 char ch = string.charAt(i);
271                 switch (ch) {
272                 case '\"':
273                     result.append("\\\"");
274                     break;
275                 case '\\':
276                     result.append("\\\\");
277                     break;
278                 case '/':
279                     result.append("\\/");
280                     break;
281                 case '\b':
282                     result.append("\\b");
283                     break;
284                 case '\f':
285                     result.append("\\f");
286                     break;
287                 case '\n':
288                     if (indent < 0) {
289                         result.append("\\n");
290                     } else {
291                         result.append('\n').append(getIndent(indent));
292                     }
293                     break;
294                 case '\r':
295                     result.append("\\r");
296                     break;
297                 case '\t':
298                     result.append("\\t");
299                     break;
300                 default:
301                     if (ch <= 0x1F || 0x7F <= ch && ch <= 0x9F) {
302                         result.append("\\u").append(Utility.hex(ch, 4));
303                     } else {
304                         result.append(ch);
305                     }
306                     break;
307                 }
308             }
309             return result.append('"');
310         }
311 
toString()312         public String toString() {
313             return print(new StringBuilder(), 0).toString();
314         }
315 
create(Type ordered)316         protected Item create(Type ordered) {
317             switch (ordered) {
318             case unorderedItem:
319                 return new TableItem(this);
320             case orderedItem:
321                 return new ArrayItem(this);
322             default:
323                 throw new UnsupportedOperationException();
324             }
325         }
326 
makeSubItem(String element, Type ordered)327         public abstract Item makeSubItem(String element, Type ordered);
328 
put(String element, String value)329         public abstract void put(String element, String value);
330 
getRoot()331         public Item getRoot() {
332             if (parent == null) {
333                 return this;
334             } else {
335                 return parent.getRoot();
336             }
337         }
338     }
339 
340     static class TableItem extends Item {
TableItem(Item parent)341         public TableItem(Item parent) {
342             super(parent);
343         }
344 
345         private Map<String, Item> map = new LinkedHashMap<String, Item>();
346 
get(String element)347         public Item get(String element) {
348             return map.get(element);
349         }
350 
put(String element, String value)351         public void put(String element, String value) {
352             Item old = map.get(element);
353             if (old != null) {
354                 if (old instanceof StringItem) {
355                     if (value.equals(((StringItem) old).value)) {
356                         return;
357                     }
358                 }
359                 throw new IllegalArgumentException("ERROR: Table already has object: " + element + ", " + old + ", "
360                     + value + ", " + getRoot().toString());
361             }
362             map.put(element, new StringItem(value));
363         }
364 
makeSubItem(String element, Type ordered)365         public Item makeSubItem(String element, Type ordered) {
366             Item result = map.get(element);
367             if (result != null) {
368                 return result;
369             }
370             result = create(ordered);
371             result.parent = this;
372 
373             map.put(element, result);
374             return result;
375         }
376 
print(Appendable result, int i)377         public Appendable print(Appendable result, int i) {
378             try {
379                 if (map.size() == 0) {
380                     result.append("{}");
381                     return result;
382                 }
383                 result.append("{\n");
384                 boolean first = true;
385                 for (String key : map.keySet()) {
386                     Item value = map.get(key);
387                     if (first) {
388                         first = false;
389                     } else {
390                         result.append(",\n");
391                     }
392                     indent(result, i + 1);
393                     appendString(result, key, -1).append(" : ");
394                     value.print(result, i + 1);
395                 }
396                 result.append("\n");
397                 indent(result, i).append("}");
398                 return result;
399             } catch (IOException e) {
400                 throw new ICUUncheckedIOException(e);
401             }
402         }
403 
404         @Override
size()405         public int size() {
406             return map.size();
407         }
408     }
409 
410     static class ArrayItem extends Item {
ArrayItem(Item parent)411         public ArrayItem(Item parent) {
412             super(parent);
413         }
414 
415         private List<Row.R2<String, Item>> list = new ArrayList<Row.R2<String, Item>>();
416 
417         @Override
print(Appendable result, int i)418         public Appendable print(Appendable result, int i) {
419             try {
420                 if (list.size() == 0) {
421                     result.append("[]");
422                     return result;
423                 }
424 
425                 result.append("[\n");
426                 for (int j = 0; j < list.size(); ++j) {
427                     if (j != 0) {
428                         result.append(",\n");
429                     }
430                     indent(result, i + 1);
431                     R2<String, Item> row = list.get(j);
432                     result.append("{");
433                     appendString(result, row.get0(), i + 1);
434                     result.append(" : ");
435                     row.get1().print(result, i + 1);
436                     result.append("}");
437                 }
438                 result.append("\n");
439                 indent(result, i).append("]");
440                 return result;
441             } catch (IOException e) {
442                 throw new IllegalArgumentException(e);
443             }
444         }
445 
makeSubItem(String element, Type ordered)446         public Item makeSubItem(String element, Type ordered) {
447             Item result = create(ordered);
448             list.add(Row.of(element, result));
449             return result;
450         }
451 
put(String element, String value)452         public void put(String element, String value) {
453             list.add(Row.of(element, (Item) new StringItem(value)));
454         }
455 
456         @Override
size()457         public int size() {
458             return list.size();
459         }
460     }
461 
462     static class StringItem extends Item {
463         private String value;
464 
StringItem(String value2)465         public StringItem(String value2) {
466             super(null);
467             value = value2;
468         }
469 
470         @Override
print(Appendable result, int i)471         public Appendable print(Appendable result, int i) {
472             try {
473                 return appendString(result, value, i + 1);
474             } catch (IOException e) {
475                 throw new IllegalArgumentException(e);
476             }
477         }
478 
479         @Override
makeSubItem(String element, Type ordered)480         public Item makeSubItem(String element, Type ordered) {
481             throw new UnsupportedOperationException();
482         }
483 
484         @Override
put(String element, String value)485         public void put(String element, String value) {
486             throw new UnsupportedOperationException();
487         }
488 
489         @Override
size()490         public int size() {
491             throw new UnsupportedOperationException();
492         }
493     }
494 }
495