xref: /external/cldr/tools/cldr-code/src/main/java/org/unicode/cldr/draft/ScriptMetadata.java

package org.unicode.cldr.draft;

import com.google.common.base.Joiner;
import com.ibm.icu.impl.Relation;
import com.ibm.icu.lang.UScript;
import com.ibm.icu.text.Transform;
import com.ibm.icu.text.UTF16;
import com.ibm.icu.util.ICUException;
import com.ibm.icu.util.VersionInfo;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
import org.unicode.cldr.tool.CountryCodeConverter;
import org.unicode.cldr.util.CldrUtility;
import org.unicode.cldr.util.Containment;
import org.unicode.cldr.util.SemiFileReader;
import org.unicode.cldr.util.StandardCodes;
import org.unicode.cldr.util.StandardCodes.LstrType;
import org.unicode.cldr.util.Validity;
import org.unicode.cldr.util.Validity.Status;
import org.unicode.cldr.util.With;

public class ScriptMetadata {
    private static final int MAX_RANK = 33;
    private static final String DATA_FILE = "/org/unicode/cldr/util/data/Script_Metadata.csv";
    private static final VersionInfo UNICODE_VERSION =
            VersionInfo.getInstance(CldrUtility.getProperty("SCRIPT_UNICODE_VERSION", "16"));

    // To get the data, go do the Script MetaData spreadsheet
    // Download As Comma Separated Items into DATA_FILE
    // Set the last string in the UNICODE_VERSION line above to the right Unicode Version (for
    // Unicode beta).
    // Run TestScriptMetadata.
    // Then run GenerateScriptMetadata.
    // See http://cldr.unicode.org/development/updating-codes/updating-script-metadata
    private enum Column {
        // must match the spreadsheet header (caseless compare) or have the alternate header as an
        // argument.
        // doesn't have to be in order
        WR,
        AGE,
        SAMPLE_CODE,
        ID_USAGE("ID Usage (UAX31)"),
        RTL("RTL?"),
        LB_LETTERS("LB letters?"),
        SHAPING_REQ("Shaping Req?"),
        IME("IME?"),
        ORIGIN_COUNTRY("Origin Country"),
        DENSITY("~Density"),
        LANG_CODE,
        HAS_CASE("Has Case?");

        int columnNumber = -1;
        final Set<String> names = new HashSet<>();

        Column(String... alternateNames) {
            names.add(this.name());
            for (String name : alternateNames) {
                names.add(name.toUpperCase(Locale.ENGLISH));
            }
        }

        static void setColumns(String[] headers) {
            for (int i = 0; i < headers.length; ++i) {
                String header = headers[i].toUpperCase(Locale.ENGLISH);
                for (Column v : values()) {
                    if (v.names.contains(header)) {
                        v.columnNumber = i;
                    }
                }
            }
            for (Column v : values()) {
                if (v.columnNumber == -1) {
                    throw new IllegalArgumentException(
                            "Missing field for " + v + ", may need to add additional column alias");
                }
            }
        }

        String getItem(String[] items) {
            return items[columnNumber];
        }

        int getInt(String[] items, int defaultValue) {
            final String item = getItem(items);
            return item.isEmpty() || item.equalsIgnoreCase("n/a")
                    ? defaultValue
                    : Integer.parseInt(item);
        }
    }

    public enum IdUsage {
        UNKNOWN("Other"),
        EXCLUSION("Historic"),
        LIMITED_USE("Limited Use"),
        ASPIRATIONAL("Aspirational"),
        RECOMMENDED("Major Use");

        public final String name;

        private IdUsage(String name) {
            this.name = name;
        }
    }

    public enum Trinary {
        UNKNOWN,
        NO,
        YES
    }

    public enum Shaping {
        UNKNOWN,
        NO,
        MIN,
        YES
    }

    static StandardCodes SC = StandardCodes.make();
    static EnumLookup<Shaping> shapingLookup =
            EnumLookup.of(Shaping.class, null, "n/a", Shaping.UNKNOWN);
    static EnumLookup<Trinary> trinaryLookup =
            EnumLookup.of(Trinary.class, null, "n/a", Trinary.UNKNOWN);
    static EnumLookup<IdUsage> idUsageLookup =
            EnumLookup.of(IdUsage.class, null, "n/a", IdUsage.UNKNOWN);

    public static void addNameToCode(String type, Map<String, String> hashMap) {
        for (String language : SC.getAvailableCodes(type)) {
            Map<String, String> fullData = StandardCodes.getLStreg().get(type).get(language);
            String name = fullData.get("Description");
            hashMap.put(name.toUpperCase(Locale.ENGLISH), language);
        }
    }

    public static final class SkipNewUnicodeException extends ICUException {}

    /**
     * Scripts that either have no known languages as yet (Cpmn) or are used for any language
     * (Brai).
     */
    public static final Set<String> SCRIPTS_WITH_NO_LANGUAGES = Set.of("Brai", "Cpmn");

    public static class Info implements Comparable<Info> {
        public final int rank;
        public final VersionInfo age;
        public final String sampleChar;
        public final IdUsage idUsage;
        public final Trinary rtl;
        public final Trinary lbLetters;
        public final Trinary hasCase;
        public final Shaping shapingReq;
        public final Trinary ime;
        public final int density;
        public final String originCountry;
        public final String likelyLanguage;

        private Info(String[] items) {
            // 3,Han,Hani,1.1,"75,963",字,5B57,China,3,Chinese,zh,Recommended,no,Yes,no,Yes,no
            rank = Math.min(Column.WR.getInt(items, 999), MAX_RANK);
            age = VersionInfo.getInstance(Column.AGE.getItem(items));
            if (age.compareTo(UNICODE_VERSION) > 0) {
                throw new SkipNewUnicodeException();
            }
            // Parse the code point of the sample character, rather than the sample character
            // itself.
            // The code point is more reliable, especially when the spreadsheet has a bug
            // for supplementary characters.
            int sampleCode = Integer.parseInt(Column.SAMPLE_CODE.getItem(items), 16);
            sampleChar = UTF16.valueOf(sampleCode);
            idUsage = idUsageLookup.forString(Column.ID_USAGE.getItem(items));
            rtl = trinaryLookup.forString(Column.RTL.getItem(items));
            lbLetters = trinaryLookup.forString(Column.LB_LETTERS.getItem(items));
            shapingReq = shapingLookup.forString(Column.SHAPING_REQ.getItem(items));
            ime = trinaryLookup.forString(Column.IME.getItem(items));
            hasCase = trinaryLookup.forString(Column.HAS_CASE.getItem(items));
            density = Column.DENSITY.getInt(items, -1);
            String script = items[2];

            final String countryRaw = Column.ORIGIN_COUNTRY.getItem(items);
            String country = CountryCodeConverter.getCodeFromName(countryRaw, false);
            if (country == null) {
                // Give context when throwing an error. Because this is run in a static init
                // context, the stack trace is typically incorrect when something goes wrong.
                errors.add(
                        "ScriptMetadata.java: Can't map "
                                + countryRaw
                                + " to country/region. Try updating external/alternate_country_names.txt");
            }
            originCountry = country == null ? "ZZ" : country;

            String langCode = Column.LANG_CODE.getItem(items);
            if (langCode.equals("n/a")) {
                langCode = null;
            }
            likelyLanguage = langCode == null ? "und" : langCode;

            // check for bad countries, bad languages

            final Status scriptStatus =
                    Validity.getInstance().getCodeToStatus(LstrType.script).get(script);
            if (!(scriptStatus == Status.special || scriptStatus == Status.unknown)) {
                final Status countryStatus =
                        Validity.getInstance().getCodeToStatus(LstrType.region).get(originCountry);
                if (countryStatus != Status.regular) {
                    errors.add(
                            "ScriptMetadata.java: the country ("
                                    + originCountry
                                    + ") for "
                                    + script
                                    + " is not valid: "
                                    + countryStatus);
                }
                final Status languageStatus =
                        Validity.getInstance()
                                .getCodeToStatus(LstrType.language)
                                .get(likelyLanguage);
                if (languageStatus != Status.regular
                        // make exception for scripts that has no known languages
                        && !SCRIPTS_WITH_NO_LANGUAGES.contains(script)) {
                    errors.add(
                            "ScriptMetadata.java: the likely language ("
                                    + likelyLanguage
                                    + ") for "
                                    + script
                                    + " is not valid: "
                                    + languageStatus);
                }
            }
        }

        public Info(Info other, String string, String sampleCharacter) {
            rank = other.rank;
            age = other.age;
            sampleChar = sampleCharacter == null ? other.sampleChar : sampleCharacter;
            idUsage = other.idUsage;
            rtl = other.rtl;
            lbLetters = other.lbLetters;
            hasCase = other.hasCase;
            shapingReq = other.shapingReq;
            ime = "IME:YES".equals(string) ? Trinary.YES : other.ime;
            density = other.density;
            originCountry = other.originCountry;
            likelyLanguage = other.likelyLanguage;
        }

        // public Trinary parseTrinary(Column title, String[] items) {
        // return Trinary.valueOf(fix(title.getItem(items)).toUpperCase(Locale.ENGLISH));
        // }
        String fix(String in) {
            return in.toUpperCase(Locale.ENGLISH)
                    .replace("N/A", "UNKNOWN")
                    .replace("?", "UNKNOWN")
                    .replace("RTL", "YES");
        }

        @Override
        public String toString() {
            return rank
                    + "\tSample: "
                    + sampleChar
                    + "\tCountry: "
                    + getName("territory", originCountry)
                    + " ("
                    + originCountry
                    + ")"
                    + "\tLanguage: "
                    + getName("language", likelyLanguage)
                    + " ("
                    + likelyLanguage
                    + ")"
                    + "\tId: "
                    + idUsage
                    + "\tRtl: "
                    + rtl
                    + "\tLb: "
                    + lbLetters
                    + "\tShape: "
                    + shapingReq
                    + "\tIme: "
                    + ime
                    + "\tCase: "
                    + hasCase
                    + "\tDensity: "
                    + density;
        }

        public Object getName(String type, String code) {
            List<String> fullData = SC.getFullData(type, code);
            if (fullData == null) {
                return "unavailable";
            }
            return fullData.get(0);
        }

        @Override
        public int compareTo(Info o) {
            // we don't actually care what the comparison value is, as long as it is transitive and
            // consistent with equals.
            return toString().compareTo(o.toString());
        }
    }

    public static Set<String> errors = new LinkedHashSet<>();
    static HashMap<String, Integer> titleToColumn = new HashMap<>();

    private static class MyFileReader extends SemiFileReader {
        private Map<String, Info> data = new HashMap<>();

        @Override
        protected boolean isCodePoint() {
            return false;
        }

        @Override
        protected String[] splitLine(String line) {
            return CldrUtility.splitCommaSeparated(line);
        }

        @Override
        protected boolean handleLine(int lineCount, int start, int end, String[] items) {
            if (items[0].startsWith("For help") || items[0].isEmpty()) {
                return true; // header lines
            }
            if (items[0].equals("WR")) {
                Column.setColumns(items);
                return true;
            }
            Info info;
            try {
                info = new Info(items);
            } catch (SkipNewUnicodeException e) {
                return true;
            } catch (Exception e) {
                errors.add(
                        e.getClass().getName()
                                + "\t"
                                + e.getMessage()
                                + "\t"
                                + Arrays.asList(items));
                return true;
            }

            String script = items[2];
            data.put(script, info);
            Set<String> extras = EXTRAS.get(script);
            if (extras != null) {
                for (String script2 : extras) {
                    Info info2 = info;
                    if (script2.equals("Jpan")) {
                        // HACK
                        info2 = new Info(info, "IME:YES", null);
                    } else if (script2.equals("Jamo")) {
                        info2 = new Info(info, null, "ᄒ");
                    }
                    data.put(script2, info2);
                }
            }
            return true;
        }

        @Override
        public MyFileReader process(Class<?> classLocation, String fileName) {
            super.process(classLocation, fileName);
            return this;
        }

        private Map<String, Info> getData() {
            if (!errors.isEmpty()) {
                throw new RuntimeException(Joiner.on("\n\t").join(errors));
            }
            return Collections.unmodifiableMap(data);
        }
    }

    public enum Groupings {
        EUROPEAN("150"),
        MIDDLE_EASTERN("145"),
        CENTRAL_ASIAN("143"),
        SOUTH_ASIAN("034"),
        SOUTHEAST_ASIAN("035"),
        EAST_ASIAN("030"),
        AFRICAN("002"),
        AMERICAN("019"),
        ;
        public final Set<String> scripts;

        private Groupings(String... regions) {
            scripts =
                    With.in(getScripts())
                            .toUnmodifiableCollection(
                                    new ScriptMetadata.RegionFilter(regions),
                                    new TreeSet<String>());
        }
    }

    static class RegionFilter implements com.ibm.icu.text.Transform<String, String> {
        final String[] containingRegion;

        RegionFilter(String... containingRegion) {
            this.containingRegion = containingRegion;
        }

        @Override
        public String transform(String script) {
            String currentRegion = getInfo(script).originCountry;
            while (true) {
                for (String s : containingRegion) {
                    if (s.equals(currentRegion)) {
                        return script;
                    }
                }
                if (currentRegion.equals("001") || currentRegion.equals("ZZ")) {
                    return null;
                }
                currentRegion = Containment.getContainer(currentRegion);
            }
        }
    }

    static Relation<String, String> EXTRAS =
            Relation.of(new HashMap<String, Set<String>>(), HashSet.class);

    static {
        EXTRAS.put("Hani", "Hans");
        EXTRAS.put("Hani", "Hant");
        EXTRAS.put("Hani", "Hanb");
        EXTRAS.put("Hang", "Kore");
        EXTRAS.put("Hang", "Jamo");
        EXTRAS.put("Hira", "Jpan");
        EXTRAS.freeze();
    }

    static final Map<String, Info> data =
            new MyFileReader().process(ScriptMetadata.class, DATA_FILE).getData();

    public static Info getInfo(String s) {
        Info result = data.get(s);
        if (result == null) {
            try {
                String name2 = UScript.getShortName(UScript.getCodeFromName(s));
                result = data.get(name2);
            } catch (Exception e) {
            }
        }
        return result;
    }

    public static Set<String> getScripts() {
        return data.keySet();
    }

    public static Info getInfo(int i) {
        return data.get(UScript.getShortName(i));
    }

    public static Set<Entry<String, Info>> iterable() {
        return data.entrySet();
    }

    /**
     * Specialized scripts
     *
     * @return
     */
    public static Set<String> getExtras() {
        return EXTRAS.values();
    }

    public static Transform<String, String> TO_SHORT_SCRIPT =
            new Transform<>() {
                @Override
                public String transform(String source) {
                    return UScript.getShortName(UScript.getCodeFromName(source));
                }
            };
    public static Transform<String, String> TO_LONG_SCRIPT =
            new Transform<>() {
                @Override
                public String transform(String source) {
                    return UScript.getName(UScript.getCodeFromName(source));
                }
            };
}