android-14.0.0_r21/search

package org.unicode.cldr.tool;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.unicode.cldr.draft.FileUtilities;
import org.unicode.cldr.test.DisplayAndInputProcessor;
import org.unicode.cldr.test.SubmissionLocales;
import org.unicode.cldr.tool.FormattedFileWriter.Anchors;
import org.unicode.cldr.tool.Option.Options;
import org.unicode.cldr.tool.Option.Params;
import org.unicode.cldr.util.CLDRConfig;
import org.unicode.cldr.util.CLDRFile;
import org.unicode.cldr.util.CLDRFile.Status;
import org.unicode.cldr.util.CLDRPaths;
import org.unicode.cldr.util.CldrUtility;
import org.unicode.cldr.util.Counter;
import org.unicode.cldr.util.DtdData;
import org.unicode.cldr.util.DtdType;
import org.unicode.cldr.util.Factory;
import org.unicode.cldr.util.LanguageTagParser;
import org.unicode.cldr.util.Level;
import org.unicode.cldr.util.LocaleIDParser;
import org.unicode.cldr.util.Organization;
import org.unicode.cldr.util.Pair;
import org.unicode.cldr.util.PathHeader;
import org.unicode.cldr.util.PathHeader.PageId;
import org.unicode.cldr.util.PathStarrer;
import org.unicode.cldr.util.PatternCache;
import org.unicode.cldr.util.SimpleXMLSource;
import org.unicode.cldr.util.StandardCodes;
import org.unicode.cldr.util.SupplementalDataInfo;
import org.unicode.cldr.util.SupplementalDataInfo.CoverageVariableInfo;
import org.unicode.cldr.util.TransliteratorUtilities;
import org.unicode.cldr.util.XMLFileReader;
import org.unicode.cldr.util.XPathParts;

import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Multimap;
import com.google.common.collect.TreeMultimap;
import com.ibm.icu.impl.Relation;
import com.ibm.icu.impl.Row.R2;
import com.ibm.icu.impl.Row.R3;
import com.ibm.icu.impl.Row.R4;
import com.ibm.icu.text.NumberFormat;
import com.ibm.icu.text.UnicodeSet;
import com.ibm.icu.util.ICUUncheckedIOException;
import com.ibm.icu.util.Output;

public class ChartDelta extends Chart {
    private static final boolean verbose_skipping = false;

    private static final String DEFAULT_DELTA_DIR_NAME = "delta";
    private static final String DEFAULT_CHURN_DIR_NAME = "churn";

    private static final boolean SKIP_REFORMAT_ANNOTATIONS = ToolConstants.PREV_CHART_VERSION.compareTo("30") >= 0;

    private static final PageId DEBUG_PAGE_ID = PageId.DayPeriod;

    private static final SupplementalDataInfo SUPPLEMENTAL_DATA_INFO = CLDRConfig.getInstance().getSupplementalDataInfo();

    private enum MyOptions {
        fileFilter(new Params().setHelp("filter files by dir/locale, eg: ^main/en$ or .*/en").setMatch(".*")),
        orgFilter(new Params().setHelp("filter files by organization").setMatch(".*")),
        Vxml(new Params().setHelp("use cldr-aux for the base directory")),
        coverageFilter(new Params().setHelp("filter files by coverage").setMatch(".*")),
        directory(new Params().setHelp("Set the output directory name").setDefault(DEFAULT_DELTA_DIR_NAME).setMatch(".*")),
        verbose(new Params().setHelp("verbose debugging messages")),
        highLevelOnly(new Params().setHelp("check high-level paths (churn) only").setFlag('H')),
        ;

        // BOILERPLATE TO COPY
        final Option option;

        private MyOptions(Params params) {
            option = new Option(this, params);
        }

        private static Options myOptions = new Options();
        static {
            for (MyOptions option : MyOptions.values()) {
                myOptions.add(option, option.option);
            }
        }

        private static Set<String> parse(String[] args) {
            return myOptions.parse(MyOptions.values()[0], args, true);
        }
    }

    private final Matcher fileFilter;
    private final String dirName; // "delta" or "churn" or set as option
    private final String chartNameCap; // capitalized, e.g., "Delta" or "Churn"
    private final String DIR; // full path of output folder
    private final Level minimumPathCoverage;
    private final boolean verbose;

    /**
     * If true, check only high-level paths, i.e., paths for which any changes
     * have high potential to cause disruptive "churn"
     */
    private final boolean highLevelOnly;

    public static void main(String[] args) {
        main(args, false);
    }

    public static void main(String[] args, boolean highLevelOnly) {
        System.out.println("use -DCHART_VERSION=36.0 -DPREV_CHART_VERSION=34.0 to generate the differences between v36 and v34.");
        MyOptions.parse(args);
        Matcher fileFilter = !MyOptions.fileFilter.option.doesOccur() ? null : PatternCache.get(MyOptions.fileFilter.option.getValue()).matcher("");
        if (MyOptions.orgFilter.option.doesOccur()) {
            if (MyOptions.fileFilter.option.doesOccur()) {
                throw new IllegalArgumentException("Can't have both fileFilter and orgFilter");
            }
            String rawOrg = MyOptions.orgFilter.option.getValue();
            Organization org = Organization.fromString(rawOrg);
            Set<String> locales = StandardCodes.make().getLocaleCoverageLocales(org);
            fileFilter = PatternCache.get("^(main|annotations)/(" + Joiner.on("|").join(locales) + ")$").matcher("");
        }
        Level coverage = !MyOptions.coverageFilter.option.doesOccur() ? null : Level.fromString(MyOptions.coverageFilter.option.getValue());
        boolean verbose = MyOptions.verbose.option.doesOccur();
        if (MyOptions.highLevelOnly.option.doesOccur()) {
            highLevelOnly = true;
        }
        String dirName = MyOptions.directory.option.getValue();
        if (highLevelOnly && DEFAULT_DELTA_DIR_NAME.equals(dirName)) {
            System.out.println("For highLevelOnly, changing directory from " + DEFAULT_DELTA_DIR_NAME
                    + " to " + DEFAULT_CHURN_DIR_NAME);
            dirName = DEFAULT_CHURN_DIR_NAME;
        }
        ChartDelta temp = new ChartDelta(fileFilter, coverage, dirName, verbose, highLevelOnly);
        temp.writeChart(null);
        temp.showTotals();
        if (highLevelOnly) {
            HighLevelPaths.reportHighLevelPathUsage();
        }
        System.out.println("Finished. Files may have been created in these directories:");
        System.out.println(temp.DIR);
        System.out.println(getTsvDir(temp.DIR, temp.dirName));
    }

    private ChartDelta(Matcher fileFilter, Level coverage, String dirName, boolean verbose, boolean highLevelOnly) {
        this.fileFilter = fileFilter;
        this.verbose = verbose;
        this.highLevelOnly = highLevelOnly;
        this.dirName = dirName;
        this.chartNameCap = dirName.substring(0, 1).toUpperCase() + dirName.substring(1);
        this.DIR = CLDRPaths.CHART_DIRECTORY + dirName;
        this.minimumPathCoverage = coverage;
    }

    private static final String SEP = "\u0001";
    private static final boolean DEBUG = false;
    private static final String DEBUG_FILE = null; // "windowsZones.xml";
    static Pattern fileMatcher = PatternCache.get(".*");

    static PathHeader.Factory phf = PathHeader.getFactory(ENGLISH);
    static final Set<String> DONT_CARE = new HashSet<>(Arrays.asList("draft", "standard", "reference"));

    @Override
    public String getDirectory() {
        return DIR;
    }

    @Override
    public String getTitle() {
        return chartNameCap + " Charts";
    }

    @Override
    public String getFileName() {
        return "index";
    }

    @Override
    public String getExplanation() {
        return "<p>Charts showing the differences from the last version. "
            + "Titles prefixed by ¤ are special: either the locale data summary or supplemental data. "
            + "Not all changed data is charted yet. For details see each chart.</p>";
    }

    @Override
    public void writeContents(FormattedFileWriter pw) throws IOException {
        FormattedFileWriter.Anchors anchors = new FormattedFileWriter.Anchors();
        FileUtilities.copyFile(ChartDelta.class, "index.css", getDirectory());
        FormattedFileWriter.copyIncludeHtmls(getDirectory(), true);
        counter.clear();
        fileCounters.clear();
        writeNonLdmlPlain(anchors);
        writeLdml(anchors);
        pw.setIndex("Main Chart Index", "../index.html");
        pw.write(anchors.toString());
    }

    private static class PathHeaderSegment extends R3<PathHeader, Integer, String> {
        public PathHeaderSegment(PathHeader b, int elementIndex, String attribute) {
            super(b, elementIndex, attribute);
        }
    }

    private static class PathDiff extends R4<PathHeaderSegment, String, String, String> {
        public PathDiff(String locale, PathHeaderSegment pathHeaderSegment, String oldValue, String newValue) {
            super(pathHeaderSegment, locale, oldValue, newValue);
        }
    }

    private static final CLDRFile EMPTY_CLDR = new CLDRFile(new SimpleXMLSource("und").freeze());

    private static final File CLDR_BASE_DIR = CLDRConfig.getInstance().getCldrBaseDirectory();

    private enum ChangeType {
        added, deleted, changed, same;
        public static ChangeType get(String oldValue, String currentValue) {
            return oldValue == null ? added
                : currentValue == null ? deleted
                    : oldValue.equals(currentValue) ? same
                        : changed;
        }
    }

    private Counter<ChangeType> counter = new Counter<>();
    private Map<String, Counter<ChangeType>> fileCounters = new TreeMap<>();
    private Set<String> badHeaders = new TreeSet<>();

    /**
     * Add the count of changed items
     */
    private void addChange(String file, ChangeType changeType, int count) {
        counter.add(changeType, count); // unified add
        Counter<ChangeType> fileCounter = fileCounters.get(file);
        if (fileCounter == null) {
            fileCounters.put(file, fileCounter = new Counter<>());
        }
        fileCounter.add(changeType, count);
    }

    private void showTotals() {
        try (PrintWriter pw = FileUtilities.openUTF8Writer(getTsvDir(DIR, dirName), dirName + "_summary.tsv")) {
            // pw.println("# percentages are of *new* total");
            pw.print("# dir\tfile");
            for (ChangeType item : ChangeType.values()) {
                pw.print("\t" + (item == ChangeType.same ? "total" : item.toString()));
            }
            pw.println();
            showTotal(pw, "TOTAL/", counter);

            for (Entry<String, Counter<ChangeType>> entry : fileCounters.entrySet()) {
                showTotal(pw, entry.getKey(), entry.getValue());
            }
            for (String s : badHeaders) {
                pw.println(s);
            }
            // pw.println("# EOF");
        } catch (IOException e) {
            throw new ICUUncheckedIOException(e);
        }
    }

    private void showTotal(PrintWriter pw, String title2, Counter<ChangeType> counter2) {
        long total = counter2.getTotal();
        NumberFormat pf = NumberFormat.getPercentInstance();
        pf.setMinimumFractionDigits(2);
        NumberFormat nf = NumberFormat.getIntegerInstance();
        pw.print(title2.replace("/", "\t"));
        for (ChangeType item : ChangeType.values()) {
            if (item == ChangeType.same) {
                pw.print("\t" + nf.format(total));
            } else {
                final long current = counter2.getCount(item);
                pw.print("\t" + nf.format(current));
            }
        }
        pw.println();
    }

    /**
     *
     * @param anchors
     * @throws IOException
     *
     * TODO: shorten the function using subroutines
     */
    private void writeLdml(Anchors anchors)  throws IOException {
        try (PrintWriter tsvFile = FileUtilities.openUTF8Writer(getTsvDir(DIR, dirName), dirName + ".tsv");
            PrintWriter tsvCountFile = FileUtilities.openUTF8Writer(getTsvDir(DIR, dirName), dirName + "_count.tsv");
            ) {
            tsvFile.println("# Section\tPage\tHeader\tCode\tLocale\tOld\tNew\tLevel");

            // set up factories
            List<Factory> factories = new ArrayList<>();
            List<Factory> oldFactories = new ArrayList<>();

            Counter<PathHeader> counts = new Counter<>();

            String dirBase = ToolConstants.getBaseDirectory(ToolConstants.CHART_VERSION);
            String prevDirBase = ToolConstants.getBaseDirectory(ToolConstants.PREV_CHART_VERSION);

            for (String dir : DtdType.ldml.directories) {
                if (dir.equals("annotationsDerived") || dir.equals("casing")) {
                    continue;
                }
                String current = dirBase + "common/" + dir;
                String past = prevDirBase + "common/" + dir;
                try {
                    factories.add(Factory.make(current, ".*"));
                } catch (Exception e1) {
                    System.out.println("Skipping: " + dir + "\t" + e1.getMessage());
                    continue; // skip where the directories don't exist in old versions
                }
                try {
                    oldFactories.add(Factory.make(past, ".*"));
                } catch (Exception e) {
                    System.out.println("Couldn't open factory: " + past);
                    past = null;
                    oldFactories.add(null);
                }
                System.out.println("Will compare: " + dir + "\t\t" + current + "\t\t" + past);
            }
            if (factories.isEmpty()) {
                throw new IllegalArgumentException("No factories found for "
                    + dirBase + ": " + DtdType.ldml.directories);
            }
            // get a list of all the locales to cycle over

            Relation<String, String> baseToLocales = Relation.of(new TreeMap<String, Set<String>>(), HashSet.class);
            Matcher m = fileMatcher.matcher("");
            Set<String> defaultContents = SDI.getDefaultContentLocales();
            LanguageTagParser ltp = new LanguageTagParser();
            LikelySubtags ls = new LikelySubtags();
            for (String file : factories.get(0).getAvailable()) {
                if (defaultContents.contains(file)) {
                    continue;
                }
                if (!m.reset(file).matches()) {
                    continue;
                }
                String base = file.equals("root") ? "root" : ltp.set(ls.minimize(file)).getLanguageScript();
                baseToLocales.put(base, file);
            }

            // do keyboards later

            Status currentStatus = new Status();
            Status oldStatus = new Status();
            Set<PathDiff> diff = new TreeSet<>();
            Set<String> paths = new HashSet<>();

            Relation<PathHeader, String> diffAll = Relation.of(new TreeMap<PathHeader, Set<String>>(), TreeSet.class);
            for (Entry<String, Set<String>> baseNLocale : baseToLocales.keyValuesSet()) {
                String base = baseNLocale.getKey();
                for (int i = 0; i < factories.size(); ++i) {
                    Factory factory = factories.get(i);
                    Factory oldFactory = oldFactories.get(i);
                    List<File> sourceDirs = Arrays.asList(factory.getSourceDirectories());
                    if (sourceDirs.size() != 1) {
                        throw new IllegalArgumentException("Internal error: expect single source dir");
                    }
                    File sourceDir = sourceDirs.get(0);
                    String sourceDirLeaf = sourceDir.getName();
                    boolean resolving = !sourceDirLeaf.contains("subdivisions")
                        && !sourceDirLeaf.contains("transforms");

                    for (String locale : baseNLocale.getValue()) {
                        String nameAndLocale = sourceDirLeaf + "/" + locale;
                        if (fileFilter != null && !fileFilter.reset(nameAndLocale).find()) {
                            if (verbose && verbose_skipping) {
                                System.out.println("SKIPPING: " + nameAndLocale);
                            }
                            continue;
                        }
                        if (verbose) {
                            System.out.println(nameAndLocale);
                        }
                        CLDRFile current = makeWithFallback(factory, locale, resolving);
                        CLDRFile old = makeWithFallback(oldFactory, locale, resolving);
                        DisplayAndInputProcessor daip = new DisplayAndInputProcessor(old);

                        if (!locale.equals("root") && current.getLocaleID().equals("root") && old.getLocaleID().equals("root")) {
                            continue;
                        }
                        if (old == EMPTY_CLDR && current == EMPTY_CLDR) {
                            continue;
                        }
                        if (highLevelOnly && !HighLevelPaths.localeIsHighLevel(locale)) {
                            continue;
                        }
                        paths.clear();
                        for (String path : current.fullIterable()) {
                            if (allowPath(locale, path)) {
                                paths.add(path);
                            }
                        }
                        for (String path : old.fullIterable()) {
                            if (!paths.contains(path) && allowPath(locale, path)) {
                                paths.add(path);
                            }
                        }

                        Output<String> reformattedValue = new Output<>();
                        Output<Boolean> hasReformattedValue = new Output<>();

                        for (String path : paths) {
                            if (path.startsWith("//ldml/identity")
                                || path.endsWith("/alias")
                                || path.startsWith("//ldml/segmentations") // do later
                                || path.startsWith("//ldml/rbnf") // do later
                                ) {
                                continue;
                            }
                            PathHeader ph = getPathHeader(path);
                            if (ph == null) {
                                continue;
                            }

                            String oldValue;
                            String currentValue;

                            {
                                String sourceLocaleCurrent = current.getSourceLocaleID(path, currentStatus);
                                String sourceLocaleOld = getReformattedPath(oldStatus, old, path, reformattedValue, hasReformattedValue);

                                // filter out stuff that differs at a higher level
                                if (!sourceLocaleCurrent.equals(locale)
                                    && !sourceLocaleOld.equals(locale)) {
                                    continue;
                                }
                                if (!path.equals(currentStatus.pathWhereFound)
                                    && !path.equals(oldStatus.pathWhereFound)) {
                                    continue;
                                }
                                // fix some incorrect cases?

                                currentValue = current.getStringValue(path);
                                if (CldrUtility.INHERITANCE_MARKER.equals(currentValue)) {
                                    currentValue = current.getBaileyValue(path, null, null);
                                }

                                String oldRawValue = hasReformattedValue.value ? reformattedValue.value : old.getStringValue(path);
                                if (CldrUtility.INHERITANCE_MARKER.equals(oldRawValue)) {
                                    oldRawValue = old.getBaileyValue(path, null, null);
                                }
                                // ignore differences due to old DAIP
                                oldValue = dontDaipValue(oldRawValue, path) ? oldRawValue : daip.processInput(path, oldRawValue, null);
                            }
                            if (highLevelOnly && new SuspiciousChange(oldValue, currentValue, path, locale).isDisruptive() == false) {
                                continue;
                            }
                            // handle non-distinguishing attributes
                            addPathDiff(sourceDir, old, current, locale, ph, diff);

                            addValueDiff(sourceDir, oldValue, currentValue, locale, ph, diff, diffAll);
                        }
                    }
                }
                writeDiffs(anchors, base, diff, tsvFile, counts);
                diff.clear();
            }
            writeDiffs(diffAll);

            writeCounter(tsvCountFile, "Count", counts);
        }
    }

    public boolean dontDaipValue(String oldRawValue, String path) {
        return oldRawValue == null || path.startsWith("//ldml/collations");
    }

    private boolean allowPath(String locale, String path) {
        if (minimumPathCoverage != null) {
            Level pathLevel = SUPPLEMENTAL_DATA_INFO.getCoverageLevel(path, locale);
            if (minimumPathCoverage.compareTo(pathLevel) < 0) {
                return false;
            }
        }
        return true;
    }

    private String getReformattedPath(Status oldStatus, CLDRFile old, String path, Output<String> value, Output<Boolean> hasReformattedValue) {
        if (SKIP_REFORMAT_ANNOTATIONS || !path.startsWith("//ldml/annotations/")) {
            hasReformattedValue.value = Boolean.FALSE;
            return old.getSourceLocaleID(path, oldStatus);
        }
        // OLD:     <annotation cp='[��]' tts='grinning face'>face; grin</annotation>
        // NEW:     <annotation cp="��">face | grin</annotation>
        //          <annotation cp="��" type="tts">grinning face</annotation>
        // from the NEW paths, get the OLD values
        XPathParts parts = XPathParts.getFrozenInstance(path).cloneAsThawed(); // not frozen, for removeAttribute
        boolean isTts = parts.getAttributeValue(-1, "type") != null;
        if (isTts) {
            parts.removeAttribute(-1, "type");
        }
        String cp = parts.getAttributeValue(-1, "cp");
        parts.setAttribute(-1, "cp", "[" + cp + "]");

        String oldStylePath = parts.toString();
        String temp = old.getStringValue(oldStylePath);
        if (temp == null) {
            hasReformattedValue.value = Boolean.FALSE;
        } else if (isTts) {
            String temp2 = old.getFullXPath(oldStylePath);
            value.value = XPathParts.getFrozenInstance(temp2).getAttributeValue(-1, "tts");
            hasReformattedValue.value = Boolean.TRUE;
        } else {
            value.value = temp.replaceAll("\\s*;\\s*", " | ");
            hasReformattedValue.value = Boolean.TRUE;
        }
        return old.getSourceLocaleID(oldStylePath, oldStatus);
    }

    PathStarrer starrer = new PathStarrer().setSubstitutionPattern("%A");

    private PathHeader getPathHeader(String path) {
        try {
            PathHeader ph = phf.fromPath(path);
            if (ph.getPageId() == PageId.Unknown) {
                String star = starrer.set(path);
                badHeaders.add(star);
                return null;
            }
            return ph;
        } catch (Exception e) {
            String star = starrer.set(path);
            badHeaders.add(star);
            // System.err.println("Skipping path with bad PathHeader: " + path);
            return null;
        }
    }

    private CLDRFile makeWithFallback(Factory oldFactory, String locale, boolean resolving) {
        if (oldFactory == null) {
            return EMPTY_CLDR;
        }
        CLDRFile old;
        String oldLocale = locale;
        while (true) { // fall back for old, maybe to root
            try {
                old = oldFactory.make(oldLocale, resolving);
                break;
            } catch (Exception e) {
                oldLocale = LocaleIDParser.getParent(oldLocale);
                if (oldLocale == null) {
                    return EMPTY_CLDR;
                }
            }
        }
        return old;
    }

    private void addPathDiff(File sourceDir, CLDRFile old, CLDRFile current, String locale, PathHeader ph, Set<PathDiff> diff2) {
        String path = ph.getOriginalPath();
        String fullPathCurrent = current.getFullXPath(path);
        String fullPathOld = old.getFullXPath(path);
        if (Objects.equals(fullPathCurrent, fullPathOld)) {
            return;
        }
        XPathParts pathPlain = XPathParts.getFrozenInstance(path);
        XPathParts pathCurrent = fullPathCurrent == null ? pathPlain : XPathParts.getFrozenInstance(fullPathCurrent);
        XPathParts pathOld = fullPathOld == null ? pathPlain : XPathParts.getFrozenInstance(fullPathOld);
        TreeSet<String> fullAttributes = null;
        int size = pathCurrent.size();
        String parentAndName = parentAndName(sourceDir, locale);
        for (int elementIndex = 0; elementIndex < size; ++elementIndex) { // will have same size
            Collection<String> distinguishing = pathPlain.getAttributeKeys(elementIndex);
            Collection<String> attributesCurrent = pathCurrent.getAttributeKeys(elementIndex);
            Collection<String> attributesOld = pathCurrent.getAttributeKeys(elementIndex);
            if (attributesCurrent.isEmpty() && attributesOld.isEmpty()) {
                continue;
            }
            if (fullAttributes == null) {
                fullAttributes = new TreeSet<>();
            } else {
                fullAttributes.clear();
            }
            fullAttributes.addAll(attributesCurrent);
            fullAttributes.addAll(attributesOld);
            fullAttributes.removeAll(distinguishing);
            fullAttributes.removeAll(DONT_CARE);

            // at this point we only have non-distinguishing
            for (String attribute : fullAttributes) {
                String attributeValueOld = pathOld.getAttributeValue(elementIndex, attribute);
                String attributeValueCurrent = pathCurrent.getAttributeValue(elementIndex, attribute);
                if (Objects.equals(attributeValueOld, attributeValueCurrent)) {
                    addChange(parentAndName, ChangeType.same, 1);
                    continue;
                }
                addChange(parentAndName, ChangeType.get(attributeValueOld, attributeValueCurrent), 1);

                PathDiff row = new PathDiff(
                    locale,
                    new PathHeaderSegment(ph, size - elementIndex - 1, attribute),
                    attributeValueOld,
                    attributeValueCurrent);
                if (DEBUG) {
                    System.out.println(row);
                }
                diff2.add(row);
            }
        }
    }

    private String parentAndName(File sourceDir, String locale) {
        return sourceDir.getName() + "/" + locale + ".xml";
    }

    private void addValueDiff(File sourceDir, String valueOld, String valueCurrent, String locale, PathHeader ph, Set<PathDiff> diff,
        Relation<PathHeader, String> diffAll) {
        // handle stuff that can be split specially
        Splitter splitter = getSplitter(ph.getOriginalPath(), valueOld, valueCurrent);
        int count = 1;
        String parentAndName = parentAndName(sourceDir, locale);
        if (Objects.equals(valueCurrent, valueOld)) {
            if (splitter != null && valueCurrent != null) {
                count = splitHandlingNull(splitter, valueCurrent).size();
            }
            addChange(parentAndName, ChangeType.same, count);
        } else {
            if (splitter != null) {
                List<String> setOld = splitHandlingNull(splitter, valueOld);
                List<String> setNew = splitHandlingNull(splitter, valueCurrent);
                int[] sameAndNotInSecond = new int[2];
                valueOld = getFilteredValue(setOld, setNew, sameAndNotInSecond);
                addChange(parentAndName, ChangeType.same, sameAndNotInSecond[0]);
                addChange(parentAndName, ChangeType.deleted, sameAndNotInSecond[1]);
                sameAndNotInSecond[0] = sameAndNotInSecond[1] = 0;
                valueCurrent = getFilteredValue(setNew, setOld, sameAndNotInSecond);
                addChange(parentAndName, ChangeType.added, sameAndNotInSecond[1]);
            } else if (hasUnicodeSetValue(ph.getOriginalPath())) {
                UnicodeSet usOld = valueOld == null ? UnicodeSet.EMPTY : new UnicodeSet(valueOld);
                UnicodeSet usCurrent = valueCurrent == null ? UnicodeSet.EMPTY : new UnicodeSet(valueCurrent);
                UnicodeSet oldOnly = new UnicodeSet(usOld).removeAll(usCurrent);
                UnicodeSet currentOnly = new UnicodeSet(usCurrent).removeAll(usOld);
                addChange(parentAndName, ChangeType.same, usOld.size()-oldOnly.size());
                addChange(parentAndName, ChangeType.deleted, oldOnly.size());
                addChange(parentAndName, ChangeType.added, currentOnly.size());
                valueOld = usOld.size()==oldOnly.size() ? oldOnly.toPattern(false) : "…" + oldOnly + "…";
                valueCurrent = usCurrent.size()==currentOnly.size() ? currentOnly.toPattern(false) : "…" + currentOnly + "…";
            } else {
                addChange(parentAndName, ChangeType.get(valueOld, valueCurrent), count);
            }
            PathDiff row = new PathDiff(locale, new PathHeaderSegment(ph, -1, ""), valueOld, valueCurrent);
            diff.add(row);
            diffAll.put(ph, locale);
        }
    }

    private boolean hasUnicodeSetValue(String xpath) {
        return xpath.startsWith("//ldml/characters/exemplar");
    }

    private List<String> splitHandlingNull(Splitter splitter, String value) {
        return value == null ? null : splitter.splitToList(value);
    }

    private Splitter getSplitter(String path, String valueOld, String valueCurrent) {
        if (path.contains("/annotation") && !path.contains("tts")) {
            return DtdData.BAR_SPLITTER;
        } else if (valueOld != null && valueOld.contains("\n") || valueCurrent != null && valueCurrent.contains("\n")) {
            return DtdData.CR_SPLITTER;
        } else {
            return null;
        }
    }

    /**
     * Return string with all lines from linesToRemove removed
     * @param toGetStringFor
     * @param linesToRemove
     * @return
     */
    private String getFilteredValue(Collection<String> toGetStringFor, Collection<String> linesToRemove,
        int[] sameAndDiff) {
        if (toGetStringFor == null) {
            return null;
        }
        StringBuilder buf = new StringBuilder();
        Set<String> toRemove = linesToRemove == null ? Collections.emptySet() : new HashSet<>(linesToRemove);
        boolean removed = false;
        for (String old : toGetStringFor) {
            if (toRemove.contains(old)) {
                removed = true;
                sameAndDiff[0]++;
            } else {
                sameAndDiff[1]++;
                if (removed) {
                    buf.append("…\n");
                    removed = false;
                }
                buf.append(old).append('\n');
            }
        }
        if (removed) {
            buf.append("…");
        } else if (buf.length() > 0) {
            buf.setLength(buf.length() - 1); // remove final \n
        }
        return buf.toString();
    }

    private void writeDiffs(Anchors anchors, String file, String title, Multimap<PathHeader, String> bcp, PrintWriter tsvFile) {
        if (bcp.isEmpty()) {
            System.out.println("\tDeleting: " + DIR + "/" + file);
            new File(DIR + file).delete();
            return;
        }
        TablePrinter tablePrinter = new TablePrinter()
            .addColumn("Section", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true)
            .addColumn("Page", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true)//.setRepeatDivider(true)
            .addColumn("Header", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true)
            .addColumn("Code", "class='source'", null, "class='source'", false)
            .addColumn("Old", "class='target'", null, "class='target'", false) //  width='20%'
            .addColumn("New", "class='target'", null, "class='target'", false); //  width='20%'
        PathHeader ph1 = phf.fromPath("//supplementalData/metadata/alias/subdivisionAlias[@type=\"TW-TXQ\"]/_reason");
        PathHeader ph2 = phf.fromPath("//supplementalData/metadata/alias/subdivisionAlias[@type=\"LA-XN\"]/_replacement");
        ph1.compareTo(ph2);
        for (Entry<PathHeader, Collection<String>> entry : bcp.asMap().entrySet()) {
            PathHeader ph = entry.getKey();
            if (ph.getPageId() == DEBUG_PAGE_ID) {
                System.out.println(ph + "\t" + ph.getOriginalPath());
            }
            for (String value : entry.getValue()) {
                String[] oldNew = value.split(SEP);
                tablePrinter.addRow()
                .addCell(ph.getSectionId())
                .addCell(ph.getPageId())
                .addCell(ph.getHeader())
                .addCell(ph.getCode())
                .addCell(oldNew[0])
                .addCell(oldNew[1])
                .finishRow();
            }
        }
        writeTable(anchors, file, tablePrinter, title, tsvFile);
    }

    private void writeDiffs(Relation<PathHeader, String> diffAll) {
        TablePrinter tablePrinter = new TablePrinter()
            .addColumn("Section", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true)
            .addColumn("Page", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true)
            .addColumn("Header", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true)
            .addColumn("Code", "class='source'", null, "class='source'", true)
            .addColumn("Locales where different", "class='target'", null, "class='target'", true);
        for (Entry<PathHeader, Set<String>> row : diffAll.keyValuesSet()) {
            PathHeader ph = row.getKey();
            Set<String> locales = row.getValue();
            tablePrinter.addRow()
            .addCell(ph.getSectionId())
            .addCell(ph.getPageId())
            .addCell(ph.getHeader())
            .addCell(ph.getCode())
            .addCell(Joiner.on(" ").join(locales))
            .finishRow();
        }
    }

    private void writeDiffs(Anchors anchors, String file, Set<PathDiff> diff, PrintWriter tsvFile, Counter<PathHeader> counts) {
        if (diff.isEmpty()) {
            return;
        }
        TablePrinter tablePrinter = new TablePrinter()
            .addColumn("Section", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true)
            .addColumn("Page", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true)
            .addColumn("Header", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true)
            .addColumn("Code", "class='source'", null, "class='source'", true)
            .addColumn("Locale", "class='source'", null, "class='source'", true)
            .addColumn("Old", "class='target'", null, "class='target'", true) //  width='20%'
            .addColumn("New", "class='target'", null, "class='target'", true) //  width='20%'
            .addColumn("Level", "class='target'", null, "class='target'", true);

        for (PathDiff row : diff) {
            PathHeaderSegment phs = row.get0();
            counts.add(phs.get0(), 1);
            String locale = row.get1();
            String oldValue = row.get2();
            String currentValue = row.get3();

            PathHeader ph = phs.get0();
            Integer pathIndex = phs.get1();
            String attribute = phs.get2();
            String specialCode = ph.getCode();

            if (!attribute.isEmpty()) {
                specialCode += "_" + attribute;
                if (pathIndex != 0) {
                    specialCode += "|" + pathIndex;
                }
            }
            Level coverageLevel = SUPPLEMENTAL_DATA_INFO.getCoverageLevel(ph.getOriginalPath(), locale);
            String fixedOldValue = oldValue == null ? "▷missing◁" : TransliteratorUtilities.toHTML.transform(oldValue);
            String fixedNewValue = currentValue == null ? "▷removed◁" : TransliteratorUtilities.toHTML.transform(currentValue);

            tablePrinter.addRow()
            .addCell(ph.getSectionId())
            .addCell(ph.getPageId())
            .addCell(ph.getHeader())
            .addCell(specialCode)
            .addCell(locale)
            .addCell(fixedOldValue)
            .addCell(fixedNewValue)
            .addCell(coverageLevel)
            .finishRow();

        }
        String title = ENGLISH.getName(file) + " " + chartNameCap;
        writeTable(anchors, file, tablePrinter, title, tsvFile);

        diff.clear();
    }

    private class ChartDeltaSub extends Chart {
        private String title;
        private String file;
        private TablePrinter tablePrinter;
        private PrintWriter tsvFile;

        private ChartDeltaSub(String title, String file, TablePrinter tablePrinter, PrintWriter tsvFile) {
            super();
            this.title = title;
            this.file = file;
            this.tablePrinter = tablePrinter;
            this.tsvFile = tsvFile;
        }

        @Override
        public String getDirectory() {
            return DIR;
        }

        @Override
        public boolean getShowDate() {
            return false;
        }

        @Override
        public String getTitle() {
            return title;
        }

        @Override
        public String getFileName() {
            return file;
        }

        @Override
        public String getExplanation() {
            return "<p>Lists data fields that differ from the last major version (see versions above)."
                + " Inherited differences in locales are suppressed, except where the source locales are different. "
                + "<p>";
        }

        @Override
        public void writeContents(FormattedFileWriter pw) throws IOException {
            pw.write(tablePrinter.toTable());
            tablePrinter.toTsv(tsvFile);
        }
    }

    private void writeTable(Anchors anchors, String file, TablePrinter tablePrinter, String title, PrintWriter tsvFile) {
        ChartDeltaSub chartDeltaSub = new ChartDeltaSub(title, file, tablePrinter, tsvFile);
        chartDeltaSub.writeChart(anchors);
    }

    private void writeNonLdmlPlain(Anchors anchors) throws IOException {
        try (PrintWriter tsvFile = FileUtilities.openUTF8Writer(getTsvDir(DIR, dirName), dirName + "_supp.tsv");
            PrintWriter tsvCountFile = FileUtilities.openUTF8Writer(getTsvDir(DIR, dirName), dirName + "_supp_count.tsv");
            ) {
            tsvFile.println("# Section\tPage\tHeader\tCode\tOld\tNew");

            Multimap<PathHeader, String> bcp = TreeMultimap.create();
            Multimap<PathHeader, String> supplemental = TreeMultimap.create();
            Multimap<PathHeader, String> transforms = TreeMultimap.create();

            Counter<PathHeader> countSame = new Counter<>();
            Counter<PathHeader> countAdded = new Counter<>();
            Counter<PathHeader> countDeleted = new Counter<>();

            for (String dir : new File(CLDRPaths.BASE_DIRECTORY + "common/").list()) {
                if (DtdType.ldml.directories.contains(dir)
                    || dir.equals(".DS_Store")
                    || dir.equals("dtd") // TODO as flat files
                    || dir.equals("properties") // TODO as flat files
                    || dir.equals("uca") // TODO as flat files
                    ) {
                    continue;
                }
                File dirOld = new File(PREV_CHART_VERSION_DIRECTORY + "common/" + dir);
                System.out.println("\tLast dir: " + dirOld);
                File dir2 = new File(CHART_VERSION_DIRECTORY + "common/" + dir);
                System.out.println("\tCurr dir: " + dir2);

                for (String file : dir2.list()) {
                    if (!file.endsWith(".xml")) {
                        continue;
                    }
                    String parentAndFile = dir + "/" + file;
                    String base = file.substring(0, file.length() - 4);
                    if (fileFilter != null && !fileFilter.reset(dir + "/" + base).find()) {
                        if (verbose) { //  && verbose_skipping
                            System.out.println("SKIPPING: " + dir + "/" + base);
                        }
                        continue;
                    }
                    if (highLevelOnly && !HighLevelPaths.localeIsHighLevel(base)) {
                        continue;
                    }
                    if (verbose) {
                        System.out.println(file);
                    }
                    Relation<PathHeader, String> contentsOld = fillData(dirOld.toString() + "/", file, base);
                    Relation<PathHeader, String> contents2 = fillData(dir2.toString() + "/", file, base);

                    Set<PathHeader> keys = new TreeSet<>(CldrUtility.ifNull(contentsOld.keySet(), Collections.<PathHeader> emptySet()));
                    keys.addAll(CldrUtility.ifNull(contents2.keySet(), Collections.<PathHeader> emptySet()));
                    DtdType dtdType = null;
                    for (PathHeader key : keys) {
                        String originalPath = key.getOriginalPath();
                        if (highLevelOnly && !HighLevelPaths.pathIsHighLevel(originalPath, base)) {
                            continue;
                        }
                        boolean isTransform = originalPath.contains("/tRule");
                        if (dtdType == null) {
                            dtdType = DtdType.fromPath(originalPath);
                        }
                        Multimap<PathHeader, String> target = dtdType == DtdType.ldmlBCP47 ? bcp
                            : isTransform ? transforms
                                : supplemental;
                        Set<String> setOld = contentsOld.get(key);
                        Set<String> set2 = contents2.get(key);

                        if (Objects.equals(setOld, set2)) {
                            if (file.equals(DEBUG_FILE)) { // for debugging
                                System.out.println("**Same: " + key + "\t" + setOld);
                            }
                            addChange(parentAndFile, ChangeType.same, setOld.size());
                            countSame.add(key, 1);
                            continue;
                        }
                        if (setOld == null) {
                            addChange(parentAndFile, ChangeType.added, set2.size());
                            for (String s : set2) {
                                addRow(target, key, "▷missing◁", s);
                                countAdded.add(key, 1);
                            }
                        } else if (set2 == null) {
                            addChange(parentAndFile, ChangeType.deleted, setOld.size());
                            for (String s : setOld) {
                                addRow(target, key, s, "▷removed◁");
                                countDeleted.add(key, 1);
                            }
                        } else {
                            Set<String> s1MOld = setOld;
                            Set<String> s2M1 = set2;
                            if (s1MOld.isEmpty()) {
                                addRow(target, key, "▷missing◁", Joiner.on(", ").join(s2M1));
                                addChange(parentAndFile, ChangeType.added, s2M1.size());
                                countAdded.add(key, 1);
                            } else if (s2M1.isEmpty()) {
                                addRow(target, key, Joiner.on(", ").join(s1MOld), "▷removed◁");
                                addChange(parentAndFile, ChangeType.deleted, s1MOld.size());
                                countDeleted.add(key, 1);
                            } else {
                                String valueOld;
                                String valueCurrent;

                                int[] sameAndNotInSecond = new int[2];
                                valueOld = getFilteredValue(s1MOld, s1MOld, sameAndNotInSecond);
                                addChange(parentAndFile, ChangeType.same, sameAndNotInSecond[0]);
                                countSame.add(key, 1);
                                addChange(parentAndFile, ChangeType.deleted, sameAndNotInSecond[1]);
                                sameAndNotInSecond[1] = 0;
                                countDeleted.add(key, 1);
                                valueCurrent = getFilteredValue(s2M1, s1MOld, sameAndNotInSecond);
                                addChange(parentAndFile, ChangeType.added, sameAndNotInSecond[1]);
                                addRow(target, key, valueOld, valueCurrent);
                                countAdded.add(key, 1);
                            }
                        }
                    }
                }
            }
            writeDiffs(anchors, "bcp47", "¤¤BCP47 " + chartNameCap, bcp, tsvFile);
            writeDiffs(anchors, "supplemental-data", "¤¤Supplemental " + chartNameCap, supplemental, tsvFile);
            writeDiffs(anchors, "transforms", "¤¤Transforms " + chartNameCap, transforms, tsvFile);

            writeCounter(tsvCountFile, "CountSame", countSame);
            tsvCountFile.println();
            writeCounter(tsvCountFile, "CountAdded", countAdded);
            tsvCountFile.println();
            writeCounter(tsvCountFile, "CountDeleted", countDeleted);

            //tsvFile.println("# EOF");
            //tsvCountFile.println("# EOF");
        }
    }

    private void writeCounter(PrintWriter tsvFile, String title, Counter<PathHeader> countDeleted) {
        tsvFile.append("# "
            + title
            + "\tSection\tPage\tSubhead\tCode\n\n");
        for (R2<Long, PathHeader> entry : countDeleted.getEntrySetSortedByCount(false, null)) {
            tsvFile.println(entry.get0() + "\t" + entry.get1());
        }
    }

    private void addRow(Multimap<PathHeader, String> target, PathHeader key, String oldItem, String newItem) {
        if (oldItem.isEmpty() || newItem.isEmpty()) {
            throw new IllegalArgumentException();
        }
        target.put(key, oldItem + SEP + newItem);
    }

    /**
     * Fill in the chart data for the specified file
     *
     * @param directory
     * @param file like "xx.xml" where "xx" may be a locale name
     * @param fileBase like "xx", same as file without ".xml"
     * @return the Relation
     */
    private Relation<PathHeader, String> fillData(String directory, String file, String fileBase) {
        Relation<PathHeader, String> results = Relation.of(new TreeMap<PathHeader, Set<String>>(), TreeSet.class);

        List<Pair<String, String>> contents1;
        try {
            contents1 = XMLFileReader.loadPathValues(directory + file, new ArrayList<Pair<String, String>>(), true);
        } catch (Exception e) {
            /*
             * This happens with e = ICUException, file = grammaticalFeatures.xml in cldr-36.0
             */
            return results;
        }
        DtdType dtdType = null;
        DtdData dtdData = null;
        Multimap<String, String> extras = TreeMultimap.create();

        for (Pair<String, String> s : contents1) {
            String path = s.getFirst();
            if (highLevelOnly && !HighLevelPaths.pathIsHighLevel(path, fileBase /* locale, or not */)) {
                continue;
            }
            String value = s.getSecond();
            if (dtdType == null) {
                /*
                 * Note: although dtdType and dtdData depend on path, they are the same for all paths
                 * in the same file, so they only need to be set the first time through this loop.
                 *
                 * Note: the current DTD in CLDR_BASE_DIR is supposed to be backward-compatible, that is, to support
                 * paths from all archived versions. Any exception to that rule (e.g., for "grammaticalState") is a bug.
                 */
                dtdType = DtdType.fromPath(path);
                dtdData = DtdData.getInstance(dtdType, CLDR_BASE_DIR);
            }
            XPathParts pathPlain = XPathParts.getFrozenInstance(path);
            try {
                if (dtdData.isMetadata(pathPlain)) {
                    continue;
                }
            } catch (NullPointerException e) {
                /*
                 * TODO: this happens for "grammaticalState" in this path from version 37:
                 * //supplementalData/grammaticalData/grammaticalFeatures[@targets="nominal"][@locales="he"]/grammaticalState[@values="definite indefinite construct"]
                 * Reference: https://unicode-org.atlassian.net/browse/CLDR-13306
                 */
                System.out.println("Caught NullPointerException in fillData calling isMetadata, path = " + path);
                continue;
            }
            Set<String> pathForValues = dtdData.getRegularizedPaths(pathPlain, extras);
            if (pathForValues != null) {
                for (String pathForValue : pathForValues) {
                    PathHeader pathHeader = phf.fromPath(pathForValue);
                    if (pathHeader.getPageId() == PageId.Suppress) {
                        continue;
                    }
                    Splitter splitter = DtdData.getValueSplitter(pathPlain);
                    for (String line : splitter.split(value)) {
                        // special case # in transforms
                        if (isComment(pathPlain, line)) {
                            continue;
                        }
                        results.put(pathHeader, line);
                    }
                }
            }
            for (Entry<String, Collection<String>> entry : extras.asMap().entrySet()) {
                final String extraPath = entry.getKey();
                final PathHeader pathHeaderExtra = phf.fromPath(extraPath);
                if (pathHeaderExtra.getPageId() == PageId.Suppress) {
                    continue;
                }
                final Collection<String> extraValue = entry.getValue();
                if (isExtraSplit(extraPath)) {
                    for (String items : extraValue) {
                        results.putAll(pathHeaderExtra, DtdData.SPACE_SPLITTER.splitToList(items));
                    }
                } else {
                    results.putAll(pathHeaderExtra, extraValue);
                }
            }
            if (pathForValues == null && !value.isEmpty()) {
                System.err.println("Shouldn't happen");
            }
        }
        return results;
    }

    private boolean isExtraSplit(String extraPath) {
        if (extraPath.endsWith("/_type") && extraPath.startsWith("//supplementalData/metaZones/mapTimezones")) {
            return true;
        }
        return false;
    }

    private static boolean isComment(XPathParts pathPlain, String line) {
        if (pathPlain.contains("transform")) {
            if (line.startsWith("#")) {
                return true;
            }
        }
        return false;
    }

    /**
     * Determine when changes to the values for paths should be treated as
     * potentially "disruptive" for the purpose of "churn" reporting
     */
    private class SuspiciousChange {
        /**
         * the old and new values, such as "HH:mm–HH:mm v" and "HH:mm – HH:mm v"
         */
        private String oldValue, newValue;

        /**
         * the path, such as //ldml/dates/calendars/calendar[@type="gregorian"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id="Hmv"]/greatestDifference[@id="H"]
         */
        private String path;

        /**
         * the locale (such as "doi") in which the path was found, or null, or possibly
         * the base file name without extension, like "xx" if the file name is "xx.xml",
         * where "xx" may or may not be a locale; e.g., "supplementalData"
         */
        private String locale;

        SuspiciousChange(String oldValue, String newValue, String path, String locale) {
            this.oldValue = oldValue;
            this.newValue = newValue;
            this.path = path;
            this.locale = locale;
        }

        /**
         * Is the change from the old value to the new value, for this path and locale, potentially disruptive?
         *
         * @return true or false
         */
        public boolean isDisruptive() {
            /*
             * OR, not AND: certain changes in value are disruptive even for paths not
             * otherwise treated as high-level, and changes for high-level paths are
             * disruptive even if the changes in values themselves are not identified
             * as disruptive.
             */
            return valueChangeIsDisruptive() || HighLevelPaths.pathIsHighLevel(path, locale);
        }

        /**
         * Is the change from the old value to the current value potentially disruptive, based (primarily) on
         * the values themselves?
         *
         * @return true or false
         */
        private boolean valueChangeIsDisruptive() {
            if (oldValue == null || newValue == null || oldValue.equals(newValue)) {
                return false;
            }
            if (valueChangeIsDisruptiveWhitespaceOnly()) {
                return true;
            }
            return false;
        }

        /**
         * Is the change disruptive whitespace only?
         * Per design doc, "Format changes: second to none on the disruptiveness scale are changes involving spaces such as SPACE -> NBSP
         * or NBSP -> Narrow NBSP. Or adding a space somewhere in the format where previously there was none."
         *
         * @return true or false
         */
        private boolean valueChangeIsDisruptiveWhitespaceOnly() {
            /*
             * annotations often have changes like "pop gorn", "popgorn", not treated as disruptive
             */
            if (path.startsWith("//ldml/annotations")) {
                return false;
            }
            if (removeWhitespace(oldValue).equals(removeWhitespace(newValue))) {
                return true;
            }
            return false;
        }

        /**
         * Remove whitespace from the given string
         *
         * Remove whitespace as defined by regex \s, and also
         * U+00A0 NO-BREAK SPACE
         * U+2007 FIGURE SPACE
         * U+202F NARROW NO-BREAK SPACE
         *
         * @param s the string
         * @return the modified string
         */
        private String removeWhitespace(String s) {
            return s.replaceAll("[\\s\\u00A0\\u2007\\u202F]", "");
        }
    }

    /**
     * Determine which paths are considered "high-level" paths, i.e.,
     * paths for which any changes have high potential to cause disruptive "churn".
     * Whether a path is high-level sometimes depends on the locale or xml file in
     * which it occurs.
     * Some paths are high-level regardless of the locale in which they are located.
     * Other paths are high-level for some locales but not others. For example,
     *    //ldml/localeDisplayNames/languages/language[@type="xx"]
     * is high level in locale "xx", and maybe "en", but not for all locales.
     */
    private static class HighLevelPaths {
        /**
         * A set of paths to be treated as "high-level".
         * These are complete paths to be matched exactly.
         * Other paths are recognized by special functions like isHighLevelTerritoryName.
         *
         * The ordering and comments are based on the design spec.
         */
        final private static Set<String> highLevelPaths = new HashSet<>(Arrays.asList(
            /*
             * Core data
             */
            "//ldml/characters/exemplarCharacters",
            "//ldml/numbers/defaultNumberingSystem",
            "//ldml/numbers/otherNumberingSystems/native",
            /*
             * Territory and Language names
             *  Country/Region names (English and Native names) -- see isHighLevelTerritoryName
             *   //ldml/localeDisplayName/territories/territory/...
             *  Language names (English and Native) -- see isHighLevelLangName
             *   //ldml/localeDisplayNames/languages/language/...
             */
            /*
             * Date
             * Note: "year", "month", etc., below, form a subset (eight) of all possible values for type,
             * excluding, for example, "fri" and "zone". If we use starred paths, we would need further complication
             * to filter out "fri", "zone", etc.
             */
            "//ldml/dates/fields/field[@type=\"year\"]/displayName",
            "//ldml/dates/fields/field[@type=\"month\"]/displayName",
            "//ldml/dates/fields/field[@type=\"week\"]/displayName",
            "//ldml/dates/fields/field[@type=\"day\"]/displayName",
            "//ldml/dates/fields/field[@type=\"hour\"]/displayName",
            "//ldml/dates/fields/field[@type=\"era\"]/displayName",
            "//ldml/dates/fields/field[@type=\"minute\"]/displayName",
            "//ldml/dates/fields/field[@type=\"second\"]/displayName",
            /*
             * First day of week: firstDay in supplementalData.xml; see isHighLevelFirstDay
             * First week of year: see isHighLevelWeekOfPreference
             */
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateFormats/dateFormatLength[@type=\"full\"]/dateFormat[@type=\"standard\"]/pattern[@type=\"standard\"]",
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateFormats/dateFormatLength[@type=\"long\"]/dateFormat[@type=\"standard\"]/pattern[@type=\"standard\"]",
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateFormats/dateFormatLength[@type=\"medium\"]/dateFormat[@type=\"standard\"]/pattern[@type=\"standard\"]",
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateFormats/dateFormatLength[@type=\"short\"]/dateFormat[@type=\"standard\"]/pattern[@type=\"standard\"]",
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\"MMMEd\"]",
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\"MEd\"]",
            /*
             * Time
             */
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/timeFormats/timeFormatLength[@type=\"full\"]/timeFormat[@type=\"standard\"]/pattern[@type=\"standard\"]",
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/timeFormats/timeFormatLength[@type=\"long\"]/timeFormat[@type=\"standard\"]/pattern[@type=\"standard\"]",
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/timeFormats/timeFormatLength[@type=\"medium\"]/timeFormat[@type=\"standard\"]/pattern[@type=\"standard\"]",
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/timeFormats/timeFormatLength[@type=\"short\"]/timeFormat[@type=\"standard\"]/pattern[@type=\"standard\"]",
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod[@type=\"am\"]",
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"abbreviated\"]/dayPeriod[@type=\"am\"]",
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod[@type=\"pm\"]",
            "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"abbreviated\"]/dayPeriod[@type=\"pm\"]",
            /*
             * Currency (English and Native) -- see isHighLevelCurrencyName
             * E.g., //ldml/numbers/currencies/currency[@type=\"KRW\"]/displayName"
             *
             * ISO Currency Code: SupplementalData.xml match <region iso3166> -- see isHighLevelCurrencyCode
             */
            /*
             * Currency Formats
             *  a. Currency thousand separator
             *  b. Currency decimal separator
             *  c. Currency Symbol //ldml/numbers/currencies/currency[@type="CNY"]/symbol
             *  d. Currency Symbol Narrow //ldml/numbers/currencies/currency[@type=\"CNY\"]/symbol[@alt=\"narrow\"]"
             *
             * See isHighLevelCurrencySeparatorOrSymbol
             */
            "//ldml/numbers/currencyFormats[@numberSystem=\"latn\"]/currencyFormatLength/currencyFormat[@type=\"standard\"]/pattern[@type=\"standard\"]",
            "//ldml/numbers/currencyFormats[@numberSystem=\"arab\"]/currencyFormatLength/currencyFormat[@type=\"standard\"]/pattern[@type=\"standard\"]",
            /*
             * Number Symbols
             */
            "//ldml/numbers/minimumGroupingDigits",
            "//ldml/numbers/symbols[@numberSystem=\"latn\"]/decimal",
            "//ldml/numbers/symbols[@numberSystem=\"latn\"]/group",
            "//ldml/numbers/symbols[@numberSystem=\"arab\"]/decimal",
            "//ldml/numbers/symbols[@numberSystem=\"arab\"]/group",
            /*
             * Number formats
             */
            "//ldml/numbers/decimalFormats[@numberSystem=\"latn\"]/decimalFormatLength/decimalFormat[@type=\"standard\"]/pattern[@type=\"standard\"]",
            "//ldml/numbers/percentFormats[@numberSystem=\"latn\"]/percentFormatLength/percentFormat[@type=\"standard\"]/pattern[@type=\"standard\"]",
            "//ldml/numbers/currencyFormats[@numberSystem=\"latn\"]/currencyFormatLength/currencyFormat[@type=\"accounting\"]/pattern[@type=\"standard\"]",
            "//ldml/numbers/decimalFormats[@numberSystem=\"arab\"]/decimalFormatLength/decimalFormat[@type=\"standard\"]/pattern[@type=\"standard\"]",
            "//ldml/numbers/percentFormats[@numberSystem=\"arab\"]/percentFormatLength/percentFormat[@type=\"standard\"]/pattern[@type=\"standard\"]"
            /*
             * "Complementary Observations"
             */
            /*
             * Changes to language aliases (supplementalMetaData) -- see isHighLevelLangAlias
             * E.g., //supplementalData/metadata/alias/languageAlias[@type="aar"]
             */
            /*
             * Changes in the containment graph -- see isHighLevelTerritoryContainment
             * Data mostly (or entirely?) from M49 standard, thus CLDR has limited control.
             * Users use the containment graph in a variety of ways.
             * E.g., //supplementalData/territoryContainment/group[@type="003"][@contains="013 021 029"]
             */
            /*
             * Format changes: second to none on the disruptiveness scale are changes involving spaces such as SPACE -> NBSP
             *  or NBSP -> Narrow NBSP. Or adding a space somewhere in the format where previously there was none.
             *  -- see SuspiciousChange.valueChangeIsDisruptiveWhitespaceOnly
             */
            /*
             * TODO: per design doc, "Adding a timezone"
             * TODO: per design doc, "Changes of symbols or codes that are cross-locale in some way such as the unknown
             *  currency symbol change '???' -> '¤'."
             * TODO: per design doc, "Change in character properties (not a CLDR but a Unicode change), and here especially
             *  newly adding or removing punctuation. Frequently irritates parsers."
             */
        ));

        static Pattern currencyPattern = Pattern.compile("^//ldml/numbers/currencies/currency.*/displayName.*");

        /**
         * Should the given path in the given locale be taken into account for generating "churn" reports?
         *
         * @param path the path of interest
         * @param locale the locale in which the path was found, or null, or possibly
         *     the base file name without extension, like "xx" if the file name is "xx.xml",
         *     where "xx" may or may not be a locale; e.g., "supplementalData"
         * @return true if it counts, else false to ignore
         */
        private static boolean pathIsHighLevel(String path, String locale) {
            if (path == null || locale == null) {
                return false;
            }
            if (!localeIsHighLevel(locale)) { // for efficiency, this should be caught at a higher level
                System.out.println("locale [" + locale + "] failed localeIsHighLevel in pathIsHighLevel; path = " + path);
                return false;
            }
            if (pathIsReallyHighLevel(path, locale)) {
                if (verboseHighLevelReporting) {
                    recordHighLevelMatch(path);
                }
                return true;
            }
            return false;
        }

        private static boolean pathIsReallyHighLevel(String path, String locale) {
            if (highLevelPaths.contains(path)) {
                return true;
            } else if (isHighLevelTerritoryName(path, locale)) {
                return true;
            } else if (isHighLevelLangName(path, locale)) {
                return true;
            } else if (isHighLevelCurrencyName(path, locale)) {
                return true;
            } else if (isHighLevelCurrencyCode(path, locale)) {
                return true;
            } else if (isHighLevelCurrencySeparatorOrSymbol(path, locale)) {
                return true;
            } else if (isHighLevelLangAlias(path, locale)) {
                return true;
            } else if (isHighLevelTerritoryContainment(path, locale)) {
                return true;
            } else if (isHighLevelFirstDay(path, locale)) {
                return true;
            } else if (isHighLevelWeekOfPreference(path, locale)) {
                return true;
            }
            return false;
        }

        /**
         * Is the given locale, or base name, to be considered for "high level" churn report?
         *
         * @param locale the locale string, or base name like "supplementalData" as in "supplementalData.xml"
         * @return true or false
         */
        private static boolean localeIsHighLevel(String locale) {
            return SubmissionLocales.CLDR_OR_HIGH_LEVEL_LOCALES.contains(locale)
                || "supplementalData".equals(locale);
        }

        /**
         * Changes to language aliases (supplemental metadata)
         * E.g., //supplementalData/metadata/alias/languageAlias[@type="aar"]
         *
         * @param path
         * @param locale must be "supplementalData" to match
         * @return true or false
         */
        private static boolean isHighLevelLangAlias(String path, String locale) {
            if ("supplementalData".equals(locale)) {
                if (path.startsWith("//supplementalData/metadata/alias/languageAlias")) {
                    return true;
                }
            }
            return false;
        }

        /**
         * Changes in the containment graph
         * Data mostly (or entirely?) from M49 standard, thus CLDR has limited control.
         * Users use the containment graph in a variety of ways.
         * E.g., //supplementalData/territoryContainment/group[@type="003"][@contains="013 021 029"]
         *
         * @param path
         * @param locale must be "supplementalData" to match
         * @return true or false
         */
        private static boolean isHighLevelTerritoryContainment(String path, String locale) {
            if ("supplementalData".equals(locale)) {
                if (path.startsWith("//supplementalData/territoryContainment")) {
                    return true;
                }
            }
            return false;
        }

        /**
         * Is the given path a high-level territory name path in the given locale?
         *
         * E.g., //ldml/localeDisplayNames/territories/territory[@type="NNN"]
         * if type "NNN" CORRESPONDS TO the locale or the locale is "en"
         *
         * English names (en.xml): match all types
         * Native: check each territory type NNN corresponding to the given locale
         *
         * Exclude "alt"
         *
         * @param path
         * @param locale
         * @return true or false
         */
        private static boolean isHighLevelTerritoryName(String path, String locale) {
            if (path.startsWith("//ldml/localeDisplayNames/territories/territory")
                    && !path.contains("[@alt=")) {
                if ("en".equals(locale)) {
                    return true;
                }
                CoverageVariableInfo cvi = SUPPLEMENTAL_DATA_INFO.getCoverageVariableInfo(locale);
                if (cvi != null) {
                    for (String type : cvi.targetTerritories) {
                        if (path.contains("[@type=\"" + type + "\"]")) {
                            return true;
                        }
                    }
                }
            }
            return false;
        }

        /**
         * Is the given path a high-level language name path in the given locale?
         *
         * E.g., //ldml/localeDisplayNames/languages/language[@type="xx"]
         * if type "xx" matches the locale or the locale is "en"
         *
         * Exclude "alt"
         *
         * @param path
         * @param locale
         * @return true or false
         */
        private static boolean isHighLevelLangName(String path, String locale) {
            if (path.startsWith("//ldml/localeDisplayNames/languages/language")
                    && !path.contains("[@alt=")) {
                if ("en".equals(locale)) {
                    /*
                     * English names (en.xml): match all types
                     */
                    return true;
                } else if (path.contains("[@type=\"" + locale + "\"]")) {
                    /*
                     * Native names: match the type=”xx” of each xml file to identify the Native. E.g., type=ko if ko.xml
                     */
                    return true;
                }
            }
            return false;
        }

        /**
         * Is the given path a high-level currency name path in the given locale?
         *
         * E.g., //ldml/numbers/currencies/currency[@type=\"AAA\"]/displayName
         * if type "AAA" CORRESPONDS TO the locale or the locale is "en"
         *
         * English names (en.xml): match all types
         * Native: check each currency type AAA corresponding to the given locale
         *
         * Do NOT exclude "alt"; e.g.,
         * //ldml/numbers/currencies/currency[@type="ADP"]/displayName[@alt="proposed-u167-1"]
         *
         * @param path
         * @param locale
         * @return true or false
         */
        private static boolean isHighLevelCurrencyName(String path, String locale) {
            if (currencyPattern.matcher(path).matches()) {
                if ("en".equals(locale)) {
                    return true;
                }
                CoverageVariableInfo cvi = SUPPLEMENTAL_DATA_INFO.getCoverageVariableInfo(locale);
                if (cvi != null) {
                    for (String type : cvi.targetCurrencies) {
                        if (path.contains("[@type=\"" + type + "\"]")) {
                            return true;
                        }
                    }
                }
            }
            return false;
        }

        /**
         * Is the given path a high-level currency code path in the given locale?
         *
         * E.g., //supplementalData/currencyData/region[@iso3166="AC"]/currency[@iso4217="SHP"][@from="1976-01-01"]
         *
         * @param path
         * @param locale must be "supplementalData" to match
         * @return true or false
         */
        private static boolean isHighLevelCurrencyCode(String path, String locale) {
            if ("supplementalData".equals(locale)) {
                if (path.contains("iso3166")) {
                    return true;
                }
            }
            return false;
        }

        /**
         * Is the given path a high-level currency thousands-separator or decimal-separator path in the given locale?
         *
         * E.g., //ldml/numbers/currencies/currency[@type="ESP"]/group
         *       //ldml/numbers/currencies/currency[@type="ESP"]/decimal
         *       //ldml/numbers/currencies/currency[@type="CNY"]/symbol
         *       //ldml/numbers/currencies/currency[@type="CNY"]/symbol[@alt="narrow"]"
         *
         * @param path
         * @param locale
         * @return true or false
         */
        private static boolean isHighLevelCurrencySeparatorOrSymbol(String path, String locale) {
            if (path.startsWith("//ldml/numbers/currencies/currency")
                && (path.contains("group") || path.contains("decimal") || path.contains("symbol"))) {
                return true;
            }
            return false;
        }

        /**
         * Is the given path a high-level weekData/firstDay in the given locale?
         *
         * E.g.,//supplementalData/weekData/firstDay[@day="fri"][@territories="MV"]
         *
         * @param path
         * @param locale must be "supplementalData" to match
         * @return true or false
         */
        private static boolean isHighLevelFirstDay(String path, String locale) {
            if ("supplementalData".equals(locale)) {
                if (path.startsWith("//supplementalData/weekData/firstDay")) {
                    return true;
                }
            }
            return false;
        }

        /**
         * Is the given path a high-level weekOfPreference in the given locale?
         *
         * E.g., //supplementalData/weekData/weekOfPreference[@ordering="weekOfYear"][@locales="und"]
         *
         * @param path
         * @param locale must be "supplementalData" to match
         * @return true or false
         */
        private static boolean isHighLevelWeekOfPreference(String path, String locale) {
            if ("supplementalData".equals(locale)) {
                if (path.startsWith("//supplementalData/weekData/weekOfPreference")) {
                    return true;
                }
            }
            return false;
        }

        /**
         * For debugging, testing
         */
        private static Set<String> highLevelPathMatched = null;
        private static boolean verboseHighLevelReporting = false;

        private static void recordHighLevelMatch(String path) {
            if (highLevelPathMatched == null) {
                highLevelPathMatched = new HashSet<>();
            }
            highLevelPathMatched.add(path);
        }

        /**
         * For debugging, report on any paths in highLevelPaths that never matched
         */
        private static void reportHighLevelPathUsage() {
            if (!verboseHighLevelReporting) {
                return;
            }
            if (highLevelPathMatched == null) {
                System.out.println("Zero high-level paths were matched!");
                return;
            }
            for (String path : highLevelPaths) {
                if (!highLevelPathMatched.contains(path)) {
                    System.out.println("Unmatched high-level path: " + path);
                }
            }
            for (String path : highLevelPathMatched) {
                if (!highLevelPaths.contains(path)) {
                    System.out.println("Special matched high-level path: " + path);
                }
            }
        }
    }
}