1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package libcore; 18 19 import java.io.IOException; 20 import java.io.PrintStream; 21 import java.nio.file.Path; 22 import java.util.ArrayList; 23 import java.util.Arrays; 24 import java.util.Collections; 25 import java.util.Comparator; 26 import java.util.LinkedHashMap; 27 import java.util.List; 28 import java.util.Locale; 29 import java.util.Map; 30 import java.util.Objects; 31 import java.util.regex.Matcher; 32 import java.util.regex.Pattern; 33 34 /** 35 * Helps compare openjdk_java_files contents against upstream file contents. 36 * 37 * Outputs a tab-separated table comparing each openjdk_java_files entry 38 * against OpenJDK upstreams. This can help verify updates to later upstreams 39 * or focus attention towards files that may have been missed in a previous 40 * update (http://b/36461944) or are otherwise surprising (http://b/36429512). 41 * 42 * - Identifies each file as identical to, different from or missing from 43 * each upstream; diffs are not produced. 44 * - Optionally, copies all openjdk_java_files from the default upstream 45 * (eg. OpenJDK8u121-b13) to a new directory, for easy directory comparison 46 * using e.g. kdiff3, which allows inspecting detailed diffs. 47 * - The ANDROID_BUILD_TOP environment variable must be set to point to the 48 * AOSP root directory (parent of libcore). 49 * 50 * To check out upstreams OpenJDK 7u40, 8u60, 8u121-b13, and 9+181, run: 51 * 52 * mkdir ~/openjdk 53 * cd ~/openjdk 54 * export OPENJDK_HOME=$PWD 55 * hg clone http://hg.openjdk.java.net/jdk7u/jdk7u40/ 7u40 56 * (cd !$ ; sh get_source.sh) 57 * hg clone http://hg.openjdk.java.net/jdk8u/jdk8u 8u121-b13 58 * (cd !$ ; hg update -r jdk8u121-b13 && sh get_source.sh && sh common/bin/hgforest.sh update -r jdk8u121-b13) 59 * hg clone http://hg.openjdk.java.net/jdk8u/jdk8u60/ 8u60 60 * (cd !$ ; sh get_source.sh) 61 * hg clone http://hg.openjdk.java.net/jdk9/jdk9/ 9+181 62 * (cd !$ ; hg update -r jdk-9+181 && sh get_source.sh && sh common/bin/hgforest.sh update -r jdk-9+181) 63 * 64 * To get the 9b113+ upstream, follow the instructions from the commit 65 * message of AOSP libcore commit 29957558cf0db700bfaae360a80c42dc3871d0e5 66 * at https://android-review.googlesource.com/c/304056/ 67 * 68 * To get OpenJDK head: hg clone http://hg.openjdk.java.net/jdk/jdk/ head 69 */ 70 public class CompareUpstreams { 71 72 private final StandardRepositories standardRepositories; 73 CompareUpstreams(StandardRepositories standardRepositories)74 public CompareUpstreams(StandardRepositories standardRepositories) { 75 this.standardRepositories = Objects.requireNonNull(standardRepositories); 76 } 77 androidChangedComments(List<String> lines)78 private static Map<String, Integer> androidChangedComments(List<String> lines) { 79 List<String> problems = new ArrayList<>(); 80 Map<String, Integer> result = new LinkedHashMap<>(); 81 Pattern pattern = Pattern.compile( 82 "// (BEGIN |END |)Android-((?:changed|added|removed|note)(?:: )?.*)$"); 83 for (String line : lines) { 84 Matcher matcher = pattern.matcher(line); 85 if (matcher.find()) { 86 String type = matcher.group(1); 87 if (type.equals("END")) { 88 continue; 89 } 90 String match = matcher.group(2); 91 if (match.isEmpty()) { 92 match = "[empty comment]"; 93 } 94 Integer oldCount = result.get(match); 95 if (oldCount == null) { 96 oldCount = 0; 97 } 98 result.put(match, oldCount + 1); 99 } else if (line.contains("Android-")) { 100 problems.add(line); 101 } 102 } 103 if (!problems.isEmpty()) { 104 throw new IllegalArgumentException(problems.toString()); 105 } 106 return result; 107 } 108 androidChangedCommentsSummary(List<String> lines)109 private static String androidChangedCommentsSummary(List<String> lines) { 110 Map<String, Integer> map = androidChangedComments(lines); 111 List<String> comments = new ArrayList<>(map.keySet()); 112 Collections.sort(comments, Comparator.comparing(map::get).reversed()); 113 List<String> result = new ArrayList<>(); 114 for (String comment : comments) { 115 int count = map.get(comment); 116 if (count == 1) { 117 result.add(comment); 118 } else { 119 result.add(comment + " (x" + count + ")"); 120 } 121 } 122 return escapeTsv(String.join("\n", result)); 123 } 124 escapeTsv(String value)125 private static String escapeTsv(String value) { 126 if (value.contains("\t")) { 127 throw new IllegalArgumentException(value); // tsv doesn't support escaping tabs 128 } 129 return "\"" + value.replace("\"", "\"\"") + "\""; 130 } 131 printTsv(PrintStream out, List<String> values)132 private static void printTsv(PrintStream out, List<String> values) { 133 out.println(String.join("\t", values)); 134 } 135 136 /** 137 * Prints tab-separated values comparing ojluni files vs. each 138 * upstream, for each of the rel_paths, suitable for human 139 * analysis in a spreadsheet. 140 * This includes whether the corresponding upstream file is 141 * missing, identical, or by how many lines it differs, and 142 * a guess as to the correct upstream based on minimal line 143 * difference (ties broken in favor of upstreams that occur 144 * earlier in the list). 145 */ run(PrintStream out, List<Path> relPaths)146 private void run(PrintStream out, List<Path> relPaths) throws IOException { 147 // upstreams are in decreasing order of preference 148 List<String> headers = new ArrayList<>(); 149 headers.addAll(Arrays.asList( 150 "rel_path", "expected_upstream", "guessed_upstream", "changes", "vs. expected")); 151 for (Repository upstream : standardRepositories.historicUpstreams()) { 152 headers.add(upstream.name()); 153 } 154 headers.add("diff"); 155 printTsv(out, headers); 156 for (Path relPath : relPaths) { 157 Repository expectedUpstream = standardRepositories.referenceUpstreamAsOfAndroidP( 158 relPath); 159 out.print(relPath + "\t"); 160 Path ojluniFile = standardRepositories.ojluni().absolutePath(relPath); 161 List<String> linesB = Util.readLines(ojluniFile); 162 int bestDistance = Integer.MAX_VALUE; 163 Repository guessedUpstream = null; 164 List<Repository> upstreams = new ArrayList<>(); 165 upstreams.add(expectedUpstream); 166 upstreams.addAll(standardRepositories.historicUpstreams()); 167 List<String> comparisons = new ArrayList<>(upstreams.size()); 168 for (Repository upstream : upstreams) { 169 final String comparison; 170 Path upstreamFile = upstream.absolutePath(relPath); 171 if (upstreamFile == null) { 172 comparison = "missing"; 173 } else { 174 List<String> linesA = Util.readLines(upstreamFile); 175 int distance = Util.editDistance(linesA, linesB); 176 if (distance == 0) { 177 comparison = "identical"; 178 } else { 179 double percentDifferent = 100.0 * distance / Math 180 .max(linesA.size(), linesB.size()); 181 comparison = String 182 .format(Locale.US, "%.1f%% different (%d lines)", percentDifferent, 183 distance); 184 } 185 if (distance < bestDistance) { 186 bestDistance = distance; 187 guessedUpstream = upstream; 188 } 189 } 190 comparisons.add(comparison); 191 } 192 String changedCommentsSummary = androidChangedCommentsSummary(linesB); 193 194 String diffCommand = ""; 195 if (!comparisons.get(0).equals("identical")) { 196 Path expectedUpstreamPath = expectedUpstream.pathFromRepository(relPath); 197 if (expectedUpstreamPath != null) { 198 diffCommand = "${ANDROID_BUILD_TOP}/libcore/tools/upstream/upstream-diff " 199 + "-r ojluni," + expectedUpstream.name() + " " + relPath; 200 } else { 201 diffCommand = "FILE MISSING"; 202 } 203 } 204 List<String> values = new ArrayList<>(); 205 values.add(expectedUpstream.name()); 206 values.add(guessedUpstream == null ? "" : guessedUpstream.name()); 207 values.add(changedCommentsSummary); 208 values.addAll(comparisons); 209 values.add(diffCommand); 210 printTsv(out, values); 211 } 212 } 213 run()214 public void run() throws IOException { 215 List<Path> relPaths = standardRepositories.ojluni().loadRelPathsFromBlueprint(); 216 run(System.out, relPaths); 217 } 218 main(String[] args)219 public static void main(String[] args) throws IOException { 220 StandardRepositories standardRepositories = StandardRepositories.fromEnv(); 221 CompareUpstreams action = new CompareUpstreams(standardRepositories); 222 action.run(); 223 } 224 } 225