1 // Copyright (c) 2017, the R8 project authors. Please see the AUTHORS file 2 // for details. All rights reserved. Use of this source code is governed by a 3 // BSD-style license that can be found in the LICENSE file. 4 package com.android.tools.r8.compatdx; 5 6 import static com.android.tools.r8.utils.FileUtils.isApkFile; 7 import static com.android.tools.r8.utils.FileUtils.isArchive; 8 import static com.android.tools.r8.utils.FileUtils.isClassFile; 9 import static com.android.tools.r8.utils.FileUtils.isDexFile; 10 import static com.android.tools.r8.utils.FileUtils.isJarFile; 11 import static com.android.tools.r8.utils.FileUtils.isZipFile; 12 13 import com.android.tools.r8.CompilationException; 14 import com.android.tools.r8.CompilationMode; 15 import com.android.tools.r8.D8; 16 import com.android.tools.r8.D8Command; 17 import com.android.tools.r8.D8Output; 18 import com.android.tools.r8.Resource; 19 import com.android.tools.r8.compatdx.CompatDx.DxCompatOptions.DxUsageMessage; 20 import com.android.tools.r8.compatdx.CompatDx.DxCompatOptions.PositionInfo; 21 import com.android.tools.r8.dex.Constants; 22 import com.android.tools.r8.errors.CompilationError; 23 import com.android.tools.r8.errors.Unimplemented; 24 import com.android.tools.r8.logging.Log; 25 import com.android.tools.r8.utils.FileUtils; 26 import com.android.tools.r8.utils.ThreadUtils; 27 import com.google.common.collect.ImmutableList; 28 import com.google.common.io.ByteStreams; 29 import com.google.common.io.Closer; 30 import java.io.File; 31 import java.io.IOException; 32 import java.io.InputStream; 33 import java.io.PrintStream; 34 import java.nio.file.Files; 35 import java.nio.file.Path; 36 import java.nio.file.Paths; 37 import java.nio.file.StandardCopyOption; 38 import java.util.ArrayList; 39 import java.util.List; 40 import java.util.concurrent.ExecutorService; 41 import java.util.zip.ZipEntry; 42 import java.util.zip.ZipException; 43 import java.util.zip.ZipInputStream; 44 import java.util.zip.ZipOutputStream; 45 import joptsimple.OptionParser; 46 import joptsimple.OptionSet; 47 import joptsimple.OptionSpec; 48 49 /** 50 * Dx compatibility interface for d8. 51 * 52 * This should become a mostly drop-in replacement for uses of the DX dexer (eg, dx --dex ...). 53 */ 54 public class CompatDx { 55 56 private static final String USAGE_HEADER = "Usage: compatdx [options] <input files>"; 57 58 /** 59 * Compatibility options parsing for the DX --dex sub-command. 60 */ 61 public static class DxCompatOptions { 62 // Final values after parsing. 63 // Note: These are ordered by their occurrence in "dx --help" 64 public final boolean help; 65 public final boolean debug; 66 public final boolean verbose; 67 public final PositionInfo positions; 68 public final boolean noLocals; 69 public final boolean noOptimize; 70 public final boolean statistics; 71 public final String optimizeList; 72 public final String noOptimizeList; 73 public final boolean noStrict; 74 public final boolean keepClasses; 75 public final String output; 76 public final String dumpTo; 77 public final int dumpWidth; 78 public final String dumpMethod; 79 public final boolean verboseDump; 80 public final boolean dump; 81 public final boolean noFiles; 82 public final boolean coreLibrary; 83 public final int numThreads; 84 public final boolean incremental; 85 public final boolean forceJumbo; 86 public final boolean noWarning; 87 public final boolean multiDex; 88 public final String mainDexList; 89 public final boolean minimalMainDex; 90 public final int minApiLevel; 91 public final String inputList; 92 public final ImmutableList<String> inputs; 93 // Undocumented option 94 public final int maxIndexNumber; 95 96 private static final String FILE_ARG = "file"; 97 private static final String NUM_ARG = "number"; 98 private static final String METHOD_ARG = "method"; 99 100 public enum PositionInfo { 101 NONE, IMPORTANT, LINES 102 } 103 104 // Exception thrown on invalid dx compat usage. 105 public static class DxUsageMessage extends Exception { 106 public final String message; 107 DxUsageMessage()108 public DxUsageMessage() { 109 this(""); 110 } 111 DxUsageMessage(String message)112 DxUsageMessage(String message) { 113 this.message = message; 114 } 115 printHelpOn(PrintStream sink)116 void printHelpOn(PrintStream sink) throws IOException { 117 sink.println(message); 118 } 119 } 120 121 // Exception thrown on options parse error. 122 private static class DxParseError extends DxUsageMessage { 123 private final OptionParser parser; 124 DxParseError(OptionParser parser)125 private DxParseError(OptionParser parser) { 126 this.parser = parser; 127 } 128 129 @Override printHelpOn(PrintStream sink)130 public void printHelpOn(PrintStream sink) throws IOException { 131 parser.printHelpOn(sink); 132 } 133 } 134 135 // Parsing specification. 136 private static class Spec { 137 final OptionParser parser; 138 139 // Note: These are ordered by their occurrence in "dx --help" 140 final OptionSpec<Void> debug; 141 final OptionSpec<Void> verbose; 142 final OptionSpec<String> positions; 143 final OptionSpec<Void> noLocals; 144 final OptionSpec<Void> noOptimize; 145 final OptionSpec<Void> statistics; 146 final OptionSpec<String> optimizeList; 147 final OptionSpec<String> noOptimizeList; 148 final OptionSpec<Void> noStrict; 149 final OptionSpec<Void> keepClasses; 150 final OptionSpec<String> output; 151 final OptionSpec<String> dumpTo; 152 final OptionSpec<Integer> dumpWidth; 153 final OptionSpec<String> dumpMethod; 154 final OptionSpec<Void> dump; 155 final OptionSpec<Void> verboseDump; 156 final OptionSpec<Void> noFiles; 157 final OptionSpec<Void> coreLibrary; 158 final OptionSpec<Integer> numThreads; 159 final OptionSpec<Void> incremental; 160 final OptionSpec<Void> forceJumbo; 161 final OptionSpec<Void> noWarning; 162 final OptionSpec<Void> multiDex; 163 final OptionSpec<String> mainDexList; 164 final OptionSpec<Void> minimalMainDex; 165 final OptionSpec<Integer> minApiLevel; 166 final OptionSpec<String> inputList; 167 final OptionSpec<String> inputs; 168 final OptionSpec<Void> help; 169 final OptionSpec<Integer> maxIndexNumber; 170 Spec()171 Spec() { 172 parser = new OptionParser(); 173 parser.accepts("dex"); 174 debug = parser.accepts("debug", "Print debug information"); 175 verbose = parser.accepts("verbose", "Print verbose information"); 176 positions = parser 177 .accepts("positions", 178 "What source-position information to keep. One of: none, lines, important") 179 .withOptionalArg() 180 .describedAs("keep") 181 .defaultsTo("lines"); 182 noLocals = parser.accepts("no-locals", "Don't keep local variable information"); 183 statistics = parser.accepts("statistics", "Print statistics information"); 184 noOptimize = parser.accepts("no-optimize", "Don't optimize"); 185 optimizeList = parser 186 .accepts("optimize-list", "File listing methods to optimize") 187 .withRequiredArg() 188 .describedAs(FILE_ARG); 189 noOptimizeList = parser 190 .accepts("no-optimize-list", "File listing methods not to optimize") 191 .withRequiredArg() 192 .describedAs(FILE_ARG); 193 noStrict = parser.accepts("no-strict", "Disable strict file/class name checks"); 194 keepClasses = parser.accepts("keep-classes", "Keep input class files in in output jar"); 195 output = parser 196 .accepts("output", "Output file or directory") 197 .withRequiredArg() 198 .describedAs(FILE_ARG); 199 dumpTo = parser 200 .accepts("dump-to", "File to dump information to") 201 .withRequiredArg() 202 .describedAs(FILE_ARG); 203 dumpWidth = parser 204 .accepts("dump-width", "Max width for columns in dump output") 205 .withRequiredArg() 206 .ofType(Integer.class) 207 .defaultsTo(0) 208 .describedAs(NUM_ARG); 209 dumpMethod = parser 210 .accepts("dump-method", "Method to dump information for") 211 .withRequiredArg() 212 .describedAs(METHOD_ARG); 213 dump = parser.accepts("dump", "Dump information"); 214 verboseDump = parser.accepts("verbose-dump", "Dump verbose information"); 215 noFiles = parser.accepts("no-files", "Don't fail if given no files"); 216 coreLibrary = parser.accepts("core-library", "Construct a core library"); 217 numThreads = parser 218 .accepts("num-threads", "Number of threads to run with") 219 .withRequiredArg() 220 .ofType(Integer.class) 221 .defaultsTo(1) 222 .describedAs(NUM_ARG); 223 incremental = parser.accepts("incremental", "Merge result with the output if it exists"); 224 forceJumbo = parser.accepts("force-jumbo", "Force use of string-jumbo instructions"); 225 noWarning = parser.accepts("no-warning", "Suppress warnings"); 226 maxIndexNumber = parser.accepts("set-max-idx-number", 227 "Undocumented: Set maximal index number to use in a dex file.") 228 .withRequiredArg() 229 .ofType(Integer.class) 230 .defaultsTo(0) 231 .describedAs("Maximum index"); 232 minimalMainDex = parser.accepts("minimal-main-dex", "Produce smallest possible main dex"); 233 mainDexList = parser 234 .accepts("main-dex-list", "File listing classes that must be in the main dex file") 235 .withRequiredArg() 236 .describedAs(FILE_ARG); 237 multiDex = 238 parser 239 .accepts("multi-dex", "Allow generation of multi-dex") 240 .requiredIf(minimalMainDex, mainDexList, maxIndexNumber); 241 minApiLevel = parser 242 .accepts("min-sdk-version", "Minimum Android API level compatibility.") 243 .withRequiredArg().ofType(Integer.class); 244 inputList = parser 245 .accepts("input-list", "File listing input files") 246 .withRequiredArg() 247 .describedAs(FILE_ARG); 248 inputs = parser.nonOptions("Input files"); 249 help = parser.accepts("help", "Print this message").forHelp(); 250 } 251 } 252 DxCompatOptions(OptionSet options, Spec spec)253 private DxCompatOptions(OptionSet options, Spec spec) throws DxParseError { 254 help = options.has(spec.help); 255 debug = options.has(spec.debug); 256 verbose = options.has(spec.verbose); 257 if (options.has(spec.positions)) { 258 switch (options.valueOf(spec.positions)) { 259 case "none": 260 positions = PositionInfo.NONE; 261 break; 262 case "important": 263 positions = PositionInfo.IMPORTANT; 264 break; 265 case "lines": 266 positions = PositionInfo.LINES; 267 break; 268 default: 269 positions = PositionInfo.IMPORTANT; 270 break; 271 } 272 } else { 273 positions = PositionInfo.LINES; 274 } 275 noLocals = options.has(spec.noLocals); 276 noOptimize = options.has(spec.noOptimize); 277 statistics = options.has(spec.statistics); 278 optimizeList = options.valueOf(spec.optimizeList); 279 noOptimizeList = options.valueOf(spec.noOptimizeList); 280 noStrict = options.has(spec.noStrict); 281 keepClasses = options.has(spec.keepClasses); 282 output = options.valueOf(spec.output); 283 dumpTo = options.valueOf(spec.dumpTo); 284 dumpWidth = options.valueOf(spec.dumpWidth); 285 dumpMethod = options.valueOf(spec.dumpMethod); 286 dump = options.has(spec.dump); 287 verboseDump = options.has(spec.verboseDump); 288 noFiles = options.has(spec.noFiles); 289 coreLibrary = options.has(spec.coreLibrary); 290 numThreads = lastIntOf(options.valuesOf(spec.numThreads)); 291 incremental = options.has(spec.incremental); 292 forceJumbo = options.has(spec.forceJumbo); 293 noWarning = options.has(spec.noWarning); 294 multiDex = options.has(spec.multiDex); 295 mainDexList = options.valueOf(spec.mainDexList); 296 minimalMainDex = options.has(spec.minimalMainDex); 297 if (options.has(spec.minApiLevel)) { 298 List<Integer> allMinApiLevels = options.valuesOf(spec.minApiLevel); 299 minApiLevel = allMinApiLevels.get(allMinApiLevels.size() - 1); 300 } else { 301 minApiLevel = Constants.DEFAULT_ANDROID_API; 302 } 303 inputList = options.valueOf(spec.inputList); 304 inputs = ImmutableList.copyOf(options.valuesOf(spec.inputs)); 305 maxIndexNumber = options.valueOf(spec.maxIndexNumber); 306 } 307 parse(String[] args)308 public static DxCompatOptions parse(String[] args) throws DxParseError { 309 Spec spec = new Spec(); 310 return new DxCompatOptions(spec.parser.parse(args), spec); 311 } 312 lastIntOf(List<Integer> values)313 private static int lastIntOf(List<Integer> values) { 314 assert !values.isEmpty(); 315 return values.get(values.size() - 1); 316 } 317 } 318 main(String[] args)319 public static void main(String[] args) throws IOException { 320 try { 321 run(args); 322 } catch (CompilationException e) { 323 System.err.println(e.getMessage()); 324 System.exit(1); 325 } catch (DxUsageMessage e) { 326 System.err.println(USAGE_HEADER); 327 e.printHelpOn(System.err); 328 System.exit(1); 329 } 330 } 331 run(String[] args)332 private static void run(String[] args) throws DxUsageMessage, IOException, CompilationException { 333 System.out.println("CompatDx " + String.join(" ", args)); 334 DxCompatOptions dexArgs = DxCompatOptions.parse(args); 335 if (dexArgs.help) { 336 printHelpOn(System.out); 337 return; 338 } 339 CompilationMode mode = CompilationMode.RELEASE; 340 Path output = null; 341 List<Path> inputs = new ArrayList<>(); 342 boolean singleDexFile = !dexArgs.multiDex; 343 Path mainDexList = null; 344 int numberOfThreads = 1; 345 346 for (String path : dexArgs.inputs) { 347 processPath(new File(path), inputs); 348 } 349 if (inputs.isEmpty()) { 350 if (dexArgs.noFiles) { 351 return; 352 } 353 throw new DxUsageMessage("No input files specified"); 354 } 355 356 if (!Log.ENABLED && (!dexArgs.noWarning || dexArgs.debug || dexArgs.verbose)) { 357 System.out.println("Warning: logging is not enabled for this build."); 358 } 359 360 if (dexArgs.dump) { 361 System.out.println("Warning: dump is not supported"); 362 } 363 364 if (dexArgs.verboseDump) { 365 throw new Unimplemented("verbose dump file not yet supported"); 366 } 367 368 if (dexArgs.dumpMethod != null) { 369 throw new Unimplemented("method-dump not yet supported"); 370 } 371 372 if (dexArgs.output != null) { 373 output = Paths.get(dexArgs.output); 374 if (FileUtils.isDexFile(output)) { 375 if (!singleDexFile) { 376 throw new DxUsageMessage("Cannot output to a single dex-file when running with multidex"); 377 } 378 } else if (!FileUtils.isArchive(output) 379 && (!output.toFile().exists() || !output.toFile().isDirectory())) { 380 throw new DxUsageMessage("Unsupported output file or output directory does not exist. " 381 + "Output must be a directory or a file of type dex, apk, jar or zip."); 382 } 383 } 384 385 if (dexArgs.dumpTo != null) { 386 throw new Unimplemented("dump-to file not yet supported"); 387 } 388 389 if (dexArgs.positions == PositionInfo.NONE) { 390 System.out.println("Warning: no support for positions none."); 391 } 392 393 if (dexArgs.positions == PositionInfo.LINES && !dexArgs.noLocals) { 394 mode = CompilationMode.DEBUG; 395 } 396 397 if (dexArgs.incremental) { 398 throw new Unimplemented("incremental merge not supported yet"); 399 } 400 401 if (dexArgs.forceJumbo) { 402 System.out.println( 403 "Warning: no support for forcing jumbo-strings.\n" 404 + "Strings will only use jumbo-string indexing if necessary.\n" 405 + "Make sure that any dex merger subsequently used " 406 + "supports correct handling of jumbo-strings (eg, D8/R8 does)."); 407 } 408 409 if (dexArgs.noOptimize) { 410 System.out.println("Warning: no support for not optimizing"); 411 } 412 413 if (dexArgs.optimizeList != null) { 414 throw new Unimplemented("no support for optimize-method list"); 415 } 416 417 if (dexArgs.noOptimizeList != null) { 418 throw new Unimplemented("no support for dont-optimize-method list"); 419 } 420 421 if (dexArgs.statistics) { 422 System.out.println("Warning: no support for printing statistics"); 423 } 424 425 if (dexArgs.numThreads > 1) { 426 numberOfThreads = dexArgs.numThreads; 427 } 428 429 if (dexArgs.mainDexList != null) { 430 mainDexList = Paths.get(dexArgs.mainDexList); 431 } 432 433 if (dexArgs.noStrict) { 434 System.out.println("Warning: conservative main-dex list not yet supported"); 435 } else { 436 System.out.println("Warning: strict name checking not yet supported"); 437 } 438 439 if (dexArgs.minimalMainDex) { 440 System.out.println("Warning: minimal main-dex support is not yet supported"); 441 } 442 443 if (dexArgs.maxIndexNumber != 0) { 444 System.out.println("Warning: internal maximum-index setting is not supported"); 445 } 446 447 if (numberOfThreads < 1) { 448 throw new DxUsageMessage("Invalid numThreads value of " + numberOfThreads); 449 } 450 ExecutorService executor = ThreadUtils.getExecutorService(numberOfThreads); 451 D8Output result; 452 try { 453 result = D8.run( 454 D8Command.builder() 455 .addProgramFiles(inputs, true) 456 .setMode(mode) 457 .setMinApiLevel( 458 dexArgs.multiDex && dexArgs.minApiLevel < 21 ? 21 : dexArgs.minApiLevel) 459 .setMainDexListFile(mainDexList) 460 .build()); 461 } finally { 462 executor.shutdown(); 463 } 464 465 if (output == null) { 466 return; 467 } 468 469 if (singleDexFile) { 470 if (result.getDexResources().size() > 1) { 471 throw new CompilationError( 472 "Compilation result could not fit into a single dex file. " 473 + "Reduce the input-program size or run with --multi-dex enabled"); 474 } 475 if (isDexFile(output)) { 476 try (Closer closer = Closer.create()) { 477 InputStream stream = result.getDexResources().get(0).getStream(closer); 478 Files.copy(stream, output, StandardCopyOption.REPLACE_EXISTING); 479 } 480 return; 481 } 482 } 483 484 if (dexArgs.keepClasses) { 485 if (!isArchive(output)) { 486 throw new DxCompatOptions.DxUsageMessage( 487 "Output must be an archive when --keep-classes is set."); 488 } 489 writeZipWithClasses(inputs, result, output); 490 } else { 491 result.write(output); 492 } 493 } 494 printHelpOn(PrintStream sink)495 static void printHelpOn(PrintStream sink) throws IOException { 496 sink.println(USAGE_HEADER); 497 new DxCompatOptions.Spec().parser.printHelpOn(sink); 498 } 499 processPath(File file, List<Path> files)500 private static void processPath(File file, List<Path> files) { 501 if (!file.exists()) { 502 throw new CompilationError("File does not exist: " + file); 503 } 504 if (file.isDirectory()) { 505 processDirectory(file, files); 506 return; 507 } 508 Path path = file.toPath(); 509 if (isZipFile(path) || isJarFile(path) || isClassFile(path)) { 510 files.add(path); 511 return; 512 } 513 if (isApkFile(path)) { 514 throw new Unimplemented("apk files not yet supported"); 515 } 516 } 517 processDirectory(File directory, List<Path> files)518 private static void processDirectory(File directory, List<Path> files) { 519 assert directory.exists(); 520 for (File file : directory.listFiles()) { 521 processPath(file, files); 522 } 523 } 524 writeZipWithClasses(List<Path> inputs, D8Output output, Path path)525 private static void writeZipWithClasses(List<Path> inputs, D8Output output, Path path) 526 throws IOException { 527 try (Closer closer = Closer.create()) { 528 try (ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(path))) { 529 // For each input archive file, add all class files within. 530 for (Path input : inputs) { 531 if (isArchive(input)) { 532 try (ZipInputStream in = new ZipInputStream(Files.newInputStream(input))) { 533 ZipEntry entry; 534 while ((entry = in.getNextEntry()) != null) { 535 if (isClassFile(Paths.get(entry.getName()))) { 536 addEntry(entry.getName(), in, out); 537 } 538 } 539 } catch (ZipException e) { 540 throw new CompilationError( 541 "Zip error while reading '" + input + "': " + e.getMessage(), e); 542 } 543 } 544 } 545 // Add dex files. 546 List<Resource> dexProgramSources = output.getDexResources(); 547 for (int i = 0; i < dexProgramSources.size(); i++) { 548 addEntry(getDexFileName(i), dexProgramSources.get(i).getStream(closer), out); 549 } 550 } 551 } 552 } 553 addEntry(String name, InputStream in, ZipOutputStream out)554 private static void addEntry(String name, InputStream in, ZipOutputStream out) 555 throws IOException { 556 ZipEntry zipEntry = new ZipEntry(name); 557 byte[] bytes = ByteStreams.toByteArray(in); 558 zipEntry.setSize(bytes.length); 559 out.putNextEntry(zipEntry); 560 out.write(bytes); 561 out.closeEntry(); 562 } 563 getDexFileName(int index)564 private static String getDexFileName(int index) { 565 return index == 0 ? "classes.dex" : "classes" + (index + 1) + ".dex"; 566 } 567 } 568