• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <inttypes.h>
18 #include <stdio.h>
19 #include <stdlib.h>
20 #include <sys/stat.h>
21 #include "base/memory_tool.h"
22 
23 #include <fstream>
24 #include <iostream>
25 #include <limits>
26 #include <sstream>
27 #include <string>
28 #include <unordered_set>
29 #include <vector>
30 
31 #if defined(__linux__) && defined(__arm__)
32 #include <sys/personality.h>
33 #include <sys/utsname.h>
34 #endif
35 
36 #include "android-base/stringprintf.h"
37 #include "android-base/strings.h"
38 
39 #include "arch/instruction_set_features.h"
40 #include "arch/mips/instruction_set_features_mips.h"
41 #include "art_method-inl.h"
42 #include "base/callee_save_type.h"
43 #include "base/dumpable.h"
44 #include "base/macros.h"
45 #include "base/scoped_flock.h"
46 #include "base/stl_util.h"
47 #include "base/stringpiece.h"
48 #include "base/time_utils.h"
49 #include "base/timing_logger.h"
50 #include "base/unix_file/fd_file.h"
51 #include "class_linker.h"
52 #include "class_loader_context.h"
53 #include "compiler.h"
54 #include "compiler_callbacks.h"
55 #include "debug/elf_debug_writer.h"
56 #include "debug/method_debug_info.h"
57 #include "dex/quick_compiler_callbacks.h"
58 #include "dex/verification_results.h"
59 #include "dex2oat_return_codes.h"
60 #include "dex_file-inl.h"
61 #include "driver/compiler_driver.h"
62 #include "driver/compiler_options.h"
63 #include "elf_file.h"
64 #include "elf_writer.h"
65 #include "elf_writer_quick.h"
66 #include "gc/space/image_space.h"
67 #include "gc/space/space-inl.h"
68 #include "gc/verification.h"
69 #include "image_writer.h"
70 #include "interpreter/unstarted_runtime.h"
71 #include "java_vm_ext.h"
72 #include "jit/profile_compilation_info.h"
73 #include "leb128.h"
74 #include "linker/buffered_output_stream.h"
75 #include "linker/file_output_stream.h"
76 #include "linker/multi_oat_relative_patcher.h"
77 #include "mirror/class-inl.h"
78 #include "mirror/class_loader.h"
79 #include "mirror/object-inl.h"
80 #include "mirror/object_array-inl.h"
81 #include "nativehelper/ScopedLocalRef.h"
82 #include "oat_file.h"
83 #include "oat_file_assistant.h"
84 #include "oat_writer.h"
85 #include "os.h"
86 #include "runtime.h"
87 #include "runtime_options.h"
88 #include "scoped_thread_state_change-inl.h"
89 #include "utils.h"
90 #include "vdex_file.h"
91 #include "verifier/verifier_deps.h"
92 #include "well_known_classes.h"
93 #include "zip_archive.h"
94 
95 namespace art {
96 
97 using android::base::StringAppendV;
98 using android::base::StringPrintf;
99 
100 static constexpr size_t kDefaultMinDexFilesForSwap = 2;
101 static constexpr size_t kDefaultMinDexFileCumulativeSizeForSwap = 20 * MB;
102 
103 // Compiler filter override for very large apps.
104 static constexpr CompilerFilter::Filter kLargeAppFilter = CompilerFilter::kVerify;
105 
106 static int original_argc;
107 static char** original_argv;
108 
CommandLine()109 static std::string CommandLine() {
110   std::vector<std::string> command;
111   for (int i = 0; i < original_argc; ++i) {
112     command.push_back(original_argv[i]);
113   }
114   return android::base::Join(command, ' ');
115 }
116 
117 // A stripped version. Remove some less essential parameters. If we see a "--zip-fd=" parameter, be
118 // even more aggressive. There won't be much reasonable data here for us in that case anyways (the
119 // locations are all staged).
StrippedCommandLine()120 static std::string StrippedCommandLine() {
121   std::vector<std::string> command;
122 
123   // Do a pre-pass to look for zip-fd and the compiler filter.
124   bool saw_zip_fd = false;
125   bool saw_compiler_filter = false;
126   for (int i = 0; i < original_argc; ++i) {
127     if (android::base::StartsWith(original_argv[i], "--zip-fd=")) {
128       saw_zip_fd = true;
129     }
130     if (android::base::StartsWith(original_argv[i], "--compiler-filter=")) {
131       saw_compiler_filter = true;
132     }
133   }
134 
135   // Now filter out things.
136   for (int i = 0; i < original_argc; ++i) {
137     // All runtime-arg parameters are dropped.
138     if (strcmp(original_argv[i], "--runtime-arg") == 0) {
139       i++;  // Drop the next part, too.
140       continue;
141     }
142 
143     // Any instruction-setXXX is dropped.
144     if (android::base::StartsWith(original_argv[i], "--instruction-set")) {
145       continue;
146     }
147 
148     // The boot image is dropped.
149     if (android::base::StartsWith(original_argv[i], "--boot-image=")) {
150       continue;
151     }
152 
153     // The image format is dropped.
154     if (android::base::StartsWith(original_argv[i], "--image-format=")) {
155       continue;
156     }
157 
158     // This should leave any dex-file and oat-file options, describing what we compiled.
159 
160     // However, we prefer to drop this when we saw --zip-fd.
161     if (saw_zip_fd) {
162       // Drop anything --zip-X, --dex-X, --oat-X, --swap-X, or --app-image-X
163       if (android::base::StartsWith(original_argv[i], "--zip-") ||
164           android::base::StartsWith(original_argv[i], "--dex-") ||
165           android::base::StartsWith(original_argv[i], "--oat-") ||
166           android::base::StartsWith(original_argv[i], "--swap-") ||
167           android::base::StartsWith(original_argv[i], "--app-image-")) {
168         continue;
169       }
170     }
171 
172     command.push_back(original_argv[i]);
173   }
174 
175   if (!saw_compiler_filter) {
176     command.push_back("--compiler-filter=" +
177         CompilerFilter::NameOfFilter(CompilerFilter::kDefaultCompilerFilter));
178   }
179 
180   // Construct the final output.
181   if (command.size() <= 1U) {
182     // It seems only "/system/bin/dex2oat" is left, or not even that. Use a pretty line.
183     return "Starting dex2oat.";
184   }
185   return android::base::Join(command, ' ');
186 }
187 
UsageErrorV(const char * fmt,va_list ap)188 static void UsageErrorV(const char* fmt, va_list ap) {
189   std::string error;
190   StringAppendV(&error, fmt, ap);
191   LOG(ERROR) << error;
192 }
193 
UsageError(const char * fmt,...)194 static void UsageError(const char* fmt, ...) {
195   va_list ap;
196   va_start(ap, fmt);
197   UsageErrorV(fmt, ap);
198   va_end(ap);
199 }
200 
Usage(const char * fmt,...)201 NO_RETURN static void Usage(const char* fmt, ...) {
202   va_list ap;
203   va_start(ap, fmt);
204   UsageErrorV(fmt, ap);
205   va_end(ap);
206 
207   UsageError("Command: %s", CommandLine().c_str());
208 
209   UsageError("Usage: dex2oat [options]...");
210   UsageError("");
211   UsageError("  -j<number>: specifies the number of threads used for compilation.");
212   UsageError("       Default is the number of detected hardware threads available on the");
213   UsageError("       host system.");
214   UsageError("      Example: -j12");
215   UsageError("");
216   UsageError("  --dex-file=<dex-file>: specifies a .dex, .jar, or .apk file to compile.");
217   UsageError("      Example: --dex-file=/system/framework/core.jar");
218   UsageError("");
219   UsageError("  --dex-location=<dex-location>: specifies an alternative dex location to");
220   UsageError("      encode in the oat file for the corresponding --dex-file argument.");
221   UsageError("      Example: --dex-file=/home/build/out/system/framework/core.jar");
222   UsageError("               --dex-location=/system/framework/core.jar");
223   UsageError("");
224   UsageError("  --zip-fd=<file-descriptor>: specifies a file descriptor of a zip file");
225   UsageError("      containing a classes.dex file to compile.");
226   UsageError("      Example: --zip-fd=5");
227   UsageError("");
228   UsageError("  --zip-location=<zip-location>: specifies a symbolic name for the file");
229   UsageError("      corresponding to the file descriptor specified by --zip-fd.");
230   UsageError("      Example: --zip-location=/system/app/Calculator.apk");
231   UsageError("");
232   UsageError("  --oat-file=<file.oat>: specifies an oat output destination via a filename.");
233   UsageError("      Example: --oat-file=/system/framework/boot.oat");
234   UsageError("");
235   UsageError("  --oat-fd=<number>: specifies the oat output destination via a file descriptor.");
236   UsageError("      Example: --oat-fd=6");
237   UsageError("");
238   UsageError("  --oat-location=<oat-name>: specifies a symbolic name for the file corresponding");
239   UsageError("      to the file descriptor specified by --oat-fd.");
240   UsageError("      Example: --oat-location=/data/dalvik-cache/system@app@Calculator.apk.oat");
241   UsageError("");
242   UsageError("  --oat-symbols=<file.oat>: specifies an oat output destination with full symbols.");
243   UsageError("      Example: --oat-symbols=/symbols/system/framework/boot.oat");
244   UsageError("");
245   UsageError("  --image=<file.art>: specifies an output image filename.");
246   UsageError("      Example: --image=/system/framework/boot.art");
247   UsageError("");
248   UsageError("  --image-format=(uncompressed|lz4|lz4hc):");
249   UsageError("      Which format to store the image.");
250   UsageError("      Example: --image-format=lz4");
251   UsageError("      Default: uncompressed");
252   UsageError("");
253   UsageError("  --image-classes=<classname-file>: specifies classes to include in an image.");
254   UsageError("      Example: --image=frameworks/base/preloaded-classes");
255   UsageError("");
256   UsageError("  --base=<hex-address>: specifies the base address when creating a boot image.");
257   UsageError("      Example: --base=0x50000000");
258   UsageError("");
259   UsageError("  --boot-image=<file.art>: provide the image file for the boot class path.");
260   UsageError("      Do not include the arch as part of the name, it is added automatically.");
261   UsageError("      Example: --boot-image=/system/framework/boot.art");
262   UsageError("               (specifies /system/framework/<arch>/boot.art as the image file)");
263   UsageError("      Default: $ANDROID_ROOT/system/framework/boot.art");
264   UsageError("");
265   UsageError("  --android-root=<path>: used to locate libraries for portable linking.");
266   UsageError("      Example: --android-root=out/host/linux-x86");
267   UsageError("      Default: $ANDROID_ROOT");
268   UsageError("");
269   UsageError("  --instruction-set=(arm|arm64|mips|mips64|x86|x86_64): compile for a particular");
270   UsageError("      instruction set.");
271   UsageError("      Example: --instruction-set=x86");
272   UsageError("      Default: arm");
273   UsageError("");
274   UsageError("  --instruction-set-features=...,: Specify instruction set features");
275   UsageError("      Example: --instruction-set-features=div");
276   UsageError("      Default: default");
277   UsageError("");
278   UsageError("  --compile-pic: Force indirect use of code, methods, and classes");
279   UsageError("      Default: disabled");
280   UsageError("");
281   UsageError("  --compiler-backend=(Quick|Optimizing): select compiler backend");
282   UsageError("      set.");
283   UsageError("      Example: --compiler-backend=Optimizing");
284   UsageError("      Default: Optimizing");
285   UsageError("");
286   UsageError("  --compiler-filter="
287                 "(assume-verified"
288                 "|extract"
289                 "|verify"
290                 "|quicken"
291                 "|space-profile"
292                 "|space"
293                 "|speed-profile"
294                 "|speed"
295                 "|everything-profile"
296                 "|everything):");
297   UsageError("      select compiler filter.");
298   UsageError("      Example: --compiler-filter=everything");
299   UsageError("      Default: speed");
300   UsageError("");
301   UsageError("  --huge-method-max=<method-instruction-count>: threshold size for a huge");
302   UsageError("      method for compiler filter tuning.");
303   UsageError("      Example: --huge-method-max=%d", CompilerOptions::kDefaultHugeMethodThreshold);
304   UsageError("      Default: %d", CompilerOptions::kDefaultHugeMethodThreshold);
305   UsageError("");
306   UsageError("  --large-method-max=<method-instruction-count>: threshold size for a large");
307   UsageError("      method for compiler filter tuning.");
308   UsageError("      Example: --large-method-max=%d", CompilerOptions::kDefaultLargeMethodThreshold);
309   UsageError("      Default: %d", CompilerOptions::kDefaultLargeMethodThreshold);
310   UsageError("");
311   UsageError("  --small-method-max=<method-instruction-count>: threshold size for a small");
312   UsageError("      method for compiler filter tuning.");
313   UsageError("      Example: --small-method-max=%d", CompilerOptions::kDefaultSmallMethodThreshold);
314   UsageError("      Default: %d", CompilerOptions::kDefaultSmallMethodThreshold);
315   UsageError("");
316   UsageError("  --tiny-method-max=<method-instruction-count>: threshold size for a tiny");
317   UsageError("      method for compiler filter tuning.");
318   UsageError("      Example: --tiny-method-max=%d", CompilerOptions::kDefaultTinyMethodThreshold);
319   UsageError("      Default: %d", CompilerOptions::kDefaultTinyMethodThreshold);
320   UsageError("");
321   UsageError("  --num-dex-methods=<method-count>: threshold size for a small dex file for");
322   UsageError("      compiler filter tuning. If the input has fewer than this many methods");
323   UsageError("      and the filter is not interpret-only or verify-none or verify-at-runtime, ");
324   UsageError("      overrides the filter to use speed");
325   UsageError("      Example: --num-dex-method=%d", CompilerOptions::kDefaultNumDexMethodsThreshold);
326   UsageError("      Default: %d", CompilerOptions::kDefaultNumDexMethodsThreshold);
327   UsageError("");
328   UsageError("  --inline-max-code-units=<code-units-count>: the maximum code units that a method");
329   UsageError("      can have to be considered for inlining. A zero value will disable inlining.");
330   UsageError("      Honored only by Optimizing. Has priority over the --compiler-filter option.");
331   UsageError("      Intended for development/experimental use.");
332   UsageError("      Example: --inline-max-code-units=%d",
333              CompilerOptions::kDefaultInlineMaxCodeUnits);
334   UsageError("      Default: %d", CompilerOptions::kDefaultInlineMaxCodeUnits);
335   UsageError("");
336   UsageError("  --dump-timing: display a breakdown of where time was spent");
337   UsageError("");
338   UsageError("  -g");
339   UsageError("  --generate-debug-info: Generate debug information for native debugging,");
340   UsageError("      such as stack unwinding information, ELF symbols and DWARF sections.");
341   UsageError("      If used without --debuggable, it will be best-effort only.");
342   UsageError("      This option does not affect the generated code. (disabled by default)");
343   UsageError("");
344   UsageError("  --no-generate-debug-info: Do not generate debug information for native debugging.");
345   UsageError("");
346   UsageError("  --generate-mini-debug-info: Generate minimal amount of LZMA-compressed");
347   UsageError("      debug information necessary to print backtraces. (disabled by default)");
348   UsageError("");
349   UsageError("  --no-generate-mini-debug-info: Do not generate backtrace info.");
350   UsageError("");
351   UsageError("  --generate-build-id: Generate GNU-compatible linker build ID ELF section with");
352   UsageError("      SHA-1 of the file content (and thus stable across identical builds)");
353   UsageError("");
354   UsageError("  --no-generate-build-id: Do not generate the build ID ELF section.");
355   UsageError("");
356   UsageError("  --debuggable: Produce code debuggable with Java debugger.");
357   UsageError("");
358   UsageError("  --avoid-storing-invocation: Avoid storing the invocation args in the key value");
359   UsageError("      store. Used to test determinism with different args.");
360   UsageError("");
361   UsageError("  --runtime-arg <argument>: used to specify various arguments for the runtime,");
362   UsageError("      such as initial heap size, maximum heap size, and verbose output.");
363   UsageError("      Use a separate --runtime-arg switch for each argument.");
364   UsageError("      Example: --runtime-arg -Xms256m");
365   UsageError("");
366   UsageError("  --profile-file=<filename>: specify profiler output file to use for compilation.");
367   UsageError("");
368   UsageError("  --profile-file-fd=<number>: same as --profile-file but accepts a file descriptor.");
369   UsageError("      Cannot be used together with --profile-file.");
370   UsageError("");
371   UsageError("  --swap-file=<file-name>: specifies a file to use for swap.");
372   UsageError("      Example: --swap-file=/data/tmp/swap.001");
373   UsageError("");
374   UsageError("  --swap-fd=<file-descriptor>: specifies a file to use for swap (by descriptor).");
375   UsageError("      Example: --swap-fd=10");
376   UsageError("");
377   UsageError("  --swap-dex-size-threshold=<size>: specifies the minimum total dex file size in");
378   UsageError("      bytes to allow the use of swap.");
379   UsageError("      Example: --swap-dex-size-threshold=1000000");
380   UsageError("      Default: %zu", kDefaultMinDexFileCumulativeSizeForSwap);
381   UsageError("");
382   UsageError("  --swap-dex-count-threshold=<count>: specifies the minimum number of dex files to");
383   UsageError("      allow the use of swap.");
384   UsageError("      Example: --swap-dex-count-threshold=10");
385   UsageError("      Default: %zu", kDefaultMinDexFilesForSwap);
386   UsageError("");
387   UsageError("  --very-large-app-threshold=<size>: specifies the minimum total dex file size in");
388   UsageError("      bytes to consider the input \"very large\" and reduce compilation done.");
389   UsageError("      Example: --very-large-app-threshold=100000000");
390   UsageError("");
391   UsageError("  --app-image-fd=<file-descriptor>: specify output file descriptor for app image.");
392   UsageError("      Example: --app-image-fd=10");
393   UsageError("");
394   UsageError("  --app-image-file=<file-name>: specify a file name for app image.");
395   UsageError("      Example: --app-image-file=/data/dalvik-cache/system@app@Calculator.apk.art");
396   UsageError("");
397   UsageError("  --multi-image: specify that separate oat and image files be generated for each "
398              "input dex file.");
399   UsageError("");
400   UsageError("  --force-determinism: force the compiler to emit a deterministic output.");
401   UsageError("");
402   UsageError("  --dump-cfg=<cfg-file>: dump control-flow graphs (CFGs) to specified file.");
403   UsageError("      Example: --dump-cfg=output.cfg");
404   UsageError("");
405   UsageError("  --dump-cfg-append: when dumping CFGs to an existing file, append new CFG data to");
406   UsageError("      existing data (instead of overwriting existing data with new data, which is");
407   UsageError("      the default behavior). This option is only meaningful when used with");
408   UsageError("      --dump-cfg.");
409   UsageError("");
410   UsageError("  --classpath-dir=<directory-path>: directory used to resolve relative class paths.");
411   UsageError("");
412   UsageError("  --class-loader-context=<string spec>: a string specifying the intended");
413   UsageError("      runtime loading context for the compiled dex files.");
414   UsageError("");
415   UsageError("      It describes how the class loader chain should be built in order to ensure");
416   UsageError("      classes are resolved during dex2aot as they would be resolved at runtime.");
417   UsageError("      This spec will be encoded in the oat file. If at runtime the dex file is");
418   UsageError("      loaded in a different context, the oat file will be rejected.");
419   UsageError("");
420   UsageError("      The chain is interpreted in the natural 'parent order', meaning that class");
421   UsageError("      loader 'i+1' will be the parent of class loader 'i'.");
422   UsageError("      The compilation sources will be appended to the classpath of the first class");
423   UsageError("      loader.");
424   UsageError("");
425   UsageError("      E.g. if the context is 'PCL[lib1.dex];DLC[lib2.dex]' and ");
426   UsageError("      --dex-file=src.dex then dex2oat will setup a PathClassLoader with classpath ");
427   UsageError("      'lib1.dex:src.dex' and set its parent to a DelegateLastClassLoader with ");
428   UsageError("      classpath 'lib2.dex'.");
429   UsageError("      ");
430   UsageError("      Note that the compiler will be tolerant if the source dex files specified");
431   UsageError("      with --dex-file are found in the classpath. The source dex files will be");
432   UsageError("      removed from any class loader's classpath possibly resulting in empty");
433   UsageError("      class loaders.");
434   UsageError("");
435   UsageError("      Example: --class-loader-context=PCL[lib1.dex:lib2.dex];DLC[lib3.dex]");
436   UsageError("");
437   UsageError("  --dirty-image-objects=<directory-path>: list of known dirty objects in the image.");
438   UsageError("      The image writer will group them together.");
439   UsageError("");
440   std::cerr << "See log for usage error information\n";
441   exit(EXIT_FAILURE);
442 }
443 
444 // The primary goal of the watchdog is to prevent stuck build servers
445 // during development when fatal aborts lead to a cascade of failures
446 // that result in a deadlock.
447 class WatchDog {
448 // WatchDog defines its own CHECK_PTHREAD_CALL to avoid using LOG which uses locks
449 #undef CHECK_PTHREAD_CALL
450 #define CHECK_WATCH_DOG_PTHREAD_CALL(call, args, what) \
451   do { \
452     int rc = call args; \
453     if (rc != 0) { \
454       errno = rc; \
455       std::string message(# call); \
456       message += " failed for "; \
457       message += reason; \
458       Fatal(message); \
459     } \
460   } while (false)
461 
462  public:
WatchDog(int64_t timeout_in_milliseconds)463   explicit WatchDog(int64_t timeout_in_milliseconds)
464       : timeout_in_milliseconds_(timeout_in_milliseconds),
465         shutting_down_(false) {
466     const char* reason = "dex2oat watch dog thread startup";
467     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_init, (&mutex_, nullptr), reason);
468 #ifndef __APPLE__
469     pthread_condattr_t condattr;
470     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_condattr_init, (&condattr), reason);
471     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_condattr_setclock, (&condattr, CLOCK_MONOTONIC), reason);
472     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_cond_init, (&cond_, &condattr), reason);
473     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_condattr_destroy, (&condattr), reason);
474 #endif
475     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_attr_init, (&attr_), reason);
476     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_create, (&pthread_, &attr_, &CallBack, this), reason);
477     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_attr_destroy, (&attr_), reason);
478   }
~WatchDog()479   ~WatchDog() {
480     const char* reason = "dex2oat watch dog thread shutdown";
481     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_lock, (&mutex_), reason);
482     shutting_down_ = true;
483     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_cond_signal, (&cond_), reason);
484     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_unlock, (&mutex_), reason);
485 
486     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_join, (pthread_, nullptr), reason);
487 
488     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_cond_destroy, (&cond_), reason);
489     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_destroy, (&mutex_), reason);
490   }
491 
492   // TODO: tune the multiplier for GC verification, the following is just to make the timeout
493   //       large.
494   static constexpr int64_t kWatchdogVerifyMultiplier =
495       kVerifyObjectSupport > kVerifyObjectModeFast ? 100 : 1;
496 
497   // When setting timeouts, keep in mind that the build server may not be as fast as your
498   // desktop. Debug builds are slower so they have larger timeouts.
499   static constexpr int64_t kWatchdogSlowdownFactor = kIsDebugBuild ? 5U : 1U;
500 
501   // 9.5 minutes scaled by kSlowdownFactor. This is slightly smaller than the Package Manager
502   // watchdog (PackageManagerService.WATCHDOG_TIMEOUT, 10 minutes), so that dex2oat will abort
503   // itself before that watchdog would take down the system server.
504   static constexpr int64_t kWatchDogTimeoutSeconds = kWatchdogSlowdownFactor * (9 * 60 + 30);
505 
506   static constexpr int64_t kDefaultWatchdogTimeoutInMS =
507       kWatchdogVerifyMultiplier * kWatchDogTimeoutSeconds * 1000;
508 
509  private:
CallBack(void * arg)510   static void* CallBack(void* arg) {
511     WatchDog* self = reinterpret_cast<WatchDog*>(arg);
512     ::art::SetThreadName("dex2oat watch dog");
513     self->Wait();
514     return nullptr;
515   }
516 
Fatal(const std::string & message)517   NO_RETURN static void Fatal(const std::string& message) {
518     // TODO: When we can guarantee it won't prevent shutdown in error cases, move to LOG. However,
519     //       it's rather easy to hang in unwinding.
520     //       LogLine also avoids ART logging lock issues, as it's really only a wrapper around
521     //       logcat logging or stderr output.
522     android::base::LogMessage::LogLine(__FILE__,
523                                        __LINE__,
524                                        android::base::LogId::DEFAULT,
525                                        LogSeverity::FATAL,
526                                        message.c_str());
527     // If we're on the host, try to dump all threads to get a sense of what's going on. This is
528     // restricted to the host as the dump may itself go bad.
529     // TODO: Use a double watchdog timeout, so we can enable this on-device.
530     if (!kIsTargetBuild && Runtime::Current() != nullptr) {
531       Runtime::Current()->AttachCurrentThread("Watchdog thread attached for dumping",
532                                               true,
533                                               nullptr,
534                                               false);
535       Runtime::Current()->DumpForSigQuit(std::cerr);
536     }
537     exit(1);
538   }
539 
Wait()540   void Wait() {
541     timespec timeout_ts;
542 #if defined(__APPLE__)
543     InitTimeSpec(true, CLOCK_REALTIME, timeout_in_milliseconds_, 0, &timeout_ts);
544 #else
545     InitTimeSpec(true, CLOCK_MONOTONIC, timeout_in_milliseconds_, 0, &timeout_ts);
546 #endif
547     const char* reason = "dex2oat watch dog thread waiting";
548     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_lock, (&mutex_), reason);
549     while (!shutting_down_) {
550       int rc = TEMP_FAILURE_RETRY(pthread_cond_timedwait(&cond_, &mutex_, &timeout_ts));
551       if (rc == ETIMEDOUT) {
552         Fatal(StringPrintf("dex2oat did not finish after %" PRId64 " seconds",
553                            timeout_in_milliseconds_/1000));
554       } else if (rc != 0) {
555         std::string message(StringPrintf("pthread_cond_timedwait failed: %s",
556                                          strerror(errno)));
557         Fatal(message.c_str());
558       }
559     }
560     CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_unlock, (&mutex_), reason);
561   }
562 
563   // TODO: Switch to Mutex when we can guarantee it won't prevent shutdown in error cases.
564   pthread_mutex_t mutex_;
565   pthread_cond_t cond_;
566   pthread_attr_t attr_;
567   pthread_t pthread_;
568 
569   const int64_t timeout_in_milliseconds_;
570   bool shutting_down_;
571 };
572 
573 class Dex2Oat FINAL {
574  public:
Dex2Oat(TimingLogger * timings)575   explicit Dex2Oat(TimingLogger* timings) :
576       compiler_kind_(Compiler::kOptimizing),
577       instruction_set_(kRuntimeISA == kArm ? kThumb2 : kRuntimeISA),
578       // Take the default set of instruction features from the build.
579       image_file_location_oat_checksum_(0),
580       image_file_location_oat_data_begin_(0),
581       image_patch_delta_(0),
582       key_value_store_(nullptr),
583       verification_results_(nullptr),
584       runtime_(nullptr),
585       thread_count_(sysconf(_SC_NPROCESSORS_CONF)),
586       start_ns_(NanoTime()),
587       start_cputime_ns_(ProcessCpuNanoTime()),
588       oat_fd_(-1),
589       input_vdex_fd_(-1),
590       output_vdex_fd_(-1),
591       input_vdex_file_(nullptr),
592       zip_fd_(-1),
593       image_base_(0U),
594       image_classes_zip_filename_(nullptr),
595       image_classes_filename_(nullptr),
596       image_storage_mode_(ImageHeader::kStorageModeUncompressed),
597       compiled_classes_zip_filename_(nullptr),
598       compiled_classes_filename_(nullptr),
599       compiled_methods_zip_filename_(nullptr),
600       compiled_methods_filename_(nullptr),
601       passes_to_run_filename_(nullptr),
602       dirty_image_objects_filename_(nullptr),
603       multi_image_(false),
604       is_host_(false),
605       elf_writers_(),
606       oat_writers_(),
607       rodata_(),
608       image_writer_(nullptr),
609       driver_(nullptr),
610       opened_dex_files_maps_(),
611       opened_dex_files_(),
612       no_inline_from_dex_files_(),
613       dump_stats_(false),
614       dump_passes_(false),
615       dump_timing_(false),
616       dump_slow_timing_(kIsDebugBuild),
617       avoid_storing_invocation_(false),
618       swap_fd_(kInvalidFd),
619       app_image_fd_(kInvalidFd),
620       profile_file_fd_(kInvalidFd),
621       timings_(timings),
622       force_determinism_(false)
623       {}
624 
~Dex2Oat()625   ~Dex2Oat() {
626     // Log completion time before deleting the runtime_, because this accesses
627     // the runtime.
628     LogCompletionTime();
629 
630     if (!kIsDebugBuild && !(RUNNING_ON_MEMORY_TOOL && kMemoryToolDetectsLeaks)) {
631       // We want to just exit on non-debug builds, not bringing the runtime down
632       // in an orderly fashion. So release the following fields.
633       driver_.release();
634       image_writer_.release();
635       for (std::unique_ptr<const DexFile>& dex_file : opened_dex_files_) {
636         dex_file.release();
637       }
638       for (std::unique_ptr<MemMap>& map : opened_dex_files_maps_) {
639         map.release();
640       }
641       for (std::unique_ptr<File>& vdex_file : vdex_files_) {
642         vdex_file.release();
643       }
644       for (std::unique_ptr<File>& oat_file : oat_files_) {
645         oat_file.release();
646       }
647       runtime_.release();
648       verification_results_.release();
649       key_value_store_.release();
650     }
651   }
652 
653   struct ParserOptions {
654     std::vector<const char*> oat_symbols;
655     std::string boot_image_filename;
656     int64_t watch_dog_timeout_in_ms = -1;
657     bool watch_dog_enabled = true;
658     bool requested_specific_compiler = false;
659     std::string error_msg;
660   };
661 
ParseZipFd(const StringPiece & option)662   void ParseZipFd(const StringPiece& option) {
663     ParseUintOption(option, "--zip-fd", &zip_fd_, Usage);
664   }
665 
ParseInputVdexFd(const StringPiece & option)666   void ParseInputVdexFd(const StringPiece& option) {
667     // Note that the input vdex fd might be -1.
668     ParseIntOption(option, "--input-vdex-fd", &input_vdex_fd_, Usage);
669   }
670 
ParseOutputVdexFd(const StringPiece & option)671   void ParseOutputVdexFd(const StringPiece& option) {
672     ParseUintOption(option, "--output-vdex-fd", &output_vdex_fd_, Usage);
673   }
674 
ParseOatFd(const StringPiece & option)675   void ParseOatFd(const StringPiece& option) {
676     ParseUintOption(option, "--oat-fd", &oat_fd_, Usage);
677   }
678 
ParseFdForCollection(const StringPiece & option,const char * arg_name,std::vector<uint32_t> * fds)679   void ParseFdForCollection(const StringPiece& option,
680                             const char* arg_name,
681                             std::vector<uint32_t>* fds) {
682     uint32_t fd;
683     ParseUintOption(option, arg_name, &fd, Usage);
684     fds->push_back(fd);
685   }
686 
ParseJ(const StringPiece & option)687   void ParseJ(const StringPiece& option) {
688     ParseUintOption(option, "-j", &thread_count_, Usage, /* is_long_option */ false);
689   }
690 
ParseBase(const StringPiece & option)691   void ParseBase(const StringPiece& option) {
692     DCHECK(option.starts_with("--base="));
693     const char* image_base_str = option.substr(strlen("--base=")).data();
694     char* end;
695     image_base_ = strtoul(image_base_str, &end, 16);
696     if (end == image_base_str || *end != '\0') {
697       Usage("Failed to parse hexadecimal value for option %s", option.data());
698     }
699   }
700 
ParseInstructionSet(const StringPiece & option)701   void ParseInstructionSet(const StringPiece& option) {
702     DCHECK(option.starts_with("--instruction-set="));
703     StringPiece instruction_set_str = option.substr(strlen("--instruction-set=")).data();
704     // StringPiece is not necessarily zero-terminated, so need to make a copy and ensure it.
705     std::unique_ptr<char[]> buf(new char[instruction_set_str.length() + 1]);
706     strncpy(buf.get(), instruction_set_str.data(), instruction_set_str.length());
707     buf.get()[instruction_set_str.length()] = 0;
708     instruction_set_ = GetInstructionSetFromString(buf.get());
709     // arm actually means thumb2.
710     if (instruction_set_ == InstructionSet::kArm) {
711       instruction_set_ = InstructionSet::kThumb2;
712     }
713   }
714 
ParseInstructionSetVariant(const StringPiece & option,ParserOptions * parser_options)715   void ParseInstructionSetVariant(const StringPiece& option, ParserOptions* parser_options) {
716     DCHECK(option.starts_with("--instruction-set-variant="));
717     StringPiece str = option.substr(strlen("--instruction-set-variant=")).data();
718     instruction_set_features_ = InstructionSetFeatures::FromVariant(
719         instruction_set_, str.as_string(), &parser_options->error_msg);
720     if (instruction_set_features_.get() == nullptr) {
721       Usage("%s", parser_options->error_msg.c_str());
722     }
723   }
724 
ParseInstructionSetFeatures(const StringPiece & option,ParserOptions * parser_options)725   void ParseInstructionSetFeatures(const StringPiece& option, ParserOptions* parser_options) {
726     DCHECK(option.starts_with("--instruction-set-features="));
727     StringPiece str = option.substr(strlen("--instruction-set-features=")).data();
728     if (instruction_set_features_ == nullptr) {
729       instruction_set_features_ = InstructionSetFeatures::FromVariant(
730           instruction_set_, "default", &parser_options->error_msg);
731       if (instruction_set_features_.get() == nullptr) {
732         Usage("Problem initializing default instruction set features variant: %s",
733               parser_options->error_msg.c_str());
734       }
735     }
736     instruction_set_features_ =
737         instruction_set_features_->AddFeaturesFromString(str.as_string(),
738                                                          &parser_options->error_msg);
739     if (instruction_set_features_ == nullptr) {
740       Usage("Error parsing '%s': %s", option.data(), parser_options->error_msg.c_str());
741     }
742   }
743 
ParseCompilerBackend(const StringPiece & option,ParserOptions * parser_options)744   void ParseCompilerBackend(const StringPiece& option, ParserOptions* parser_options) {
745     DCHECK(option.starts_with("--compiler-backend="));
746     parser_options->requested_specific_compiler = true;
747     StringPiece backend_str = option.substr(strlen("--compiler-backend=")).data();
748     if (backend_str == "Quick") {
749       compiler_kind_ = Compiler::kQuick;
750     } else if (backend_str == "Optimizing") {
751       compiler_kind_ = Compiler::kOptimizing;
752     } else {
753       Usage("Unknown compiler backend: %s", backend_str.data());
754     }
755   }
756 
ParseImageFormat(const StringPiece & option)757   void ParseImageFormat(const StringPiece& option) {
758     const StringPiece substr("--image-format=");
759     DCHECK(option.starts_with(substr));
760     const StringPiece format_str = option.substr(substr.length());
761     if (format_str == "lz4") {
762       image_storage_mode_ = ImageHeader::kStorageModeLZ4;
763     } else if (format_str == "lz4hc") {
764       image_storage_mode_ = ImageHeader::kStorageModeLZ4HC;
765     } else if (format_str == "uncompressed") {
766       image_storage_mode_ = ImageHeader::kStorageModeUncompressed;
767     } else {
768       Usage("Unknown image format: %s", format_str.data());
769     }
770   }
771 
ProcessOptions(ParserOptions * parser_options)772   void ProcessOptions(ParserOptions* parser_options) {
773     compiler_options_->boot_image_ = !image_filenames_.empty();
774     compiler_options_->app_image_ = app_image_fd_ != -1 || !app_image_file_name_.empty();
775 
776     if (IsAppImage() && IsBootImage()) {
777       Usage("Can't have both --image and (--app-image-fd or --app-image-file)");
778     }
779 
780     if (oat_filenames_.empty() && oat_fd_ == -1) {
781       Usage("Output must be supplied with either --oat-file or --oat-fd");
782     }
783 
784     if (input_vdex_fd_ != -1 && !input_vdex_.empty()) {
785       Usage("Can't have both --input-vdex-fd and --input-vdex");
786     }
787 
788     if (output_vdex_fd_ != -1 && !output_vdex_.empty()) {
789       Usage("Can't have both --output-vdex-fd and --output-vdex");
790     }
791 
792     if (!oat_filenames_.empty() && oat_fd_ != -1) {
793       Usage("--oat-file should not be used with --oat-fd");
794     }
795 
796     if ((output_vdex_fd_ == -1) != (oat_fd_ == -1)) {
797       Usage("VDEX and OAT output must be specified either with one --oat-filename "
798             "or with --oat-fd and --output-vdex-fd file descriptors");
799     }
800 
801     if (!parser_options->oat_symbols.empty() && oat_fd_ != -1) {
802       Usage("--oat-symbols should not be used with --oat-fd");
803     }
804 
805     if (!parser_options->oat_symbols.empty() && is_host_) {
806       Usage("--oat-symbols should not be used with --host");
807     }
808 
809     if (output_vdex_fd_ != -1 && !image_filenames_.empty()) {
810       Usage("--output-vdex-fd should not be used with --image");
811     }
812 
813     if (oat_fd_ != -1 && !image_filenames_.empty()) {
814       Usage("--oat-fd should not be used with --image");
815     }
816 
817     if (!parser_options->oat_symbols.empty() &&
818         parser_options->oat_symbols.size() != oat_filenames_.size()) {
819       Usage("--oat-file arguments do not match --oat-symbols arguments");
820     }
821 
822     if (!image_filenames_.empty() && image_filenames_.size() != oat_filenames_.size()) {
823       Usage("--oat-file arguments do not match --image arguments");
824     }
825 
826     if (android_root_.empty()) {
827       const char* android_root_env_var = getenv("ANDROID_ROOT");
828       if (android_root_env_var == nullptr) {
829         Usage("--android-root unspecified and ANDROID_ROOT not set");
830       }
831       android_root_ += android_root_env_var;
832     }
833 
834     if (!IsBootImage() && parser_options->boot_image_filename.empty()) {
835       parser_options->boot_image_filename += android_root_;
836       parser_options->boot_image_filename += "/framework/boot.art";
837     }
838     if (!parser_options->boot_image_filename.empty()) {
839       boot_image_filename_ = parser_options->boot_image_filename;
840     }
841 
842     if (image_classes_filename_ != nullptr && !IsBootImage()) {
843       Usage("--image-classes should only be used with --image");
844     }
845 
846     if (image_classes_filename_ != nullptr && !boot_image_filename_.empty()) {
847       Usage("--image-classes should not be used with --boot-image");
848     }
849 
850     if (image_classes_zip_filename_ != nullptr && image_classes_filename_ == nullptr) {
851       Usage("--image-classes-zip should be used with --image-classes");
852     }
853 
854     if (compiled_classes_filename_ != nullptr && !IsBootImage()) {
855       Usage("--compiled-classes should only be used with --image");
856     }
857 
858     if (compiled_classes_filename_ != nullptr && !boot_image_filename_.empty()) {
859       Usage("--compiled-classes should not be used with --boot-image");
860     }
861 
862     if (compiled_classes_zip_filename_ != nullptr && compiled_classes_filename_ == nullptr) {
863       Usage("--compiled-classes-zip should be used with --compiled-classes");
864     }
865 
866     if (dex_filenames_.empty() && zip_fd_ == -1) {
867       Usage("Input must be supplied with either --dex-file or --zip-fd");
868     }
869 
870     if (!dex_filenames_.empty() && zip_fd_ != -1) {
871       Usage("--dex-file should not be used with --zip-fd");
872     }
873 
874     if (!dex_filenames_.empty() && !zip_location_.empty()) {
875       Usage("--dex-file should not be used with --zip-location");
876     }
877 
878     if (dex_locations_.empty()) {
879       for (const char* dex_file_name : dex_filenames_) {
880         dex_locations_.push_back(dex_file_name);
881       }
882     } else if (dex_locations_.size() != dex_filenames_.size()) {
883       Usage("--dex-location arguments do not match --dex-file arguments");
884     }
885 
886     if (!dex_filenames_.empty() && !oat_filenames_.empty()) {
887       if (oat_filenames_.size() != 1 && oat_filenames_.size() != dex_filenames_.size()) {
888         Usage("--oat-file arguments must be singular or match --dex-file arguments");
889       }
890     }
891 
892     if (zip_fd_ != -1 && zip_location_.empty()) {
893       Usage("--zip-location should be supplied with --zip-fd");
894     }
895 
896     if (boot_image_filename_.empty()) {
897       if (image_base_ == 0) {
898         Usage("Non-zero --base not specified");
899       }
900     }
901 
902     const bool have_profile_file = !profile_file_.empty();
903     const bool have_profile_fd = profile_file_fd_ != kInvalidFd;
904     if (have_profile_file && have_profile_fd) {
905       Usage("Profile file should not be specified with both --profile-file-fd and --profile-file");
906     }
907 
908     if (have_profile_file || have_profile_fd) {
909       if (compiled_classes_filename_ != nullptr ||
910           compiled_classes_zip_filename_ != nullptr ||
911           image_classes_filename_ != nullptr ||
912           image_classes_zip_filename_ != nullptr) {
913         Usage("Profile based image creation is not supported with image or compiled classes");
914       }
915     }
916 
917     if (!parser_options->oat_symbols.empty()) {
918       oat_unstripped_ = std::move(parser_options->oat_symbols);
919     }
920 
921     // If no instruction set feature was given, use the default one for the target
922     // instruction set.
923     if (instruction_set_features_.get() == nullptr) {
924       instruction_set_features_ = InstructionSetFeatures::FromVariant(
925          instruction_set_, "default", &parser_options->error_msg);
926       if (instruction_set_features_.get() == nullptr) {
927         Usage("Problem initializing default instruction set features variant: %s",
928               parser_options->error_msg.c_str());
929       }
930     }
931 
932     if (instruction_set_ == kRuntimeISA) {
933       std::unique_ptr<const InstructionSetFeatures> runtime_features(
934           InstructionSetFeatures::FromCppDefines());
935       if (!instruction_set_features_->Equals(runtime_features.get())) {
936         LOG(WARNING) << "Mismatch between dex2oat instruction set features ("
937             << *instruction_set_features_ << ") and those of dex2oat executable ("
938             << *runtime_features <<") for the command line:\n"
939             << CommandLine();
940       }
941     }
942 
943     if (compiler_options_->inline_max_code_units_ == CompilerOptions::kUnsetInlineMaxCodeUnits) {
944       compiler_options_->inline_max_code_units_ = CompilerOptions::kDefaultInlineMaxCodeUnits;
945     }
946 
947     // Checks are all explicit until we know the architecture.
948     // Set the compilation target's implicit checks options.
949     switch (instruction_set_) {
950       case kArm:
951       case kThumb2:
952       case kArm64:
953       case kX86:
954       case kX86_64:
955       case kMips:
956       case kMips64:
957         compiler_options_->implicit_null_checks_ = true;
958         compiler_options_->implicit_so_checks_ = true;
959         break;
960 
961       default:
962         // Defaults are correct.
963         break;
964     }
965 
966     if (!IsBootImage() && multi_image_) {
967       Usage("--multi-image can only be used when creating boot images");
968     }
969     if (IsBootImage() && multi_image_ && image_filenames_.size() > 1) {
970       Usage("--multi-image cannot be used with multiple image names");
971     }
972 
973     // For now, if we're on the host and compile the boot image, *always* use multiple image files.
974     if (!kIsTargetBuild && IsBootImage()) {
975       if (image_filenames_.size() == 1) {
976         multi_image_ = true;
977       }
978     }
979 
980     // Done with usage checks, enable watchdog if requested
981     if (parser_options->watch_dog_enabled) {
982       int64_t timeout = parser_options->watch_dog_timeout_in_ms > 0
983                             ? parser_options->watch_dog_timeout_in_ms
984                             : WatchDog::kDefaultWatchdogTimeoutInMS;
985       watchdog_.reset(new WatchDog(timeout));
986     }
987 
988     // Fill some values into the key-value store for the oat header.
989     key_value_store_.reset(new SafeMap<std::string, std::string>());
990 
991     // Automatically force determinism for the boot image in a host build if read barriers
992     // are enabled, or if the default GC is CMS or MS. When the default GC is CMS
993     // (Concurrent Mark-Sweep), the GC is switched to a non-concurrent one by passing the
994     // option `-Xgc:nonconcurrent` (see below).
995     if (!kIsTargetBuild && IsBootImage()) {
996       if (SupportsDeterministicCompilation()) {
997         force_determinism_ = true;
998       } else {
999         LOG(WARNING) << "Deterministic compilation is disabled.";
1000       }
1001     }
1002     compiler_options_->force_determinism_ = force_determinism_;
1003 
1004     if (passes_to_run_filename_ != nullptr) {
1005       passes_to_run_.reset(ReadCommentedInputFromFile<std::vector<std::string>>(
1006           passes_to_run_filename_,
1007           nullptr));         // No post-processing.
1008       if (passes_to_run_.get() == nullptr) {
1009         Usage("Failed to read list of passes to run.");
1010       }
1011     }
1012     compiler_options_->passes_to_run_ = passes_to_run_.get();
1013   }
1014 
SupportsDeterministicCompilation()1015   static bool SupportsDeterministicCompilation() {
1016     return (kUseReadBarrier ||
1017             gc::kCollectorTypeDefault == gc::kCollectorTypeCMS ||
1018             gc::kCollectorTypeDefault == gc::kCollectorTypeMS);
1019   }
1020 
ExpandOatAndImageFilenames()1021   void ExpandOatAndImageFilenames() {
1022     std::string base_oat = oat_filenames_[0];
1023     size_t last_oat_slash = base_oat.rfind('/');
1024     if (last_oat_slash == std::string::npos) {
1025       Usage("--multi-image used with unusable oat filename %s", base_oat.c_str());
1026     }
1027     // We also need to honor path components that were encoded through '@'. Otherwise the loading
1028     // code won't be able to find the images.
1029     if (base_oat.find('@', last_oat_slash) != std::string::npos) {
1030       last_oat_slash = base_oat.rfind('@');
1031     }
1032     base_oat = base_oat.substr(0, last_oat_slash + 1);
1033 
1034     std::string base_img = image_filenames_[0];
1035     size_t last_img_slash = base_img.rfind('/');
1036     if (last_img_slash == std::string::npos) {
1037       Usage("--multi-image used with unusable image filename %s", base_img.c_str());
1038     }
1039     // We also need to honor path components that were encoded through '@'. Otherwise the loading
1040     // code won't be able to find the images.
1041     if (base_img.find('@', last_img_slash) != std::string::npos) {
1042       last_img_slash = base_img.rfind('@');
1043     }
1044 
1045     // Get the prefix, which is the primary image name (without path components). Strip the
1046     // extension.
1047     std::string prefix = base_img.substr(last_img_slash + 1);
1048     if (prefix.rfind('.') != std::string::npos) {
1049       prefix = prefix.substr(0, prefix.rfind('.'));
1050     }
1051     if (!prefix.empty()) {
1052       prefix = prefix + "-";
1053     }
1054 
1055     base_img = base_img.substr(0, last_img_slash + 1);
1056 
1057     // Note: we have some special case here for our testing. We have to inject the differentiating
1058     //       parts for the different core images.
1059     std::string infix;  // Empty infix by default.
1060     {
1061       // Check the first name.
1062       std::string dex_file = oat_filenames_[0];
1063       size_t last_dex_slash = dex_file.rfind('/');
1064       if (last_dex_slash != std::string::npos) {
1065         dex_file = dex_file.substr(last_dex_slash + 1);
1066       }
1067       size_t last_dex_dot = dex_file.rfind('.');
1068       if (last_dex_dot != std::string::npos) {
1069         dex_file = dex_file.substr(0, last_dex_dot);
1070       }
1071       if (android::base::StartsWith(dex_file, "core-")) {
1072         infix = dex_file.substr(strlen("core"));
1073       }
1074     }
1075 
1076     std::string base_symbol_oat;
1077     if (!oat_unstripped_.empty()) {
1078       base_symbol_oat = oat_unstripped_[0];
1079       size_t last_symbol_oat_slash = base_symbol_oat.rfind('/');
1080       if (last_symbol_oat_slash == std::string::npos) {
1081         Usage("--multi-image used with unusable symbol filename %s", base_symbol_oat.c_str());
1082       }
1083       base_symbol_oat = base_symbol_oat.substr(0, last_symbol_oat_slash + 1);
1084     }
1085 
1086     const size_t num_expanded_files = 2 + (base_symbol_oat.empty() ? 0 : 1);
1087     char_backing_storage_.reserve((dex_locations_.size() - 1) * num_expanded_files);
1088 
1089     // Now create the other names. Use a counted loop to skip the first one.
1090     for (size_t i = 1; i < dex_locations_.size(); ++i) {
1091       // TODO: Make everything properly std::string.
1092       std::string image_name = CreateMultiImageName(dex_locations_[i], prefix, infix, ".art");
1093       char_backing_storage_.push_back(base_img + image_name);
1094       image_filenames_.push_back((char_backing_storage_.end() - 1)->c_str());
1095 
1096       std::string oat_name = CreateMultiImageName(dex_locations_[i], prefix, infix, ".oat");
1097       char_backing_storage_.push_back(base_oat + oat_name);
1098       oat_filenames_.push_back((char_backing_storage_.end() - 1)->c_str());
1099 
1100       if (!base_symbol_oat.empty()) {
1101         char_backing_storage_.push_back(base_symbol_oat + oat_name);
1102         oat_unstripped_.push_back((char_backing_storage_.end() - 1)->c_str());
1103       }
1104     }
1105   }
1106 
1107   // Modify the input string in the following way:
1108   //   0) Assume input is /a/b/c.d
1109   //   1) Strip the path  -> c.d
1110   //   2) Inject prefix p -> pc.d
1111   //   3) Inject infix i  -> pci.d
1112   //   4) Replace suffix with s if it's "jar"  -> d == "jar" -> pci.s
CreateMultiImageName(std::string in,const std::string & prefix,const std::string & infix,const char * replace_suffix)1113   static std::string CreateMultiImageName(std::string in,
1114                                           const std::string& prefix,
1115                                           const std::string& infix,
1116                                           const char* replace_suffix) {
1117     size_t last_dex_slash = in.rfind('/');
1118     if (last_dex_slash != std::string::npos) {
1119       in = in.substr(last_dex_slash + 1);
1120     }
1121     if (!prefix.empty()) {
1122       in = prefix + in;
1123     }
1124     if (!infix.empty()) {
1125       // Inject infix.
1126       size_t last_dot = in.rfind('.');
1127       if (last_dot != std::string::npos) {
1128         in.insert(last_dot, infix);
1129       }
1130     }
1131     if (android::base::EndsWith(in, ".jar")) {
1132       in = in.substr(0, in.length() - strlen(".jar")) +
1133           (replace_suffix != nullptr ? replace_suffix : "");
1134     }
1135     return in;
1136   }
1137 
InsertCompileOptions(int argc,char ** argv)1138   void InsertCompileOptions(int argc, char** argv) {
1139     std::ostringstream oss;
1140     if (!avoid_storing_invocation_) {
1141       for (int i = 0; i < argc; ++i) {
1142         if (i > 0) {
1143           oss << ' ';
1144         }
1145         oss << argv[i];
1146       }
1147       key_value_store_->Put(OatHeader::kDex2OatCmdLineKey, oss.str());
1148       oss.str("");  // Reset.
1149     }
1150     oss << kRuntimeISA;
1151     key_value_store_->Put(OatHeader::kDex2OatHostKey, oss.str());
1152     key_value_store_->Put(
1153         OatHeader::kPicKey,
1154         compiler_options_->compile_pic_ ? OatHeader::kTrueValue : OatHeader::kFalseValue);
1155     key_value_store_->Put(
1156         OatHeader::kDebuggableKey,
1157         compiler_options_->debuggable_ ? OatHeader::kTrueValue : OatHeader::kFalseValue);
1158     key_value_store_->Put(
1159         OatHeader::kNativeDebuggableKey,
1160         compiler_options_->GetNativeDebuggable() ? OatHeader::kTrueValue : OatHeader::kFalseValue);
1161     key_value_store_->Put(OatHeader::kCompilerFilter,
1162         CompilerFilter::NameOfFilter(compiler_options_->GetCompilerFilter()));
1163     key_value_store_->Put(OatHeader::kConcurrentCopying,
1164                           kUseReadBarrier ? OatHeader::kTrueValue : OatHeader::kFalseValue);
1165   }
1166 
1167   // Parse the arguments from the command line. In case of an unrecognized option or impossible
1168   // values/combinations, a usage error will be displayed and exit() is called. Thus, if the method
1169   // returns, arguments have been successfully parsed.
ParseArgs(int argc,char ** argv)1170   void ParseArgs(int argc, char** argv) {
1171     original_argc = argc;
1172     original_argv = argv;
1173 
1174     InitLogging(argv, Runtime::Abort);
1175 
1176     // Skip over argv[0].
1177     argv++;
1178     argc--;
1179 
1180     if (argc == 0) {
1181       Usage("No arguments specified");
1182     }
1183 
1184     std::unique_ptr<ParserOptions> parser_options(new ParserOptions());
1185     compiler_options_.reset(new CompilerOptions());
1186 
1187     for (int i = 0; i < argc; i++) {
1188       const StringPiece option(argv[i]);
1189       const bool log_options = false;
1190       if (log_options) {
1191         LOG(INFO) << "dex2oat: option[" << i << "]=" << argv[i];
1192       }
1193       if (option.starts_with("--dex-file=")) {
1194         dex_filenames_.push_back(option.substr(strlen("--dex-file=")).data());
1195       } else if (option.starts_with("--dex-location=")) {
1196         dex_locations_.push_back(option.substr(strlen("--dex-location=")).data());
1197       } else if (option.starts_with("--zip-fd=")) {
1198         ParseZipFd(option);
1199       } else if (option.starts_with("--zip-location=")) {
1200         zip_location_ = option.substr(strlen("--zip-location=")).data();
1201       } else if (option.starts_with("--input-vdex-fd=")) {
1202         ParseInputVdexFd(option);
1203       } else if (option.starts_with("--input-vdex=")) {
1204         input_vdex_ = option.substr(strlen("--input-vdex=")).data();
1205       } else if (option.starts_with("--output-vdex=")) {
1206         output_vdex_ = option.substr(strlen("--output-vdex=")).data();
1207       } else if (option.starts_with("--output-vdex-fd=")) {
1208         ParseOutputVdexFd(option);
1209       } else if (option.starts_with("--oat-file=")) {
1210         oat_filenames_.push_back(option.substr(strlen("--oat-file=")).data());
1211       } else if (option.starts_with("--oat-symbols=")) {
1212         parser_options->oat_symbols.push_back(option.substr(strlen("--oat-symbols=")).data());
1213       } else if (option.starts_with("--oat-fd=")) {
1214         ParseOatFd(option);
1215       } else if (option.starts_with("--oat-location=")) {
1216         oat_location_ = option.substr(strlen("--oat-location=")).data();
1217       } else if (option == "--watch-dog") {
1218         parser_options->watch_dog_enabled = true;
1219       } else if (option == "--no-watch-dog") {
1220         parser_options->watch_dog_enabled = false;
1221       } else if (option.starts_with("--watchdog-timeout=")) {
1222         ParseIntOption(option,
1223                        "--watchdog-timeout",
1224                        &parser_options->watch_dog_timeout_in_ms,
1225                        Usage);
1226       } else if (option.starts_with("-j")) {
1227         ParseJ(option);
1228       } else if (option.starts_with("--image=")) {
1229         image_filenames_.push_back(option.substr(strlen("--image=")).data());
1230       } else if (option.starts_with("--image-classes=")) {
1231         image_classes_filename_ = option.substr(strlen("--image-classes=")).data();
1232       } else if (option.starts_with("--image-classes-zip=")) {
1233         image_classes_zip_filename_ = option.substr(strlen("--image-classes-zip=")).data();
1234       } else if (option.starts_with("--image-format=")) {
1235         ParseImageFormat(option);
1236       } else if (option.starts_with("--compiled-classes=")) {
1237         compiled_classes_filename_ = option.substr(strlen("--compiled-classes=")).data();
1238       } else if (option.starts_with("--compiled-classes-zip=")) {
1239         compiled_classes_zip_filename_ = option.substr(strlen("--compiled-classes-zip=")).data();
1240       } else if (option.starts_with("--compiled-methods=")) {
1241         compiled_methods_filename_ = option.substr(strlen("--compiled-methods=")).data();
1242       } else if (option.starts_with("--compiled-methods-zip=")) {
1243         compiled_methods_zip_filename_ = option.substr(strlen("--compiled-methods-zip=")).data();
1244       } else if (option.starts_with("--run-passes=")) {
1245         passes_to_run_filename_ = option.substr(strlen("--run-passes=")).data();
1246       } else if (option.starts_with("--base=")) {
1247         ParseBase(option);
1248       } else if (option.starts_with("--boot-image=")) {
1249         parser_options->boot_image_filename = option.substr(strlen("--boot-image=")).data();
1250       } else if (option.starts_with("--android-root=")) {
1251         android_root_ = option.substr(strlen("--android-root=")).data();
1252       } else if (option.starts_with("--instruction-set=")) {
1253         ParseInstructionSet(option);
1254       } else if (option.starts_with("--instruction-set-variant=")) {
1255         ParseInstructionSetVariant(option, parser_options.get());
1256       } else if (option.starts_with("--instruction-set-features=")) {
1257         ParseInstructionSetFeatures(option, parser_options.get());
1258       } else if (option.starts_with("--compiler-backend=")) {
1259         ParseCompilerBackend(option, parser_options.get());
1260       } else if (option.starts_with("--profile-file=")) {
1261         profile_file_ = option.substr(strlen("--profile-file=")).ToString();
1262       } else if (option.starts_with("--profile-file-fd=")) {
1263         ParseUintOption(option, "--profile-file-fd", &profile_file_fd_, Usage);
1264       } else if (option == "--host") {
1265         is_host_ = true;
1266       } else if (option == "--runtime-arg") {
1267         if (++i >= argc) {
1268           Usage("Missing required argument for --runtime-arg");
1269         }
1270         if (log_options) {
1271           LOG(INFO) << "dex2oat: option[" << i << "]=" << argv[i];
1272         }
1273         runtime_args_.push_back(argv[i]);
1274       } else if (option == "--dump-timing") {
1275         dump_timing_ = true;
1276       } else if (option == "--dump-passes") {
1277         dump_passes_ = true;
1278       } else if (option == "--dump-stats") {
1279         dump_stats_ = true;
1280       } else if (option == "--avoid-storing-invocation") {
1281         avoid_storing_invocation_ = true;
1282       } else if (option.starts_with("--swap-file=")) {
1283         swap_file_name_ = option.substr(strlen("--swap-file=")).data();
1284       } else if (option.starts_with("--swap-fd=")) {
1285         ParseUintOption(option, "--swap-fd", &swap_fd_, Usage);
1286       } else if (option.starts_with("--swap-dex-size-threshold=")) {
1287         ParseUintOption(option,
1288                         "--swap-dex-size-threshold",
1289                         &min_dex_file_cumulative_size_for_swap_,
1290                         Usage);
1291       } else if (option.starts_with("--swap-dex-count-threshold=")) {
1292         ParseUintOption(option,
1293                         "--swap-dex-count-threshold",
1294                         &min_dex_files_for_swap_,
1295                         Usage);
1296       } else if (option.starts_with("--very-large-app-threshold=")) {
1297         ParseUintOption(option,
1298                         "--very-large-app-threshold",
1299                         &very_large_threshold_,
1300                         Usage);
1301       } else if (option.starts_with("--app-image-file=")) {
1302         app_image_file_name_ = option.substr(strlen("--app-image-file=")).data();
1303       } else if (option.starts_with("--app-image-fd=")) {
1304         ParseUintOption(option, "--app-image-fd", &app_image_fd_, Usage);
1305       } else if (option == "--multi-image") {
1306         multi_image_ = true;
1307       } else if (option.starts_with("--no-inline-from=")) {
1308         no_inline_from_string_ = option.substr(strlen("--no-inline-from=")).data();
1309       } else if (option == "--force-determinism") {
1310         if (!SupportsDeterministicCompilation()) {
1311           Usage("Option --force-determinism requires read barriers or a CMS/MS garbage collector");
1312         }
1313         force_determinism_ = true;
1314       } else if (option.starts_with("--classpath-dir=")) {
1315         classpath_dir_ = option.substr(strlen("--classpath-dir=")).data();
1316       } else if (option.starts_with("--class-loader-context=")) {
1317         class_loader_context_ = ClassLoaderContext::Create(
1318             option.substr(strlen("--class-loader-context=")).data());
1319         if (class_loader_context_ == nullptr) {
1320           Usage("Option --class-loader-context has an incorrect format: %s", option.data());
1321         }
1322       } else if (option.starts_with("--dirty-image-objects=")) {
1323         dirty_image_objects_filename_ = option.substr(strlen("--dirty-image-objects=")).data();
1324       } else if (!compiler_options_->ParseCompilerOption(option, Usage)) {
1325         Usage("Unknown argument %s", option.data());
1326       }
1327     }
1328 
1329     ProcessOptions(parser_options.get());
1330 
1331     // Insert some compiler things.
1332     InsertCompileOptions(argc, argv);
1333   }
1334 
1335   // Check whether the oat output files are writable, and open them for later. Also open a swap
1336   // file, if a name is given.
OpenFile()1337   bool OpenFile() {
1338     // Prune non-existent dex files now so that we don't create empty oat files for multi-image.
1339     PruneNonExistentDexFiles();
1340 
1341     // Expand oat and image filenames for multi image.
1342     if (IsBootImage() && multi_image_) {
1343       ExpandOatAndImageFilenames();
1344     }
1345 
1346     // OAT and VDEX file handling
1347     if (oat_fd_ == -1) {
1348       DCHECK(!oat_filenames_.empty());
1349       for (const char* oat_filename : oat_filenames_) {
1350         std::unique_ptr<File> oat_file(OS::CreateEmptyFile(oat_filename));
1351         if (oat_file == nullptr) {
1352           PLOG(ERROR) << "Failed to create oat file: " << oat_filename;
1353           return false;
1354         }
1355         if (fchmod(oat_file->Fd(), 0644) != 0) {
1356           PLOG(ERROR) << "Failed to make oat file world readable: " << oat_filename;
1357           oat_file->Erase();
1358           return false;
1359         }
1360         oat_files_.push_back(std::move(oat_file));
1361         DCHECK_EQ(input_vdex_fd_, -1);
1362         if (!input_vdex_.empty()) {
1363           std::string error_msg;
1364           input_vdex_file_ = VdexFile::Open(input_vdex_,
1365                                             /* writable */ false,
1366                                             /* low_4gb */ false,
1367                                             DoEagerUnquickeningOfVdex(),
1368                                             &error_msg);
1369         }
1370 
1371         DCHECK_EQ(output_vdex_fd_, -1);
1372         std::string vdex_filename = output_vdex_.empty()
1373             ? ReplaceFileExtension(oat_filename, "vdex")
1374             : output_vdex_;
1375         if (vdex_filename == input_vdex_ && output_vdex_.empty()) {
1376           update_input_vdex_ = true;
1377           std::unique_ptr<File> vdex_file(OS::OpenFileReadWrite(vdex_filename.c_str()));
1378           vdex_files_.push_back(std::move(vdex_file));
1379         } else {
1380           std::unique_ptr<File> vdex_file(OS::CreateEmptyFile(vdex_filename.c_str()));
1381           if (vdex_file == nullptr) {
1382             PLOG(ERROR) << "Failed to open vdex file: " << vdex_filename;
1383             return false;
1384           }
1385           if (fchmod(vdex_file->Fd(), 0644) != 0) {
1386             PLOG(ERROR) << "Failed to make vdex file world readable: " << vdex_filename;
1387             vdex_file->Erase();
1388             return false;
1389           }
1390           vdex_files_.push_back(std::move(vdex_file));
1391         }
1392       }
1393     } else {
1394       std::unique_ptr<File> oat_file(new File(oat_fd_, oat_location_, /* check_usage */ true));
1395       if (oat_file == nullptr) {
1396         PLOG(ERROR) << "Failed to create oat file: " << oat_location_;
1397         return false;
1398       }
1399       oat_file->DisableAutoClose();
1400       if (oat_file->SetLength(0) != 0) {
1401         PLOG(WARNING) << "Truncating oat file " << oat_location_ << " failed.";
1402         oat_file->Erase();
1403         return false;
1404       }
1405       oat_files_.push_back(std::move(oat_file));
1406 
1407       if (input_vdex_fd_ != -1) {
1408         struct stat s;
1409         int rc = TEMP_FAILURE_RETRY(fstat(input_vdex_fd_, &s));
1410         if (rc == -1) {
1411           PLOG(WARNING) << "Failed getting length of vdex file";
1412         } else {
1413           std::string error_msg;
1414           input_vdex_file_ = VdexFile::Open(input_vdex_fd_,
1415                                             s.st_size,
1416                                             "vdex",
1417                                             /* writable */ false,
1418                                             /* low_4gb */ false,
1419                                             DoEagerUnquickeningOfVdex(),
1420                                             &error_msg);
1421           // If there's any problem with the passed vdex, just warn and proceed
1422           // without it.
1423           if (input_vdex_file_ == nullptr) {
1424             PLOG(WARNING) << "Failed opening vdex file: " << error_msg;
1425           }
1426         }
1427       }
1428 
1429       DCHECK_NE(output_vdex_fd_, -1);
1430       std::string vdex_location = ReplaceFileExtension(oat_location_, "vdex");
1431       std::unique_ptr<File> vdex_file(new File(output_vdex_fd_, vdex_location, /* check_usage */ true));
1432       if (vdex_file == nullptr) {
1433         PLOG(ERROR) << "Failed to create vdex file: " << vdex_location;
1434         return false;
1435       }
1436       vdex_file->DisableAutoClose();
1437       if (input_vdex_file_ != nullptr && output_vdex_fd_ == input_vdex_fd_) {
1438         update_input_vdex_ = true;
1439       } else {
1440         if (vdex_file->SetLength(0) != 0) {
1441           PLOG(ERROR) << "Truncating vdex file " << vdex_location << " failed.";
1442           vdex_file->Erase();
1443           return false;
1444         }
1445       }
1446       vdex_files_.push_back(std::move(vdex_file));
1447 
1448       oat_filenames_.push_back(oat_location_.c_str());
1449     }
1450 
1451     // If we're updating in place a vdex file, be defensive and put an invalid vdex magic in case
1452     // dex2oat gets killed.
1453     // Note: we're only invalidating the magic data in the file, as dex2oat needs the rest of
1454     // the information to remain valid.
1455     if (update_input_vdex_) {
1456       std::unique_ptr<BufferedOutputStream> vdex_out = std::make_unique<BufferedOutputStream>(
1457           std::make_unique<FileOutputStream>(vdex_files_.back().get()));
1458       if (!vdex_out->WriteFully(&VdexFile::Header::kVdexInvalidMagic,
1459                                 arraysize(VdexFile::Header::kVdexInvalidMagic))) {
1460         PLOG(ERROR) << "Failed to invalidate vdex header. File: " << vdex_out->GetLocation();
1461         return false;
1462       }
1463 
1464       if (!vdex_out->Flush()) {
1465         PLOG(ERROR) << "Failed to flush stream after invalidating header of vdex file."
1466                     << " File: " << vdex_out->GetLocation();
1467         return false;
1468       }
1469     }
1470 
1471     // Swap file handling
1472     //
1473     // If the swap fd is not -1, we assume this is the file descriptor of an open but unlinked file
1474     // that we can use for swap.
1475     //
1476     // If the swap fd is -1 and we have a swap-file string, open the given file as a swap file. We
1477     // will immediately unlink to satisfy the swap fd assumption.
1478     if (swap_fd_ == -1 && !swap_file_name_.empty()) {
1479       std::unique_ptr<File> swap_file(OS::CreateEmptyFile(swap_file_name_.c_str()));
1480       if (swap_file.get() == nullptr) {
1481         PLOG(ERROR) << "Failed to create swap file: " << swap_file_name_;
1482         return false;
1483       }
1484       swap_fd_ = swap_file->Fd();
1485       swap_file->MarkUnchecked();     // We don't we to track this, it will be unlinked immediately.
1486       swap_file->DisableAutoClose();  // We'll handle it ourselves, the File object will be
1487                                       // released immediately.
1488       unlink(swap_file_name_.c_str());
1489     }
1490 
1491     return true;
1492   }
1493 
EraseOutputFiles()1494   void EraseOutputFiles() {
1495     for (auto& files : { &vdex_files_, &oat_files_ }) {
1496       for (size_t i = 0; i < files->size(); ++i) {
1497         if ((*files)[i].get() != nullptr) {
1498           (*files)[i]->Erase();
1499           (*files)[i].reset();
1500         }
1501       }
1502     }
1503   }
1504 
LoadClassProfileDescriptors()1505   void LoadClassProfileDescriptors() {
1506     if (profile_compilation_info_ != nullptr && IsImage()) {
1507       Runtime* runtime = Runtime::Current();
1508       CHECK(runtime != nullptr);
1509       // Filter out class path classes since we don't want to include these in the image.
1510       image_classes_.reset(
1511           new std::unordered_set<std::string>(
1512               profile_compilation_info_->GetClassDescriptors(dex_files_)));
1513       VLOG(compiler) << "Loaded " << image_classes_->size()
1514                      << " image class descriptors from profile";
1515       if (VLOG_IS_ON(compiler)) {
1516         for (const std::string& s : *image_classes_) {
1517           LOG(INFO) << "Image class " << s;
1518         }
1519       }
1520     }
1521   }
1522 
1523   // Set up the environment for compilation. Includes starting the runtime and loading/opening the
1524   // boot class path.
Setup()1525   dex2oat::ReturnCode Setup() {
1526     TimingLogger::ScopedTiming t("dex2oat Setup", timings_);
1527 
1528     if (!PrepareImageClasses() || !PrepareCompiledClasses() || !PrepareCompiledMethods() ||
1529         !PrepareDirtyObjects()) {
1530       return dex2oat::ReturnCode::kOther;
1531     }
1532 
1533     // Verification results are null since we don't know if we will need them yet as the compler
1534     // filter may change.
1535     // This needs to be done before PrepareRuntimeOptions since the callbacks are passed to the
1536     // runtime.
1537     callbacks_.reset(new QuickCompilerCallbacks(
1538         IsBootImage() ?
1539             CompilerCallbacks::CallbackMode::kCompileBootImage :
1540             CompilerCallbacks::CallbackMode::kCompileApp));
1541 
1542     RuntimeArgumentMap runtime_options;
1543     if (!PrepareRuntimeOptions(&runtime_options)) {
1544       return dex2oat::ReturnCode::kOther;
1545     }
1546 
1547     CreateOatWriters();
1548     if (!AddDexFileSources()) {
1549       return dex2oat::ReturnCode::kOther;
1550     }
1551 
1552     if (IsBootImage() && image_filenames_.size() > 1) {
1553       // If we're compiling the boot image, store the boot classpath into the Key-Value store.
1554       // We need this for the multi-image case.
1555       key_value_store_->Put(OatHeader::kBootClassPathKey,
1556                             gc::space::ImageSpace::GetMultiImageBootClassPath(dex_locations_,
1557                                                                               oat_filenames_,
1558                                                                               image_filenames_));
1559     }
1560 
1561     if (!IsBootImage()) {
1562       // When compiling an app, create the runtime early to retrieve
1563       // the image location key needed for the oat header.
1564       if (!CreateRuntime(std::move(runtime_options))) {
1565         return dex2oat::ReturnCode::kCreateRuntime;
1566       }
1567 
1568       if (CompilerFilter::DependsOnImageChecksum(compiler_options_->GetCompilerFilter())) {
1569         TimingLogger::ScopedTiming t3("Loading image checksum", timings_);
1570         std::vector<gc::space::ImageSpace*> image_spaces =
1571             Runtime::Current()->GetHeap()->GetBootImageSpaces();
1572         image_file_location_oat_checksum_ = image_spaces[0]->GetImageHeader().GetOatChecksum();
1573         image_file_location_oat_data_begin_ =
1574             reinterpret_cast<uintptr_t>(image_spaces[0]->GetImageHeader().GetOatDataBegin());
1575         image_patch_delta_ = image_spaces[0]->GetImageHeader().GetPatchDelta();
1576         // Store the boot image filename(s).
1577         std::vector<std::string> image_filenames;
1578         for (const gc::space::ImageSpace* image_space : image_spaces) {
1579           image_filenames.push_back(image_space->GetImageFilename());
1580         }
1581         std::string image_file_location = android::base::Join(image_filenames, ':');
1582         if (!image_file_location.empty()) {
1583           key_value_store_->Put(OatHeader::kImageLocationKey, image_file_location);
1584         }
1585       } else {
1586         image_file_location_oat_checksum_ = 0u;
1587         image_file_location_oat_data_begin_ = 0u;
1588         image_patch_delta_ = 0;
1589       }
1590 
1591       // Open dex files for class path.
1592 
1593       if (class_loader_context_ == nullptr) {
1594         // If no context was specified use the default one (which is an empty PathClassLoader).
1595         class_loader_context_ = std::unique_ptr<ClassLoaderContext>(ClassLoaderContext::Default());
1596       }
1597 
1598       DCHECK_EQ(oat_writers_.size(), 1u);
1599 
1600       // Note: Ideally we would reject context where the source dex files are also
1601       // specified in the classpath (as it doesn't make sense). However this is currently
1602       // needed for non-prebuild tests and benchmarks which expects on the fly compilation.
1603       // Also, for secondary dex files we do not have control on the actual classpath.
1604       // Instead of aborting, remove all the source location from the context classpaths.
1605       if (class_loader_context_->RemoveLocationsFromClassPaths(
1606             oat_writers_[0]->GetSourceLocations())) {
1607         LOG(WARNING) << "The source files to be compiled are also in the classpath.";
1608       }
1609 
1610       // We need to open the dex files before encoding the context in the oat file.
1611       // (because the encoding adds the dex checksum...)
1612       // TODO(calin): consider redesigning this so we don't have to open the dex files before
1613       // creating the actual class loader.
1614       if (!class_loader_context_->OpenDexFiles(runtime_->GetInstructionSet(), classpath_dir_)) {
1615         // Do not abort if we couldn't open files from the classpath. They might be
1616         // apks without dex files and right now are opening flow will fail them.
1617         LOG(WARNING) << "Failed to open classpath dex files";
1618       }
1619 
1620       // Store the class loader context in the oat header.
1621       key_value_store_->Put(OatHeader::kClassPathKey,
1622                             class_loader_context_->EncodeContextForOatFile(classpath_dir_));
1623     }
1624 
1625     // Now that we have finalized key_value_store_, start writing the oat file.
1626     {
1627       TimingLogger::ScopedTiming t_dex("Writing and opening dex files", timings_);
1628       rodata_.reserve(oat_writers_.size());
1629       for (size_t i = 0, size = oat_writers_.size(); i != size; ++i) {
1630         rodata_.push_back(elf_writers_[i]->StartRoData());
1631         // Unzip or copy dex files straight to the oat file.
1632         std::unique_ptr<MemMap> opened_dex_files_map;
1633         std::vector<std::unique_ptr<const DexFile>> opened_dex_files;
1634         // No need to verify the dex file for:
1635         // 1) Dexlayout since it does the verification. It also may not pass the verification since
1636         // we don't update the dex checksum.
1637         // 2) when we have a vdex file, which means it was already verified.
1638         const bool verify = !DoDexLayoutOptimizations() && (input_vdex_file_ == nullptr);
1639         if (!oat_writers_[i]->WriteAndOpenDexFiles(
1640             kIsVdexEnabled ? vdex_files_[i].get() : oat_files_[i].get(),
1641             rodata_.back(),
1642             instruction_set_,
1643             instruction_set_features_.get(),
1644             key_value_store_.get(),
1645             verify,
1646             update_input_vdex_,
1647             &opened_dex_files_map,
1648             &opened_dex_files)) {
1649           return dex2oat::ReturnCode::kOther;
1650         }
1651         dex_files_per_oat_file_.push_back(MakeNonOwningPointerVector(opened_dex_files));
1652         if (opened_dex_files_map != nullptr) {
1653           opened_dex_files_maps_.push_back(std::move(opened_dex_files_map));
1654           for (std::unique_ptr<const DexFile>& dex_file : opened_dex_files) {
1655             dex_file_oat_index_map_.emplace(dex_file.get(), i);
1656             opened_dex_files_.push_back(std::move(dex_file));
1657           }
1658         } else {
1659           DCHECK(opened_dex_files.empty());
1660         }
1661       }
1662     }
1663 
1664     dex_files_ = MakeNonOwningPointerVector(opened_dex_files_);
1665 
1666     // If we need to downgrade the compiler-filter for size reasons.
1667     if (!IsBootImage() && IsVeryLarge(dex_files_)) {
1668       // Disable app image to make sure dex2oat unloading is enabled.
1669       compiler_options_->DisableAppImage();
1670 
1671       // If we need to downgrade the compiler-filter for size reasons, do that early before we read
1672       // it below for creating verification callbacks.
1673       if (!CompilerFilter::IsAsGoodAs(kLargeAppFilter, compiler_options_->GetCompilerFilter())) {
1674         LOG(INFO) << "Very large app, downgrading to verify.";
1675         // Note: this change won't be reflected in the key-value store, as that had to be
1676         //       finalized before loading the dex files. This setup is currently required
1677         //       to get the size from the DexFile objects.
1678         // TODO: refactor. b/29790079
1679         compiler_options_->SetCompilerFilter(kLargeAppFilter);
1680       }
1681     }
1682 
1683     if (CompilerFilter::IsAnyCompilationEnabled(compiler_options_->GetCompilerFilter())) {
1684       // Only modes with compilation require verification results, do this here instead of when we
1685       // create the compilation callbacks since the compilation mode may have been changed by the
1686       // very large app logic.
1687       // Avoiding setting the verification results saves RAM by not adding the dex files later in
1688       // the function.
1689       verification_results_.reset(new VerificationResults(compiler_options_.get()));
1690       callbacks_->SetVerificationResults(verification_results_.get());
1691     }
1692 
1693     // We had to postpone the swap decision till now, as this is the point when we actually
1694     // know about the dex files we're going to use.
1695 
1696     // Make sure that we didn't create the driver, yet.
1697     CHECK(driver_ == nullptr);
1698     // If we use a swap file, ensure we are above the threshold to make it necessary.
1699     if (swap_fd_ != -1) {
1700       if (!UseSwap(IsBootImage(), dex_files_)) {
1701         close(swap_fd_);
1702         swap_fd_ = -1;
1703         VLOG(compiler) << "Decided to run without swap.";
1704       } else {
1705         LOG(INFO) << "Large app, accepted running with swap.";
1706       }
1707     }
1708     // Note that dex2oat won't close the swap_fd_. The compiler driver's swap space will do that.
1709     if (IsBootImage()) {
1710       // For boot image, pass opened dex files to the Runtime::Create().
1711       // Note: Runtime acquires ownership of these dex files.
1712       runtime_options.Set(RuntimeArgumentMap::BootClassPathDexList, &opened_dex_files_);
1713       if (!CreateRuntime(std::move(runtime_options))) {
1714         return dex2oat::ReturnCode::kOther;
1715       }
1716     }
1717 
1718     // If we're doing the image, override the compiler filter to force full compilation. Must be
1719     // done ahead of WellKnownClasses::Init that causes verification.  Note: doesn't force
1720     // compilation of class initializers.
1721     // Whilst we're in native take the opportunity to initialize well known classes.
1722     Thread* self = Thread::Current();
1723     WellKnownClasses::Init(self->GetJniEnv());
1724 
1725     if (!IsBootImage()) {
1726       constexpr bool kSaveDexInput = false;
1727       if (kSaveDexInput) {
1728         SaveDexInput();
1729       }
1730     }
1731 
1732     // Ensure opened dex files are writable for dex-to-dex transformations.
1733     for (const std::unique_ptr<MemMap>& map : opened_dex_files_maps_) {
1734       if (!map->Protect(PROT_READ | PROT_WRITE)) {
1735         PLOG(ERROR) << "Failed to make .dex files writeable.";
1736         return dex2oat::ReturnCode::kOther;
1737       }
1738     }
1739 
1740     // Verification results are only required for modes that have any compilation. Avoid
1741     // adding the dex files if possible to prevent allocating large arrays.
1742     if (verification_results_ != nullptr) {
1743       for (const auto& dex_file : dex_files_) {
1744         // Pre-register dex files so that we can access verification results without locks during
1745         // compilation and verification.
1746         verification_results_->AddDexFile(dex_file);
1747       }
1748     }
1749 
1750     return dex2oat::ReturnCode::kNoFailure;
1751   }
1752 
1753   // If we need to keep the oat file open for the image writer.
ShouldKeepOatFileOpen() const1754   bool ShouldKeepOatFileOpen() const {
1755     return IsImage() && oat_fd_ != kInvalidFd;
1756   }
1757 
1758   // Doesn't return the class loader since it's not meant to be used for image compilation.
CompileDexFilesIndividually()1759   void CompileDexFilesIndividually() {
1760     CHECK(!IsImage()) << "Not supported with image";
1761     for (const DexFile* dex_file : dex_files_) {
1762       std::vector<const DexFile*> dex_files(1u, dex_file);
1763       VLOG(compiler) << "Compiling " << dex_file->GetLocation();
1764       jobject class_loader = CompileDexFiles(dex_files);
1765       CHECK(class_loader != nullptr);
1766       ScopedObjectAccess soa(Thread::Current());
1767       // Unload class loader to free RAM.
1768       jweak weak_class_loader = soa.Env()->vm->AddWeakGlobalRef(
1769           soa.Self(),
1770           soa.Decode<mirror::ClassLoader>(class_loader));
1771       soa.Env()->vm->DeleteGlobalRef(soa.Self(), class_loader);
1772       runtime_->GetHeap()->CollectGarbage(/*clear_soft_references*/ true);
1773       ObjPtr<mirror::ClassLoader> decoded_weak = soa.Decode<mirror::ClassLoader>(weak_class_loader);
1774       if (decoded_weak != nullptr) {
1775         LOG(FATAL) << "Failed to unload class loader, path from root set: "
1776                    << runtime_->GetHeap()->GetVerification()->FirstPathFromRootSet(decoded_weak);
1777       }
1778       VLOG(compiler) << "Unloaded classloader";
1779     }
1780   }
1781 
ShouldCompileDexFilesIndividually() const1782   bool ShouldCompileDexFilesIndividually() const {
1783     // Compile individually if we are:
1784     // 1. not building an image,
1785     // 2. not verifying a vdex file,
1786     // 3. using multidex,
1787     // 4. not doing any AOT compilation.
1788     // This means extract, no-vdex verify, and quicken, will use the individual compilation
1789     // mode (to reduce RAM used by the compiler).
1790     return !IsImage() &&
1791         !update_input_vdex_ &&
1792         dex_files_.size() > 1 &&
1793         !CompilerFilter::IsAotCompilationEnabled(compiler_options_->GetCompilerFilter());
1794   }
1795 
1796   // Set up and create the compiler driver and then invoke it to compile all the dex files.
Compile()1797   jobject Compile() {
1798     ClassLinker* const class_linker = Runtime::Current()->GetClassLinker();
1799 
1800     TimingLogger::ScopedTiming t("dex2oat Compile", timings_);
1801     compiler_phases_timings_.reset(new CumulativeLogger("compilation times"));
1802 
1803     // Find the dex files we should not inline from.
1804     std::vector<std::string> no_inline_filters;
1805     Split(no_inline_from_string_, ',', &no_inline_filters);
1806 
1807     // For now, on the host always have core-oj removed.
1808     const std::string core_oj = "core-oj";
1809     if (!kIsTargetBuild && !ContainsElement(no_inline_filters, core_oj)) {
1810       no_inline_filters.push_back(core_oj);
1811     }
1812 
1813     if (!no_inline_filters.empty()) {
1814       std::vector<const DexFile*> class_path_files;
1815       if (!IsBootImage()) {
1816         // The class loader context is used only for apps.
1817         class_path_files = class_loader_context_->FlattenOpenedDexFiles();
1818       }
1819 
1820       std::vector<const std::vector<const DexFile*>*> dex_file_vectors = {
1821           &class_linker->GetBootClassPath(),
1822           &class_path_files,
1823           &dex_files_
1824       };
1825       for (const std::vector<const DexFile*>* dex_file_vector : dex_file_vectors) {
1826         for (const DexFile* dex_file : *dex_file_vector) {
1827           for (const std::string& filter : no_inline_filters) {
1828             // Use dex_file->GetLocation() rather than dex_file->GetBaseLocation(). This
1829             // allows tests to specify <test-dexfile>!classes2.dex if needed but if the
1830             // base location passes the StartsWith() test, so do all extra locations.
1831             std::string dex_location = dex_file->GetLocation();
1832             if (filter.find('/') == std::string::npos) {
1833               // The filter does not contain the path. Remove the path from dex_location as well.
1834               size_t last_slash = dex_file->GetLocation().rfind('/');
1835               if (last_slash != std::string::npos) {
1836                 dex_location = dex_location.substr(last_slash + 1);
1837               }
1838             }
1839 
1840             if (android::base::StartsWith(dex_location, filter.c_str())) {
1841               VLOG(compiler) << "Disabling inlining from " << dex_file->GetLocation();
1842               no_inline_from_dex_files_.push_back(dex_file);
1843               break;
1844             }
1845           }
1846         }
1847       }
1848       if (!no_inline_from_dex_files_.empty()) {
1849         compiler_options_->no_inline_from_ = &no_inline_from_dex_files_;
1850       }
1851     }
1852 
1853     driver_.reset(new CompilerDriver(compiler_options_.get(),
1854                                      verification_results_.get(),
1855                                      compiler_kind_,
1856                                      instruction_set_,
1857                                      instruction_set_features_.get(),
1858                                      image_classes_.release(),
1859                                      compiled_classes_.release(),
1860                                      compiled_methods_.release(),
1861                                      thread_count_,
1862                                      dump_stats_,
1863                                      dump_passes_,
1864                                      compiler_phases_timings_.get(),
1865                                      swap_fd_,
1866                                      profile_compilation_info_.get()));
1867     driver_->SetDexFilesForOatFile(dex_files_);
1868 
1869     const bool compile_individually = ShouldCompileDexFilesIndividually();
1870     if (compile_individually) {
1871       // Set the compiler driver in the callbacks so that we can avoid re-verification. This not
1872       // only helps performance but also prevents reverifying quickened bytecodes. Attempting
1873       // verify quickened bytecode causes verification failures.
1874       // Only set the compiler filter if we are doing separate compilation since there is a bit
1875       // of overhead when checking if a class was previously verified.
1876       callbacks_->SetDoesClassUnloading(true, driver_.get());
1877     }
1878 
1879     // Setup vdex for compilation.
1880     if (!DoEagerUnquickeningOfVdex() && input_vdex_file_ != nullptr) {
1881       callbacks_->SetVerifierDeps(
1882           new verifier::VerifierDeps(dex_files_, input_vdex_file_->GetVerifierDepsData()));
1883 
1884       // TODO: we unquicken unconditionally, as we don't know
1885       // if the boot image has changed. How exactly we'll know is under
1886       // experimentation.
1887       TimingLogger::ScopedTiming time_unquicken("Unquicken", timings_);
1888       VdexFile::Unquicken(dex_files_, input_vdex_file_->GetQuickeningInfo());
1889     } else {
1890       // Create the main VerifierDeps, here instead of in the compiler since we want to aggregate
1891       // the results for all the dex files, not just the results for the current dex file.
1892       callbacks_->SetVerifierDeps(new verifier::VerifierDeps(dex_files_));
1893     }
1894     // Invoke the compilation.
1895     if (compile_individually) {
1896       CompileDexFilesIndividually();
1897       // Return a null classloader since we already freed released it.
1898       return nullptr;
1899     }
1900     return CompileDexFiles(dex_files_);
1901   }
1902 
1903   // Create the class loader, use it to compile, and return.
CompileDexFiles(const std::vector<const DexFile * > & dex_files)1904   jobject CompileDexFiles(const std::vector<const DexFile*>& dex_files) {
1905     ClassLinker* const class_linker = Runtime::Current()->GetClassLinker();
1906 
1907     jobject class_loader = nullptr;
1908     if (!IsBootImage()) {
1909       class_loader = class_loader_context_->CreateClassLoader(dex_files_);
1910     }
1911 
1912     // Register dex caches and key them to the class loader so that they only unload when the
1913     // class loader unloads.
1914     for (const auto& dex_file : dex_files) {
1915       ScopedObjectAccess soa(Thread::Current());
1916       // Registering the dex cache adds a strong root in the class loader that prevents the dex
1917       // cache from being unloaded early.
1918       ObjPtr<mirror::DexCache> dex_cache = class_linker->RegisterDexFile(
1919           *dex_file,
1920           soa.Decode<mirror::ClassLoader>(class_loader));
1921       if (dex_cache == nullptr) {
1922         soa.Self()->AssertPendingException();
1923         LOG(FATAL) << "Failed to register dex file " << dex_file->GetLocation() << " "
1924                    << soa.Self()->GetException()->Dump();
1925       }
1926     }
1927     driver_->CompileAll(class_loader, dex_files, timings_);
1928     return class_loader;
1929   }
1930 
1931   // Notes on the interleaving of creating the images and oat files to
1932   // ensure the references between the two are correct.
1933   //
1934   // Currently we have a memory layout that looks something like this:
1935   //
1936   // +--------------+
1937   // | images       |
1938   // +--------------+
1939   // | oat files    |
1940   // +--------------+
1941   // | alloc spaces |
1942   // +--------------+
1943   //
1944   // There are several constraints on the loading of the images and oat files.
1945   //
1946   // 1. The images are expected to be loaded at an absolute address and
1947   // contain Objects with absolute pointers within the images.
1948   //
1949   // 2. There are absolute pointers from Methods in the images to their
1950   // code in the oat files.
1951   //
1952   // 3. There are absolute pointers from the code in the oat files to Methods
1953   // in the images.
1954   //
1955   // 4. There are absolute pointers from code in the oat files to other code
1956   // in the oat files.
1957   //
1958   // To get this all correct, we go through several steps.
1959   //
1960   // 1. We prepare offsets for all data in the oat files and calculate
1961   // the oat data size and code size. During this stage, we also set
1962   // oat code offsets in methods for use by the image writer.
1963   //
1964   // 2. We prepare offsets for the objects in the images and calculate
1965   // the image sizes.
1966   //
1967   // 3. We create the oat files. Originally this was just our own proprietary
1968   // file but now it is contained within an ELF dynamic object (aka an .so
1969   // file). Since we know the image sizes and oat data sizes and code sizes we
1970   // can prepare the ELF headers and we then know the ELF memory segment
1971   // layout and we can now resolve all references. The compiler provides
1972   // LinkerPatch information in each CompiledMethod and we resolve these,
1973   // using the layout information and image object locations provided by
1974   // image writer, as we're writing the method code.
1975   //
1976   // 4. We create the image files. They need to know where the oat files
1977   // will be loaded after itself. Originally oat files were simply
1978   // memory mapped so we could predict where their contents were based
1979   // on the file size. Now that they are ELF files, we need to inspect
1980   // the ELF files to understand the in memory segment layout including
1981   // where the oat header is located within.
1982   // TODO: We could just remember this information from step 3.
1983   //
1984   // 5. We fixup the ELF program headers so that dlopen will try to
1985   // load the .so at the desired location at runtime by offsetting the
1986   // Elf32_Phdr.p_vaddr values by the desired base address.
1987   // TODO: Do this in step 3. We already know the layout there.
1988   //
1989   // Steps 1.-3. are done by the CreateOatFile() above, steps 4.-5.
1990   // are done by the CreateImageFile() below.
1991 
1992   // Write out the generated code part. Calls the OatWriter and ElfBuilder. Also prepares the
1993   // ImageWriter, if necessary.
1994   // Note: Flushing (and closing) the file is the caller's responsibility, except for the failure
1995   //       case (when the file will be explicitly erased).
WriteOutputFiles()1996   bool WriteOutputFiles() {
1997     TimingLogger::ScopedTiming t("dex2oat Oat", timings_);
1998 
1999     // Sync the data to the file, in case we did dex2dex transformations.
2000     for (const std::unique_ptr<MemMap>& map : opened_dex_files_maps_) {
2001       if (!map->Sync()) {
2002         PLOG(ERROR) << "Failed to Sync() dex2dex output. Map: " << map->GetName();
2003         return false;
2004       }
2005     }
2006 
2007     if (IsImage()) {
2008       if (IsAppImage() && image_base_ == 0) {
2009         gc::Heap* const heap = Runtime::Current()->GetHeap();
2010         for (gc::space::ImageSpace* image_space : heap->GetBootImageSpaces()) {
2011           image_base_ = std::max(image_base_, RoundUp(
2012               reinterpret_cast<uintptr_t>(image_space->GetImageHeader().GetOatFileEnd()),
2013               kPageSize));
2014         }
2015         // The non moving space is right after the oat file. Put the preferred app image location
2016         // right after the non moving space so that we ideally get a continuous immune region for
2017         // the GC.
2018         // Use the default non moving space capacity since dex2oat does not have a separate non-
2019         // moving space. This means the runtime's non moving space space size will be as large
2020         // as the growth limit for dex2oat, but smaller in the zygote.
2021         const size_t non_moving_space_capacity = gc::Heap::kDefaultNonMovingSpaceCapacity;
2022         image_base_ += non_moving_space_capacity;
2023         VLOG(compiler) << "App image base=" << reinterpret_cast<void*>(image_base_);
2024       }
2025 
2026       image_writer_.reset(new ImageWriter(*driver_,
2027                                           image_base_,
2028                                           compiler_options_->GetCompilePic(),
2029                                           IsAppImage(),
2030                                           image_storage_mode_,
2031                                           oat_filenames_,
2032                                           dex_file_oat_index_map_,
2033                                           dirty_image_objects_.get()));
2034 
2035       // We need to prepare method offsets in the image address space for direct method patching.
2036       TimingLogger::ScopedTiming t2("dex2oat Prepare image address space", timings_);
2037       if (!image_writer_->PrepareImageAddressSpace()) {
2038         LOG(ERROR) << "Failed to prepare image address space.";
2039         return false;
2040       }
2041     }
2042 
2043     // Initialize the writers with the compiler driver, image writer, and their
2044     // dex files. The writers were created without those being there yet.
2045     for (size_t i = 0, size = oat_files_.size(); i != size; ++i) {
2046       std::unique_ptr<OatWriter>& oat_writer = oat_writers_[i];
2047       std::vector<const DexFile*>& dex_files = dex_files_per_oat_file_[i];
2048       oat_writer->Initialize(driver_.get(), image_writer_.get(), dex_files);
2049     }
2050 
2051     {
2052       TimingLogger::ScopedTiming t2("dex2oat Write VDEX", timings_);
2053       DCHECK(IsBootImage() || oat_files_.size() == 1u);
2054       verifier::VerifierDeps* verifier_deps = callbacks_->GetVerifierDeps();
2055       for (size_t i = 0, size = oat_files_.size(); i != size; ++i) {
2056         File* vdex_file = vdex_files_[i].get();
2057         std::unique_ptr<BufferedOutputStream> vdex_out =
2058             std::make_unique<BufferedOutputStream>(std::make_unique<FileOutputStream>(vdex_file));
2059 
2060         if (!oat_writers_[i]->WriteVerifierDeps(vdex_out.get(), verifier_deps)) {
2061           LOG(ERROR) << "Failed to write verifier dependencies into VDEX " << vdex_file->GetPath();
2062           return false;
2063         }
2064 
2065         if (!oat_writers_[i]->WriteQuickeningInfo(vdex_out.get())) {
2066           LOG(ERROR) << "Failed to write quickening info into VDEX " << vdex_file->GetPath();
2067           return false;
2068         }
2069 
2070         // VDEX finalized, seek back to the beginning and write checksums and the header.
2071         if (!oat_writers_[i]->WriteChecksumsAndVdexHeader(vdex_out.get())) {
2072           LOG(ERROR) << "Failed to write vdex header into VDEX " << vdex_file->GetPath();
2073           return false;
2074         }
2075       }
2076     }
2077 
2078     {
2079       TimingLogger::ScopedTiming t2("dex2oat Write ELF", timings_);
2080       linker::MultiOatRelativePatcher patcher(instruction_set_, instruction_set_features_.get());
2081       for (size_t i = 0, size = oat_files_.size(); i != size; ++i) {
2082         std::unique_ptr<ElfWriter>& elf_writer = elf_writers_[i];
2083         std::unique_ptr<OatWriter>& oat_writer = oat_writers_[i];
2084 
2085         oat_writer->PrepareLayout(&patcher);
2086 
2087         size_t rodata_size = oat_writer->GetOatHeader().GetExecutableOffset();
2088         size_t text_size = oat_writer->GetOatSize() - rodata_size;
2089         elf_writer->PrepareDynamicSection(rodata_size,
2090                                           text_size,
2091                                           oat_writer->GetBssSize(),
2092                                           oat_writer->GetBssMethodsOffset(),
2093                                           oat_writer->GetBssRootsOffset());
2094 
2095         if (IsImage()) {
2096           // Update oat layout.
2097           DCHECK(image_writer_ != nullptr);
2098           DCHECK_LT(i, oat_filenames_.size());
2099           image_writer_->UpdateOatFileLayout(i,
2100                                              elf_writer->GetLoadedSize(),
2101                                              oat_writer->GetOatDataOffset(),
2102                                              oat_writer->GetOatSize());
2103         }
2104 
2105         if (IsBootImage()) {
2106           // Have the image_file_location_oat_checksum_ for boot oat files
2107           // depend on the contents of all the boot oat files. This way only
2108           // the primary image checksum needs to be checked to determine
2109           // whether any of the images are out of date.
2110           image_file_location_oat_checksum_ ^= oat_writer->GetOatHeader().GetChecksum();
2111         }
2112       }
2113 
2114       for (size_t i = 0, size = oat_files_.size(); i != size; ++i) {
2115         std::unique_ptr<File>& oat_file = oat_files_[i];
2116         std::unique_ptr<ElfWriter>& elf_writer = elf_writers_[i];
2117         std::unique_ptr<OatWriter>& oat_writer = oat_writers_[i];
2118 
2119         oat_writer->AddMethodDebugInfos(debug::MakeTrampolineInfos(oat_writer->GetOatHeader()));
2120 
2121         // We need to mirror the layout of the ELF file in the compressed debug-info.
2122         // Therefore PrepareDebugInfo() relies on the SetLoadedSectionSizes() call further above.
2123         elf_writer->PrepareDebugInfo(oat_writer->GetMethodDebugInfo());
2124 
2125         OutputStream*& rodata = rodata_[i];
2126         DCHECK(rodata != nullptr);
2127         if (!oat_writer->WriteRodata(rodata)) {
2128           LOG(ERROR) << "Failed to write .rodata section to the ELF file " << oat_file->GetPath();
2129           return false;
2130         }
2131         elf_writer->EndRoData(rodata);
2132         rodata = nullptr;
2133 
2134         OutputStream* text = elf_writer->StartText();
2135         if (!oat_writer->WriteCode(text)) {
2136           LOG(ERROR) << "Failed to write .text section to the ELF file " << oat_file->GetPath();
2137           return false;
2138         }
2139         elf_writer->EndText(text);
2140 
2141         if (!oat_writer->WriteHeader(elf_writer->GetStream(),
2142                                      image_file_location_oat_checksum_,
2143                                      image_file_location_oat_data_begin_,
2144                                      image_patch_delta_)) {
2145           LOG(ERROR) << "Failed to write oat header to the ELF file " << oat_file->GetPath();
2146           return false;
2147         }
2148 
2149         if (IsImage()) {
2150           // Update oat header information.
2151           DCHECK(image_writer_ != nullptr);
2152           DCHECK_LT(i, oat_filenames_.size());
2153           image_writer_->UpdateOatFileHeader(i, oat_writer->GetOatHeader());
2154         }
2155 
2156         elf_writer->WriteDynamicSection();
2157         elf_writer->WriteDebugInfo(oat_writer->GetMethodDebugInfo());
2158 
2159         if (!elf_writer->End()) {
2160           LOG(ERROR) << "Failed to write ELF file " << oat_file->GetPath();
2161           return false;
2162         }
2163 
2164         if (!FlushOutputFile(&vdex_files_[i]) || !FlushOutputFile(&oat_files_[i])) {
2165           return false;
2166         }
2167 
2168         VLOG(compiler) << "Oat file written successfully: " << oat_filenames_[i];
2169 
2170         oat_writer.reset();
2171         elf_writer.reset();
2172       }
2173     }
2174 
2175     return true;
2176   }
2177 
2178   // If we are compiling an image, invoke the image creation routine. Else just skip.
HandleImage()2179   bool HandleImage() {
2180     if (IsImage()) {
2181       TimingLogger::ScopedTiming t("dex2oat ImageWriter", timings_);
2182       if (!CreateImageFile()) {
2183         return false;
2184       }
2185       VLOG(compiler) << "Images written successfully";
2186     }
2187     return true;
2188   }
2189 
2190   // Create a copy from stripped to unstripped.
CopyStrippedToUnstripped()2191   bool CopyStrippedToUnstripped() {
2192     for (size_t i = 0; i < oat_unstripped_.size(); ++i) {
2193       // If we don't want to strip in place, copy from stripped location to unstripped location.
2194       // We need to strip after image creation because FixupElf needs to use .strtab.
2195       if (strcmp(oat_unstripped_[i], oat_filenames_[i]) != 0) {
2196         // If the oat file is still open, flush it.
2197         if (oat_files_[i].get() != nullptr && oat_files_[i]->IsOpened()) {
2198           if (!FlushCloseOutputFile(&oat_files_[i])) {
2199             return false;
2200           }
2201         }
2202 
2203         TimingLogger::ScopedTiming t("dex2oat OatFile copy", timings_);
2204         std::unique_ptr<File> in(OS::OpenFileForReading(oat_filenames_[i]));
2205         std::unique_ptr<File> out(OS::CreateEmptyFile(oat_unstripped_[i]));
2206         int64_t in_length = in->GetLength();
2207         if (in_length < 0) {
2208           PLOG(ERROR) << "Failed to get the length of oat file: " << in->GetPath();
2209           return false;
2210         }
2211         if (!out->Copy(in.get(), 0, in_length)) {
2212           PLOG(ERROR) << "Failed to copy oat file to file: " << out->GetPath();
2213           return false;
2214         }
2215         if (out->FlushCloseOrErase() != 0) {
2216           PLOG(ERROR) << "Failed to flush and close copied oat file: " << oat_unstripped_[i];
2217           return false;
2218         }
2219         VLOG(compiler) << "Oat file copied successfully (unstripped): " << oat_unstripped_[i];
2220       }
2221     }
2222     return true;
2223   }
2224 
FlushOutputFile(std::unique_ptr<File> * file)2225   bool FlushOutputFile(std::unique_ptr<File>* file) {
2226     if (file->get() != nullptr) {
2227       if (file->get()->Flush() != 0) {
2228         PLOG(ERROR) << "Failed to flush output file: " << file->get()->GetPath();
2229         return false;
2230       }
2231     }
2232     return true;
2233   }
2234 
FlushCloseOutputFile(std::unique_ptr<File> * file)2235   bool FlushCloseOutputFile(std::unique_ptr<File>* file) {
2236     if (file->get() != nullptr) {
2237       std::unique_ptr<File> tmp(file->release());
2238       if (tmp->FlushCloseOrErase() != 0) {
2239         PLOG(ERROR) << "Failed to flush and close output file: " << tmp->GetPath();
2240         return false;
2241       }
2242     }
2243     return true;
2244   }
2245 
FlushOutputFiles()2246   bool FlushOutputFiles() {
2247     TimingLogger::ScopedTiming t2("dex2oat Flush Output Files", timings_);
2248     for (auto& files : { &vdex_files_, &oat_files_ }) {
2249       for (size_t i = 0; i < files->size(); ++i) {
2250         if (!FlushOutputFile(&(*files)[i])) {
2251           return false;
2252         }
2253       }
2254     }
2255     return true;
2256   }
2257 
FlushCloseOutputFiles()2258   bool FlushCloseOutputFiles() {
2259     bool result = true;
2260     for (auto& files : { &vdex_files_, &oat_files_ }) {
2261       for (size_t i = 0; i < files->size(); ++i) {
2262         result &= FlushCloseOutputFile(&(*files)[i]);
2263       }
2264     }
2265     return result;
2266   }
2267 
DumpTiming()2268   void DumpTiming() {
2269     if (dump_timing_ || (dump_slow_timing_ && timings_->GetTotalNs() > MsToNs(1000))) {
2270       LOG(INFO) << Dumpable<TimingLogger>(*timings_);
2271     }
2272     if (dump_passes_) {
2273       LOG(INFO) << Dumpable<CumulativeLogger>(*driver_->GetTimingsLogger());
2274     }
2275   }
2276 
IsImage() const2277   bool IsImage() const {
2278     return IsAppImage() || IsBootImage();
2279   }
2280 
IsAppImage() const2281   bool IsAppImage() const {
2282     return compiler_options_->IsAppImage();
2283   }
2284 
IsBootImage() const2285   bool IsBootImage() const {
2286     return compiler_options_->IsBootImage();
2287   }
2288 
IsHost() const2289   bool IsHost() const {
2290     return is_host_;
2291   }
2292 
UseProfile() const2293   bool UseProfile() const {
2294     return profile_file_fd_ != -1 || !profile_file_.empty();
2295   }
2296 
DoProfileGuidedOptimizations() const2297   bool DoProfileGuidedOptimizations() const {
2298     return UseProfile();
2299   }
2300 
DoDexLayoutOptimizations() const2301   bool DoDexLayoutOptimizations() const {
2302     return DoProfileGuidedOptimizations();
2303   }
2304 
DoEagerUnquickeningOfVdex() const2305   bool DoEagerUnquickeningOfVdex() const {
2306     // DexLayout can invalidate the vdex metadata, so we need to unquicken
2307     // the vdex file eagerly, before passing it to dexlayout.
2308     return DoDexLayoutOptimizations();
2309   }
2310 
LoadProfile()2311   bool LoadProfile() {
2312     DCHECK(UseProfile());
2313     // TODO(calin): We should be using the runtime arena pool (instead of the
2314     // default profile arena). However the setup logic is messy and needs
2315     // cleaning up before that (e.g. the oat writers are created before the
2316     // runtime).
2317     profile_compilation_info_.reset(new ProfileCompilationInfo());
2318     ScopedFlock profile_file;
2319     std::string error;
2320     if (profile_file_fd_ != -1) {
2321       profile_file = LockedFile::DupOf(profile_file_fd_, "profile",
2322                                        true /* read_only_mode */, &error);
2323     } else if (profile_file_ != "") {
2324       profile_file = LockedFile::Open(profile_file_.c_str(), O_RDONLY, true, &error);
2325     }
2326 
2327     // Return early if we're unable to obtain a lock on the profile.
2328     if (profile_file.get() == nullptr) {
2329       LOG(ERROR) << "Cannot lock profiles: " << error;
2330       return false;
2331     }
2332 
2333     if (!profile_compilation_info_->Load(profile_file->Fd())) {
2334       profile_compilation_info_.reset(nullptr);
2335       return false;
2336     }
2337 
2338     return true;
2339   }
2340 
2341  private:
UseSwap(bool is_image,const std::vector<const DexFile * > & dex_files)2342   bool UseSwap(bool is_image, const std::vector<const DexFile*>& dex_files) {
2343     if (is_image) {
2344       // Don't use swap, we know generation should succeed, and we don't want to slow it down.
2345       return false;
2346     }
2347     if (dex_files.size() < min_dex_files_for_swap_) {
2348       // If there are less dex files than the threshold, assume it's gonna be fine.
2349       return false;
2350     }
2351     size_t dex_files_size = 0;
2352     for (const auto* dex_file : dex_files) {
2353       dex_files_size += dex_file->GetHeader().file_size_;
2354     }
2355     return dex_files_size >= min_dex_file_cumulative_size_for_swap_;
2356   }
2357 
IsVeryLarge(std::vector<const DexFile * > & dex_files)2358   bool IsVeryLarge(std::vector<const DexFile*>& dex_files) {
2359     size_t dex_files_size = 0;
2360     for (const auto* dex_file : dex_files) {
2361       dex_files_size += dex_file->GetHeader().file_size_;
2362     }
2363     return dex_files_size >= very_large_threshold_;
2364   }
2365 
GetClassPathLocations(const std::string & class_path)2366   std::vector<std::string> GetClassPathLocations(const std::string& class_path) {
2367     // This function is used only for apps and for an app we have exactly one oat file.
2368     DCHECK(!IsBootImage());
2369     DCHECK_EQ(oat_writers_.size(), 1u);
2370     std::vector<std::string> dex_files_canonical_locations;
2371     for (const std::string& location : oat_writers_[0]->GetSourceLocations()) {
2372       dex_files_canonical_locations.push_back(DexFile::GetDexCanonicalLocation(location.c_str()));
2373     }
2374 
2375     std::vector<std::string> parsed;
2376     Split(class_path, ':', &parsed);
2377     auto kept_it = std::remove_if(parsed.begin(),
2378                                   parsed.end(),
2379                                   [dex_files_canonical_locations](const std::string& location) {
2380       return ContainsElement(dex_files_canonical_locations,
2381                              DexFile::GetDexCanonicalLocation(location.c_str()));
2382     });
2383     parsed.erase(kept_it, parsed.end());
2384     return parsed;
2385   }
2386 
PrepareImageClasses()2387   bool PrepareImageClasses() {
2388     // If --image-classes was specified, calculate the full list of classes to include in the image.
2389     if (image_classes_filename_ != nullptr) {
2390       image_classes_ =
2391           ReadClasses(image_classes_zip_filename_, image_classes_filename_, "image");
2392       if (image_classes_ == nullptr) {
2393         return false;
2394       }
2395     } else if (IsBootImage()) {
2396       image_classes_.reset(new std::unordered_set<std::string>);
2397     }
2398     return true;
2399   }
2400 
PrepareCompiledClasses()2401   bool PrepareCompiledClasses() {
2402     // If --compiled-classes was specified, calculate the full list of classes to compile in the
2403     // image.
2404     if (compiled_classes_filename_ != nullptr) {
2405       compiled_classes_ =
2406           ReadClasses(compiled_classes_zip_filename_, compiled_classes_filename_, "compiled");
2407       if (compiled_classes_ == nullptr) {
2408         return false;
2409       }
2410     } else {
2411       compiled_classes_.reset(nullptr);  // By default compile everything.
2412     }
2413     return true;
2414   }
2415 
ReadClasses(const char * zip_filename,const char * classes_filename,const char * tag)2416   static std::unique_ptr<std::unordered_set<std::string>> ReadClasses(const char* zip_filename,
2417                                                                       const char* classes_filename,
2418                                                                       const char* tag) {
2419     std::unique_ptr<std::unordered_set<std::string>> classes;
2420     std::string error_msg;
2421     if (zip_filename != nullptr) {
2422       classes.reset(ReadImageClassesFromZip(zip_filename, classes_filename, &error_msg));
2423     } else {
2424       classes.reset(ReadImageClassesFromFile(classes_filename));
2425     }
2426     if (classes == nullptr) {
2427       LOG(ERROR) << "Failed to create list of " << tag << " classes from '"
2428                  << classes_filename << "': " << error_msg;
2429     }
2430     return classes;
2431   }
2432 
PrepareCompiledMethods()2433   bool PrepareCompiledMethods() {
2434     // If --compiled-methods was specified, read the methods to compile from the given file(s).
2435     if (compiled_methods_filename_ != nullptr) {
2436       std::string error_msg;
2437       if (compiled_methods_zip_filename_ != nullptr) {
2438         compiled_methods_.reset(ReadCommentedInputFromZip<std::unordered_set<std::string>>(
2439             compiled_methods_zip_filename_,
2440             compiled_methods_filename_,
2441             nullptr,            // No post-processing.
2442             &error_msg));
2443       } else {
2444         compiled_methods_.reset(ReadCommentedInputFromFile<std::unordered_set<std::string>>(
2445             compiled_methods_filename_,
2446             nullptr));          // No post-processing.
2447       }
2448       if (compiled_methods_.get() == nullptr) {
2449         LOG(ERROR) << "Failed to create list of compiled methods from '"
2450             << compiled_methods_filename_ << "': " << error_msg;
2451         return false;
2452       }
2453     } else {
2454       compiled_methods_.reset(nullptr);  // By default compile everything.
2455     }
2456     return true;
2457   }
2458 
PrepareDirtyObjects()2459   bool PrepareDirtyObjects() {
2460     if (dirty_image_objects_filename_ != nullptr) {
2461       dirty_image_objects_.reset(ReadCommentedInputFromFile<std::unordered_set<std::string>>(
2462           dirty_image_objects_filename_,
2463           nullptr));
2464       if (dirty_image_objects_ == nullptr) {
2465         LOG(ERROR) << "Failed to create list of dirty objects from '"
2466             << dirty_image_objects_filename_ << "'";
2467         return false;
2468       }
2469     } else {
2470       dirty_image_objects_.reset(nullptr);
2471     }
2472     return true;
2473   }
2474 
PruneNonExistentDexFiles()2475   void PruneNonExistentDexFiles() {
2476     DCHECK_EQ(dex_filenames_.size(), dex_locations_.size());
2477     size_t kept = 0u;
2478     for (size_t i = 0, size = dex_filenames_.size(); i != size; ++i) {
2479       if (!OS::FileExists(dex_filenames_[i])) {
2480         LOG(WARNING) << "Skipping non-existent dex file '" << dex_filenames_[i] << "'";
2481       } else {
2482         dex_filenames_[kept] = dex_filenames_[i];
2483         dex_locations_[kept] = dex_locations_[i];
2484         ++kept;
2485       }
2486     }
2487     dex_filenames_.resize(kept);
2488     dex_locations_.resize(kept);
2489   }
2490 
AddDexFileSources()2491   bool AddDexFileSources() {
2492     TimingLogger::ScopedTiming t2("AddDexFileSources", timings_);
2493     if (input_vdex_file_ != nullptr) {
2494       DCHECK_EQ(oat_writers_.size(), 1u);
2495       const std::string& name = zip_location_.empty() ? dex_locations_[0] : zip_location_;
2496       DCHECK(!name.empty());
2497       if (!oat_writers_[0]->AddVdexDexFilesSource(*input_vdex_file_.get(), name.c_str())) {
2498         return false;
2499       }
2500     } else if (zip_fd_ != -1) {
2501       DCHECK_EQ(oat_writers_.size(), 1u);
2502       if (!oat_writers_[0]->AddZippedDexFilesSource(File(zip_fd_, /* check_usage */ false),
2503                                                     zip_location_.c_str())) {
2504         return false;
2505       }
2506     } else if (oat_writers_.size() > 1u) {
2507       // Multi-image.
2508       DCHECK_EQ(oat_writers_.size(), dex_filenames_.size());
2509       DCHECK_EQ(oat_writers_.size(), dex_locations_.size());
2510       for (size_t i = 0, size = oat_writers_.size(); i != size; ++i) {
2511         if (!oat_writers_[i]->AddDexFileSource(dex_filenames_[i], dex_locations_[i])) {
2512           return false;
2513         }
2514       }
2515     } else {
2516       DCHECK_EQ(oat_writers_.size(), 1u);
2517       DCHECK_EQ(dex_filenames_.size(), dex_locations_.size());
2518       DCHECK_NE(dex_filenames_.size(), 0u);
2519       for (size_t i = 0; i != dex_filenames_.size(); ++i) {
2520         if (!oat_writers_[0]->AddDexFileSource(dex_filenames_[i], dex_locations_[i])) {
2521           return false;
2522         }
2523       }
2524     }
2525     return true;
2526   }
2527 
CreateOatWriters()2528   void CreateOatWriters() {
2529     TimingLogger::ScopedTiming t2("CreateOatWriters", timings_);
2530     elf_writers_.reserve(oat_files_.size());
2531     oat_writers_.reserve(oat_files_.size());
2532     for (const std::unique_ptr<File>& oat_file : oat_files_) {
2533       elf_writers_.emplace_back(CreateElfWriterQuick(instruction_set_,
2534                                                      instruction_set_features_.get(),
2535                                                      compiler_options_.get(),
2536                                                      oat_file.get()));
2537       elf_writers_.back()->Start();
2538       const bool do_dexlayout = DoDexLayoutOptimizations();
2539       oat_writers_.emplace_back(new OatWriter(
2540           IsBootImage(), timings_, do_dexlayout ? profile_compilation_info_.get() : nullptr));
2541     }
2542   }
2543 
SaveDexInput()2544   void SaveDexInput() {
2545     for (size_t i = 0; i < dex_files_.size(); ++i) {
2546       const DexFile* dex_file = dex_files_[i];
2547       std::string tmp_file_name(StringPrintf("/data/local/tmp/dex2oat.%d.%zd.dex",
2548                                              getpid(), i));
2549       std::unique_ptr<File> tmp_file(OS::CreateEmptyFile(tmp_file_name.c_str()));
2550       if (tmp_file.get() == nullptr) {
2551         PLOG(ERROR) << "Failed to open file " << tmp_file_name
2552             << ". Try: adb shell chmod 777 /data/local/tmp";
2553         continue;
2554       }
2555       // This is just dumping files for debugging. Ignore errors, and leave remnants.
2556       UNUSED(tmp_file->WriteFully(dex_file->Begin(), dex_file->Size()));
2557       UNUSED(tmp_file->Flush());
2558       UNUSED(tmp_file->Close());
2559       LOG(INFO) << "Wrote input to " << tmp_file_name;
2560     }
2561   }
2562 
PrepareRuntimeOptions(RuntimeArgumentMap * runtime_options)2563   bool PrepareRuntimeOptions(RuntimeArgumentMap* runtime_options) {
2564     RuntimeOptions raw_options;
2565     if (boot_image_filename_.empty()) {
2566       std::string boot_class_path = "-Xbootclasspath:";
2567       boot_class_path += android::base::Join(dex_filenames_, ':');
2568       raw_options.push_back(std::make_pair(boot_class_path, nullptr));
2569       std::string boot_class_path_locations = "-Xbootclasspath-locations:";
2570       boot_class_path_locations += android::base::Join(dex_locations_, ':');
2571       raw_options.push_back(std::make_pair(boot_class_path_locations, nullptr));
2572     } else {
2573       std::string boot_image_option = "-Ximage:";
2574       boot_image_option += boot_image_filename_;
2575       raw_options.push_back(std::make_pair(boot_image_option, nullptr));
2576     }
2577     for (size_t i = 0; i < runtime_args_.size(); i++) {
2578       raw_options.push_back(std::make_pair(runtime_args_[i], nullptr));
2579     }
2580 
2581     raw_options.push_back(std::make_pair("compilercallbacks", callbacks_.get()));
2582     raw_options.push_back(
2583         std::make_pair("imageinstructionset", GetInstructionSetString(instruction_set_)));
2584 
2585     // Only allow no boot image for the runtime if we're compiling one. When we compile an app,
2586     // we don't want fallback mode, it will abort as we do not push a boot classpath (it might
2587     // have been stripped in preopting, anyways).
2588     if (!IsBootImage()) {
2589       raw_options.push_back(std::make_pair("-Xno-dex-file-fallback", nullptr));
2590     }
2591     // Never allow implicit image compilation.
2592     raw_options.push_back(std::make_pair("-Xnoimage-dex2oat", nullptr));
2593     // Disable libsigchain. We don't don't need it during compilation and it prevents us
2594     // from getting a statically linked version of dex2oat (because of dlsym and RTLD_NEXT).
2595     raw_options.push_back(std::make_pair("-Xno-sig-chain", nullptr));
2596     // Disable Hspace compaction to save heap size virtual space.
2597     // Only need disable Hspace for OOM becasue background collector is equal to
2598     // foreground collector by default for dex2oat.
2599     raw_options.push_back(std::make_pair("-XX:DisableHSpaceCompactForOOM", nullptr));
2600 
2601     if (compiler_options_->IsForceDeterminism()) {
2602       // If we're asked to be deterministic, ensure non-concurrent GC for determinism.
2603       //
2604       // Note that with read barriers, this option is ignored, because Runtime::Init
2605       // overrides the foreground GC to be gc::kCollectorTypeCC when instantiating
2606       // gc::Heap. This is fine, as concurrent GC requests are not honored in dex2oat,
2607       // which uses an unstarted runtime.
2608       raw_options.push_back(std::make_pair("-Xgc:nonconcurrent", nullptr));
2609 
2610       // The default LOS implementation (map) is not deterministic. So disable it.
2611       raw_options.push_back(std::make_pair("-XX:LargeObjectSpace=disabled", nullptr));
2612 
2613       // We also need to turn off the nonmoving space. For that, we need to disable HSpace
2614       // compaction (done above) and ensure that neither foreground nor background collectors
2615       // are concurrent.
2616       //
2617       // Likewise, this option is ignored with read barriers because Runtime::Init
2618       // overrides the background GC to be gc::kCollectorTypeCCBackground, but that's
2619       // fine too, for the same reason (see above).
2620       raw_options.push_back(std::make_pair("-XX:BackgroundGC=nonconcurrent", nullptr));
2621 
2622       // To make identity hashcode deterministic, set a known seed.
2623       mirror::Object::SetHashCodeSeed(987654321U);
2624     }
2625 
2626     if (!Runtime::ParseOptions(raw_options, false, runtime_options)) {
2627       LOG(ERROR) << "Failed to parse runtime options";
2628       return false;
2629     }
2630     return true;
2631   }
2632 
2633   // Create a runtime necessary for compilation.
CreateRuntime(RuntimeArgumentMap && runtime_options)2634   bool CreateRuntime(RuntimeArgumentMap&& runtime_options) {
2635     TimingLogger::ScopedTiming t_runtime("Create runtime", timings_);
2636     if (!Runtime::Create(std::move(runtime_options))) {
2637       LOG(ERROR) << "Failed to create runtime";
2638       return false;
2639     }
2640 
2641     // Runtime::Init will rename this thread to be "main". Prefer "dex2oat" so that "top" and
2642     // "ps -a" don't change to non-descript "main."
2643     SetThreadName(kIsDebugBuild ? "dex2oatd" : "dex2oat");
2644 
2645     runtime_.reset(Runtime::Current());
2646     runtime_->SetInstructionSet(instruction_set_);
2647     for (uint32_t i = 0; i < static_cast<uint32_t>(CalleeSaveType::kLastCalleeSaveType); ++i) {
2648       CalleeSaveType type = CalleeSaveType(i);
2649       if (!runtime_->HasCalleeSaveMethod(type)) {
2650         runtime_->SetCalleeSaveMethod(runtime_->CreateCalleeSaveMethod(), type);
2651       }
2652     }
2653 
2654     // Initialize maps for unstarted runtime. This needs to be here, as running clinits needs this
2655     // set up.
2656     interpreter::UnstartedRuntime::Initialize();
2657 
2658     runtime_->GetClassLinker()->RunRootClinits();
2659 
2660     // Runtime::Create acquired the mutator_lock_ that is normally given away when we
2661     // Runtime::Start, give it away now so that we don't starve GC.
2662     Thread* self = Thread::Current();
2663     self->TransitionFromRunnableToSuspended(kNative);
2664 
2665     return true;
2666   }
2667 
2668   // Let the ImageWriter write the image files. If we do not compile PIC, also fix up the oat files.
CreateImageFile()2669   bool CreateImageFile()
2670       REQUIRES(!Locks::mutator_lock_) {
2671     CHECK(image_writer_ != nullptr);
2672     if (!IsBootImage()) {
2673       CHECK(image_filenames_.empty());
2674       image_filenames_.push_back(app_image_file_name_.c_str());
2675     }
2676     if (!image_writer_->Write(app_image_fd_,
2677                               image_filenames_,
2678                               oat_filenames_)) {
2679       LOG(ERROR) << "Failure during image file creation";
2680       return false;
2681     }
2682 
2683     // We need the OatDataBegin entries.
2684     dchecked_vector<uintptr_t> oat_data_begins;
2685     for (size_t i = 0, size = oat_filenames_.size(); i != size; ++i) {
2686       oat_data_begins.push_back(image_writer_->GetOatDataBegin(i));
2687     }
2688     // Destroy ImageWriter before doing FixupElf.
2689     image_writer_.reset();
2690 
2691     for (size_t i = 0, size = oat_filenames_.size(); i != size; ++i) {
2692       const char* oat_filename = oat_filenames_[i];
2693       // Do not fix up the ELF file if we are --compile-pic or compiling the app image
2694       if (!compiler_options_->GetCompilePic() && IsBootImage()) {
2695         std::unique_ptr<File> oat_file(OS::OpenFileReadWrite(oat_filename));
2696         if (oat_file.get() == nullptr) {
2697           PLOG(ERROR) << "Failed to open ELF file: " << oat_filename;
2698           return false;
2699         }
2700 
2701         if (!ElfWriter::Fixup(oat_file.get(), oat_data_begins[i])) {
2702           oat_file->Erase();
2703           LOG(ERROR) << "Failed to fixup ELF file " << oat_file->GetPath();
2704           return false;
2705         }
2706 
2707         if (oat_file->FlushCloseOrErase()) {
2708           PLOG(ERROR) << "Failed to flush and close fixed ELF file " << oat_file->GetPath();
2709           return false;
2710         }
2711       }
2712     }
2713 
2714     return true;
2715   }
2716 
2717   // Reads the class names (java.lang.Object) and returns a set of descriptors (Ljava/lang/Object;)
ReadImageClassesFromFile(const char * image_classes_filename)2718   static std::unordered_set<std::string>* ReadImageClassesFromFile(
2719       const char* image_classes_filename) {
2720     std::function<std::string(const char*)> process = DotToDescriptor;
2721     return ReadCommentedInputFromFile<std::unordered_set<std::string>>(image_classes_filename,
2722                                                                        &process);
2723   }
2724 
2725   // Reads the class names (java.lang.Object) and returns a set of descriptors (Ljava/lang/Object;)
ReadImageClassesFromZip(const char * zip_filename,const char * image_classes_filename,std::string * error_msg)2726   static std::unordered_set<std::string>* ReadImageClassesFromZip(
2727         const char* zip_filename,
2728         const char* image_classes_filename,
2729         std::string* error_msg) {
2730     std::function<std::string(const char*)> process = DotToDescriptor;
2731     return ReadCommentedInputFromZip<std::unordered_set<std::string>>(zip_filename,
2732                                                                       image_classes_filename,
2733                                                                       &process,
2734                                                                       error_msg);
2735   }
2736 
2737   // Read lines from the given file, dropping comments and empty lines. Post-process each line with
2738   // the given function.
2739   template <typename T>
ReadCommentedInputFromFile(const char * input_filename,std::function<std::string (const char *)> * process)2740   static T* ReadCommentedInputFromFile(
2741       const char* input_filename, std::function<std::string(const char*)>* process) {
2742     std::unique_ptr<std::ifstream> input_file(new std::ifstream(input_filename, std::ifstream::in));
2743     if (input_file.get() == nullptr) {
2744       LOG(ERROR) << "Failed to open input file " << input_filename;
2745       return nullptr;
2746     }
2747     std::unique_ptr<T> result(
2748         ReadCommentedInputStream<T>(*input_file, process));
2749     input_file->close();
2750     return result.release();
2751   }
2752 
2753   // Read lines from the given file from the given zip file, dropping comments and empty lines.
2754   // Post-process each line with the given function.
2755   template <typename T>
ReadCommentedInputFromZip(const char * zip_filename,const char * input_filename,std::function<std::string (const char *)> * process,std::string * error_msg)2756   static T* ReadCommentedInputFromZip(
2757       const char* zip_filename,
2758       const char* input_filename,
2759       std::function<std::string(const char*)>* process,
2760       std::string* error_msg) {
2761     std::unique_ptr<ZipArchive> zip_archive(ZipArchive::Open(zip_filename, error_msg));
2762     if (zip_archive.get() == nullptr) {
2763       return nullptr;
2764     }
2765     std::unique_ptr<ZipEntry> zip_entry(zip_archive->Find(input_filename, error_msg));
2766     if (zip_entry.get() == nullptr) {
2767       *error_msg = StringPrintf("Failed to find '%s' within '%s': %s", input_filename,
2768                                 zip_filename, error_msg->c_str());
2769       return nullptr;
2770     }
2771     std::unique_ptr<MemMap> input_file(zip_entry->ExtractToMemMap(zip_filename,
2772                                                                   input_filename,
2773                                                                   error_msg));
2774     if (input_file.get() == nullptr) {
2775       *error_msg = StringPrintf("Failed to extract '%s' from '%s': %s", input_filename,
2776                                 zip_filename, error_msg->c_str());
2777       return nullptr;
2778     }
2779     const std::string input_string(reinterpret_cast<char*>(input_file->Begin()),
2780                                    input_file->Size());
2781     std::istringstream input_stream(input_string);
2782     return ReadCommentedInputStream<T>(input_stream, process);
2783   }
2784 
2785   // Read lines from the given stream, dropping comments and empty lines. Post-process each line
2786   // with the given function.
2787   template <typename T>
ReadCommentedInputStream(std::istream & in_stream,std::function<std::string (const char *)> * process)2788   static T* ReadCommentedInputStream(
2789       std::istream& in_stream,
2790       std::function<std::string(const char*)>* process) {
2791     std::unique_ptr<T> output(new T());
2792     while (in_stream.good()) {
2793       std::string dot;
2794       std::getline(in_stream, dot);
2795       if (android::base::StartsWith(dot, "#") || dot.empty()) {
2796         continue;
2797       }
2798       if (process != nullptr) {
2799         std::string descriptor((*process)(dot.c_str()));
2800         output->insert(output->end(), descriptor);
2801       } else {
2802         output->insert(output->end(), dot);
2803       }
2804     }
2805     return output.release();
2806   }
2807 
LogCompletionTime()2808   void LogCompletionTime() {
2809     // Note: when creation of a runtime fails, e.g., when trying to compile an app but when there
2810     //       is no image, there won't be a Runtime::Current().
2811     // Note: driver creation can fail when loading an invalid dex file.
2812     LOG(INFO) << "dex2oat took "
2813               << PrettyDuration(NanoTime() - start_ns_)
2814               << " (" << PrettyDuration(ProcessCpuNanoTime() - start_cputime_ns_) << " cpu)"
2815               << " (threads: " << thread_count_ << ") "
2816               << ((Runtime::Current() != nullptr && driver_ != nullptr) ?
2817                   driver_->GetMemoryUsageString(kIsDebugBuild || VLOG_IS_ON(compiler)) :
2818                   "");
2819   }
2820 
StripIsaFrom(const char * image_filename,InstructionSet isa)2821   std::string StripIsaFrom(const char* image_filename, InstructionSet isa) {
2822     std::string res(image_filename);
2823     size_t last_slash = res.rfind('/');
2824     if (last_slash == std::string::npos || last_slash == 0) {
2825       return res;
2826     }
2827     size_t penultimate_slash = res.rfind('/', last_slash - 1);
2828     if (penultimate_slash == std::string::npos) {
2829       return res;
2830     }
2831     // Check that the string in-between is the expected one.
2832     if (res.substr(penultimate_slash + 1, last_slash - penultimate_slash - 1) !=
2833             GetInstructionSetString(isa)) {
2834       LOG(WARNING) << "Unexpected string when trying to strip isa: " << res;
2835       return res;
2836     }
2837     return res.substr(0, penultimate_slash) + res.substr(last_slash);
2838   }
2839 
2840   std::unique_ptr<CompilerOptions> compiler_options_;
2841   Compiler::Kind compiler_kind_;
2842 
2843   InstructionSet instruction_set_;
2844   std::unique_ptr<const InstructionSetFeatures> instruction_set_features_;
2845 
2846   uint32_t image_file_location_oat_checksum_;
2847   uintptr_t image_file_location_oat_data_begin_;
2848   int32_t image_patch_delta_;
2849   std::unique_ptr<SafeMap<std::string, std::string> > key_value_store_;
2850 
2851   std::unique_ptr<VerificationResults> verification_results_;
2852 
2853   std::unique_ptr<QuickCompilerCallbacks> callbacks_;
2854 
2855   std::unique_ptr<Runtime> runtime_;
2856 
2857   // The spec describing how the class loader should be setup for compilation.
2858   std::unique_ptr<ClassLoaderContext> class_loader_context_;
2859 
2860   size_t thread_count_;
2861   uint64_t start_ns_;
2862   uint64_t start_cputime_ns_;
2863   std::unique_ptr<WatchDog> watchdog_;
2864   std::vector<std::unique_ptr<File>> oat_files_;
2865   std::vector<std::unique_ptr<File>> vdex_files_;
2866   std::string oat_location_;
2867   std::vector<const char*> oat_filenames_;
2868   std::vector<const char*> oat_unstripped_;
2869   int oat_fd_;
2870   int input_vdex_fd_;
2871   int output_vdex_fd_;
2872   std::string input_vdex_;
2873   std::string output_vdex_;
2874   std::unique_ptr<VdexFile> input_vdex_file_;
2875   std::vector<const char*> dex_filenames_;
2876   std::vector<const char*> dex_locations_;
2877   int zip_fd_;
2878   std::string zip_location_;
2879   std::string boot_image_filename_;
2880   std::vector<const char*> runtime_args_;
2881   std::vector<const char*> image_filenames_;
2882   uintptr_t image_base_;
2883   const char* image_classes_zip_filename_;
2884   const char* image_classes_filename_;
2885   ImageHeader::StorageMode image_storage_mode_;
2886   const char* compiled_classes_zip_filename_;
2887   const char* compiled_classes_filename_;
2888   const char* compiled_methods_zip_filename_;
2889   const char* compiled_methods_filename_;
2890   const char* passes_to_run_filename_;
2891   const char* dirty_image_objects_filename_;
2892   std::unique_ptr<std::unordered_set<std::string>> image_classes_;
2893   std::unique_ptr<std::unordered_set<std::string>> compiled_classes_;
2894   std::unique_ptr<std::unordered_set<std::string>> compiled_methods_;
2895   std::unique_ptr<std::unordered_set<std::string>> dirty_image_objects_;
2896   std::unique_ptr<std::vector<std::string>> passes_to_run_;
2897   bool multi_image_;
2898   bool is_host_;
2899   std::string android_root_;
2900   // Dex files we are compiling, does not include the class path dex files.
2901   std::vector<const DexFile*> dex_files_;
2902   std::string no_inline_from_string_;
2903 
2904   std::vector<std::unique_ptr<ElfWriter>> elf_writers_;
2905   std::vector<std::unique_ptr<OatWriter>> oat_writers_;
2906   std::vector<OutputStream*> rodata_;
2907   std::vector<std::unique_ptr<OutputStream>> vdex_out_;
2908   std::unique_ptr<ImageWriter> image_writer_;
2909   std::unique_ptr<CompilerDriver> driver_;
2910 
2911   std::vector<std::unique_ptr<MemMap>> opened_dex_files_maps_;
2912   std::vector<std::unique_ptr<const DexFile>> opened_dex_files_;
2913 
2914   // Note that this might contain pointers owned by class_loader_context_.
2915   std::vector<const DexFile*> no_inline_from_dex_files_;
2916 
2917   bool dump_stats_;
2918   bool dump_passes_;
2919   bool dump_timing_;
2920   bool dump_slow_timing_;
2921   bool avoid_storing_invocation_;
2922   std::string swap_file_name_;
2923   int swap_fd_;
2924   size_t min_dex_files_for_swap_ = kDefaultMinDexFilesForSwap;
2925   size_t min_dex_file_cumulative_size_for_swap_ = kDefaultMinDexFileCumulativeSizeForSwap;
2926   size_t very_large_threshold_ = std::numeric_limits<size_t>::max();
2927   std::string app_image_file_name_;
2928   int app_image_fd_;
2929   std::string profile_file_;
2930   int profile_file_fd_;
2931   std::unique_ptr<ProfileCompilationInfo> profile_compilation_info_;
2932   TimingLogger* timings_;
2933   std::unique_ptr<CumulativeLogger> compiler_phases_timings_;
2934   std::vector<std::vector<const DexFile*>> dex_files_per_oat_file_;
2935   std::unordered_map<const DexFile*, size_t> dex_file_oat_index_map_;
2936 
2937   // Backing storage.
2938   std::vector<std::string> char_backing_storage_;
2939 
2940   // See CompilerOptions.force_determinism_.
2941   bool force_determinism_;
2942 
2943   // Directory of relative classpaths.
2944   std::string classpath_dir_;
2945 
2946   // Whether the given input vdex is also the output.
2947   bool update_input_vdex_ = false;
2948 
2949   DISALLOW_IMPLICIT_CONSTRUCTORS(Dex2Oat);
2950 };
2951 
b13564922()2952 static void b13564922() {
2953 #if defined(__linux__) && defined(__arm__)
2954   int major, minor;
2955   struct utsname uts;
2956   if (uname(&uts) != -1 &&
2957       sscanf(uts.release, "%d.%d", &major, &minor) == 2 &&
2958       ((major < 3) || ((major == 3) && (minor < 4)))) {
2959     // Kernels before 3.4 don't handle the ASLR well and we can run out of address
2960     // space (http://b/13564922). Work around the issue by inhibiting further mmap() randomization.
2961     int old_personality = personality(0xffffffff);
2962     if ((old_personality & ADDR_NO_RANDOMIZE) == 0) {
2963       int new_personality = personality(old_personality | ADDR_NO_RANDOMIZE);
2964       if (new_personality == -1) {
2965         LOG(WARNING) << "personality(. | ADDR_NO_RANDOMIZE) failed.";
2966       }
2967     }
2968   }
2969 #endif
2970 }
2971 
2972 class ScopedGlobalRef {
2973  public:
ScopedGlobalRef(jobject obj)2974   explicit ScopedGlobalRef(jobject obj) : obj_(obj) {}
~ScopedGlobalRef()2975   ~ScopedGlobalRef() {
2976     if (obj_ != nullptr) {
2977       ScopedObjectAccess soa(Thread::Current());
2978       soa.Env()->vm->DeleteGlobalRef(soa.Self(), obj_);
2979     }
2980   }
2981 
2982  private:
2983   jobject obj_;
2984 };
2985 
CompileImage(Dex2Oat & dex2oat)2986 static dex2oat::ReturnCode CompileImage(Dex2Oat& dex2oat) {
2987   dex2oat.LoadClassProfileDescriptors();
2988   // Keep the class loader that was used for compilation live for the rest of the compilation
2989   // process.
2990   ScopedGlobalRef class_loader(dex2oat.Compile());
2991 
2992   if (!dex2oat.WriteOutputFiles()) {
2993     dex2oat.EraseOutputFiles();
2994     return dex2oat::ReturnCode::kOther;
2995   }
2996 
2997   // Flush boot.oat. We always expect the output file by name, and it will be re-opened from the
2998   // unstripped name. Do not close the file if we are compiling the image with an oat fd since the
2999   // image writer will require this fd to generate the image.
3000   if (dex2oat.ShouldKeepOatFileOpen()) {
3001     if (!dex2oat.FlushOutputFiles()) {
3002       dex2oat.EraseOutputFiles();
3003       return dex2oat::ReturnCode::kOther;
3004     }
3005   } else if (!dex2oat.FlushCloseOutputFiles()) {
3006     return dex2oat::ReturnCode::kOther;
3007   }
3008 
3009   // Creates the boot.art and patches the oat files.
3010   if (!dex2oat.HandleImage()) {
3011     return dex2oat::ReturnCode::kOther;
3012   }
3013 
3014   // When given --host, finish early without stripping.
3015   if (dex2oat.IsHost()) {
3016     if (!dex2oat.FlushCloseOutputFiles()) {
3017       return dex2oat::ReturnCode::kOther;
3018     }
3019     dex2oat.DumpTiming();
3020     return dex2oat::ReturnCode::kNoFailure;
3021   }
3022 
3023   // Copy stripped to unstripped location, if necessary.
3024   if (!dex2oat.CopyStrippedToUnstripped()) {
3025     return dex2oat::ReturnCode::kOther;
3026   }
3027 
3028   // FlushClose again, as stripping might have re-opened the oat files.
3029   if (!dex2oat.FlushCloseOutputFiles()) {
3030     return dex2oat::ReturnCode::kOther;
3031   }
3032 
3033   dex2oat.DumpTiming();
3034   return dex2oat::ReturnCode::kNoFailure;
3035 }
3036 
CompileApp(Dex2Oat & dex2oat)3037 static dex2oat::ReturnCode CompileApp(Dex2Oat& dex2oat) {
3038   // Keep the class loader that was used for compilation live for the rest of the compilation
3039   // process.
3040   ScopedGlobalRef class_loader(dex2oat.Compile());
3041 
3042   if (!dex2oat.WriteOutputFiles()) {
3043     dex2oat.EraseOutputFiles();
3044     return dex2oat::ReturnCode::kOther;
3045   }
3046 
3047   // Do not close the oat files here. We might have gotten the output file by file descriptor,
3048   // which we would lose.
3049 
3050   // When given --host, finish early without stripping.
3051   if (dex2oat.IsHost()) {
3052     if (!dex2oat.FlushCloseOutputFiles()) {
3053       return dex2oat::ReturnCode::kOther;
3054     }
3055 
3056     dex2oat.DumpTiming();
3057     return dex2oat::ReturnCode::kNoFailure;
3058   }
3059 
3060   // Copy stripped to unstripped location, if necessary. This will implicitly flush & close the
3061   // stripped versions. If this is given, we expect to be able to open writable files by name.
3062   if (!dex2oat.CopyStrippedToUnstripped()) {
3063     return dex2oat::ReturnCode::kOther;
3064   }
3065 
3066   // Flush and close the files.
3067   if (!dex2oat.FlushCloseOutputFiles()) {
3068     return dex2oat::ReturnCode::kOther;
3069   }
3070 
3071   dex2oat.DumpTiming();
3072   return dex2oat::ReturnCode::kNoFailure;
3073 }
3074 
Dex2oat(int argc,char ** argv)3075 static dex2oat::ReturnCode Dex2oat(int argc, char** argv) {
3076   b13564922();
3077 
3078   TimingLogger timings("compiler", false, false);
3079 
3080   // Allocate `dex2oat` on the heap instead of on the stack, as Clang
3081   // might produce a stack frame too large for this function or for
3082   // functions inlining it (such as main), that would not fit the
3083   // requirements of the `-Wframe-larger-than` option.
3084   std::unique_ptr<Dex2Oat> dex2oat = std::make_unique<Dex2Oat>(&timings);
3085 
3086   // Parse arguments. Argument mistakes will lead to exit(EXIT_FAILURE) in UsageError.
3087   dex2oat->ParseArgs(argc, argv);
3088 
3089   // If needed, process profile information for profile guided compilation.
3090   // This operation involves I/O.
3091   if (dex2oat->UseProfile()) {
3092     if (!dex2oat->LoadProfile()) {
3093       LOG(ERROR) << "Failed to process profile file";
3094       return dex2oat::ReturnCode::kOther;
3095     }
3096   }
3097 
3098   art::MemMap::Init();  // For ZipEntry::ExtractToMemMap, and vdex.
3099 
3100   // Check early that the result of compilation can be written
3101   if (!dex2oat->OpenFile()) {
3102     return dex2oat::ReturnCode::kOther;
3103   }
3104 
3105   // Print the complete line when any of the following is true:
3106   //   1) Debug build
3107   //   2) Compiling an image
3108   //   3) Compiling with --host
3109   //   4) Compiling on the host (not a target build)
3110   // Otherwise, print a stripped command line.
3111   if (kIsDebugBuild || dex2oat->IsBootImage() || dex2oat->IsHost() || !kIsTargetBuild) {
3112     LOG(INFO) << CommandLine();
3113   } else {
3114     LOG(INFO) << StrippedCommandLine();
3115   }
3116 
3117   dex2oat::ReturnCode setup_code = dex2oat->Setup();
3118   if (setup_code != dex2oat::ReturnCode::kNoFailure) {
3119     dex2oat->EraseOutputFiles();
3120     return setup_code;
3121   }
3122 
3123   // Helps debugging on device. Can be used to determine which dalvikvm instance invoked a dex2oat
3124   // instance. Used by tools/bisection_search/bisection_search.py.
3125   VLOG(compiler) << "Running dex2oat (parent PID = " << getppid() << ")";
3126 
3127   dex2oat::ReturnCode result;
3128   if (dex2oat->IsImage()) {
3129     result = CompileImage(*dex2oat);
3130   } else {
3131     result = CompileApp(*dex2oat);
3132   }
3133 
3134   return result;
3135 }
3136 }  // namespace art
3137 
main(int argc,char ** argv)3138 int main(int argc, char** argv) {
3139   int result = static_cast<int>(art::Dex2oat(argc, argv));
3140   // Everything was done, do an explicit exit here to avoid running Runtime destructors that take
3141   // time (bug 10645725) unless we're a debug build or running on valgrind. Note: The Dex2Oat class
3142   // should not destruct the runtime in this case.
3143   if (!art::kIsDebugBuild && (RUNNING_ON_MEMORY_TOOL == 0)) {
3144     _exit(result);
3145   }
3146   return result;
3147 }
3148