1// Copyright 2017 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package java 16 17import ( 18 "strconv" 19 "strings" 20 21 "github.com/google/blueprint" 22 "github.com/google/blueprint/proptools" 23 24 "android/soong/android" 25 "android/soong/remoteexec" 26) 27 28type DexProperties struct { 29 // If set to true, compile dex regardless of installable. Defaults to false. 30 Compile_dex *bool 31 32 // list of module-specific flags that will be used for dex compiles 33 Dxflags []string `android:"arch_variant"` 34 35 // A list of files containing rules that specify the classes to keep in the main dex file. 36 Main_dex_rules []string `android:"path"` 37 38 Optimize struct { 39 // If false, disable all optimization. Defaults to true for android_app and 40 // android_test_helper_app modules, false for android_test, java_library, and java_test modules. 41 Enabled *bool 42 // True if the module containing this has it set by default. 43 EnabledByDefault bool `blueprint:"mutated"` 44 45 // Whether to continue building even if warnings are emitted. Defaults to true. 46 Ignore_warnings *bool 47 48 // If true, runs R8 in Proguard compatibility mode (default). 49 // Otherwise, runs R8 in full mode. 50 Proguard_compatibility *bool 51 52 // If true, optimize for size by removing unused code. Defaults to true for apps, 53 // false for libraries and tests. 54 Shrink *bool 55 56 // If true, optimize bytecode. Defaults to false. 57 Optimize *bool 58 59 // If true, obfuscate bytecode. Defaults to false. 60 Obfuscate *bool 61 62 // If true, do not use the flag files generated by aapt that automatically keep 63 // classes referenced by the app manifest. Defaults to false. 64 No_aapt_flags *bool 65 66 // If true, optimize for size by removing unused resources. Defaults to false. 67 Shrink_resources *bool 68 69 // Flags to pass to proguard. 70 Proguard_flags []string 71 72 // Specifies the locations of files containing proguard flags. 73 Proguard_flags_files []string `android:"path"` 74 } 75 76 // Keep the data uncompressed. We always need uncompressed dex for execution, 77 // so this might actually save space by avoiding storing the same data twice. 78 // This defaults to reasonable value based on module and should not be set. 79 // It exists only to support ART tests. 80 Uncompress_dex *bool 81 82 // Exclude kotlinc generate files: *.kotlin_module, *.kotlin_builtins. Defaults to false. 83 Exclude_kotlinc_generated_files *bool 84} 85 86type dexer struct { 87 dexProperties DexProperties 88 89 // list of extra proguard flag files 90 extraProguardFlagFiles android.Paths 91 proguardDictionary android.OptionalPath 92 proguardConfiguration android.OptionalPath 93 proguardUsageZip android.OptionalPath 94 95 providesTransitiveHeaderJars 96} 97 98func (d *dexer) effectiveOptimizeEnabled() bool { 99 return BoolDefault(d.dexProperties.Optimize.Enabled, d.dexProperties.Optimize.EnabledByDefault) 100} 101 102var d8, d8RE = pctx.MultiCommandRemoteStaticRules("d8", 103 blueprint.RuleParams{ 104 Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + 105 `mkdir -p $$(dirname $tmpJar) && ` + 106 `${config.Zip2ZipCmd} -i $in -o $tmpJar -x '**/*.dex' && ` + 107 `$d8Template${config.D8Cmd} ${config.D8Flags} --output $outDir $d8Flags $tmpJar && ` + 108 `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + 109 `${config.MergeZipsCmd} -D -stripFile "**/*.class" $mergeZipsFlags $out $outDir/classes.dex.jar $in`, 110 CommandDeps: []string{ 111 "${config.D8Cmd}", 112 "${config.Zip2ZipCmd}", 113 "${config.SoongZipCmd}", 114 "${config.MergeZipsCmd}", 115 }, 116 }, map[string]*remoteexec.REParams{ 117 "$d8Template": &remoteexec.REParams{ 118 Labels: map[string]string{"type": "compile", "compiler": "d8"}, 119 Inputs: []string{"${config.D8Jar}"}, 120 ExecStrategy: "${config.RED8ExecStrategy}", 121 ToolchainInputs: []string{"${config.JavaCmd}"}, 122 Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, 123 }, 124 "$zipTemplate": &remoteexec.REParams{ 125 Labels: map[string]string{"type": "tool", "name": "soong_zip"}, 126 Inputs: []string{"${config.SoongZipCmd}", "$outDir"}, 127 OutputFiles: []string{"$outDir/classes.dex.jar"}, 128 ExecStrategy: "${config.RED8ExecStrategy}", 129 Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, 130 }, 131 }, []string{"outDir", "d8Flags", "zipFlags", "tmpJar", "mergeZipsFlags"}, nil) 132 133var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8", 134 blueprint.RuleParams{ 135 Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + 136 `rm -f "$outDict" && rm -f "$outConfig" && rm -rf "${outUsageDir}" && ` + 137 `mkdir -p $$(dirname ${outUsage}) && ` + 138 `mkdir -p $$(dirname $tmpJar) && ` + 139 `${config.Zip2ZipCmd} -i $in -o $tmpJar -x '**/*.dex' && ` + 140 `$r8Template${config.R8Cmd} ${config.R8Flags} -injars $tmpJar --output $outDir ` + 141 `--no-data-resources ` + 142 `-printmapping ${outDict} ` + 143 `--pg-conf-output ${outConfig} ` + 144 `-printusage ${outUsage} ` + 145 `--deps-file ${out}.d ` + 146 `$r8Flags && ` + 147 `touch "${outDict}" "${outConfig}" "${outUsage}" && ` + 148 `${config.SoongZipCmd} -o ${outUsageZip} -C ${outUsageDir} -f ${outUsage} && ` + 149 `rm -rf ${outUsageDir} && ` + 150 `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + 151 `${config.MergeZipsCmd} -D -stripFile "**/*.class" $mergeZipsFlags $out $outDir/classes.dex.jar $in`, 152 Depfile: "${out}.d", 153 Deps: blueprint.DepsGCC, 154 CommandDeps: []string{ 155 "${config.R8Cmd}", 156 "${config.Zip2ZipCmd}", 157 "${config.SoongZipCmd}", 158 "${config.MergeZipsCmd}", 159 }, 160 }, map[string]*remoteexec.REParams{ 161 "$r8Template": &remoteexec.REParams{ 162 Labels: map[string]string{"type": "compile", "compiler": "r8"}, 163 Inputs: []string{"$implicits", "${config.R8Jar}"}, 164 OutputFiles: []string{"${outUsage}"}, 165 ExecStrategy: "${config.RER8ExecStrategy}", 166 ToolchainInputs: []string{"${config.JavaCmd}"}, 167 Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, 168 }, 169 "$zipTemplate": &remoteexec.REParams{ 170 Labels: map[string]string{"type": "tool", "name": "soong_zip"}, 171 Inputs: []string{"${config.SoongZipCmd}", "$outDir"}, 172 OutputFiles: []string{"$outDir/classes.dex.jar"}, 173 ExecStrategy: "${config.RER8ExecStrategy}", 174 Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, 175 }, 176 "$zipUsageTemplate": &remoteexec.REParams{ 177 Labels: map[string]string{"type": "tool", "name": "soong_zip"}, 178 Inputs: []string{"${config.SoongZipCmd}", "${outUsage}"}, 179 OutputFiles: []string{"${outUsageZip}"}, 180 ExecStrategy: "${config.RER8ExecStrategy}", 181 Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, 182 }, 183 }, []string{"outDir", "outDict", "outConfig", "outUsage", "outUsageZip", "outUsageDir", 184 "r8Flags", "zipFlags", "tmpJar", "mergeZipsFlags"}, []string{"implicits"}) 185 186func (d *dexer) dexCommonFlags(ctx android.ModuleContext, 187 dexParams *compileDexParams) (flags []string, deps android.Paths) { 188 189 flags = d.dexProperties.Dxflags 190 // Translate all the DX flags to D8 ones until all the build files have been migrated 191 // to D8 flags. See: b/69377755 192 flags = android.RemoveListFromList(flags, 193 []string{"--core-library", "--dex", "--multi-dex"}) 194 195 for _, f := range android.PathsForModuleSrc(ctx, d.dexProperties.Main_dex_rules) { 196 flags = append(flags, "--main-dex-rules", f.String()) 197 deps = append(deps, f) 198 } 199 200 if ctx.Config().Getenv("NO_OPTIMIZE_DX") != "" { 201 flags = append(flags, "--debug") 202 } 203 204 if ctx.Config().Getenv("GENERATE_DEX_DEBUG") != "" { 205 flags = append(flags, 206 "--debug", 207 "--verbose") 208 } 209 210 // Supplying the platform build flag disables various features like API modeling and desugaring. 211 // For targets with a stable min SDK version (i.e., when the min SDK is both explicitly specified 212 // and managed+versioned), we suppress this flag to ensure portability. 213 // Note: Targets with a min SDK kind of core_platform (e.g., framework.jar) or unspecified (e.g., 214 // services.jar), are not classified as stable, which is WAI. 215 // TODO(b/232073181): Expand to additional min SDK cases after validation. 216 if !dexParams.sdkVersion.Stable() { 217 flags = append(flags, "--android-platform-build") 218 } 219 220 effectiveVersion, err := dexParams.minSdkVersion.EffectiveVersion(ctx) 221 if err != nil { 222 ctx.PropertyErrorf("min_sdk_version", "%s", err) 223 } 224 225 flags = append(flags, "--min-api "+strconv.Itoa(effectiveVersion.FinalOrFutureInt())) 226 return flags, deps 227} 228 229func d8Flags(flags javaBuilderFlags) (d8Flags []string, d8Deps android.Paths) { 230 d8Flags = append(d8Flags, flags.bootClasspath.FormRepeatedClassPath("--lib ")...) 231 d8Flags = append(d8Flags, flags.dexClasspath.FormRepeatedClassPath("--lib ")...) 232 233 d8Deps = append(d8Deps, flags.bootClasspath...) 234 d8Deps = append(d8Deps, flags.dexClasspath...) 235 236 return d8Flags, d8Deps 237} 238 239func (d *dexer) r8Flags(ctx android.ModuleContext, flags javaBuilderFlags) (r8Flags []string, r8Deps android.Paths) { 240 opt := d.dexProperties.Optimize 241 242 // When an app contains references to APIs that are not in the SDK specified by 243 // its LOCAL_SDK_VERSION for example added by support library or by runtime 244 // classes added by desugaring, we artifically raise the "SDK version" "linked" by 245 // ProGuard, to 246 // - suppress ProGuard warnings of referencing symbols unknown to the lower SDK version. 247 // - prevent ProGuard stripping subclass in the support library that extends class added in the higher SDK version. 248 // See b/20667396 249 var proguardRaiseDeps classpath 250 ctx.VisitDirectDepsWithTag(proguardRaiseTag, func(m android.Module) { 251 dep := ctx.OtherModuleProvider(m, JavaInfoProvider).(JavaInfo) 252 proguardRaiseDeps = append(proguardRaiseDeps, dep.HeaderJars...) 253 }) 254 255 r8Flags = append(r8Flags, proguardRaiseDeps.FormJavaClassPath("-libraryjars")) 256 r8Deps = append(r8Deps, proguardRaiseDeps...) 257 r8Flags = append(r8Flags, flags.bootClasspath.FormJavaClassPath("-libraryjars")) 258 r8Deps = append(r8Deps, flags.bootClasspath...) 259 r8Flags = append(r8Flags, flags.dexClasspath.FormJavaClassPath("-libraryjars")) 260 r8Deps = append(r8Deps, flags.dexClasspath...) 261 262 transitiveStaticLibsLookupMap := map[android.Path]bool{} 263 if d.transitiveStaticLibsHeaderJars != nil { 264 for _, jar := range d.transitiveStaticLibsHeaderJars.ToList() { 265 transitiveStaticLibsLookupMap[jar] = true 266 } 267 } 268 transitiveHeaderJars := android.Paths{} 269 if d.transitiveLibsHeaderJars != nil { 270 for _, jar := range d.transitiveLibsHeaderJars.ToList() { 271 if _, ok := transitiveStaticLibsLookupMap[jar]; ok { 272 // don't include a lib if it is already packaged in the current JAR as a static lib 273 continue 274 } 275 transitiveHeaderJars = append(transitiveHeaderJars, jar) 276 } 277 } 278 transitiveClasspath := classpath(transitiveHeaderJars) 279 r8Flags = append(r8Flags, transitiveClasspath.FormJavaClassPath("-libraryjars")) 280 r8Deps = append(r8Deps, transitiveClasspath...) 281 282 flagFiles := android.Paths{ 283 android.PathForSource(ctx, "build/make/core/proguard.flags"), 284 } 285 286 flagFiles = append(flagFiles, d.extraProguardFlagFiles...) 287 // TODO(ccross): static android library proguard files 288 289 flagFiles = append(flagFiles, android.PathsForModuleSrc(ctx, opt.Proguard_flags_files)...) 290 291 r8Flags = append(r8Flags, android.JoinWithPrefix(flagFiles.Strings(), "-include ")) 292 r8Deps = append(r8Deps, flagFiles...) 293 294 // TODO(b/70942988): This is included from build/make/core/proguard.flags 295 r8Deps = append(r8Deps, android.PathForSource(ctx, 296 "build/make/core/proguard_basic_keeps.flags")) 297 298 r8Flags = append(r8Flags, opt.Proguard_flags...) 299 300 if BoolDefault(opt.Proguard_compatibility, true) { 301 r8Flags = append(r8Flags, "--force-proguard-compatibility") 302 } else { 303 // TODO(b/213833843): Allow configuration of the prefix via a build variable. 304 var sourceFilePrefix = "go/retraceme " 305 var sourceFileTemplate = "\"" + sourceFilePrefix + "%MAP_ID\"" 306 // TODO(b/200967150): Also tag the source file in compat builds. 307 if Bool(opt.Optimize) || Bool(opt.Obfuscate) { 308 r8Flags = append(r8Flags, "--map-id-template", "%MAP_HASH") 309 r8Flags = append(r8Flags, "--source-file-template", sourceFileTemplate) 310 } 311 } 312 313 // TODO(ccross): Don't shrink app instrumentation tests by default. 314 if !Bool(opt.Shrink) { 315 r8Flags = append(r8Flags, "-dontshrink") 316 } 317 318 if !Bool(opt.Optimize) { 319 r8Flags = append(r8Flags, "-dontoptimize") 320 } 321 322 // TODO(ccross): error if obufscation + app instrumentation test. 323 if !Bool(opt.Obfuscate) { 324 r8Flags = append(r8Flags, "-dontobfuscate") 325 } 326 // TODO(ccross): if this is an instrumentation test of an obfuscated app, use the 327 // dictionary of the app and move the app from libraryjars to injars. 328 329 // Don't strip out debug information for eng builds. 330 if ctx.Config().Eng() { 331 r8Flags = append(r8Flags, "--debug") 332 } 333 334 // TODO(b/180878971): missing classes should be added to the relevant builds. 335 // TODO(b/229727645): do not use true as default for Android platform builds. 336 if proptools.BoolDefault(opt.Ignore_warnings, true) { 337 r8Flags = append(r8Flags, "-ignorewarnings") 338 } 339 340 return r8Flags, r8Deps 341} 342 343type compileDexParams struct { 344 flags javaBuilderFlags 345 sdkVersion android.SdkSpec 346 minSdkVersion android.ApiLevel 347 classesJar android.Path 348 jarName string 349} 350 351func (d *dexer) compileDex(ctx android.ModuleContext, dexParams *compileDexParams) android.OutputPath { 352 353 // Compile classes.jar into classes.dex and then javalib.jar 354 javalibJar := android.PathForModuleOut(ctx, "dex", dexParams.jarName).OutputPath 355 outDir := android.PathForModuleOut(ctx, "dex") 356 tmpJar := android.PathForModuleOut(ctx, "withres-withoutdex", dexParams.jarName) 357 358 zipFlags := "--ignore_missing_files" 359 if proptools.Bool(d.dexProperties.Uncompress_dex) { 360 zipFlags += " -L 0" 361 } 362 363 commonFlags, commonDeps := d.dexCommonFlags(ctx, dexParams) 364 365 // Exclude kotlinc generated files when "exclude_kotlinc_generated_files" is set to true. 366 mergeZipsFlags := "" 367 if proptools.BoolDefault(d.dexProperties.Exclude_kotlinc_generated_files, false) { 368 mergeZipsFlags = "-stripFile META-INF/*.kotlin_module -stripFile **/*.kotlin_builtins" 369 } 370 371 useR8 := d.effectiveOptimizeEnabled() 372 if useR8 { 373 proguardDictionary := android.PathForModuleOut(ctx, "proguard_dictionary") 374 d.proguardDictionary = android.OptionalPathForPath(proguardDictionary) 375 proguardConfiguration := android.PathForModuleOut(ctx, "proguard_configuration") 376 d.proguardConfiguration = android.OptionalPathForPath(proguardConfiguration) 377 proguardUsageDir := android.PathForModuleOut(ctx, "proguard_usage") 378 proguardUsage := proguardUsageDir.Join(ctx, ctx.Namespace().Path, 379 android.ModuleNameWithPossibleOverride(ctx), "unused.txt") 380 proguardUsageZip := android.PathForModuleOut(ctx, "proguard_usage.zip") 381 d.proguardUsageZip = android.OptionalPathForPath(proguardUsageZip) 382 r8Flags, r8Deps := d.r8Flags(ctx, dexParams.flags) 383 r8Deps = append(r8Deps, commonDeps...) 384 rule := r8 385 args := map[string]string{ 386 "r8Flags": strings.Join(append(commonFlags, r8Flags...), " "), 387 "zipFlags": zipFlags, 388 "outDict": proguardDictionary.String(), 389 "outConfig": proguardConfiguration.String(), 390 "outUsageDir": proguardUsageDir.String(), 391 "outUsage": proguardUsage.String(), 392 "outUsageZip": proguardUsageZip.String(), 393 "outDir": outDir.String(), 394 "tmpJar": tmpJar.String(), 395 "mergeZipsFlags": mergeZipsFlags, 396 } 397 if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_R8") { 398 rule = r8RE 399 args["implicits"] = strings.Join(r8Deps.Strings(), ",") 400 } 401 ctx.Build(pctx, android.BuildParams{ 402 Rule: rule, 403 Description: "r8", 404 Output: javalibJar, 405 ImplicitOutputs: android.WritablePaths{proguardDictionary, proguardUsageZip}, 406 Input: dexParams.classesJar, 407 Implicits: r8Deps, 408 Args: args, 409 }) 410 } else { 411 d8Flags, d8Deps := d8Flags(dexParams.flags) 412 d8Deps = append(d8Deps, commonDeps...) 413 rule := d8 414 if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_D8") { 415 rule = d8RE 416 } 417 ctx.Build(pctx, android.BuildParams{ 418 Rule: rule, 419 Description: "d8", 420 Output: javalibJar, 421 Input: dexParams.classesJar, 422 Implicits: d8Deps, 423 Args: map[string]string{ 424 "d8Flags": strings.Join(append(commonFlags, d8Flags...), " "), 425 "zipFlags": zipFlags, 426 "outDir": outDir.String(), 427 "tmpJar": tmpJar.String(), 428 "mergeZipsFlags": mergeZipsFlags, 429 }, 430 }) 431 } 432 if proptools.Bool(d.dexProperties.Uncompress_dex) { 433 alignedJavalibJar := android.PathForModuleOut(ctx, "aligned", dexParams.jarName).OutputPath 434 TransformZipAlign(ctx, alignedJavalibJar, javalibJar) 435 javalibJar = alignedJavalibJar 436 } 437 438 return javalibJar 439} 440