1// Copyright 2017 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package java 16 17import ( 18 "strconv" 19 "strings" 20 21 "github.com/google/blueprint" 22 "github.com/google/blueprint/proptools" 23 24 "android/soong/android" 25 "android/soong/remoteexec" 26) 27 28type DexProperties struct { 29 // If set to true, compile dex regardless of installable. Defaults to false. 30 Compile_dex *bool 31 32 // list of module-specific flags that will be used for dex compiles 33 Dxflags []string `android:"arch_variant"` 34 35 // A list of files containing rules that specify the classes to keep in the main dex file. 36 Main_dex_rules []string `android:"path"` 37 38 Optimize struct { 39 // If false, disable all optimization. Defaults to true for android_app and android_test 40 // modules, false for java_library and java_test modules. 41 Enabled *bool 42 // True if the module containing this has it set by default. 43 EnabledByDefault bool `blueprint:"mutated"` 44 45 // If true, runs R8 in Proguard compatibility mode (default). 46 // Otherwise, runs R8 in full mode. 47 Proguard_compatibility *bool 48 49 // If true, optimize for size by removing unused code. Defaults to true for apps, 50 // false for libraries and tests. 51 Shrink *bool 52 53 // If true, optimize bytecode. Defaults to false. 54 Optimize *bool 55 56 // If true, obfuscate bytecode. Defaults to false. 57 Obfuscate *bool 58 59 // If true, do not use the flag files generated by aapt that automatically keep 60 // classes referenced by the app manifest. Defaults to false. 61 No_aapt_flags *bool 62 63 // Flags to pass to proguard. 64 Proguard_flags []string 65 66 // Specifies the locations of files containing proguard flags. 67 Proguard_flags_files []string `android:"path"` 68 } 69 70 // Keep the data uncompressed. We always need uncompressed dex for execution, 71 // so this might actually save space by avoiding storing the same data twice. 72 // This defaults to reasonable value based on module and should not be set. 73 // It exists only to support ART tests. 74 Uncompress_dex *bool 75 76 // Exclude kotlinc generate files: *.kotlin_module, *.kotlin_builtins. Defaults to false. 77 Exclude_kotlinc_generated_files *bool 78} 79 80type dexer struct { 81 dexProperties DexProperties 82 83 // list of extra proguard flag files 84 extraProguardFlagFiles android.Paths 85 proguardDictionary android.OptionalPath 86 proguardUsageZip android.OptionalPath 87} 88 89func (d *dexer) effectiveOptimizeEnabled() bool { 90 return BoolDefault(d.dexProperties.Optimize.Enabled, d.dexProperties.Optimize.EnabledByDefault) 91} 92 93var d8, d8RE = pctx.MultiCommandRemoteStaticRules("d8", 94 blueprint.RuleParams{ 95 Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + 96 `mkdir -p $$(dirname $tmpJar) && ` + 97 `${config.Zip2ZipCmd} -i $in -o $tmpJar -x '**/*.dex' && ` + 98 `$d8Template${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $tmpJar && ` + 99 `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + 100 `${config.MergeZipsCmd} -D -stripFile "**/*.class" $mergeZipsFlags $out $outDir/classes.dex.jar $in`, 101 CommandDeps: []string{ 102 "${config.D8Cmd}", 103 "${config.Zip2ZipCmd}", 104 "${config.SoongZipCmd}", 105 "${config.MergeZipsCmd}", 106 }, 107 }, map[string]*remoteexec.REParams{ 108 "$d8Template": &remoteexec.REParams{ 109 Labels: map[string]string{"type": "compile", "compiler": "d8"}, 110 Inputs: []string{"${config.D8Jar}"}, 111 ExecStrategy: "${config.RED8ExecStrategy}", 112 ToolchainInputs: []string{"${config.JavaCmd}"}, 113 Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, 114 }, 115 "$zipTemplate": &remoteexec.REParams{ 116 Labels: map[string]string{"type": "tool", "name": "soong_zip"}, 117 Inputs: []string{"${config.SoongZipCmd}", "$outDir"}, 118 OutputFiles: []string{"$outDir/classes.dex.jar"}, 119 ExecStrategy: "${config.RED8ExecStrategy}", 120 Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, 121 }, 122 }, []string{"outDir", "d8Flags", "zipFlags", "tmpJar", "mergeZipsFlags"}, nil) 123 124var r8, r8RE = pctx.MultiCommandRemoteStaticRules("r8", 125 blueprint.RuleParams{ 126 Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` + 127 `rm -f "$outDict" && rm -rf "${outUsageDir}" && ` + 128 `mkdir -p $$(dirname ${outUsage}) && ` + 129 `mkdir -p $$(dirname $tmpJar) && ` + 130 `${config.Zip2ZipCmd} -i $in -o $tmpJar -x '**/*.dex' && ` + 131 `$r8Template${config.R8Cmd} ${config.DexFlags} -injars $tmpJar --output $outDir ` + 132 `--no-data-resources ` + 133 `-printmapping ${outDict} ` + 134 `-printusage ${outUsage} ` + 135 `--deps-file ${out}.d ` + 136 `$r8Flags && ` + 137 `touch "${outDict}" "${outUsage}" && ` + 138 `${config.SoongZipCmd} -o ${outUsageZip} -C ${outUsageDir} -f ${outUsage} && ` + 139 `rm -rf ${outUsageDir} && ` + 140 `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` + 141 `${config.MergeZipsCmd} -D -stripFile "**/*.class" $mergeZipsFlags $out $outDir/classes.dex.jar $in`, 142 Depfile: "${out}.d", 143 Deps: blueprint.DepsGCC, 144 CommandDeps: []string{ 145 "${config.R8Cmd}", 146 "${config.Zip2ZipCmd}", 147 "${config.SoongZipCmd}", 148 "${config.MergeZipsCmd}", 149 }, 150 }, map[string]*remoteexec.REParams{ 151 "$r8Template": &remoteexec.REParams{ 152 Labels: map[string]string{"type": "compile", "compiler": "r8"}, 153 Inputs: []string{"$implicits", "${config.R8Jar}"}, 154 OutputFiles: []string{"${outUsage}"}, 155 ExecStrategy: "${config.RER8ExecStrategy}", 156 ToolchainInputs: []string{"${config.JavaCmd}"}, 157 Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, 158 }, 159 "$zipTemplate": &remoteexec.REParams{ 160 Labels: map[string]string{"type": "tool", "name": "soong_zip"}, 161 Inputs: []string{"${config.SoongZipCmd}", "$outDir"}, 162 OutputFiles: []string{"$outDir/classes.dex.jar"}, 163 ExecStrategy: "${config.RER8ExecStrategy}", 164 Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, 165 }, 166 "$zipUsageTemplate": &remoteexec.REParams{ 167 Labels: map[string]string{"type": "tool", "name": "soong_zip"}, 168 Inputs: []string{"${config.SoongZipCmd}", "${outUsage}"}, 169 OutputFiles: []string{"${outUsageZip}"}, 170 ExecStrategy: "${config.RER8ExecStrategy}", 171 Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"}, 172 }, 173 }, []string{"outDir", "outDict", "outUsage", "outUsageZip", "outUsageDir", 174 "r8Flags", "zipFlags", "tmpJar", "mergeZipsFlags"}, []string{"implicits"}) 175 176func (d *dexer) dexCommonFlags(ctx android.ModuleContext, 177 minSdkVersion android.SdkSpec) (flags []string, deps android.Paths) { 178 179 flags = d.dexProperties.Dxflags 180 // Translate all the DX flags to D8 ones until all the build files have been migrated 181 // to D8 flags. See: b/69377755 182 flags = android.RemoveListFromList(flags, 183 []string{"--core-library", "--dex", "--multi-dex"}) 184 185 for _, f := range android.PathsForModuleSrc(ctx, d.dexProperties.Main_dex_rules) { 186 flags = append(flags, "--main-dex-rules", f.String()) 187 deps = append(deps, f) 188 } 189 190 if ctx.Config().Getenv("NO_OPTIMIZE_DX") != "" { 191 flags = append(flags, "--debug") 192 } 193 194 if ctx.Config().Getenv("GENERATE_DEX_DEBUG") != "" { 195 flags = append(flags, 196 "--debug", 197 "--verbose") 198 } 199 200 effectiveVersion, err := minSdkVersion.EffectiveVersion(ctx) 201 if err != nil { 202 ctx.PropertyErrorf("min_sdk_version", "%s", err) 203 } 204 205 flags = append(flags, "--min-api "+strconv.Itoa(effectiveVersion.FinalOrFutureInt())) 206 return flags, deps 207} 208 209func d8Flags(flags javaBuilderFlags) (d8Flags []string, d8Deps android.Paths) { 210 d8Flags = append(d8Flags, flags.bootClasspath.FormRepeatedClassPath("--lib ")...) 211 d8Flags = append(d8Flags, flags.dexClasspath.FormRepeatedClassPath("--lib ")...) 212 213 d8Deps = append(d8Deps, flags.bootClasspath...) 214 d8Deps = append(d8Deps, flags.dexClasspath...) 215 216 return d8Flags, d8Deps 217} 218 219func (d *dexer) r8Flags(ctx android.ModuleContext, flags javaBuilderFlags) (r8Flags []string, r8Deps android.Paths) { 220 opt := d.dexProperties.Optimize 221 222 // When an app contains references to APIs that are not in the SDK specified by 223 // its LOCAL_SDK_VERSION for example added by support library or by runtime 224 // classes added by desugaring, we artifically raise the "SDK version" "linked" by 225 // ProGuard, to 226 // - suppress ProGuard warnings of referencing symbols unknown to the lower SDK version. 227 // - prevent ProGuard stripping subclass in the support library that extends class added in the higher SDK version. 228 // See b/20667396 229 var proguardRaiseDeps classpath 230 ctx.VisitDirectDepsWithTag(proguardRaiseTag, func(m android.Module) { 231 dep := ctx.OtherModuleProvider(m, JavaInfoProvider).(JavaInfo) 232 proguardRaiseDeps = append(proguardRaiseDeps, dep.HeaderJars...) 233 }) 234 235 r8Flags = append(r8Flags, proguardRaiseDeps.FormJavaClassPath("-libraryjars")) 236 r8Flags = append(r8Flags, flags.bootClasspath.FormJavaClassPath("-libraryjars")) 237 r8Flags = append(r8Flags, flags.dexClasspath.FormJavaClassPath("-libraryjars")) 238 239 r8Deps = append(r8Deps, proguardRaiseDeps...) 240 r8Deps = append(r8Deps, flags.bootClasspath...) 241 r8Deps = append(r8Deps, flags.dexClasspath...) 242 243 flagFiles := android.Paths{ 244 android.PathForSource(ctx, "build/make/core/proguard.flags"), 245 } 246 247 flagFiles = append(flagFiles, d.extraProguardFlagFiles...) 248 // TODO(ccross): static android library proguard files 249 250 flagFiles = append(flagFiles, android.PathsForModuleSrc(ctx, opt.Proguard_flags_files)...) 251 252 r8Flags = append(r8Flags, android.JoinWithPrefix(flagFiles.Strings(), "-include ")) 253 r8Deps = append(r8Deps, flagFiles...) 254 255 // TODO(b/70942988): This is included from build/make/core/proguard.flags 256 r8Deps = append(r8Deps, android.PathForSource(ctx, 257 "build/make/core/proguard_basic_keeps.flags")) 258 259 r8Flags = append(r8Flags, opt.Proguard_flags...) 260 261 if BoolDefault(opt.Proguard_compatibility, true) { 262 r8Flags = append(r8Flags, "--force-proguard-compatibility") 263 } else { 264 // TODO(b/213833843): Allow configuration of the prefix via a build variable. 265 var sourceFilePrefix = "go/retraceme " 266 var sourceFileTemplate = "\"" + sourceFilePrefix + "%MAP_ID\"" 267 // TODO(b/200967150): Also tag the source file in compat builds. 268 if Bool(opt.Optimize) || Bool(opt.Obfuscate) { 269 r8Flags = append(r8Flags, "--map-id-template", "%MAP_HASH") 270 r8Flags = append(r8Flags, "--source-file-template", sourceFileTemplate) 271 } 272 } 273 274 // TODO(ccross): Don't shrink app instrumentation tests by default. 275 if !Bool(opt.Shrink) { 276 r8Flags = append(r8Flags, "-dontshrink") 277 } 278 279 if !Bool(opt.Optimize) { 280 r8Flags = append(r8Flags, "-dontoptimize") 281 } 282 283 // TODO(ccross): error if obufscation + app instrumentation test. 284 if !Bool(opt.Obfuscate) { 285 r8Flags = append(r8Flags, "-dontobfuscate") 286 } 287 // TODO(ccross): if this is an instrumentation test of an obfuscated app, use the 288 // dictionary of the app and move the app from libraryjars to injars. 289 290 // Don't strip out debug information for eng builds. 291 if ctx.Config().Eng() { 292 r8Flags = append(r8Flags, "--debug") 293 } 294 295 // TODO(b/180878971): missing classes should be added to the relevant builds. 296 r8Flags = append(r8Flags, "-ignorewarnings") 297 298 return r8Flags, r8Deps 299} 300 301func (d *dexer) compileDex(ctx android.ModuleContext, flags javaBuilderFlags, minSdkVersion android.SdkSpec, 302 classesJar android.Path, jarName string) android.OutputPath { 303 304 // Compile classes.jar into classes.dex and then javalib.jar 305 javalibJar := android.PathForModuleOut(ctx, "dex", jarName).OutputPath 306 outDir := android.PathForModuleOut(ctx, "dex") 307 tmpJar := android.PathForModuleOut(ctx, "withres-withoutdex", jarName) 308 309 zipFlags := "--ignore_missing_files" 310 if proptools.Bool(d.dexProperties.Uncompress_dex) { 311 zipFlags += " -L 0" 312 } 313 314 commonFlags, commonDeps := d.dexCommonFlags(ctx, minSdkVersion) 315 316 // Exclude kotlinc generated files when "exclude_kotlinc_generated_files" is set to true. 317 mergeZipsFlags := "" 318 if proptools.BoolDefault(d.dexProperties.Exclude_kotlinc_generated_files, false) { 319 mergeZipsFlags = "-stripFile META-INF/*.kotlin_module -stripFile **/*.kotlin_builtins" 320 } 321 322 useR8 := d.effectiveOptimizeEnabled() 323 if useR8 { 324 proguardDictionary := android.PathForModuleOut(ctx, "proguard_dictionary") 325 d.proguardDictionary = android.OptionalPathForPath(proguardDictionary) 326 proguardUsageDir := android.PathForModuleOut(ctx, "proguard_usage") 327 proguardUsage := proguardUsageDir.Join(ctx, ctx.Namespace().Path, 328 android.ModuleNameWithPossibleOverride(ctx), "unused.txt") 329 proguardUsageZip := android.PathForModuleOut(ctx, "proguard_usage.zip") 330 d.proguardUsageZip = android.OptionalPathForPath(proguardUsageZip) 331 r8Flags, r8Deps := d.r8Flags(ctx, flags) 332 r8Deps = append(r8Deps, commonDeps...) 333 rule := r8 334 args := map[string]string{ 335 "r8Flags": strings.Join(append(commonFlags, r8Flags...), " "), 336 "zipFlags": zipFlags, 337 "outDict": proguardDictionary.String(), 338 "outUsageDir": proguardUsageDir.String(), 339 "outUsage": proguardUsage.String(), 340 "outUsageZip": proguardUsageZip.String(), 341 "outDir": outDir.String(), 342 "tmpJar": tmpJar.String(), 343 "mergeZipsFlags": mergeZipsFlags, 344 } 345 if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_R8") { 346 rule = r8RE 347 args["implicits"] = strings.Join(r8Deps.Strings(), ",") 348 } 349 ctx.Build(pctx, android.BuildParams{ 350 Rule: rule, 351 Description: "r8", 352 Output: javalibJar, 353 ImplicitOutputs: android.WritablePaths{proguardDictionary, proguardUsageZip}, 354 Input: classesJar, 355 Implicits: r8Deps, 356 Args: args, 357 }) 358 } else { 359 d8Flags, d8Deps := d8Flags(flags) 360 d8Deps = append(d8Deps, commonDeps...) 361 rule := d8 362 if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_D8") { 363 rule = d8RE 364 } 365 ctx.Build(pctx, android.BuildParams{ 366 Rule: rule, 367 Description: "d8", 368 Output: javalibJar, 369 Input: classesJar, 370 Implicits: d8Deps, 371 Args: map[string]string{ 372 "d8Flags": strings.Join(append(commonFlags, d8Flags...), " "), 373 "zipFlags": zipFlags, 374 "outDir": outDir.String(), 375 "tmpJar": tmpJar.String(), 376 "mergeZipsFlags": mergeZipsFlags, 377 }, 378 }) 379 } 380 if proptools.Bool(d.dexProperties.Uncompress_dex) { 381 alignedJavalibJar := android.PathForModuleOut(ctx, "aligned", jarName).OutputPath 382 TransformZipAlign(ctx, alignedJavalibJar, javalibJar) 383 javalibJar = alignedJavalibJar 384 } 385 386 return javalibJar 387} 388