1// Copyright 2017 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package build 16 17import ( 18 "fmt" 19 "os" 20 "path/filepath" 21 "runtime" 22 "strconv" 23 "strings" 24 "time" 25 26 "android/soong/shared" 27 28 "github.com/golang/protobuf/proto" 29 30 smpb "android/soong/ui/metrics/metrics_proto" 31) 32 33type Config struct{ *configImpl } 34 35type configImpl struct { 36 // From the environment 37 arguments []string 38 goma bool 39 environ *Environment 40 distDir string 41 buildDateTime string 42 43 // From the arguments 44 parallel int 45 keepGoing int 46 verbose bool 47 checkbuild bool 48 dist bool 49 skipConfig bool 50 skipKati bool 51 skipKatiNinja bool 52 skipNinja bool 53 skipSoongTests bool 54 55 // From the product config 56 katiArgs []string 57 ninjaArgs []string 58 katiSuffix string 59 targetDevice string 60 targetDeviceDir string 61 62 // Autodetected 63 totalRAM uint64 64 65 brokenDupRules bool 66 brokenUsesNetwork bool 67 brokenNinjaEnvVars []string 68 69 pathReplaced bool 70 71 useBazel bool 72 73 // During Bazel execution, Bazel cannot write outside OUT_DIR. 74 // So if DIST_DIR is set to an external dir (outside of OUT_DIR), we need to rig it temporarily and then migrate files at the end of the build. 75 riggedDistDirForBazel string 76 77 // Set by multiproduct_kati 78 emptyNinjaFile bool 79} 80 81const srcDirFileCheck = "build/soong/root.bp" 82 83var buildFiles = []string{"Android.mk", "Android.bp"} 84 85type BuildAction uint 86 87const ( 88 // Builds all of the modules and their dependencies of a specified directory, relative to the root 89 // directory of the source tree. 90 BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota 91 92 // Builds all of the modules and their dependencies of a list of specified directories. All specified 93 // directories are relative to the root directory of the source tree. 94 BUILD_MODULES_IN_DIRECTORIES 95 96 // Build a list of specified modules. If none was specified, simply build the whole source tree. 97 BUILD_MODULES 98) 99 100type bazelBuildMode int 101 102// Bazel-related build modes. 103const ( 104 // Don't use bazel at all. 105 noBazel bazelBuildMode = iota 106 107 // Only generate build files (in a subdirectory of the out directory) and exit. 108 generateBuildFiles 109 110 // Generate synthetic build files and incorporate these files into a build which 111 // partially uses Bazel. Build metadata may come from Android.bp or BUILD files. 112 mixedBuild 113) 114 115// checkTopDir validates that the current directory is at the root directory of the source tree. 116func checkTopDir(ctx Context) { 117 if _, err := os.Stat(srcDirFileCheck); err != nil { 118 if os.IsNotExist(err) { 119 ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck) 120 } 121 ctx.Fatalln("Error verifying tree state:", err) 122 } 123} 124 125func NewConfig(ctx Context, args ...string) Config { 126 ret := &configImpl{ 127 environ: OsEnvironment(), 128 } 129 130 // Default matching ninja 131 ret.parallel = runtime.NumCPU() + 2 132 ret.keepGoing = 1 133 134 ret.totalRAM = detectTotalRAM(ctx) 135 136 ret.parseArgs(ctx, args) 137 138 // Make sure OUT_DIR is set appropriately 139 if outDir, ok := ret.environ.Get("OUT_DIR"); ok { 140 ret.environ.Set("OUT_DIR", filepath.Clean(outDir)) 141 } else { 142 outDir := "out" 143 if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok { 144 if wd, err := os.Getwd(); err != nil { 145 ctx.Fatalln("Failed to get working directory:", err) 146 } else { 147 outDir = filepath.Join(baseDir, filepath.Base(wd)) 148 } 149 } 150 ret.environ.Set("OUT_DIR", outDir) 151 } 152 153 if distDir, ok := ret.environ.Get("DIST_DIR"); ok { 154 ret.distDir = filepath.Clean(distDir) 155 } else { 156 ret.distDir = filepath.Join(ret.OutDir(), "dist") 157 } 158 159 ret.environ.Unset( 160 // We're already using it 161 "USE_SOONG_UI", 162 163 // We should never use GOROOT/GOPATH from the shell environment 164 "GOROOT", 165 "GOPATH", 166 167 // These should only come from Soong, not the environment. 168 "CLANG", 169 "CLANG_CXX", 170 "CCC_CC", 171 "CCC_CXX", 172 173 // Used by the goma compiler wrapper, but should only be set by 174 // gomacc 175 "GOMACC_PATH", 176 177 // We handle this above 178 "OUT_DIR_COMMON_BASE", 179 180 // This is handled above too, and set for individual commands later 181 "DIST_DIR", 182 183 // Variables that have caused problems in the past 184 "BASH_ENV", 185 "CDPATH", 186 "DISPLAY", 187 "GREP_OPTIONS", 188 "NDK_ROOT", 189 "POSIXLY_CORRECT", 190 191 // Drop make flags 192 "MAKEFLAGS", 193 "MAKELEVEL", 194 "MFLAGS", 195 196 // Set in envsetup.sh, reset in makefiles 197 "ANDROID_JAVA_TOOLCHAIN", 198 199 // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional 200 "ANDROID_BUILD_TOP", 201 "ANDROID_HOST_OUT", 202 "ANDROID_PRODUCT_OUT", 203 "ANDROID_HOST_OUT_TESTCASES", 204 "ANDROID_TARGET_OUT_TESTCASES", 205 "ANDROID_TOOLCHAIN", 206 "ANDROID_TOOLCHAIN_2ND_ARCH", 207 "ANDROID_DEV_SCRIPTS", 208 "ANDROID_EMULATOR_PREBUILTS", 209 "ANDROID_PRE_BUILD_PATHS", 210 ) 211 212 if ret.UseGoma() || ret.ForceUseGoma() { 213 ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.") 214 ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.") 215 } 216 217 // Tell python not to spam the source tree with .pyc files. 218 ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1") 219 220 tmpDir := absPath(ctx, ret.TempDir()) 221 ret.environ.Set("TMPDIR", tmpDir) 222 223 // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes 224 symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(), 225 "llvm-binutils-stable/llvm-symbolizer") 226 ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath)) 227 228 // Precondition: the current directory is the top of the source tree 229 checkTopDir(ctx) 230 231 if srcDir := absPath(ctx, "."); strings.ContainsRune(srcDir, ' ') { 232 ctx.Println("You are building in a directory whose absolute path contains a space character:") 233 ctx.Println() 234 ctx.Printf("%q\n", srcDir) 235 ctx.Println() 236 ctx.Fatalln("Directory names containing spaces are not supported") 237 } 238 239 if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') { 240 ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:") 241 ctx.Println() 242 ctx.Printf("%q\n", outDir) 243 ctx.Println() 244 ctx.Fatalln("Directory names containing spaces are not supported") 245 } 246 247 if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') { 248 ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:") 249 ctx.Println() 250 ctx.Printf("%q\n", distDir) 251 ctx.Println() 252 ctx.Fatalln("Directory names containing spaces are not supported") 253 } 254 255 // Configure Java-related variables, including adding it to $PATH 256 java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag()) 257 java9Home := filepath.Join("prebuilts/jdk/jdk9", ret.HostPrebuiltTag()) 258 java11Home := filepath.Join("prebuilts/jdk/jdk11", ret.HostPrebuiltTag()) 259 javaHome := func() string { 260 if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok { 261 return override 262 } 263 if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" { 264 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 11 toolchain is now the global default.") 265 } 266 return java11Home 267 }() 268 absJavaHome := absPath(ctx, javaHome) 269 270 ret.configureLocale(ctx) 271 272 newPath := []string{filepath.Join(absJavaHome, "bin")} 273 if path, ok := ret.environ.Get("PATH"); ok && path != "" { 274 newPath = append(newPath, path) 275 } 276 277 ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME") 278 ret.environ.Set("JAVA_HOME", absJavaHome) 279 ret.environ.Set("ANDROID_JAVA_HOME", javaHome) 280 ret.environ.Set("ANDROID_JAVA8_HOME", java8Home) 281 ret.environ.Set("ANDROID_JAVA9_HOME", java9Home) 282 ret.environ.Set("ANDROID_JAVA11_HOME", java11Home) 283 ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator))) 284 285 outDir := ret.OutDir() 286 buildDateTimeFile := filepath.Join(outDir, "build_date.txt") 287 if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" { 288 ret.buildDateTime = buildDateTime 289 } else { 290 ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10) 291 } 292 293 ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile) 294 295 if ret.UseRBE() { 296 for k, v := range getRBEVars(ctx, Config{ret}) { 297 ret.environ.Set(k, v) 298 } 299 } 300 301 bpd := ret.BazelMetricsDir() 302 if err := os.RemoveAll(bpd); err != nil { 303 ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err) 304 } 305 306 ret.useBazel = ret.environ.IsEnvTrue("USE_BAZEL") 307 308 if ret.UseBazel() { 309 if err := os.MkdirAll(bpd, 0777); err != nil { 310 ctx.Fatalf("Failed to create bazel profile directory %q: %v", bpd, err) 311 } 312 } 313 314 if ret.UseBazel() { 315 ret.riggedDistDirForBazel = filepath.Join(ret.OutDir(), "dist") 316 } else { 317 // Not rigged 318 ret.riggedDistDirForBazel = ret.distDir 319 } 320 321 c := Config{ret} 322 storeConfigMetrics(ctx, c) 323 return c 324} 325 326// NewBuildActionConfig returns a build configuration based on the build action. The arguments are 327// processed based on the build action and extracts any arguments that belongs to the build action. 328func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config { 329 return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...) 330} 331 332// storeConfigMetrics selects a set of configuration information and store in 333// the metrics system for further analysis. 334func storeConfigMetrics(ctx Context, config Config) { 335 if ctx.Metrics == nil { 336 return 337 } 338 339 b := &smpb.BuildConfig{ 340 ForceUseGoma: proto.Bool(config.ForceUseGoma()), 341 UseGoma: proto.Bool(config.UseGoma()), 342 UseRbe: proto.Bool(config.UseRBE()), 343 } 344 ctx.Metrics.BuildConfig(b) 345 346 s := &smpb.SystemResourceInfo{ 347 TotalPhysicalMemory: proto.Uint64(config.TotalRAM()), 348 AvailableCpus: proto.Int32(int32(runtime.NumCPU())), 349 } 350 ctx.Metrics.SystemResourceInfo(s) 351} 352 353// getConfigArgs processes the command arguments based on the build action and creates a set of new 354// arguments to be accepted by Config. 355func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string { 356 // The next block of code verifies that the current directory is the root directory of the source 357 // tree. It then finds the relative path of dir based on the root directory of the source tree 358 // and verify that dir is inside of the source tree. 359 checkTopDir(ctx) 360 topDir, err := os.Getwd() 361 if err != nil { 362 ctx.Fatalf("Error retrieving top directory: %v", err) 363 } 364 dir, err = filepath.EvalSymlinks(dir) 365 if err != nil { 366 ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err) 367 } 368 dir, err = filepath.Abs(dir) 369 if err != nil { 370 ctx.Fatalf("Unable to find absolute path %s: %v", dir, err) 371 } 372 relDir, err := filepath.Rel(topDir, dir) 373 if err != nil { 374 ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err) 375 } 376 // If there are ".." in the path, it's not in the source tree. 377 if strings.Contains(relDir, "..") { 378 ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir) 379 } 380 381 configArgs := args[:] 382 383 // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to 384 // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules. 385 targetNamePrefix := "MODULES-IN-" 386 if inList("GET-INSTALL-PATH", configArgs) { 387 targetNamePrefix = "GET-INSTALL-PATH-IN-" 388 configArgs = removeFromList("GET-INSTALL-PATH", configArgs) 389 } 390 391 var targets []string 392 393 switch action { 394 case BUILD_MODULES: 395 // No additional processing is required when building a list of specific modules or all modules. 396 case BUILD_MODULES_IN_A_DIRECTORY: 397 // If dir is the root source tree, all the modules are built of the source tree are built so 398 // no need to find the build file. 399 if topDir == dir { 400 break 401 } 402 403 buildFile := findBuildFile(ctx, relDir) 404 if buildFile == "" { 405 ctx.Fatalf("Build file not found for %s directory", relDir) 406 } 407 targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} 408 case BUILD_MODULES_IN_DIRECTORIES: 409 newConfigArgs, dirs := splitArgs(configArgs) 410 configArgs = newConfigArgs 411 targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix) 412 } 413 414 // Tidy only override all other specified targets. 415 tidyOnly := os.Getenv("WITH_TIDY_ONLY") 416 if tidyOnly == "true" || tidyOnly == "1" { 417 configArgs = append(configArgs, "tidy_only") 418 } else { 419 configArgs = append(configArgs, targets...) 420 } 421 422 return configArgs 423} 424 425// convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name. 426func convertToTarget(dir string, targetNamePrefix string) string { 427 return targetNamePrefix + strings.ReplaceAll(dir, "/", "-") 428} 429 430// hasBuildFile returns true if dir contains an Android build file. 431func hasBuildFile(ctx Context, dir string) bool { 432 for _, buildFile := range buildFiles { 433 _, err := os.Stat(filepath.Join(dir, buildFile)) 434 if err == nil { 435 return true 436 } 437 if !os.IsNotExist(err) { 438 ctx.Fatalf("Error retrieving the build file stats: %v", err) 439 } 440 } 441 return false 442} 443 444// findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file 445// in the current and any sub directory of dir. If a build file is not found, traverse the path 446// up by one directory and repeat again until either a build file is found or reached to the root 447// source tree. The returned filename of build file is "Android.mk". If one was not found, a blank 448// string is returned. 449func findBuildFile(ctx Context, dir string) string { 450 // If the string is empty or ".", assume it is top directory of the source tree. 451 if dir == "" || dir == "." { 452 return "" 453 } 454 455 found := false 456 for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) { 457 err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error { 458 if err != nil { 459 return err 460 } 461 if found { 462 return filepath.SkipDir 463 } 464 if info.IsDir() { 465 return nil 466 } 467 for _, buildFile := range buildFiles { 468 if info.Name() == buildFile { 469 found = true 470 return filepath.SkipDir 471 } 472 } 473 return nil 474 }) 475 if err != nil { 476 ctx.Fatalf("Error finding Android build file: %v", err) 477 } 478 479 if found { 480 return filepath.Join(buildDir, "Android.mk") 481 } 482 } 483 484 return "" 485} 486 487// splitArgs iterates over the arguments list and splits into two lists: arguments and directories. 488func splitArgs(args []string) (newArgs []string, dirs []string) { 489 specialArgs := map[string]bool{ 490 "showcommands": true, 491 "snod": true, 492 "dist": true, 493 "checkbuild": true, 494 } 495 496 newArgs = []string{} 497 dirs = []string{} 498 499 for _, arg := range args { 500 // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory. 501 if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 { 502 newArgs = append(newArgs, arg) 503 continue 504 } 505 506 if _, ok := specialArgs[arg]; ok { 507 newArgs = append(newArgs, arg) 508 continue 509 } 510 511 dirs = append(dirs, arg) 512 } 513 514 return newArgs, dirs 515} 516 517// getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a 518// directory from the dirs list does not exist, a fatal error is raised. relDir is related to the 519// source root tree where the build action command was invoked. Each directory is validated if the 520// build file can be found and follows the format "dir1:target1,target2,...". Target is optional. 521func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) { 522 for _, dir := range dirs { 523 // The directory may have specified specific modules to build. ":" is the separator to separate 524 // the directory and the list of modules. 525 s := strings.Split(dir, ":") 526 l := len(s) 527 if l > 2 { // more than one ":" was specified. 528 ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir) 529 } 530 531 dir = filepath.Join(relDir, s[0]) 532 if _, err := os.Stat(dir); err != nil { 533 ctx.Fatalf("couldn't find directory %s", dir) 534 } 535 536 // Verify that if there are any targets specified after ":". Each target is separated by ",". 537 var newTargets []string 538 if l == 2 && s[1] != "" { 539 newTargets = strings.Split(s[1], ",") 540 if inList("", newTargets) { 541 ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir) 542 } 543 } 544 545 // If there are specified targets to build in dir, an android build file must exist for the one 546 // shot build. For the non-targets case, find the appropriate build file and build all the 547 // modules in dir (or the closest one in the dir path). 548 if len(newTargets) > 0 { 549 if !hasBuildFile(ctx, dir) { 550 ctx.Fatalf("Couldn't locate a build file from %s directory", dir) 551 } 552 } else { 553 buildFile := findBuildFile(ctx, dir) 554 if buildFile == "" { 555 ctx.Fatalf("Build file not found for %s directory", dir) 556 } 557 newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} 558 } 559 560 targets = append(targets, newTargets...) 561 } 562 563 return targets 564} 565 566func (c *configImpl) parseArgs(ctx Context, args []string) { 567 for i := 0; i < len(args); i++ { 568 arg := strings.TrimSpace(args[i]) 569 if arg == "showcommands" { 570 c.verbose = true 571 } else if arg == "--skip-ninja" { 572 c.skipNinja = true 573 } else if arg == "--skip-make" { 574 c.skipConfig = true 575 c.skipKati = true 576 } else if arg == "--skip-kati" { 577 // TODO: remove --skip-kati once module builds have been migrated to --song-only 578 c.skipKati = true 579 } else if arg == "--soong-only" { 580 c.skipKati = true 581 c.skipKatiNinja = true 582 } else if arg == "--skip-soong-tests" { 583 c.skipSoongTests = true 584 } else if len(arg) > 0 && arg[0] == '-' { 585 parseArgNum := func(def int) int { 586 if len(arg) > 2 { 587 p, err := strconv.ParseUint(arg[2:], 10, 31) 588 if err != nil { 589 ctx.Fatalf("Failed to parse %q: %v", arg, err) 590 } 591 return int(p) 592 } else if i+1 < len(args) { 593 p, err := strconv.ParseUint(args[i+1], 10, 31) 594 if err == nil { 595 i++ 596 return int(p) 597 } 598 } 599 return def 600 } 601 602 if len(arg) > 1 && arg[1] == 'j' { 603 c.parallel = parseArgNum(c.parallel) 604 } else if len(arg) > 1 && arg[1] == 'k' { 605 c.keepGoing = parseArgNum(0) 606 } else { 607 ctx.Fatalln("Unknown option:", arg) 608 } 609 } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 { 610 if k == "OUT_DIR" { 611 ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.") 612 } 613 c.environ.Set(k, v) 614 } else if arg == "dist" { 615 c.dist = true 616 } else { 617 if arg == "checkbuild" { 618 c.checkbuild = true 619 } 620 c.arguments = append(c.arguments, arg) 621 } 622 } 623} 624 625func (c *configImpl) configureLocale(ctx Context) { 626 cmd := Command(ctx, Config{c}, "locale", "locale", "-a") 627 output, err := cmd.Output() 628 629 var locales []string 630 if err == nil { 631 locales = strings.Split(string(output), "\n") 632 } else { 633 // If we're unable to list the locales, let's assume en_US.UTF-8 634 locales = []string{"en_US.UTF-8"} 635 ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales) 636 } 637 638 // gettext uses LANGUAGE, which is passed directly through 639 640 // For LANG and LC_*, only preserve the evaluated version of 641 // LC_MESSAGES 642 userLang := "" 643 if lc_all, ok := c.environ.Get("LC_ALL"); ok { 644 userLang = lc_all 645 } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok { 646 userLang = lc_messages 647 } else if lang, ok := c.environ.Get("LANG"); ok { 648 userLang = lang 649 } 650 651 c.environ.UnsetWithPrefix("LC_") 652 653 if userLang != "" { 654 c.environ.Set("LC_MESSAGES", userLang) 655 } 656 657 // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed 658 // for others) 659 if inList("C.UTF-8", locales) { 660 c.environ.Set("LANG", "C.UTF-8") 661 } else if inList("C.utf8", locales) { 662 // These normalize to the same thing 663 c.environ.Set("LANG", "C.UTF-8") 664 } else if inList("en_US.UTF-8", locales) { 665 c.environ.Set("LANG", "en_US.UTF-8") 666 } else if inList("en_US.utf8", locales) { 667 // These normalize to the same thing 668 c.environ.Set("LANG", "en_US.UTF-8") 669 } else { 670 ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8") 671 } 672} 673 674// Lunch configures the environment for a specific product similarly to the 675// `lunch` bash function. 676func (c *configImpl) Lunch(ctx Context, product, variant string) { 677 if variant != "eng" && variant != "userdebug" && variant != "user" { 678 ctx.Fatalf("Invalid variant %q. Must be one of 'user', 'userdebug' or 'eng'", variant) 679 } 680 681 c.environ.Set("TARGET_PRODUCT", product) 682 c.environ.Set("TARGET_BUILD_VARIANT", variant) 683 c.environ.Set("TARGET_BUILD_TYPE", "release") 684 c.environ.Unset("TARGET_BUILD_APPS") 685 c.environ.Unset("TARGET_BUILD_UNBUNDLED") 686} 687 688// Tapas configures the environment to build one or more unbundled apps, 689// similarly to the `tapas` bash function. 690func (c *configImpl) Tapas(ctx Context, apps []string, arch, variant string) { 691 if len(apps) == 0 { 692 apps = []string{"all"} 693 } 694 if variant == "" { 695 variant = "eng" 696 } 697 698 if variant != "eng" && variant != "userdebug" && variant != "user" { 699 ctx.Fatalf("Invalid variant %q. Must be one of 'user', 'userdebug' or 'eng'", variant) 700 } 701 702 var product string 703 switch arch { 704 case "arm", "": 705 product = "aosp_arm" 706 case "arm64": 707 product = "aosm_arm64" 708 case "x86": 709 product = "aosp_x86" 710 case "x86_64": 711 product = "aosp_x86_64" 712 default: 713 ctx.Fatalf("Invalid architecture: %q", arch) 714 } 715 716 c.environ.Set("TARGET_PRODUCT", product) 717 c.environ.Set("TARGET_BUILD_VARIANT", variant) 718 c.environ.Set("TARGET_BUILD_TYPE", "release") 719 c.environ.Set("TARGET_BUILD_APPS", strings.Join(apps, " ")) 720} 721 722func (c *configImpl) Environment() *Environment { 723 return c.environ 724} 725 726func (c *configImpl) Arguments() []string { 727 return c.arguments 728} 729 730func (c *configImpl) OutDir() string { 731 if outDir, ok := c.environ.Get("OUT_DIR"); ok { 732 return outDir 733 } 734 return "out" 735} 736 737func (c *configImpl) DistDir() string { 738 if c.UseBazel() { 739 return c.riggedDistDirForBazel 740 } else { 741 return c.distDir 742 } 743} 744 745func (c *configImpl) RealDistDir() string { 746 return c.distDir 747} 748 749func (c *configImpl) NinjaArgs() []string { 750 if c.skipKati { 751 return c.arguments 752 } 753 return c.ninjaArgs 754} 755 756func (c *configImpl) BazelOutDir() string { 757 return filepath.Join(c.OutDir(), "bazel") 758} 759 760func (c *configImpl) SoongOutDir() string { 761 return filepath.Join(c.OutDir(), "soong") 762} 763 764func (c *configImpl) TempDir() string { 765 return shared.TempDirForOutDir(c.SoongOutDir()) 766} 767 768func (c *configImpl) FileListDir() string { 769 return filepath.Join(c.OutDir(), ".module_paths") 770} 771 772func (c *configImpl) KatiSuffix() string { 773 if c.katiSuffix != "" { 774 return c.katiSuffix 775 } 776 panic("SetKatiSuffix has not been called") 777} 778 779// Checkbuild returns true if "checkbuild" was one of the build goals, which means that the 780// user is interested in additional checks at the expense of build time. 781func (c *configImpl) Checkbuild() bool { 782 return c.checkbuild 783} 784 785func (c *configImpl) Dist() bool { 786 return c.dist 787} 788 789func (c *configImpl) IsVerbose() bool { 790 return c.verbose 791} 792 793func (c *configImpl) SkipKati() bool { 794 return c.skipKati 795} 796 797func (c *configImpl) SkipKatiNinja() bool { 798 return c.skipKatiNinja 799} 800 801func (c *configImpl) SkipNinja() bool { 802 return c.skipNinja 803} 804 805func (c *configImpl) SetSkipNinja(v bool) { 806 c.skipNinja = v 807} 808 809func (c *configImpl) SkipConfig() bool { 810 return c.skipConfig 811} 812 813func (c *configImpl) TargetProduct() string { 814 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { 815 return v 816 } 817 panic("TARGET_PRODUCT is not defined") 818} 819 820func (c *configImpl) TargetDevice() string { 821 return c.targetDevice 822} 823 824func (c *configImpl) SetTargetDevice(device string) { 825 c.targetDevice = device 826} 827 828func (c *configImpl) TargetBuildVariant() string { 829 if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok { 830 return v 831 } 832 panic("TARGET_BUILD_VARIANT is not defined") 833} 834 835func (c *configImpl) KatiArgs() []string { 836 return c.katiArgs 837} 838 839func (c *configImpl) Parallel() int { 840 return c.parallel 841} 842 843func (c *configImpl) HighmemParallel() int { 844 if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok { 845 return i 846 } 847 848 const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024 849 parallel := c.Parallel() 850 if c.UseRemoteBuild() { 851 // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism 852 // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs 853 // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention. 854 // Return 1/16th of the size of the local pool, rounding up. 855 return (parallel + 15) / 16 856 } else if c.totalRAM == 0 { 857 // Couldn't detect the total RAM, don't restrict highmem processes. 858 return parallel 859 } else if c.totalRAM <= 16*1024*1024*1024 { 860 // Less than 16GB of ram, restrict to 1 highmem processes 861 return 1 862 } else if c.totalRAM <= 32*1024*1024*1024 { 863 // Less than 32GB of ram, restrict to 2 highmem processes 864 return 2 865 } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel { 866 // If less than 8GB total RAM per process, reduce the number of highmem processes 867 return p 868 } 869 // No restriction on highmem processes 870 return parallel 871} 872 873func (c *configImpl) TotalRAM() uint64 { 874 return c.totalRAM 875} 876 877// ForceUseGoma determines whether we should override Goma deprecation 878// and use Goma for the current build or not. 879func (c *configImpl) ForceUseGoma() bool { 880 if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok { 881 v = strings.TrimSpace(v) 882 if v != "" && v != "false" { 883 return true 884 } 885 } 886 return false 887} 888 889func (c *configImpl) UseGoma() bool { 890 if v, ok := c.environ.Get("USE_GOMA"); ok { 891 v = strings.TrimSpace(v) 892 if v != "" && v != "false" { 893 return true 894 } 895 } 896 return false 897} 898 899func (c *configImpl) StartGoma() bool { 900 if !c.UseGoma() { 901 return false 902 } 903 904 if v, ok := c.environ.Get("NOSTART_GOMA"); ok { 905 v = strings.TrimSpace(v) 906 if v != "" && v != "false" { 907 return false 908 } 909 } 910 return true 911} 912 913func (c *configImpl) UseRBE() bool { 914 if v, ok := c.environ.Get("USE_RBE"); ok { 915 v = strings.TrimSpace(v) 916 if v != "" && v != "false" { 917 return true 918 } 919 } 920 return false 921} 922 923func (c *configImpl) UseBazel() bool { 924 return c.useBazel 925} 926 927func (c *configImpl) bazelBuildMode() bazelBuildMode { 928 if c.Environment().IsEnvTrue("USE_BAZEL_ANALYSIS") { 929 return mixedBuild 930 } else if c.Environment().IsEnvTrue("GENERATE_BAZEL_FILES") { 931 return generateBuildFiles 932 } else { 933 return noBazel 934 } 935} 936 937func (c *configImpl) StartRBE() bool { 938 if !c.UseRBE() { 939 return false 940 } 941 942 if v, ok := c.environ.Get("NOSTART_RBE"); ok { 943 v = strings.TrimSpace(v) 944 if v != "" && v != "false" { 945 return false 946 } 947 } 948 return true 949} 950 951func (c *configImpl) rbeLogDir() string { 952 for _, f := range []string{"RBE_log_dir", "FLAG_log_dir"} { 953 if v, ok := c.environ.Get(f); ok { 954 return v 955 } 956 } 957 if c.Dist() { 958 return c.LogsDir() 959 } 960 return c.OutDir() 961} 962 963func (c *configImpl) rbeStatsOutputDir() string { 964 for _, f := range []string{"RBE_output_dir", "FLAG_output_dir"} { 965 if v, ok := c.environ.Get(f); ok { 966 return v 967 } 968 } 969 return c.rbeLogDir() 970} 971 972func (c *configImpl) rbeLogPath() string { 973 for _, f := range []string{"RBE_log_path", "FLAG_log_path"} { 974 if v, ok := c.environ.Get(f); ok { 975 return v 976 } 977 } 978 return fmt.Sprintf("text://%v/reproxy_log.txt", c.rbeLogDir()) 979} 980 981func (c *configImpl) rbeExecRoot() string { 982 for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} { 983 if v, ok := c.environ.Get(f); ok { 984 return v 985 } 986 } 987 wd, err := os.Getwd() 988 if err != nil { 989 return "" 990 } 991 return wd 992} 993 994func (c *configImpl) rbeDir() string { 995 if v, ok := c.environ.Get("RBE_DIR"); ok { 996 return v 997 } 998 return "prebuilts/remoteexecution-client/live/" 999} 1000 1001func (c *configImpl) rbeReproxy() string { 1002 for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} { 1003 if v, ok := c.environ.Get(f); ok { 1004 return v 1005 } 1006 } 1007 return filepath.Join(c.rbeDir(), "reproxy") 1008} 1009 1010func (c *configImpl) rbeAuth() (string, string) { 1011 credFlags := []string{"use_application_default_credentials", "use_gce_credentials", "credential_file"} 1012 for _, cf := range credFlags { 1013 for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} { 1014 if v, ok := c.environ.Get(f); ok { 1015 v = strings.TrimSpace(v) 1016 if v != "" && v != "false" && v != "0" { 1017 return "RBE_" + cf, v 1018 } 1019 } 1020 } 1021 } 1022 return "RBE_use_application_default_credentials", "true" 1023} 1024 1025func (c *configImpl) UseRemoteBuild() bool { 1026 return c.UseGoma() || c.UseRBE() 1027} 1028 1029// RemoteParallel controls how many remote jobs (i.e., commands which contain 1030// gomacc) are run in parallel. Note the parallelism of all other jobs is 1031// still limited by Parallel() 1032func (c *configImpl) RemoteParallel() int { 1033 if !c.UseRemoteBuild() { 1034 return 0 1035 } 1036 if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok { 1037 return i 1038 } 1039 return 500 1040} 1041 1042func (c *configImpl) SetKatiArgs(args []string) { 1043 c.katiArgs = args 1044} 1045 1046func (c *configImpl) SetNinjaArgs(args []string) { 1047 c.ninjaArgs = args 1048} 1049 1050func (c *configImpl) SetKatiSuffix(suffix string) { 1051 c.katiSuffix = suffix 1052} 1053 1054func (c *configImpl) LastKatiSuffixFile() string { 1055 return filepath.Join(c.OutDir(), "last_kati_suffix") 1056} 1057 1058func (c *configImpl) HasKatiSuffix() bool { 1059 return c.katiSuffix != "" 1060} 1061 1062func (c *configImpl) KatiEnvFile() string { 1063 return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh") 1064} 1065 1066func (c *configImpl) KatiBuildNinjaFile() string { 1067 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja") 1068} 1069 1070func (c *configImpl) KatiPackageNinjaFile() string { 1071 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja") 1072} 1073 1074func (c *configImpl) SoongNinjaFile() string { 1075 return filepath.Join(c.SoongOutDir(), "build.ninja") 1076} 1077 1078func (c *configImpl) CombinedNinjaFile() string { 1079 if c.katiSuffix == "" { 1080 return filepath.Join(c.OutDir(), "combined.ninja") 1081 } 1082 return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja") 1083} 1084 1085func (c *configImpl) SoongAndroidMk() string { 1086 return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk") 1087} 1088 1089func (c *configImpl) SoongMakeVarsMk() string { 1090 return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk") 1091} 1092 1093func (c *configImpl) ProductOut() string { 1094 return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice()) 1095} 1096 1097func (c *configImpl) DevicePreviousProductConfig() string { 1098 return filepath.Join(c.ProductOut(), "previous_build_config.mk") 1099} 1100 1101func (c *configImpl) KatiPackageMkDir() string { 1102 return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging") 1103} 1104 1105func (c *configImpl) hostOutRoot() string { 1106 return filepath.Join(c.OutDir(), "host") 1107} 1108 1109func (c *configImpl) HostOut() string { 1110 return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag()) 1111} 1112 1113// This probably needs to be multi-valued, so not exporting it for now 1114func (c *configImpl) hostCrossOut() string { 1115 if runtime.GOOS == "linux" { 1116 return filepath.Join(c.hostOutRoot(), "windows-x86") 1117 } else { 1118 return "" 1119 } 1120} 1121 1122func (c *configImpl) HostPrebuiltTag() string { 1123 if runtime.GOOS == "linux" { 1124 return "linux-x86" 1125 } else if runtime.GOOS == "darwin" { 1126 return "darwin-x86" 1127 } else { 1128 panic("Unsupported OS") 1129 } 1130} 1131 1132func (c *configImpl) PrebuiltBuildTool(name string) string { 1133 if v, ok := c.environ.Get("SANITIZE_HOST"); ok { 1134 if sanitize := strings.Fields(v); inList("address", sanitize) { 1135 asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name) 1136 if _, err := os.Stat(asan); err == nil { 1137 return asan 1138 } 1139 } 1140 } 1141 return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name) 1142} 1143 1144func (c *configImpl) SetBuildBrokenDupRules(val bool) { 1145 c.brokenDupRules = val 1146} 1147 1148func (c *configImpl) BuildBrokenDupRules() bool { 1149 return c.brokenDupRules 1150} 1151 1152func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) { 1153 c.brokenUsesNetwork = val 1154} 1155 1156func (c *configImpl) BuildBrokenUsesNetwork() bool { 1157 return c.brokenUsesNetwork 1158} 1159 1160func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) { 1161 c.brokenNinjaEnvVars = val 1162} 1163 1164func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string { 1165 return c.brokenNinjaEnvVars 1166} 1167 1168func (c *configImpl) SetTargetDeviceDir(dir string) { 1169 c.targetDeviceDir = dir 1170} 1171 1172func (c *configImpl) TargetDeviceDir() string { 1173 return c.targetDeviceDir 1174} 1175 1176func (c *configImpl) BuildDateTime() string { 1177 return c.buildDateTime 1178} 1179 1180func (c *configImpl) MetricsUploaderApp() string { 1181 if p, ok := c.environ.Get("ANDROID_ENABLE_METRICS_UPLOAD"); ok { 1182 return p 1183 } 1184 return "" 1185} 1186 1187// LogsDir returns the logs directory where build log and metrics 1188// files are located. By default, the logs directory is the out 1189// directory. If the argument dist is specified, the logs directory 1190// is <dist_dir>/logs. 1191func (c *configImpl) LogsDir() string { 1192 if c.Dist() { 1193 // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files 1194 return filepath.Join(c.RealDistDir(), "logs") 1195 } 1196 return c.OutDir() 1197} 1198 1199// BazelMetricsDir returns the <logs dir>/bazel_metrics directory 1200// where the bazel profiles are located. 1201func (c *configImpl) BazelMetricsDir() string { 1202 return filepath.Join(c.LogsDir(), "bazel_metrics") 1203} 1204 1205func (c *configImpl) SetEmptyNinjaFile(v bool) { 1206 c.emptyNinjaFile = v 1207} 1208 1209func (c *configImpl) EmptyNinjaFile() bool { 1210 return c.emptyNinjaFile 1211} 1212