1// Copyright 2017 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package build 16 17import ( 18 "context" 19 "encoding/json" 20 "fmt" 21 "io/ioutil" 22 "math/rand" 23 "os" 24 "os/exec" 25 "path/filepath" 26 "runtime" 27 "strconv" 28 "strings" 29 "syscall" 30 "time" 31 32 "android/soong/shared" 33 34 "google.golang.org/protobuf/proto" 35 36 smpb "android/soong/ui/metrics/metrics_proto" 37) 38 39const ( 40 envConfigDir = "vendor/google/tools/soong_config" 41 jsonSuffix = "json" 42 43 configFetcher = "vendor/google/tools/soong/expconfigfetcher" 44 envConfigFetchTimeout = 10 * time.Second 45) 46 47var ( 48 rbeRandPrefix int 49 googleProdCredsExistCache bool 50) 51 52func init() { 53 rand.Seed(time.Now().UnixNano()) 54 rbeRandPrefix = rand.Intn(1000) 55} 56 57type Config struct{ *configImpl } 58 59type configImpl struct { 60 // Some targets that are implemented in soong_build 61 // (bp2build, json-module-graph) are not here and have their own bits below. 62 arguments []string 63 goma bool 64 environ *Environment 65 distDir string 66 buildDateTime string 67 logsPrefix string 68 69 // From the arguments 70 parallel int 71 keepGoing int 72 verbose bool 73 checkbuild bool 74 dist bool 75 jsonModuleGraph bool 76 apiBp2build bool // Generate BUILD files for Soong modules that contribute APIs 77 bp2build bool 78 queryview bool 79 reportMkMetrics bool // Collect and report mk2bp migration progress metrics. 80 soongDocs bool 81 multitreeBuild bool // This is a multitree build. 82 skipConfig bool 83 skipKati bool 84 skipKatiNinja bool 85 skipSoong bool 86 skipNinja bool 87 skipSoongTests bool 88 searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces 89 skipMetricsUpload bool 90 buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time 91 buildFromTextStub bool 92 93 // From the product config 94 katiArgs []string 95 ninjaArgs []string 96 katiSuffix string 97 targetDevice string 98 targetDeviceDir string 99 sandboxConfig *SandboxConfig 100 101 // Autodetected 102 totalRAM uint64 103 104 brokenDupRules bool 105 brokenUsesNetwork bool 106 brokenNinjaEnvVars []string 107 108 pathReplaced bool 109 110 bazelProdMode bool 111 bazelDevMode bool 112 bazelStagingMode bool 113 114 // Set by multiproduct_kati 115 emptyNinjaFile bool 116 117 metricsUploader string 118 119 bazelForceEnabledModules string 120 121 includeTags []string 122 sourceRootDirs []string 123 124 // Data source to write ninja weight list 125 ninjaWeightListSource NinjaWeightListSource 126} 127 128type NinjaWeightListSource uint 129 130const ( 131 // ninja doesn't use weight list. 132 NOT_USED NinjaWeightListSource = iota 133 // ninja uses weight list based on previous builds by ninja log 134 NINJA_LOG 135 // ninja thinks every task has the same weight. 136 EVENLY_DISTRIBUTED 137 // ninja uses an external custom weight list 138 EXTERNAL_FILE 139 // ninja uses a prioritized module list from Soong 140 HINT_FROM_SOONG 141) 142const srcDirFileCheck = "build/soong/root.bp" 143 144var buildFiles = []string{"Android.mk", "Android.bp"} 145 146type BuildAction uint 147 148const ( 149 // Builds all of the modules and their dependencies of a specified directory, relative to the root 150 // directory of the source tree. 151 BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota 152 153 // Builds all of the modules and their dependencies of a list of specified directories. All specified 154 // directories are relative to the root directory of the source tree. 155 BUILD_MODULES_IN_DIRECTORIES 156 157 // Build a list of specified modules. If none was specified, simply build the whole source tree. 158 BUILD_MODULES 159) 160 161// checkTopDir validates that the current directory is at the root directory of the source tree. 162func checkTopDir(ctx Context) { 163 if _, err := os.Stat(srcDirFileCheck); err != nil { 164 if os.IsNotExist(err) { 165 ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck) 166 } 167 ctx.Fatalln("Error verifying tree state:", err) 168 } 169} 170 171// fetchEnvConfig optionally fetches a configuration file that can then subsequently be 172// loaded into Soong environment to control certain aspects of build behavior (e.g., enabling RBE). 173// If a configuration file already exists on disk, the fetch is run in the background 174// so as to NOT block the rest of the build execution. 175func fetchEnvConfig(ctx Context, config *configImpl, envConfigName string) error { 176 configName := envConfigName + "." + jsonSuffix 177 expConfigFetcher := &smpb.ExpConfigFetcher{Filename: &configName} 178 defer func() { 179 ctx.Metrics.ExpConfigFetcher(expConfigFetcher) 180 }() 181 if !config.GoogleProdCredsExist() { 182 status := smpb.ExpConfigFetcher_MISSING_GCERT 183 expConfigFetcher.Status = &status 184 return nil 185 } 186 187 s, err := os.Stat(configFetcher) 188 if err != nil { 189 if os.IsNotExist(err) { 190 return nil 191 } 192 return err 193 } 194 if s.Mode()&0111 == 0 { 195 status := smpb.ExpConfigFetcher_ERROR 196 expConfigFetcher.Status = &status 197 return fmt.Errorf("configuration fetcher binary %v is not executable: %v", configFetcher, s.Mode()) 198 } 199 200 configExists := false 201 outConfigFilePath := filepath.Join(config.OutDir(), configName) 202 if _, err := os.Stat(outConfigFilePath); err == nil { 203 configExists = true 204 } 205 206 tCtx, cancel := context.WithTimeout(ctx, envConfigFetchTimeout) 207 fetchStart := time.Now() 208 cmd := exec.CommandContext(tCtx, configFetcher, "-output_config_dir", config.OutDir(), 209 "-output_config_name", configName) 210 if err := cmd.Start(); err != nil { 211 status := smpb.ExpConfigFetcher_ERROR 212 expConfigFetcher.Status = &status 213 return err 214 } 215 216 fetchCfg := func() error { 217 if err := cmd.Wait(); err != nil { 218 status := smpb.ExpConfigFetcher_ERROR 219 expConfigFetcher.Status = &status 220 return err 221 } 222 fetchEnd := time.Now() 223 expConfigFetcher.Micros = proto.Uint64(uint64(fetchEnd.Sub(fetchStart).Microseconds())) 224 expConfigFetcher.Filename = proto.String(outConfigFilePath) 225 226 if _, err := os.Stat(outConfigFilePath); err != nil { 227 status := smpb.ExpConfigFetcher_NO_CONFIG 228 expConfigFetcher.Status = &status 229 return err 230 } 231 status := smpb.ExpConfigFetcher_CONFIG 232 expConfigFetcher.Status = &status 233 return nil 234 } 235 236 // If a config file does not exist, wait for the config file to be fetched. Otherwise 237 // fetch the config file in the background and return immediately. 238 if !configExists { 239 defer cancel() 240 return fetchCfg() 241 } 242 243 go func() { 244 defer cancel() 245 if err := fetchCfg(); err != nil { 246 ctx.Verbosef("Failed to fetch config file %v: %v\n", configName, err) 247 } 248 }() 249 return nil 250} 251 252func loadEnvConfig(ctx Context, config *configImpl, bc string) error { 253 if bc == "" { 254 return nil 255 } 256 257 configDirs := []string{ 258 config.OutDir(), 259 os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"), 260 envConfigDir, 261 } 262 for _, dir := range configDirs { 263 cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix)) 264 envVarsJSON, err := ioutil.ReadFile(cfgFile) 265 if err != nil { 266 continue 267 } 268 ctx.Verbosef("Loading config file %v\n", cfgFile) 269 var envVars map[string]map[string]string 270 if err := json.Unmarshal(envVarsJSON, &envVars); err != nil { 271 fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error()) 272 continue 273 } 274 for k, v := range envVars["env"] { 275 if os.Getenv(k) != "" { 276 continue 277 } 278 config.environ.Set(k, v) 279 } 280 ctx.Verbosef("Finished loading config file %v\n", cfgFile) 281 break 282 } 283 284 return nil 285} 286 287func defaultBazelProdMode(cfg *configImpl) bool { 288 // Environment flag to disable Bazel for users which experience 289 // broken bazel-handled builds, or significant performance regressions. 290 if cfg.IsBazelMixedBuildForceDisabled() { 291 return false 292 } 293 // Darwin-host builds are currently untested with Bazel. 294 if runtime.GOOS == "darwin" { 295 return false 296 } 297 return true 298} 299 300func UploadOnlyConfig(ctx Context, _ ...string) Config { 301 ret := &configImpl{ 302 environ: OsEnvironment(), 303 sandboxConfig: &SandboxConfig{}, 304 } 305 srcDir := absPath(ctx, ".") 306 bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG") 307 if err := loadEnvConfig(ctx, ret, bc); err != nil { 308 ctx.Fatalln("Failed to parse env config files: %v", err) 309 } 310 ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ) 311 return Config{ret} 312} 313 314func NewConfig(ctx Context, args ...string) Config { 315 ret := &configImpl{ 316 environ: OsEnvironment(), 317 sandboxConfig: &SandboxConfig{}, 318 } 319 320 // Default matching ninja 321 ret.parallel = runtime.NumCPU() + 2 322 ret.keepGoing = 1 323 324 ret.totalRAM = detectTotalRAM(ctx) 325 ret.parseArgs(ctx, args) 326 327 if ret.ninjaWeightListSource == HINT_FROM_SOONG { 328 ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "true") 329 } 330 // Make sure OUT_DIR is set appropriately 331 if outDir, ok := ret.environ.Get("OUT_DIR"); ok { 332 ret.environ.Set("OUT_DIR", filepath.Clean(outDir)) 333 } else { 334 outDir := "out" 335 if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok { 336 if wd, err := os.Getwd(); err != nil { 337 ctx.Fatalln("Failed to get working directory:", err) 338 } else { 339 outDir = filepath.Join(baseDir, filepath.Base(wd)) 340 } 341 } 342 ret.environ.Set("OUT_DIR", outDir) 343 } 344 345 // loadEnvConfig needs to know what the OUT_DIR is, so it should 346 // be called after we determine the appropriate out directory. 347 bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG") 348 349 if bc != "" { 350 if err := fetchEnvConfig(ctx, ret, bc); err != nil { 351 ctx.Verbosef("Failed to fetch config file: %v\n", err) 352 } 353 if err := loadEnvConfig(ctx, ret, bc); err != nil { 354 ctx.Fatalln("Failed to parse env config files: %v", err) 355 } 356 } 357 358 if distDir, ok := ret.environ.Get("DIST_DIR"); ok { 359 ret.distDir = filepath.Clean(distDir) 360 } else { 361 ret.distDir = filepath.Join(ret.OutDir(), "dist") 362 } 363 364 if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok { 365 ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false") 366 } 367 368 ret.environ.Unset( 369 // We're already using it 370 "USE_SOONG_UI", 371 372 // We should never use GOROOT/GOPATH from the shell environment 373 "GOROOT", 374 "GOPATH", 375 376 // These should only come from Soong, not the environment. 377 "CLANG", 378 "CLANG_CXX", 379 "CCC_CC", 380 "CCC_CXX", 381 382 // Used by the goma compiler wrapper, but should only be set by 383 // gomacc 384 "GOMACC_PATH", 385 386 // We handle this above 387 "OUT_DIR_COMMON_BASE", 388 389 // This is handled above too, and set for individual commands later 390 "DIST_DIR", 391 392 // Variables that have caused problems in the past 393 "BASH_ENV", 394 "CDPATH", 395 "DISPLAY", 396 "GREP_OPTIONS", 397 "JAVAC", 398 "NDK_ROOT", 399 "POSIXLY_CORRECT", 400 401 // Drop make flags 402 "MAKEFLAGS", 403 "MAKELEVEL", 404 "MFLAGS", 405 406 // Set in envsetup.sh, reset in makefiles 407 "ANDROID_JAVA_TOOLCHAIN", 408 409 // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional 410 "ANDROID_BUILD_TOP", 411 "ANDROID_HOST_OUT", 412 "ANDROID_PRODUCT_OUT", 413 "ANDROID_HOST_OUT_TESTCASES", 414 "ANDROID_TARGET_OUT_TESTCASES", 415 "ANDROID_TOOLCHAIN", 416 "ANDROID_TOOLCHAIN_2ND_ARCH", 417 "ANDROID_DEV_SCRIPTS", 418 "ANDROID_EMULATOR_PREBUILTS", 419 "ANDROID_PRE_BUILD_PATHS", 420 ) 421 422 if ret.UseGoma() || ret.ForceUseGoma() { 423 ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.") 424 ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.") 425 } 426 427 // Tell python not to spam the source tree with .pyc files. 428 ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1") 429 430 tmpDir := absPath(ctx, ret.TempDir()) 431 ret.environ.Set("TMPDIR", tmpDir) 432 433 // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes 434 symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(), 435 "llvm-binutils-stable/llvm-symbolizer") 436 ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath)) 437 438 // Precondition: the current directory is the top of the source tree 439 checkTopDir(ctx) 440 441 srcDir := absPath(ctx, ".") 442 if strings.ContainsRune(srcDir, ' ') { 443 ctx.Println("You are building in a directory whose absolute path contains a space character:") 444 ctx.Println() 445 ctx.Printf("%q\n", srcDir) 446 ctx.Println() 447 ctx.Fatalln("Directory names containing spaces are not supported") 448 } 449 450 ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ) 451 452 if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') { 453 ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:") 454 ctx.Println() 455 ctx.Printf("%q\n", outDir) 456 ctx.Println() 457 ctx.Fatalln("Directory names containing spaces are not supported") 458 } 459 460 if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') { 461 ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:") 462 ctx.Println() 463 ctx.Printf("%q\n", distDir) 464 ctx.Println() 465 ctx.Fatalln("Directory names containing spaces are not supported") 466 } 467 468 // Configure Java-related variables, including adding it to $PATH 469 java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag()) 470 java9Home := filepath.Join("prebuilts/jdk/jdk9", ret.HostPrebuiltTag()) 471 java11Home := filepath.Join("prebuilts/jdk/jdk11", ret.HostPrebuiltTag()) 472 java17Home := filepath.Join("prebuilts/jdk/jdk17", ret.HostPrebuiltTag()) 473 javaHome := func() string { 474 if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok { 475 return override 476 } 477 if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" { 478 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 11 toolchain is now the global default.") 479 } 480 if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" { 481 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 17 toolchain is now the global default.") 482 } 483 return java17Home 484 }() 485 absJavaHome := absPath(ctx, javaHome) 486 487 ret.configureLocale(ctx) 488 489 newPath := []string{filepath.Join(absJavaHome, "bin")} 490 if path, ok := ret.environ.Get("PATH"); ok && path != "" { 491 newPath = append(newPath, path) 492 } 493 494 ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME") 495 ret.environ.Set("JAVA_HOME", absJavaHome) 496 ret.environ.Set("ANDROID_JAVA_HOME", javaHome) 497 ret.environ.Set("ANDROID_JAVA8_HOME", java8Home) 498 ret.environ.Set("ANDROID_JAVA9_HOME", java9Home) 499 ret.environ.Set("ANDROID_JAVA11_HOME", java11Home) 500 ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator))) 501 502 if ret.MultitreeBuild() { 503 ret.environ.Set("MULTITREE_BUILD", "true") 504 } 505 506 outDir := ret.OutDir() 507 buildDateTimeFile := filepath.Join(outDir, "build_date.txt") 508 if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" { 509 ret.buildDateTime = buildDateTime 510 } else { 511 ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10) 512 } 513 514 ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile) 515 516 if ret.UseRBE() { 517 for k, v := range getRBEVars(ctx, Config{ret}) { 518 ret.environ.Set(k, v) 519 } 520 } 521 522 if ret.BuildFromTextStub() { 523 // TODO(b/271443071): support hidden api check for from-text stub build 524 ret.environ.Set("UNSAFE_DISABLE_HIDDENAPI_FLAGS", "true") 525 } 526 527 bpd := ret.BazelMetricsDir() 528 if err := os.RemoveAll(bpd); err != nil { 529 ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err) 530 } 531 532 c := Config{ret} 533 storeConfigMetrics(ctx, c) 534 return c 535} 536 537// NewBuildActionConfig returns a build configuration based on the build action. The arguments are 538// processed based on the build action and extracts any arguments that belongs to the build action. 539func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config { 540 return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...) 541} 542 543// storeConfigMetrics selects a set of configuration information and store in 544// the metrics system for further analysis. 545func storeConfigMetrics(ctx Context, config Config) { 546 if ctx.Metrics == nil { 547 return 548 } 549 550 ctx.Metrics.BuildConfig(buildConfig(config)) 551 552 s := &smpb.SystemResourceInfo{ 553 TotalPhysicalMemory: proto.Uint64(config.TotalRAM()), 554 AvailableCpus: proto.Int32(int32(runtime.NumCPU())), 555 } 556 ctx.Metrics.SystemResourceInfo(s) 557} 558 559func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource { 560 switch s { 561 case NINJA_LOG: 562 return smpb.BuildConfig_NINJA_LOG.Enum() 563 case EVENLY_DISTRIBUTED: 564 return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum() 565 case EXTERNAL_FILE: 566 return smpb.BuildConfig_EXTERNAL_FILE.Enum() 567 case HINT_FROM_SOONG: 568 return smpb.BuildConfig_HINT_FROM_SOONG.Enum() 569 default: 570 return smpb.BuildConfig_NOT_USED.Enum() 571 } 572} 573 574func buildConfig(config Config) *smpb.BuildConfig { 575 c := &smpb.BuildConfig{ 576 ForceUseGoma: proto.Bool(config.ForceUseGoma()), 577 UseGoma: proto.Bool(config.UseGoma()), 578 UseRbe: proto.Bool(config.UseRBE()), 579 BazelMixedBuild: proto.Bool(config.BazelBuildEnabled()), 580 ForceDisableBazelMixedBuild: proto.Bool(config.IsBazelMixedBuildForceDisabled()), 581 NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()), 582 } 583 c.Targets = append(c.Targets, config.arguments...) 584 585 return c 586} 587 588// getConfigArgs processes the command arguments based on the build action and creates a set of new 589// arguments to be accepted by Config. 590func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string { 591 // The next block of code verifies that the current directory is the root directory of the source 592 // tree. It then finds the relative path of dir based on the root directory of the source tree 593 // and verify that dir is inside of the source tree. 594 checkTopDir(ctx) 595 topDir, err := os.Getwd() 596 if err != nil { 597 ctx.Fatalf("Error retrieving top directory: %v", err) 598 } 599 dir, err = filepath.EvalSymlinks(dir) 600 if err != nil { 601 ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err) 602 } 603 dir, err = filepath.Abs(dir) 604 if err != nil { 605 ctx.Fatalf("Unable to find absolute path %s: %v", dir, err) 606 } 607 relDir, err := filepath.Rel(topDir, dir) 608 if err != nil { 609 ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err) 610 } 611 // If there are ".." in the path, it's not in the source tree. 612 if strings.Contains(relDir, "..") { 613 ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir) 614 } 615 616 configArgs := args[:] 617 618 // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to 619 // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules. 620 targetNamePrefix := "MODULES-IN-" 621 if inList("GET-INSTALL-PATH", configArgs) { 622 targetNamePrefix = "GET-INSTALL-PATH-IN-" 623 configArgs = removeFromList("GET-INSTALL-PATH", configArgs) 624 } 625 626 var targets []string 627 628 switch action { 629 case BUILD_MODULES: 630 // No additional processing is required when building a list of specific modules or all modules. 631 case BUILD_MODULES_IN_A_DIRECTORY: 632 // If dir is the root source tree, all the modules are built of the source tree are built so 633 // no need to find the build file. 634 if topDir == dir { 635 break 636 } 637 638 buildFile := findBuildFile(ctx, relDir) 639 if buildFile == "" { 640 ctx.Fatalf("Build file not found for %s directory", relDir) 641 } 642 targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} 643 case BUILD_MODULES_IN_DIRECTORIES: 644 newConfigArgs, dirs := splitArgs(configArgs) 645 configArgs = newConfigArgs 646 targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix) 647 } 648 649 // Tidy only override all other specified targets. 650 tidyOnly := os.Getenv("WITH_TIDY_ONLY") 651 if tidyOnly == "true" || tidyOnly == "1" { 652 configArgs = append(configArgs, "tidy_only") 653 } else { 654 configArgs = append(configArgs, targets...) 655 } 656 657 return configArgs 658} 659 660// convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name. 661func convertToTarget(dir string, targetNamePrefix string) string { 662 return targetNamePrefix + strings.ReplaceAll(dir, "/", "-") 663} 664 665// hasBuildFile returns true if dir contains an Android build file. 666func hasBuildFile(ctx Context, dir string) bool { 667 for _, buildFile := range buildFiles { 668 _, err := os.Stat(filepath.Join(dir, buildFile)) 669 if err == nil { 670 return true 671 } 672 if !os.IsNotExist(err) { 673 ctx.Fatalf("Error retrieving the build file stats: %v", err) 674 } 675 } 676 return false 677} 678 679// findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file 680// in the current and any sub directory of dir. If a build file is not found, traverse the path 681// up by one directory and repeat again until either a build file is found or reached to the root 682// source tree. The returned filename of build file is "Android.mk". If one was not found, a blank 683// string is returned. 684func findBuildFile(ctx Context, dir string) string { 685 // If the string is empty or ".", assume it is top directory of the source tree. 686 if dir == "" || dir == "." { 687 return "" 688 } 689 690 found := false 691 for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) { 692 err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error { 693 if err != nil { 694 return err 695 } 696 if found { 697 return filepath.SkipDir 698 } 699 if info.IsDir() { 700 return nil 701 } 702 for _, buildFile := range buildFiles { 703 if info.Name() == buildFile { 704 found = true 705 return filepath.SkipDir 706 } 707 } 708 return nil 709 }) 710 if err != nil { 711 ctx.Fatalf("Error finding Android build file: %v", err) 712 } 713 714 if found { 715 return filepath.Join(buildDir, "Android.mk") 716 } 717 } 718 719 return "" 720} 721 722// splitArgs iterates over the arguments list and splits into two lists: arguments and directories. 723func splitArgs(args []string) (newArgs []string, dirs []string) { 724 specialArgs := map[string]bool{ 725 "showcommands": true, 726 "snod": true, 727 "dist": true, 728 "checkbuild": true, 729 } 730 731 newArgs = []string{} 732 dirs = []string{} 733 734 for _, arg := range args { 735 // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory. 736 if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 { 737 newArgs = append(newArgs, arg) 738 continue 739 } 740 741 if _, ok := specialArgs[arg]; ok { 742 newArgs = append(newArgs, arg) 743 continue 744 } 745 746 dirs = append(dirs, arg) 747 } 748 749 return newArgs, dirs 750} 751 752// getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a 753// directory from the dirs list does not exist, a fatal error is raised. relDir is related to the 754// source root tree where the build action command was invoked. Each directory is validated if the 755// build file can be found and follows the format "dir1:target1,target2,...". Target is optional. 756func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) { 757 for _, dir := range dirs { 758 // The directory may have specified specific modules to build. ":" is the separator to separate 759 // the directory and the list of modules. 760 s := strings.Split(dir, ":") 761 l := len(s) 762 if l > 2 { // more than one ":" was specified. 763 ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir) 764 } 765 766 dir = filepath.Join(relDir, s[0]) 767 if _, err := os.Stat(dir); err != nil { 768 ctx.Fatalf("couldn't find directory %s", dir) 769 } 770 771 // Verify that if there are any targets specified after ":". Each target is separated by ",". 772 var newTargets []string 773 if l == 2 && s[1] != "" { 774 newTargets = strings.Split(s[1], ",") 775 if inList("", newTargets) { 776 ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir) 777 } 778 } 779 780 // If there are specified targets to build in dir, an android build file must exist for the one 781 // shot build. For the non-targets case, find the appropriate build file and build all the 782 // modules in dir (or the closest one in the dir path). 783 if len(newTargets) > 0 { 784 if !hasBuildFile(ctx, dir) { 785 ctx.Fatalf("Couldn't locate a build file from %s directory", dir) 786 } 787 } else { 788 buildFile := findBuildFile(ctx, dir) 789 if buildFile == "" { 790 ctx.Fatalf("Build file not found for %s directory", dir) 791 } 792 newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} 793 } 794 795 targets = append(targets, newTargets...) 796 } 797 798 return targets 799} 800 801func (c *configImpl) parseArgs(ctx Context, args []string) { 802 for i := 0; i < len(args); i++ { 803 arg := strings.TrimSpace(args[i]) 804 if arg == "showcommands" { 805 c.verbose = true 806 } else if arg == "--empty-ninja-file" { 807 c.emptyNinjaFile = true 808 } else if arg == "--skip-ninja" { 809 c.skipNinja = true 810 } else if arg == "--skip-make" { 811 // TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati, 812 // but it does run a Kati ninja file if the .kati_enabled marker file was created 813 // by a previous build. 814 c.skipConfig = true 815 c.skipKati = true 816 } else if arg == "--skip-kati" { 817 // TODO: remove --skip-kati once module builds have been migrated to --song-only 818 c.skipKati = true 819 } else if arg == "--soong-only" { 820 c.skipKati = true 821 c.skipKatiNinja = true 822 } else if arg == "--config-only" { 823 c.skipKati = true 824 c.skipKatiNinja = true 825 c.skipSoong = true 826 } else if arg == "--skip-config" { 827 c.skipConfig = true 828 } else if arg == "--skip-soong-tests" { 829 c.skipSoongTests = true 830 } else if arg == "--skip-metrics-upload" { 831 c.skipMetricsUpload = true 832 } else if arg == "--mk-metrics" { 833 c.reportMkMetrics = true 834 } else if arg == "--multitree-build" { 835 c.multitreeBuild = true 836 } else if arg == "--bazel-mode" { 837 c.bazelProdMode = true 838 } else if arg == "--bazel-mode-dev" { 839 c.bazelDevMode = true 840 } else if arg == "--bazel-mode-staging" { 841 c.bazelStagingMode = true 842 } else if arg == "--search-api-dir" { 843 c.searchApiDir = true 844 } else if strings.HasPrefix(arg, "--ninja_weight_source=") { 845 source := strings.TrimPrefix(arg, "--ninja_weight_source=") 846 if source == "ninja_log" { 847 c.ninjaWeightListSource = NINJA_LOG 848 } else if source == "evenly_distributed" { 849 c.ninjaWeightListSource = EVENLY_DISTRIBUTED 850 } else if source == "not_used" { 851 c.ninjaWeightListSource = NOT_USED 852 } else if source == "soong" { 853 c.ninjaWeightListSource = HINT_FROM_SOONG 854 } else if strings.HasPrefix(source, "file,") { 855 c.ninjaWeightListSource = EXTERNAL_FILE 856 filePath := strings.TrimPrefix(source, "file,") 857 err := validateNinjaWeightList(filePath) 858 if err != nil { 859 ctx.Fatalf("Malformed weight list from %s: %s", filePath, err) 860 } 861 _, err = copyFile(filePath, filepath.Join(c.OutDir(), ".ninja_weight_list")) 862 if err != nil { 863 ctx.Fatalf("Error to copy ninja weight list from %s: %s", filePath, err) 864 } 865 } else { 866 ctx.Fatalf("unknown option for ninja_weight_source: %s", source) 867 } 868 } else if arg == "--build-from-text-stub" { 869 c.buildFromTextStub = true 870 } else if strings.HasPrefix(arg, "--build-command=") { 871 buildCmd := strings.TrimPrefix(arg, "--build-command=") 872 // remove quotations 873 buildCmd = strings.TrimPrefix(buildCmd, "\"") 874 buildCmd = strings.TrimSuffix(buildCmd, "\"") 875 ctx.Metrics.SetBuildCommand([]string{buildCmd}) 876 } else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") { 877 c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=") 878 } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") { 879 buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=") 880 val, err := strconv.ParseInt(buildTimeStr, 10, 64) 881 if err == nil { 882 c.buildStartedTime = val 883 } else { 884 ctx.Fatalf("Error parsing build-time-started-unix-millis", err) 885 } 886 } else if len(arg) > 0 && arg[0] == '-' { 887 parseArgNum := func(def int) int { 888 if len(arg) > 2 { 889 p, err := strconv.ParseUint(arg[2:], 10, 31) 890 if err != nil { 891 ctx.Fatalf("Failed to parse %q: %v", arg, err) 892 } 893 return int(p) 894 } else if i+1 < len(args) { 895 p, err := strconv.ParseUint(args[i+1], 10, 31) 896 if err == nil { 897 i++ 898 return int(p) 899 } 900 } 901 return def 902 } 903 904 if len(arg) > 1 && arg[1] == 'j' { 905 c.parallel = parseArgNum(c.parallel) 906 } else if len(arg) > 1 && arg[1] == 'k' { 907 c.keepGoing = parseArgNum(0) 908 } else { 909 ctx.Fatalln("Unknown option:", arg) 910 } 911 } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 { 912 if k == "OUT_DIR" { 913 ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.") 914 } 915 c.environ.Set(k, v) 916 } else if arg == "dist" { 917 c.dist = true 918 } else if arg == "json-module-graph" { 919 c.jsonModuleGraph = true 920 } else if arg == "bp2build" { 921 c.bp2build = true 922 } else if arg == "api_bp2build" { 923 c.apiBp2build = true 924 } else if arg == "queryview" { 925 c.queryview = true 926 } else if arg == "soong_docs" { 927 c.soongDocs = true 928 } else { 929 if arg == "checkbuild" { 930 c.checkbuild = true 931 } 932 c.arguments = append(c.arguments, arg) 933 } 934 } 935 if (!c.bazelProdMode) && (!c.bazelDevMode) && (!c.bazelStagingMode) { 936 c.bazelProdMode = defaultBazelProdMode(c) 937 } 938} 939 940func validateNinjaWeightList(weightListFilePath string) (err error) { 941 data, err := os.ReadFile(weightListFilePath) 942 if err != nil { 943 return 944 } 945 lines := strings.Split(strings.TrimSpace(string(data)), "\n") 946 for _, line := range lines { 947 fields := strings.Split(line, ",") 948 if len(fields) != 2 { 949 return fmt.Errorf("wrong format, each line should have two fields, but '%s'", line) 950 } 951 _, err = strconv.Atoi(fields[1]) 952 if err != nil { 953 return 954 } 955 } 956 return 957} 958 959func (c *configImpl) configureLocale(ctx Context) { 960 cmd := Command(ctx, Config{c}, "locale", "locale", "-a") 961 output, err := cmd.Output() 962 963 var locales []string 964 if err == nil { 965 locales = strings.Split(string(output), "\n") 966 } else { 967 // If we're unable to list the locales, let's assume en_US.UTF-8 968 locales = []string{"en_US.UTF-8"} 969 ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales) 970 } 971 972 // gettext uses LANGUAGE, which is passed directly through 973 974 // For LANG and LC_*, only preserve the evaluated version of 975 // LC_MESSAGES 976 userLang := "" 977 if lc_all, ok := c.environ.Get("LC_ALL"); ok { 978 userLang = lc_all 979 } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok { 980 userLang = lc_messages 981 } else if lang, ok := c.environ.Get("LANG"); ok { 982 userLang = lang 983 } 984 985 c.environ.UnsetWithPrefix("LC_") 986 987 if userLang != "" { 988 c.environ.Set("LC_MESSAGES", userLang) 989 } 990 991 // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed 992 // for others) 993 if inList("C.UTF-8", locales) { 994 c.environ.Set("LANG", "C.UTF-8") 995 } else if inList("C.utf8", locales) { 996 // These normalize to the same thing 997 c.environ.Set("LANG", "C.UTF-8") 998 } else if inList("en_US.UTF-8", locales) { 999 c.environ.Set("LANG", "en_US.UTF-8") 1000 } else if inList("en_US.utf8", locales) { 1001 // These normalize to the same thing 1002 c.environ.Set("LANG", "en_US.UTF-8") 1003 } else { 1004 ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8") 1005 } 1006} 1007 1008func (c *configImpl) Environment() *Environment { 1009 return c.environ 1010} 1011 1012func (c *configImpl) Arguments() []string { 1013 return c.arguments 1014} 1015 1016func (c *configImpl) SoongBuildInvocationNeeded() bool { 1017 if len(c.Arguments()) > 0 { 1018 // Explicit targets requested that are not special targets like b2pbuild 1019 // or the JSON module graph 1020 return true 1021 } 1022 1023 if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() && !c.ApiBp2build() { 1024 // Command line was empty, the default Ninja target is built 1025 return true 1026 } 1027 1028 // bp2build + dist may be used to dist bp2build logs but does not require SoongBuildInvocation 1029 if c.Dist() && !c.Bp2Build() { 1030 return true 1031 } 1032 1033 // build.ninja doesn't need to be generated 1034 return false 1035} 1036 1037func (c *configImpl) OutDir() string { 1038 if outDir, ok := c.environ.Get("OUT_DIR"); ok { 1039 return outDir 1040 } 1041 return "out" 1042} 1043 1044func (c *configImpl) DistDir() string { 1045 return c.distDir 1046} 1047 1048func (c *configImpl) RealDistDir() string { 1049 return c.distDir 1050} 1051 1052func (c *configImpl) NinjaArgs() []string { 1053 if c.skipKati { 1054 return c.arguments 1055 } 1056 return c.ninjaArgs 1057} 1058 1059func (c *configImpl) BazelOutDir() string { 1060 return filepath.Join(c.OutDir(), "bazel") 1061} 1062 1063func (c *configImpl) bazelOutputBase() string { 1064 return filepath.Join(c.BazelOutDir(), "output") 1065} 1066 1067func (c *configImpl) SoongOutDir() string { 1068 return filepath.Join(c.OutDir(), "soong") 1069} 1070 1071func (c *configImpl) ApiSurfacesOutDir() string { 1072 return filepath.Join(c.OutDir(), "api_surfaces") 1073} 1074 1075func (c *configImpl) PrebuiltOS() string { 1076 switch runtime.GOOS { 1077 case "linux": 1078 return "linux-x86" 1079 case "darwin": 1080 return "darwin-x86" 1081 default: 1082 panic("Unknown GOOS") 1083 } 1084} 1085 1086func (c *configImpl) HostToolDir() string { 1087 if c.SkipKatiNinja() { 1088 return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin") 1089 } else { 1090 return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin") 1091 } 1092} 1093 1094func (c *configImpl) NamedGlobFile(name string) string { 1095 return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja") 1096} 1097 1098func (c *configImpl) UsedEnvFile(tag string) string { 1099 return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag) 1100} 1101 1102func (c *configImpl) Bp2BuildFilesMarkerFile() string { 1103 return shared.JoinPath(c.SoongOutDir(), "bp2build_files_marker") 1104} 1105 1106func (c *configImpl) Bp2BuildWorkspaceMarkerFile() string { 1107 return shared.JoinPath(c.SoongOutDir(), "bp2build_workspace_marker") 1108} 1109 1110func (c *configImpl) SoongDocsHtml() string { 1111 return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html") 1112} 1113 1114func (c *configImpl) QueryviewMarkerFile() string { 1115 return shared.JoinPath(c.SoongOutDir(), "queryview.marker") 1116} 1117 1118func (c *configImpl) ApiBp2buildMarkerFile() string { 1119 return shared.JoinPath(c.SoongOutDir(), "api_bp2build.marker") 1120} 1121 1122func (c *configImpl) ModuleGraphFile() string { 1123 return shared.JoinPath(c.SoongOutDir(), "module-graph.json") 1124} 1125 1126func (c *configImpl) ModuleActionsFile() string { 1127 return shared.JoinPath(c.SoongOutDir(), "module-actions.json") 1128} 1129 1130func (c *configImpl) TempDir() string { 1131 return shared.TempDirForOutDir(c.SoongOutDir()) 1132} 1133 1134func (c *configImpl) FileListDir() string { 1135 return filepath.Join(c.OutDir(), ".module_paths") 1136} 1137 1138func (c *configImpl) KatiSuffix() string { 1139 if c.katiSuffix != "" { 1140 return c.katiSuffix 1141 } 1142 panic("SetKatiSuffix has not been called") 1143} 1144 1145// Checkbuild returns true if "checkbuild" was one of the build goals, which means that the 1146// user is interested in additional checks at the expense of build time. 1147func (c *configImpl) Checkbuild() bool { 1148 return c.checkbuild 1149} 1150 1151func (c *configImpl) Dist() bool { 1152 return c.dist 1153} 1154 1155func (c *configImpl) JsonModuleGraph() bool { 1156 return c.jsonModuleGraph 1157} 1158 1159func (c *configImpl) Bp2Build() bool { 1160 return c.bp2build 1161} 1162 1163func (c *configImpl) ApiBp2build() bool { 1164 return c.apiBp2build 1165} 1166 1167func (c *configImpl) Queryview() bool { 1168 return c.queryview 1169} 1170 1171func (c *configImpl) SoongDocs() bool { 1172 return c.soongDocs 1173} 1174 1175func (c *configImpl) IsVerbose() bool { 1176 return c.verbose 1177} 1178 1179func (c *configImpl) MultitreeBuild() bool { 1180 return c.multitreeBuild 1181} 1182 1183func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource { 1184 return c.ninjaWeightListSource 1185} 1186 1187func (c *configImpl) SkipKati() bool { 1188 return c.skipKati 1189} 1190 1191func (c *configImpl) SkipKatiNinja() bool { 1192 return c.skipKatiNinja 1193} 1194 1195func (c *configImpl) SkipSoong() bool { 1196 return c.skipSoong 1197} 1198 1199func (c *configImpl) SkipNinja() bool { 1200 return c.skipNinja 1201} 1202 1203func (c *configImpl) SetSkipNinja(v bool) { 1204 c.skipNinja = v 1205} 1206 1207func (c *configImpl) SkipConfig() bool { 1208 return c.skipConfig 1209} 1210 1211func (c *configImpl) BuildFromTextStub() bool { 1212 return c.buildFromTextStub 1213} 1214 1215func (c *configImpl) TargetProduct() string { 1216 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { 1217 return v 1218 } 1219 panic("TARGET_PRODUCT is not defined") 1220} 1221 1222func (c *configImpl) TargetDevice() string { 1223 return c.targetDevice 1224} 1225 1226func (c *configImpl) SetTargetDevice(device string) { 1227 c.targetDevice = device 1228} 1229 1230func (c *configImpl) TargetBuildVariant() string { 1231 if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok { 1232 return v 1233 } 1234 panic("TARGET_BUILD_VARIANT is not defined") 1235} 1236 1237func (c *configImpl) KatiArgs() []string { 1238 return c.katiArgs 1239} 1240 1241func (c *configImpl) Parallel() int { 1242 return c.parallel 1243} 1244 1245func (c *configImpl) GetSourceRootDirs() []string { 1246 return c.sourceRootDirs 1247} 1248 1249func (c *configImpl) SetSourceRootDirs(i []string) { 1250 c.sourceRootDirs = i 1251} 1252 1253func (c *configImpl) GetIncludeTags() []string { 1254 return c.includeTags 1255} 1256 1257func (c *configImpl) SetIncludeTags(i []string) { 1258 c.includeTags = i 1259} 1260 1261func (c *configImpl) GetLogsPrefix() string { 1262 return c.logsPrefix 1263} 1264 1265func (c *configImpl) SetLogsPrefix(prefix string) { 1266 c.logsPrefix = prefix 1267} 1268 1269func (c *configImpl) HighmemParallel() int { 1270 if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok { 1271 return i 1272 } 1273 1274 const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024 1275 parallel := c.Parallel() 1276 if c.UseRemoteBuild() { 1277 // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism 1278 // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs 1279 // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention. 1280 // Return 1/16th of the size of the local pool, rounding up. 1281 return (parallel + 15) / 16 1282 } else if c.totalRAM == 0 { 1283 // Couldn't detect the total RAM, don't restrict highmem processes. 1284 return parallel 1285 } else if c.totalRAM <= 16*1024*1024*1024 { 1286 // Less than 16GB of ram, restrict to 1 highmem processes 1287 return 1 1288 } else if c.totalRAM <= 32*1024*1024*1024 { 1289 // Less than 32GB of ram, restrict to 2 highmem processes 1290 return 2 1291 } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel { 1292 // If less than 8GB total RAM per process, reduce the number of highmem processes 1293 return p 1294 } 1295 // No restriction on highmem processes 1296 return parallel 1297} 1298 1299func (c *configImpl) TotalRAM() uint64 { 1300 return c.totalRAM 1301} 1302 1303// ForceUseGoma determines whether we should override Goma deprecation 1304// and use Goma for the current build or not. 1305func (c *configImpl) ForceUseGoma() bool { 1306 if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok { 1307 v = strings.TrimSpace(v) 1308 if v != "" && v != "false" { 1309 return true 1310 } 1311 } 1312 return false 1313} 1314 1315func (c *configImpl) UseGoma() bool { 1316 if v, ok := c.environ.Get("USE_GOMA"); ok { 1317 v = strings.TrimSpace(v) 1318 if v != "" && v != "false" { 1319 return true 1320 } 1321 } 1322 return false 1323} 1324 1325func (c *configImpl) StartGoma() bool { 1326 if !c.UseGoma() { 1327 return false 1328 } 1329 1330 if v, ok := c.environ.Get("NOSTART_GOMA"); ok { 1331 v = strings.TrimSpace(v) 1332 if v != "" && v != "false" { 1333 return false 1334 } 1335 } 1336 return true 1337} 1338 1339func (c *configImpl) UseRBE() bool { 1340 if v, ok := c.Environment().Get("USE_RBE"); ok { 1341 v = strings.TrimSpace(v) 1342 if v != "" && v != "false" { 1343 return true 1344 } 1345 } 1346 return false 1347} 1348 1349func (c *configImpl) BazelBuildEnabled() bool { 1350 return c.bazelProdMode || c.bazelDevMode || c.bazelStagingMode 1351} 1352 1353func (c *configImpl) StartRBE() bool { 1354 if !c.UseRBE() { 1355 return false 1356 } 1357 1358 if v, ok := c.environ.Get("NOSTART_RBE"); ok { 1359 v = strings.TrimSpace(v) 1360 if v != "" && v != "false" { 1361 return false 1362 } 1363 } 1364 return true 1365} 1366 1367func (c *configImpl) rbeProxyLogsDir() string { 1368 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} { 1369 if v, ok := c.environ.Get(f); ok { 1370 return v 1371 } 1372 } 1373 buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir()) 1374 return filepath.Join(buildTmpDir, "rbe") 1375} 1376 1377func (c *configImpl) rbeCacheDir() string { 1378 for _, f := range []string{"RBE_cache_dir", "FLAG_cache_dir"} { 1379 if v, ok := c.environ.Get(f); ok { 1380 return v 1381 } 1382 } 1383 return shared.JoinPath(c.SoongOutDir(), "rbe") 1384} 1385 1386func (c *configImpl) shouldCleanupRBELogsDir() bool { 1387 // Perform a log directory cleanup only when the log directory 1388 // is auto created by the build rather than user-specified. 1389 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} { 1390 if _, ok := c.environ.Get(f); ok { 1391 return false 1392 } 1393 } 1394 return true 1395} 1396 1397func (c *configImpl) rbeExecRoot() string { 1398 for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} { 1399 if v, ok := c.environ.Get(f); ok { 1400 return v 1401 } 1402 } 1403 wd, err := os.Getwd() 1404 if err != nil { 1405 return "" 1406 } 1407 return wd 1408} 1409 1410func (c *configImpl) rbeDir() string { 1411 if v, ok := c.environ.Get("RBE_DIR"); ok { 1412 return v 1413 } 1414 return "prebuilts/remoteexecution-client/live/" 1415} 1416 1417func (c *configImpl) rbeReproxy() string { 1418 for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} { 1419 if v, ok := c.environ.Get(f); ok { 1420 return v 1421 } 1422 } 1423 return filepath.Join(c.rbeDir(), "reproxy") 1424} 1425 1426func (c *configImpl) rbeAuth() (string, string) { 1427 credFlags := []string{ 1428 "use_application_default_credentials", 1429 "use_gce_credentials", 1430 "credential_file", 1431 "use_google_prod_creds", 1432 } 1433 for _, cf := range credFlags { 1434 for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} { 1435 if v, ok := c.environ.Get(f); ok { 1436 v = strings.TrimSpace(v) 1437 if v != "" && v != "false" && v != "0" { 1438 return "RBE_" + cf, v 1439 } 1440 } 1441 } 1442 } 1443 return "RBE_use_application_default_credentials", "true" 1444} 1445 1446func (c *configImpl) rbeSockAddr(dir string) (string, error) { 1447 maxNameLen := len(syscall.RawSockaddrUnix{}.Path) 1448 base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix) 1449 1450 name := filepath.Join(dir, base) 1451 if len(name) < maxNameLen { 1452 return name, nil 1453 } 1454 1455 name = filepath.Join("/tmp", base) 1456 if len(name) < maxNameLen { 1457 return name, nil 1458 } 1459 1460 return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen) 1461} 1462 1463// IsGooglerEnvironment returns true if the current build is running 1464// on a Google developer machine and false otherwise. 1465func (c *configImpl) IsGooglerEnvironment() bool { 1466 cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG" 1467 if v, ok := c.environ.Get(cf); ok { 1468 return v == "googler" 1469 } 1470 return false 1471} 1472 1473// GoogleProdCredsExist determine whether credentials exist on the 1474// Googler machine to use remote execution. 1475func (c *configImpl) GoogleProdCredsExist() bool { 1476 if googleProdCredsExistCache { 1477 return googleProdCredsExistCache 1478 } 1479 if _, err := exec.Command("/usr/bin/prodcertstatus", "--simple_output", "--nocheck_loas").Output(); err != nil { 1480 return false 1481 } 1482 googleProdCredsExistCache = true 1483 return true 1484} 1485 1486// UseRemoteBuild indicates whether to use a remote build acceleration system 1487// to speed up the build. 1488func (c *configImpl) UseRemoteBuild() bool { 1489 return c.UseGoma() || c.UseRBE() 1490} 1491 1492// StubbyExists checks whether the stubby binary exists on the machine running 1493// the build. 1494func (c *configImpl) StubbyExists() bool { 1495 if _, err := exec.LookPath("stubby"); err != nil { 1496 return false 1497 } 1498 return true 1499} 1500 1501// RemoteParallel controls how many remote jobs (i.e., commands which contain 1502// gomacc) are run in parallel. Note the parallelism of all other jobs is 1503// still limited by Parallel() 1504func (c *configImpl) RemoteParallel() int { 1505 if !c.UseRemoteBuild() { 1506 return 0 1507 } 1508 if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok { 1509 return i 1510 } 1511 return 500 1512} 1513 1514func (c *configImpl) SetKatiArgs(args []string) { 1515 c.katiArgs = args 1516} 1517 1518func (c *configImpl) SetNinjaArgs(args []string) { 1519 c.ninjaArgs = args 1520} 1521 1522func (c *configImpl) SetKatiSuffix(suffix string) { 1523 c.katiSuffix = suffix 1524} 1525 1526func (c *configImpl) LastKatiSuffixFile() string { 1527 return filepath.Join(c.OutDir(), "last_kati_suffix") 1528} 1529 1530func (c *configImpl) HasKatiSuffix() bool { 1531 return c.katiSuffix != "" 1532} 1533 1534func (c *configImpl) KatiEnvFile() string { 1535 return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh") 1536} 1537 1538func (c *configImpl) KatiBuildNinjaFile() string { 1539 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja") 1540} 1541 1542func (c *configImpl) KatiPackageNinjaFile() string { 1543 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja") 1544} 1545 1546func (c *configImpl) SoongVarsFile() string { 1547 return filepath.Join(c.SoongOutDir(), "soong.variables") 1548} 1549 1550func (c *configImpl) SoongNinjaFile() string { 1551 return filepath.Join(c.SoongOutDir(), "build.ninja") 1552} 1553 1554func (c *configImpl) CombinedNinjaFile() string { 1555 if c.katiSuffix == "" { 1556 return filepath.Join(c.OutDir(), "combined.ninja") 1557 } 1558 return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja") 1559} 1560 1561func (c *configImpl) SoongAndroidMk() string { 1562 return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk") 1563} 1564 1565func (c *configImpl) SoongMakeVarsMk() string { 1566 return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk") 1567} 1568 1569func (c *configImpl) ProductOut() string { 1570 return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice()) 1571} 1572 1573func (c *configImpl) DevicePreviousProductConfig() string { 1574 return filepath.Join(c.ProductOut(), "previous_build_config.mk") 1575} 1576 1577func (c *configImpl) KatiPackageMkDir() string { 1578 return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging") 1579} 1580 1581func (c *configImpl) hostOutRoot() string { 1582 return filepath.Join(c.OutDir(), "host") 1583} 1584 1585func (c *configImpl) HostOut() string { 1586 return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag()) 1587} 1588 1589// This probably needs to be multi-valued, so not exporting it for now 1590func (c *configImpl) hostCrossOut() string { 1591 if runtime.GOOS == "linux" { 1592 return filepath.Join(c.hostOutRoot(), "windows-x86") 1593 } else { 1594 return "" 1595 } 1596} 1597 1598func (c *configImpl) HostPrebuiltTag() string { 1599 if runtime.GOOS == "linux" { 1600 return "linux-x86" 1601 } else if runtime.GOOS == "darwin" { 1602 return "darwin-x86" 1603 } else { 1604 panic("Unsupported OS") 1605 } 1606} 1607 1608func (c *configImpl) PrebuiltBuildTool(name string) string { 1609 if v, ok := c.environ.Get("SANITIZE_HOST"); ok { 1610 if sanitize := strings.Fields(v); inList("address", sanitize) { 1611 asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name) 1612 if _, err := os.Stat(asan); err == nil { 1613 return asan 1614 } 1615 } 1616 } 1617 return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name) 1618} 1619 1620func (c *configImpl) SetBuildBrokenDupRules(val bool) { 1621 c.brokenDupRules = val 1622} 1623 1624func (c *configImpl) BuildBrokenDupRules() bool { 1625 return c.brokenDupRules 1626} 1627 1628func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) { 1629 c.brokenUsesNetwork = val 1630} 1631 1632func (c *configImpl) BuildBrokenUsesNetwork() bool { 1633 return c.brokenUsesNetwork 1634} 1635 1636func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) { 1637 c.brokenNinjaEnvVars = val 1638} 1639 1640func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string { 1641 return c.brokenNinjaEnvVars 1642} 1643 1644func (c *configImpl) SetTargetDeviceDir(dir string) { 1645 c.targetDeviceDir = dir 1646} 1647 1648func (c *configImpl) TargetDeviceDir() string { 1649 return c.targetDeviceDir 1650} 1651 1652func (c *configImpl) BuildDateTime() string { 1653 return c.buildDateTime 1654} 1655 1656func (c *configImpl) MetricsUploaderApp() string { 1657 return c.metricsUploader 1658} 1659 1660// LogsDir returns the absolute path to the logs directory where build log and 1661// metrics files are located. By default, the logs directory is the out 1662// directory. If the argument dist is specified, the logs directory 1663// is <dist_dir>/logs. 1664func (c *configImpl) LogsDir() string { 1665 dir := c.OutDir() 1666 if c.Dist() { 1667 // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files 1668 dir = filepath.Join(c.RealDistDir(), "logs") 1669 } 1670 absDir, err := filepath.Abs(dir) 1671 if err != nil { 1672 fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error()) 1673 os.Exit(1) 1674 } 1675 return absDir 1676} 1677 1678// BazelMetricsDir returns the <logs dir>/bazel_metrics directory 1679// where the bazel profiles are located. 1680func (c *configImpl) BazelMetricsDir() string { 1681 return filepath.Join(c.LogsDir(), "bazel_metrics") 1682} 1683 1684// MkFileMetrics returns the file path for make-related metrics. 1685func (c *configImpl) MkMetrics() string { 1686 return filepath.Join(c.LogsDir(), "mk_metrics.pb") 1687} 1688 1689func (c *configImpl) SetEmptyNinjaFile(v bool) { 1690 c.emptyNinjaFile = v 1691} 1692 1693func (c *configImpl) EmptyNinjaFile() bool { 1694 return c.emptyNinjaFile 1695} 1696 1697func (c *configImpl) IsBazelMixedBuildForceDisabled() bool { 1698 return c.Environment().IsEnvTrue("BUILD_BROKEN_DISABLE_BAZEL") 1699} 1700 1701func (c *configImpl) IsPersistentBazelEnabled() bool { 1702 return c.Environment().IsEnvTrue("USE_PERSISTENT_BAZEL") 1703} 1704 1705func (c *configImpl) BazelModulesForceEnabledByFlag() string { 1706 return c.bazelForceEnabledModules 1707} 1708 1709func (c *configImpl) SkipMetricsUpload() bool { 1710 return c.skipMetricsUpload 1711} 1712 1713// Returns a Time object if one was passed via a command-line flag. 1714// Otherwise returns the passed default. 1715func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time { 1716 if c.buildStartedTime == 0 { 1717 return defaultTime 1718 } 1719 return time.UnixMilli(c.buildStartedTime) 1720} 1721 1722func GetMetricsUploader(topDir string, env *Environment) string { 1723 if p, ok := env.Get("METRICS_UPLOADER"); ok { 1724 metricsUploader := filepath.Join(topDir, p) 1725 if _, err := os.Stat(metricsUploader); err == nil { 1726 return metricsUploader 1727 } 1728 } 1729 1730 return "" 1731} 1732