1// Copyright 2017 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package build 16 17import ( 18 "encoding/json" 19 "errors" 20 "fmt" 21 "io/ioutil" 22 "math/rand" 23 "os" 24 "os/exec" 25 "os/user" 26 "path/filepath" 27 "runtime" 28 "strconv" 29 "strings" 30 "syscall" 31 "time" 32 33 "android/soong/finder/fs" 34 "android/soong/shared" 35 "android/soong/ui/metrics" 36 37 "google.golang.org/protobuf/proto" 38 39 smpb "android/soong/ui/metrics/metrics_proto" 40) 41 42const ( 43 envConfigDir = "vendor/google/tools/soong_config" 44 jsonSuffix = "json" 45 abfsSrcDir = "/src" 46) 47 48var ( 49 rbeRandPrefix int 50 googleProdCredsExistCache bool 51) 52 53func init() { 54 rand.Seed(time.Now().UnixNano()) 55 rbeRandPrefix = rand.Intn(1000) 56} 57 58// Which builder are we using? 59type ninjaCommandType = int 60 61const ( 62 _ = iota 63 NINJA_NINJA 64 NINJA_N2 65 NINJA_SISO 66 NINJA_NINJAGO 67) 68 69type Config struct{ *configImpl } 70 71type configImpl struct { 72 // Some targets that are implemented in soong_build 73 arguments []string 74 goma bool 75 environ *Environment 76 distDir string 77 buildDateTime string 78 logsPrefix string 79 80 // From the arguments 81 parallel int 82 keepGoing int 83 verbose bool 84 checkbuild bool 85 dist bool 86 jsonModuleGraph bool 87 reportMkMetrics bool // Collect and report mk2bp migration progress metrics. 88 soongDocs bool 89 skipConfig bool 90 // Either the user or product config requested that we skip soong (for the banner). The other 91 // skip flags tell whether *this* soong_ui invocation will skip kati - which will be true 92 // during lunch. 93 soongOnlyRequested bool 94 skipKati bool 95 skipKatiControlledByFlags bool 96 skipKatiNinja bool 97 skipSoong bool 98 skipNinja bool 99 skipSoongTests bool 100 searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces 101 skipMetricsUpload bool 102 buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time 103 buildFromSourceStub bool 104 incrementalBuildActions bool 105 ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built 106 107 // From the product config 108 katiArgs []string 109 ninjaArgs []string 110 katiSuffix string 111 targetDevice string 112 targetDeviceDir string 113 sandboxConfig *SandboxConfig 114 115 // Autodetected 116 totalRAM uint64 117 systemCpuInfo *metrics.CpuInfo 118 systemMemInfo *metrics.MemInfo 119 120 brokenDupRules bool 121 brokenUsesNetwork bool 122 brokenNinjaEnvVars []string 123 brokenMissingOutputs bool 124 125 pathReplaced bool 126 127 // Set by multiproduct_kati 128 emptyNinjaFile bool 129 130 metricsUploader string 131 132 includeTags []string 133 sourceRootDirs []string 134 135 // Data source to write ninja weight list 136 ninjaWeightListSource NinjaWeightListSource 137 138 // This file is a detailed dump of all soong-defined modules for debugging purposes. 139 // There's quite a bit of overlap with module-info.json and soong module graph. We 140 // could consider merging them. 141 moduleDebugFile string 142 143 // Which builder are we using 144 ninjaCommand ninjaCommandType 145} 146 147type NinjaWeightListSource uint 148 149const ( 150 // ninja doesn't use weight list. 151 NOT_USED NinjaWeightListSource = iota 152 // ninja uses weight list based on previous builds by ninja log 153 NINJA_LOG 154 // ninja thinks every task has the same weight. 155 EVENLY_DISTRIBUTED 156 // ninja uses an external custom weight list 157 EXTERNAL_FILE 158 // ninja uses a prioritized module list from Soong 159 HINT_FROM_SOONG 160 // If ninja log exists, use NINJA_LOG, if not, use HINT_FROM_SOONG instead. 161 // We can assume it is an incremental build if ninja log exists. 162 DEFAULT 163) 164const srcDirFileCheck = "build/soong/root.bp" 165 166var buildFiles = []string{"Android.mk", "Android.bp"} 167 168type BuildAction uint 169 170const ( 171 // Builds all of the modules and their dependencies of a specified directory, relative to the root 172 // directory of the source tree. 173 BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota 174 175 // Builds all of the modules and their dependencies of a list of specified directories. All specified 176 // directories are relative to the root directory of the source tree. 177 BUILD_MODULES_IN_DIRECTORIES 178 179 // Build a list of specified modules. If none was specified, simply build the whole source tree. 180 BUILD_MODULES 181) 182 183// checkTopDir validates that the current directory is at the root directory of the source tree. 184func checkTopDir(ctx Context) { 185 if _, err := os.Stat(srcDirFileCheck); err != nil { 186 if os.IsNotExist(err) { 187 ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck) 188 } 189 ctx.Fatalln("Error verifying tree state:", err) 190 } 191} 192 193func loadEnvConfig(ctx Context, config *configImpl, bc string) error { 194 if bc == "" { 195 return nil 196 } 197 198 configDirs := []string{ 199 config.OutDir(), 200 os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"), 201 envConfigDir, 202 } 203 for _, dir := range configDirs { 204 cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix)) 205 envVarsJSON, err := ioutil.ReadFile(cfgFile) 206 if err != nil { 207 continue 208 } 209 ctx.Verbosef("Loading config file %v\n", cfgFile) 210 var envVars map[string]map[string]string 211 if err := json.Unmarshal(envVarsJSON, &envVars); err != nil { 212 fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error()) 213 continue 214 } 215 for k, v := range envVars["env"] { 216 if os.Getenv(k) != "" { 217 continue 218 } 219 config.environ.Set(k, v) 220 } 221 ctx.Verbosef("Finished loading config file %v\n", cfgFile) 222 break 223 } 224 225 return nil 226} 227 228func NewConfig(ctx Context, args ...string) Config { 229 ret := &configImpl{ 230 environ: OsEnvironment(), 231 sandboxConfig: &SandboxConfig{}, 232 ninjaWeightListSource: DEFAULT, 233 } 234 wd, err := os.Getwd() 235 if err != nil { 236 ctx.Fatalln("Failed to get working directory:", err) 237 } 238 239 // Skip soong tests by default on Linux 240 if runtime.GOOS == "linux" { 241 ret.skipSoongTests = true 242 } 243 244 // Default matching ninja 245 ret.parallel = runtime.NumCPU() + 2 246 ret.keepGoing = 1 247 248 ret.totalRAM = detectTotalRAM(ctx) 249 ret.systemCpuInfo, err = metrics.NewCpuInfo(fs.OsFs) 250 if err != nil { 251 ctx.Fatalln("Failed to get cpuinfo:", err) 252 } 253 ret.systemMemInfo, err = metrics.NewMemInfo(fs.OsFs) 254 if err != nil { 255 ctx.Fatalln("Failed to get meminfo:", err) 256 } 257 ret.parseArgs(ctx, args) 258 259 if value, ok := ret.environ.Get("SOONG_ONLY"); ok && !ret.skipKatiControlledByFlags { 260 if value == "true" || value == "1" || value == "y" || value == "yes" { 261 ret.soongOnlyRequested = true 262 ret.skipKatiControlledByFlags = true 263 ret.skipKati = true 264 ret.skipKatiNinja = true 265 } else { 266 ret.skipKatiControlledByFlags = true 267 ret.skipKati = false 268 ret.skipKatiNinja = false 269 } 270 } 271 272 if ret.ninjaWeightListSource == HINT_FROM_SOONG { 273 ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "always") 274 } else if ret.ninjaWeightListSource == DEFAULT { 275 defaultNinjaWeightListSource := NINJA_LOG 276 if _, err := os.Stat(filepath.Join(ret.OutDir(), ninjaLogFileName)); errors.Is(err, os.ErrNotExist) { 277 ctx.Verboseln("$OUT/.ninja_log doesn't exist, use HINT_FROM_SOONG instead") 278 defaultNinjaWeightListSource = HINT_FROM_SOONG 279 } else { 280 ctx.Verboseln("$OUT/.ninja_log exist, use NINJA_LOG") 281 } 282 ret.ninjaWeightListSource = defaultNinjaWeightListSource 283 // soong_build generates ninja hint depending on ninja log existence. 284 // Set it "depend" to avoid soong re-run due to env variable change. 285 ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "depend") 286 } 287 288 // Make sure OUT_DIR is set appropriately 289 if outDir, ok := ret.environ.Get("OUT_DIR"); ok { 290 ret.environ.Set("OUT_DIR", ret.sandboxPath(wd, filepath.Clean(outDir))) 291 } else { 292 outDir := "out" 293 if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok { 294 outDir = filepath.Join(baseDir, filepath.Base(wd)) 295 } 296 ret.environ.Set("OUT_DIR", ret.sandboxPath(wd, outDir)) 297 } 298 299 // loadEnvConfig needs to know what the OUT_DIR is, so it should 300 // be called after we determine the appropriate out directory. 301 bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG") 302 303 if bc != "" { 304 if err := loadEnvConfig(ctx, ret, bc); err != nil { 305 ctx.Fatalln("Failed to parse env config files: %v", err) 306 } 307 if !ret.canSupportRBE() { 308 // Explicitly set USE_RBE env variable to false when we cannot run 309 // an RBE build to avoid ninja local execution pool issues. 310 ret.environ.Set("USE_RBE", "false") 311 } 312 } 313 314 if distDir, ok := ret.environ.Get("DIST_DIR"); ok { 315 ret.distDir = filepath.Clean(distDir) 316 } else { 317 ret.distDir = filepath.Join(ret.OutDir(), "dist") 318 } 319 320 if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok { 321 ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false") 322 } 323 324 if os.Getenv("GENERATE_SOONG_DEBUG") == "true" { 325 ret.moduleDebugFile, _ = filepath.Abs(shared.JoinPath(ret.SoongOutDir(), "soong-debug-info.json")) 326 } 327 328 // If SOONG_USE_PARTIAL_COMPILE is set, make it one of "true" or the empty string. 329 // This simplifies the generated Ninja rules, so that they only need to check for the empty string. 330 if value, ok := ret.environ.Get("SOONG_USE_PARTIAL_COMPILE"); ok { 331 if value == "true" || value == "1" || value == "y" || value == "yes" { 332 value = "true" 333 } else { 334 value = "" 335 } 336 ret.environ.Set("SOONG_USE_PARTIAL_COMPILE", value) 337 } 338 339 ret.ninjaCommand = NINJA_NINJA 340 switch os.Getenv("SOONG_NINJA") { 341 case "n2": 342 ret.ninjaCommand = NINJA_N2 343 case "siso": 344 ret.ninjaCommand = NINJA_SISO 345 case "ninjago": 346 ret.ninjaCommand = NINJA_NINJAGO 347 default: 348 if os.Getenv("SOONG_USE_N2") == "true" { 349 ret.ninjaCommand = NINJA_N2 350 } 351 } 352 353 ret.environ.Unset( 354 // We're already using it 355 "USE_SOONG_UI", 356 357 // We should never use GOROOT/GOPATH from the shell environment 358 "GOROOT", 359 "GOPATH", 360 361 // These should only come from Soong, not the environment. 362 "CLANG", 363 "CLANG_CXX", 364 "CCC_CC", 365 "CCC_CXX", 366 367 // Used by the goma compiler wrapper, but should only be set by 368 // gomacc 369 "GOMACC_PATH", 370 371 // We handle this above 372 "OUT_DIR_COMMON_BASE", 373 374 // This is handled above too, and set for individual commands later 375 "DIST_DIR", 376 377 // Variables that have caused problems in the past 378 "BASH_ENV", 379 "CDPATH", 380 "DISPLAY", 381 "GREP_OPTIONS", 382 "JAVAC", 383 "LEX", 384 "NDK_ROOT", 385 "POSIXLY_CORRECT", 386 387 // Drop make flags 388 "MAKEFLAGS", 389 "MAKELEVEL", 390 "MFLAGS", 391 392 // Set in envsetup.sh, reset in makefiles 393 "ANDROID_JAVA_TOOLCHAIN", 394 395 // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional 396 "ANDROID_BUILD_TOP", 397 "ANDROID_HOST_OUT", 398 "ANDROID_PRODUCT_OUT", 399 "ANDROID_HOST_OUT_TESTCASES", 400 "ANDROID_TARGET_OUT_TESTCASES", 401 "ANDROID_TOOLCHAIN", 402 "ANDROID_TOOLCHAIN_2ND_ARCH", 403 "ANDROID_DEV_SCRIPTS", 404 "ANDROID_EMULATOR_PREBUILTS", 405 "ANDROID_PRE_BUILD_PATHS", 406 407 // We read it here already, don't let others share in the fun 408 "GENERATE_SOONG_DEBUG", 409 410 // Use config.ninjaCommand instead. 411 "SOONG_NINJA", 412 "SOONG_USE_N2", 413 414 // Already incorporated into the config object 415 "SOONG_ONLY", 416 ) 417 418 if ret.UseGoma() || ret.ForceUseGoma() { 419 ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.") 420 ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.") 421 } 422 423 // Tell python not to spam the source tree with .pyc files. 424 ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1") 425 426 tmpDir := absPath(ctx, ret.TempDir()) 427 ret.environ.Set("TMPDIR", ret.sandboxPath(wd, tmpDir)) 428 429 // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes 430 symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(), 431 "llvm-binutils-stable/llvm-symbolizer") 432 ret.environ.Set("ASAN_SYMBOLIZER_PATH", ret.sandboxPath(wd, absPath(ctx, symbolizerPath))) 433 434 // Precondition: the current directory is the top of the source tree 435 checkTopDir(ctx) 436 437 srcDir := absPath(ctx, ".") 438 if strings.ContainsRune(srcDir, ' ') { 439 ctx.Println("You are building in a directory whose absolute path contains a space character:") 440 ctx.Println() 441 ctx.Printf("%q\n", srcDir) 442 ctx.Println() 443 ctx.Fatalln("Directory names containing spaces are not supported") 444 } 445 446 ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ) 447 448 if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') { 449 ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:") 450 ctx.Println() 451 ctx.Printf("%q\n", outDir) 452 ctx.Println() 453 ctx.Fatalln("Directory names containing spaces are not supported") 454 } 455 456 if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') { 457 ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:") 458 ctx.Println() 459 ctx.Printf("%q\n", distDir) 460 ctx.Println() 461 ctx.Fatalln("Directory names containing spaces are not supported") 462 } 463 464 // Configure Java-related variables, including adding it to $PATH 465 java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag()) 466 java21Home := filepath.Join("prebuilts/jdk/jdk21", ret.HostPrebuiltTag()) 467 javaHome := func() string { 468 if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok { 469 return override 470 } 471 if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" { 472 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.") 473 } 474 if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" { 475 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.") 476 } 477 if toolchain21, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK21_TOOLCHAIN"); ok && toolchain21 != "true" { 478 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK21_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.") 479 } 480 return java21Home 481 }() 482 absJavaHome := absPath(ctx, javaHome) 483 484 ret.configureLocale(ctx) 485 486 newPath := []string{filepath.Join(absJavaHome, "bin")} 487 if path, ok := ret.environ.Get("PATH"); ok && path != "" { 488 newPath = append(newPath, path) 489 } 490 491 ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME") 492 ret.environ.Set("JAVA_HOME", ret.sandboxPath(wd, absJavaHome)) 493 ret.environ.Set("ANDROID_JAVA_HOME", ret.sandboxPath(wd, javaHome)) 494 ret.environ.Set("ANDROID_JAVA8_HOME", ret.sandboxPath(wd, java8Home)) 495 ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator))) 496 497 // b/286885495, https://bugzilla.redhat.com/show_bug.cgi?id=2227130: some versions of Fedora include patches 498 // to unzip to enable zipbomb detection that incorrectly handle zip64 and data descriptors and fail on large 499 // zip files produced by soong_zip. Disable zipbomb detection. 500 ret.environ.Set("UNZIP_DISABLE_ZIPBOMB_DETECTION", "TRUE") 501 502 outDir := ret.OutDir() 503 buildDateTimeFile := filepath.Join(outDir, "build_date.txt") 504 if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" { 505 ret.buildDateTime = buildDateTime 506 } else { 507 ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10) 508 } 509 510 ret.environ.Set("BUILD_DATETIME_FILE", ret.sandboxPath(wd, buildDateTimeFile)) 511 512 if _, ok := ret.environ.Get("BUILD_USERNAME"); !ok { 513 username := "unknown" 514 if u, err := user.Current(); err == nil { 515 username = u.Username 516 } else { 517 ctx.Println("Failed to get current user:", err) 518 } 519 ret.environ.Set("BUILD_USERNAME", username) 520 } 521 ret.environ.Set("PWD", ret.sandboxPath(wd, wd)) 522 523 if ret.UseRBE() { 524 for k, v := range getRBEVars(ctx, Config{ret}) { 525 ret.environ.Set(k, v) 526 } 527 } 528 529 c := Config{ret} 530 storeConfigMetrics(ctx, c) 531 return c 532} 533 534// NewBuildActionConfig returns a build configuration based on the build action. The arguments are 535// processed based on the build action and extracts any arguments that belongs to the build action. 536func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config { 537 return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...) 538} 539 540// Prepare for getting make variables. For them to be accurate, we need to have 541// obtained PRODUCT_RELEASE_CONFIG_MAPS. 542// 543// Returns: 544// 545// Whether config should be called again. 546// 547// TODO: when converting product config to a declarative language, make sure 548// that PRODUCT_RELEASE_CONFIG_MAPS is properly handled as a separate step in 549// that process. 550func SetProductReleaseConfigMaps(ctx Context, config Config) bool { 551 ctx.BeginTrace(metrics.RunKati, "SetProductReleaseConfigMaps") 552 defer ctx.EndTrace() 553 554 if config.SkipConfig() { 555 // This duplicates the logic from Build to skip product config 556 // if the user has explicitly said to. 557 return false 558 } 559 560 releaseConfigVars := []string{ 561 "PRODUCT_RELEASE_CONFIG_MAPS", 562 } 563 564 origValue, _ := config.environ.Get("PRODUCT_RELEASE_CONFIG_MAPS") 565 // Get the PRODUCT_RELEASE_CONFIG_MAPS for this product, to avoid polluting the environment 566 // when we run product config to get the rest of the make vars. 567 releaseMapVars, err := dumpMakeVars(ctx, config, nil, releaseConfigVars, false, "") 568 if err != nil { 569 ctx.Fatalln("Error getting PRODUCT_RELEASE_CONFIG_MAPS:", err) 570 } 571 productReleaseConfigMaps := releaseMapVars["PRODUCT_RELEASE_CONFIG_MAPS"] 572 os.Setenv("PRODUCT_RELEASE_CONFIG_MAPS", productReleaseConfigMaps) 573 return origValue != productReleaseConfigMaps 574} 575 576// storeConfigMetrics selects a set of configuration information and store in 577// the metrics system for further analysis. 578func storeConfigMetrics(ctx Context, config Config) { 579 if ctx.Metrics == nil { 580 return 581 } 582 583 ctx.Metrics.BuildConfig(buildConfig(config)) 584 585 cpuInfo := &smpb.SystemCpuInfo{ 586 VendorId: proto.String(config.systemCpuInfo.VendorId), 587 ModelName: proto.String(config.systemCpuInfo.ModelName), 588 CpuCores: proto.Int32(config.systemCpuInfo.CpuCores), 589 Flags: proto.String(config.systemCpuInfo.Flags), 590 } 591 memInfo := &smpb.SystemMemInfo{ 592 MemTotal: proto.Uint64(config.systemMemInfo.MemTotal), 593 MemFree: proto.Uint64(config.systemMemInfo.MemFree), 594 MemAvailable: proto.Uint64(config.systemMemInfo.MemAvailable), 595 } 596 597 s := &smpb.SystemResourceInfo{ 598 TotalPhysicalMemory: proto.Uint64(config.TotalRAM()), 599 AvailableCpus: proto.Int32(int32(runtime.NumCPU())), 600 CpuInfo: cpuInfo, 601 MemInfo: memInfo, 602 } 603 ctx.Metrics.SystemResourceInfo(s) 604} 605 606func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource { 607 switch s { 608 case NINJA_LOG: 609 return smpb.BuildConfig_NINJA_LOG.Enum() 610 case EVENLY_DISTRIBUTED: 611 return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum() 612 case EXTERNAL_FILE: 613 return smpb.BuildConfig_EXTERNAL_FILE.Enum() 614 case HINT_FROM_SOONG: 615 return smpb.BuildConfig_HINT_FROM_SOONG.Enum() 616 default: 617 return smpb.BuildConfig_NOT_USED.Enum() 618 } 619} 620 621func buildConfig(config Config) *smpb.BuildConfig { 622 var soongEnvVars *smpb.SoongEnvVars 623 ensure := func() *smpb.SoongEnvVars { 624 // Create soongEnvVars if it doesn't already exist. 625 if soongEnvVars == nil { 626 soongEnvVars = &smpb.SoongEnvVars{} 627 } 628 return soongEnvVars 629 } 630 if value, ok := config.environ.Get("SOONG_PARTIAL_COMPILE"); ok { 631 ensure().PartialCompile = proto.String(value) 632 } 633 if value, ok := config.environ.Get("SOONG_USE_PARTIAL_COMPILE"); ok { 634 ensure().UsePartialCompile = proto.String(value) 635 } 636 c := &smpb.BuildConfig{ 637 ForceUseGoma: proto.Bool(config.ForceUseGoma()), 638 UseGoma: proto.Bool(config.UseGoma()), 639 UseRbe: proto.Bool(config.UseRBE()), 640 NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()), 641 SoongEnvVars: soongEnvVars, 642 SoongOnly: proto.Bool(config.soongOnlyRequested), 643 } 644 c.Targets = append(c.Targets, config.arguments...) 645 646 return c 647} 648 649// getConfigArgs processes the command arguments based on the build action and creates a set of new 650// arguments to be accepted by Config. 651func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string { 652 // The next block of code verifies that the current directory is the root directory of the source 653 // tree. It then finds the relative path of dir based on the root directory of the source tree 654 // and verify that dir is inside of the source tree. 655 checkTopDir(ctx) 656 topDir, err := os.Getwd() 657 if err != nil { 658 ctx.Fatalf("Error retrieving top directory: %v", err) 659 } 660 dir, err = filepath.EvalSymlinks(dir) 661 if err != nil { 662 ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err) 663 } 664 dir, err = filepath.Abs(dir) 665 if err != nil { 666 ctx.Fatalf("Unable to find absolute path %s: %v", dir, err) 667 } 668 relDir, err := filepath.Rel(topDir, dir) 669 if err != nil { 670 ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err) 671 } 672 // If there are ".." in the path, it's not in the source tree. 673 if strings.Contains(relDir, "..") { 674 ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir) 675 } 676 677 configArgs := args[:] 678 679 // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to 680 // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules. 681 targetNamePrefix := "MODULES-IN-" 682 if inList("GET-INSTALL-PATH", configArgs) { 683 targetNamePrefix = "GET-INSTALL-PATH-IN-" 684 configArgs = removeFromList("GET-INSTALL-PATH", configArgs) 685 } 686 687 var targets []string 688 689 switch action { 690 case BUILD_MODULES: 691 // No additional processing is required when building a list of specific modules or all modules. 692 case BUILD_MODULES_IN_A_DIRECTORY: 693 // If dir is the root source tree, all the modules are built of the source tree are built so 694 // no need to find the build file. 695 if topDir == dir { 696 break 697 } 698 699 buildFile := findBuildFile(ctx, relDir) 700 if buildFile == "" { 701 ctx.Fatalf("Build file not found for %s directory", relDir) 702 } 703 targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} 704 case BUILD_MODULES_IN_DIRECTORIES: 705 newConfigArgs, dirs := splitArgs(configArgs) 706 configArgs = newConfigArgs 707 targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix) 708 } 709 710 // Tidy only override all other specified targets. 711 tidyOnly := os.Getenv("WITH_TIDY_ONLY") 712 if tidyOnly == "true" || tidyOnly == "1" { 713 configArgs = append(configArgs, "tidy_only") 714 } else { 715 configArgs = append(configArgs, targets...) 716 } 717 718 return configArgs 719} 720 721// convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name. 722func convertToTarget(dir string, targetNamePrefix string) string { 723 return targetNamePrefix + strings.ReplaceAll(dir, "/", "-") 724} 725 726// hasBuildFile returns true if dir contains an Android build file. 727func hasBuildFile(ctx Context, dir string) bool { 728 for _, buildFile := range buildFiles { 729 _, err := os.Stat(filepath.Join(dir, buildFile)) 730 if err == nil { 731 return true 732 } 733 if !os.IsNotExist(err) { 734 ctx.Fatalf("Error retrieving the build file stats: %v", err) 735 } 736 } 737 return false 738} 739 740// findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file 741// in the current and any sub directory of dir. If a build file is not found, traverse the path 742// up by one directory and repeat again until either a build file is found or reached to the root 743// source tree. The returned filename of build file is "Android.mk". If one was not found, a blank 744// string is returned. 745func findBuildFile(ctx Context, dir string) string { 746 // If the string is empty or ".", assume it is top directory of the source tree. 747 if dir == "" || dir == "." { 748 return "" 749 } 750 751 found := false 752 for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) { 753 err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error { 754 if err != nil { 755 return err 756 } 757 if found { 758 return filepath.SkipDir 759 } 760 if info.IsDir() { 761 return nil 762 } 763 for _, buildFile := range buildFiles { 764 if info.Name() == buildFile { 765 found = true 766 return filepath.SkipDir 767 } 768 } 769 return nil 770 }) 771 if err != nil { 772 ctx.Fatalf("Error finding Android build file: %v", err) 773 } 774 775 if found { 776 return filepath.Join(buildDir, "Android.mk") 777 } 778 } 779 780 return "" 781} 782 783// splitArgs iterates over the arguments list and splits into two lists: arguments and directories. 784func splitArgs(args []string) (newArgs []string, dirs []string) { 785 specialArgs := map[string]bool{ 786 "showcommands": true, 787 "snod": true, 788 "dist": true, 789 "checkbuild": true, 790 } 791 792 newArgs = []string{} 793 dirs = []string{} 794 795 for _, arg := range args { 796 // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory. 797 if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 { 798 newArgs = append(newArgs, arg) 799 continue 800 } 801 802 if _, ok := specialArgs[arg]; ok { 803 newArgs = append(newArgs, arg) 804 continue 805 } 806 807 dirs = append(dirs, arg) 808 } 809 810 return newArgs, dirs 811} 812 813// getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a 814// directory from the dirs list does not exist, a fatal error is raised. relDir is related to the 815// source root tree where the build action command was invoked. Each directory is validated if the 816// build file can be found and follows the format "dir1:target1,target2,...". Target is optional. 817func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) { 818 for _, dir := range dirs { 819 // The directory may have specified specific modules to build. ":" is the separator to separate 820 // the directory and the list of modules. 821 s := strings.Split(dir, ":") 822 l := len(s) 823 if l > 2 { // more than one ":" was specified. 824 ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir) 825 } 826 827 dir = filepath.Join(relDir, s[0]) 828 if _, err := os.Stat(dir); err != nil { 829 ctx.Fatalf("couldn't find directory %s", dir) 830 } 831 832 // Verify that if there are any targets specified after ":". Each target is separated by ",". 833 var newTargets []string 834 if l == 2 && s[1] != "" { 835 newTargets = strings.Split(s[1], ",") 836 if inList("", newTargets) { 837 ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir) 838 } 839 } 840 841 // If there are specified targets to build in dir, an android build file must exist for the one 842 // shot build. For the non-targets case, find the appropriate build file and build all the 843 // modules in dir (or the closest one in the dir path). 844 if len(newTargets) > 0 { 845 if !hasBuildFile(ctx, dir) { 846 ctx.Fatalf("Couldn't locate a build file from %s directory", dir) 847 } 848 } else { 849 buildFile := findBuildFile(ctx, dir) 850 if buildFile == "" { 851 ctx.Fatalf("Build file not found for %s directory", dir) 852 } 853 newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} 854 } 855 856 targets = append(targets, newTargets...) 857 } 858 859 return targets 860} 861 862func (c *configImpl) parseArgs(ctx Context, args []string) { 863 for i := 0; i < len(args); i++ { 864 arg := strings.TrimSpace(args[i]) 865 if arg == "showcommands" { 866 c.verbose = true 867 } else if arg == "--empty-ninja-file" { 868 c.emptyNinjaFile = true 869 } else if arg == "--skip-ninja" { 870 c.skipNinja = true 871 } else if arg == "--soong-only" { 872 if c.skipKatiControlledByFlags { 873 ctx.Fatalf("Cannot specify both --soong-only and --no-soong-only") 874 } 875 c.soongOnlyRequested = true 876 c.skipKatiControlledByFlags = true 877 c.skipKati = true 878 c.skipKatiNinja = true 879 } else if arg == "--no-soong-only" { 880 if c.skipKatiControlledByFlags { 881 ctx.Fatalf("Cannot specify both --soong-only and --no-soong-only") 882 } 883 c.skipKatiControlledByFlags = true 884 c.skipKati = false 885 c.skipKatiNinja = false 886 } else if arg == "--config-only" { 887 c.skipKati = true 888 c.skipKatiNinja = true 889 c.skipSoong = true 890 } else if arg == "--skip-config" { 891 c.skipConfig = true 892 } else if arg == "--skip-soong-tests" { 893 c.skipSoongTests = true 894 } else if arg == "--no-skip-soong-tests" { 895 c.skipSoongTests = false 896 } else if arg == "--skip-metrics-upload" { 897 c.skipMetricsUpload = true 898 } else if arg == "--mk-metrics" { 899 c.reportMkMetrics = true 900 } else if arg == "--search-api-dir" { 901 c.searchApiDir = true 902 } else if strings.HasPrefix(arg, "--ninja_weight_source=") { 903 source := strings.TrimPrefix(arg, "--ninja_weight_source=") 904 if source == "ninja_log" { 905 c.ninjaWeightListSource = NINJA_LOG 906 } else if source == "evenly_distributed" { 907 c.ninjaWeightListSource = EVENLY_DISTRIBUTED 908 } else if source == "not_used" { 909 c.ninjaWeightListSource = NOT_USED 910 } else if source == "soong" { 911 c.ninjaWeightListSource = HINT_FROM_SOONG 912 } else if strings.HasPrefix(source, "file,") { 913 c.ninjaWeightListSource = EXTERNAL_FILE 914 filePath := strings.TrimPrefix(source, "file,") 915 err := validateNinjaWeightList(filePath) 916 if err != nil { 917 ctx.Fatalf("Malformed weight list from %s: %s", filePath, err) 918 } 919 _, err = copyFile(filePath, filepath.Join(c.OutDir(), ".ninja_weight_list")) 920 if err != nil { 921 ctx.Fatalf("Error to copy ninja weight list from %s: %s", filePath, err) 922 } 923 } else { 924 ctx.Fatalf("unknown option for ninja_weight_source: %s", source) 925 } 926 } else if arg == "--build-from-source-stub" { 927 c.buildFromSourceStub = true 928 } else if arg == "--incremental-build-actions" { 929 c.incrementalBuildActions = true 930 } else if strings.HasPrefix(arg, "--build-command=") { 931 buildCmd := strings.TrimPrefix(arg, "--build-command=") 932 // remove quotations 933 buildCmd = strings.TrimPrefix(buildCmd, "\"") 934 buildCmd = strings.TrimSuffix(buildCmd, "\"") 935 ctx.Metrics.SetBuildCommand([]string{buildCmd}) 936 } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") { 937 buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=") 938 val, err := strconv.ParseInt(buildTimeStr, 10, 64) 939 if err == nil { 940 c.buildStartedTime = val 941 } else { 942 ctx.Fatalf("Error parsing build-time-started-unix-millis", err) 943 } 944 } else if arg == "--ensure-allowlist-integrity" { 945 c.ensureAllowlistIntegrity = true 946 } else if len(arg) > 0 && arg[0] == '-' { 947 parseArgNum := func(def int) int { 948 if len(arg) > 2 { 949 p, err := strconv.ParseUint(arg[2:], 10, 31) 950 if err != nil { 951 ctx.Fatalf("Failed to parse %q: %v", arg, err) 952 } 953 return int(p) 954 } else if i+1 < len(args) { 955 p, err := strconv.ParseUint(args[i+1], 10, 31) 956 if err == nil { 957 i++ 958 return int(p) 959 } 960 } 961 return def 962 } 963 964 if len(arg) > 1 && arg[1] == 'j' { 965 c.parallel = parseArgNum(c.parallel) 966 } else if len(arg) > 1 && arg[1] == 'k' { 967 c.keepGoing = parseArgNum(0) 968 } else { 969 ctx.Fatalln("Unknown option:", arg) 970 } 971 } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 { 972 if k == "OUT_DIR" { 973 ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.") 974 } 975 c.environ.Set(k, v) 976 } else if arg == "dist" { 977 c.dist = true 978 } else if arg == "json-module-graph" { 979 c.jsonModuleGraph = true 980 } else if arg == "soong_docs" { 981 c.soongDocs = true 982 } else { 983 if arg == "checkbuild" { 984 c.checkbuild = true 985 } 986 c.arguments = append(c.arguments, arg) 987 } 988 } 989} 990 991func validateNinjaWeightList(weightListFilePath string) (err error) { 992 data, err := os.ReadFile(weightListFilePath) 993 if err != nil { 994 return 995 } 996 lines := strings.Split(strings.TrimSpace(string(data)), "\n") 997 for _, line := range lines { 998 fields := strings.Split(line, ",") 999 if len(fields) != 2 { 1000 return fmt.Errorf("wrong format, each line should have two fields, but '%s'", line) 1001 } 1002 _, err = strconv.Atoi(fields[1]) 1003 if err != nil { 1004 return 1005 } 1006 } 1007 return 1008} 1009 1010func (c *configImpl) configureLocale(ctx Context) { 1011 cmd := Command(ctx, Config{c}, "locale", "locale", "-a") 1012 output, err := cmd.Output() 1013 1014 var locales []string 1015 if err == nil { 1016 locales = strings.Split(string(output), "\n") 1017 } else { 1018 // If we're unable to list the locales, let's assume en_US.UTF-8 1019 locales = []string{"en_US.UTF-8"} 1020 ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales) 1021 } 1022 1023 // gettext uses LANGUAGE, which is passed directly through 1024 1025 // For LANG and LC_*, only preserve the evaluated version of 1026 // LC_MESSAGES 1027 userLang := "" 1028 if lc_all, ok := c.environ.Get("LC_ALL"); ok { 1029 userLang = lc_all 1030 } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok { 1031 userLang = lc_messages 1032 } else if lang, ok := c.environ.Get("LANG"); ok { 1033 userLang = lang 1034 } 1035 1036 c.environ.UnsetWithPrefix("LC_") 1037 1038 if userLang != "" { 1039 c.environ.Set("LC_MESSAGES", userLang) 1040 } 1041 1042 // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed 1043 // for others) 1044 if inList("C.UTF-8", locales) { 1045 c.environ.Set("LANG", "C.UTF-8") 1046 } else if inList("C.utf8", locales) { 1047 // These normalize to the same thing 1048 c.environ.Set("LANG", "C.UTF-8") 1049 } else if inList("en_US.UTF-8", locales) { 1050 c.environ.Set("LANG", "en_US.UTF-8") 1051 } else if inList("en_US.utf8", locales) { 1052 // These normalize to the same thing 1053 c.environ.Set("LANG", "en_US.UTF-8") 1054 } else { 1055 ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8") 1056 } 1057} 1058 1059func (c *configImpl) Environment() *Environment { 1060 return c.environ 1061} 1062 1063func (c *configImpl) Arguments() []string { 1064 return c.arguments 1065} 1066 1067func (c *configImpl) SoongBuildInvocationNeeded() bool { 1068 if len(c.Arguments()) > 0 { 1069 // Explicit targets requested that are not special targets like b2pbuild 1070 // or the JSON module graph 1071 return true 1072 } 1073 1074 if !c.JsonModuleGraph() && !c.SoongDocs() { 1075 // Command line was empty, the default Ninja target is built 1076 return true 1077 } 1078 1079 if c.Dist() { 1080 return true 1081 } 1082 1083 // build.ninja doesn't need to be generated 1084 return false 1085} 1086 1087func (c *configImpl) OutDir() string { 1088 if outDir, ok := c.environ.Get("OUT_DIR"); ok { 1089 return outDir 1090 } 1091 return "out" 1092} 1093 1094func (c *configImpl) DistDir() string { 1095 return c.distDir 1096} 1097 1098func (c *configImpl) RealDistDir() string { 1099 return c.distDir 1100} 1101 1102func (c *configImpl) NinjaArgs() []string { 1103 if c.skipKati { 1104 return append(c.arguments, c.ninjaArgs...) 1105 } 1106 return c.ninjaArgs 1107} 1108 1109func (c *configImpl) SoongOutDir() string { 1110 return filepath.Join(c.OutDir(), "soong") 1111} 1112 1113func (c *configImpl) ApiSurfacesOutDir() string { 1114 return filepath.Join(c.OutDir(), "api_surfaces") 1115} 1116 1117func (c *configImpl) PrebuiltOS() string { 1118 switch runtime.GOOS { 1119 case "linux": 1120 switch runtime.GOARCH { 1121 case "amd64": 1122 return "linux-x86" 1123 case "arm64": 1124 return "linux-arm64" 1125 default: 1126 panic(fmt.Errorf("Unknown GOARCH %s", runtime.GOARCH)) 1127 } 1128 case "darwin": 1129 return "darwin-x86" 1130 default: 1131 panic(fmt.Errorf("Unknown GOOS %s", runtime.GOOS)) 1132 } 1133} 1134 1135func (c *configImpl) HostToolDir() string { 1136 if c.SkipKatiNinja() { 1137 return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin") 1138 } else { 1139 return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin") 1140 } 1141} 1142 1143func (c *configImpl) UsedEnvFile(tag string) string { 1144 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { 1145 return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+c.CoverageSuffix()+"."+tag) 1146 } 1147 return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag) 1148} 1149 1150func (c *configImpl) SoongDocsHtml() string { 1151 return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html") 1152} 1153 1154func (c *configImpl) ModuleGraphFile() string { 1155 return shared.JoinPath(c.SoongOutDir(), "module-graph.json") 1156} 1157 1158func (c *configImpl) ModuleActionsFile() string { 1159 return shared.JoinPath(c.SoongOutDir(), "module-actions.json") 1160} 1161 1162func (c *configImpl) TempDir() string { 1163 return shared.TempDirForOutDir(c.SoongOutDir()) 1164} 1165 1166func (c *configImpl) FileListDir() string { 1167 return filepath.Join(c.OutDir(), ".module_paths") 1168} 1169 1170func (c *configImpl) KatiSuffix() string { 1171 if c.katiSuffix != "" { 1172 return c.katiSuffix 1173 } 1174 panic("SetKatiSuffix has not been called") 1175} 1176 1177// Checkbuild returns true if "checkbuild" was one of the build goals, which means that the 1178// user is interested in additional checks at the expense of build time. 1179func (c *configImpl) Checkbuild() bool { 1180 return c.checkbuild 1181} 1182 1183func (c *configImpl) Dist() bool { 1184 return c.dist 1185} 1186 1187func (c *configImpl) JsonModuleGraph() bool { 1188 return c.jsonModuleGraph 1189} 1190 1191func (c *configImpl) SoongDocs() bool { 1192 return c.soongDocs 1193} 1194 1195func (c *configImpl) IsVerbose() bool { 1196 return c.verbose 1197} 1198 1199func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource { 1200 return c.ninjaWeightListSource 1201} 1202 1203func (c *configImpl) SkipKati() bool { 1204 return c.skipKati 1205} 1206 1207func (c *configImpl) SkipKatiNinja() bool { 1208 return c.skipKatiNinja 1209} 1210 1211func (c *configImpl) SkipSoong() bool { 1212 return c.skipSoong 1213} 1214 1215func (c *configImpl) SkipNinja() bool { 1216 return c.skipNinja 1217} 1218 1219func (c *configImpl) SetSkipNinja(v bool) { 1220 c.skipNinja = v 1221} 1222 1223func (c *configImpl) SkipConfig() bool { 1224 return c.skipConfig 1225} 1226 1227func (c *configImpl) BuildFromTextStub() bool { 1228 return !c.buildFromSourceStub 1229} 1230 1231func (c *configImpl) TargetProduct() string { 1232 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { 1233 return v 1234 } 1235 panic("TARGET_PRODUCT is not defined") 1236} 1237 1238func (c *configImpl) TargetProductOrErr() (string, error) { 1239 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { 1240 return v, nil 1241 } 1242 return "", fmt.Errorf("TARGET_PRODUCT is not defined") 1243} 1244 1245func (c *configImpl) CoverageSuffix() string { 1246 if v := c.environ.IsEnvTrue("EMMA_INSTRUMENT"); v { 1247 return ".coverage" 1248 } 1249 return "" 1250} 1251 1252func (c *configImpl) TargetDevice() string { 1253 return c.targetDevice 1254} 1255 1256func (c *configImpl) SetTargetDevice(device string) { 1257 c.targetDevice = device 1258} 1259 1260func (c *configImpl) TargetBuildVariant() string { 1261 if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok { 1262 return v 1263 } 1264 panic("TARGET_BUILD_VARIANT is not defined") 1265} 1266 1267func (c *configImpl) KatiArgs() []string { 1268 return c.katiArgs 1269} 1270 1271func (c *configImpl) Parallel() int { 1272 return c.parallel 1273} 1274 1275func (c *configImpl) GetSourceRootDirs() []string { 1276 return c.sourceRootDirs 1277} 1278 1279func (c *configImpl) SetSourceRootDirs(i []string) { 1280 c.sourceRootDirs = i 1281} 1282 1283func (c *configImpl) GetLogsPrefix() string { 1284 return c.logsPrefix 1285} 1286 1287func (c *configImpl) SetLogsPrefix(prefix string) { 1288 c.logsPrefix = prefix 1289} 1290 1291func (c *configImpl) HighmemParallel() int { 1292 if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok { 1293 return i 1294 } 1295 1296 const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024 1297 parallel := c.Parallel() 1298 if c.UseRemoteBuild() { 1299 // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism 1300 // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs 1301 // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention. 1302 // Return 1/16th of the size of the local pool, rounding up. 1303 return (parallel + 15) / 16 1304 } else if c.totalRAM == 0 { 1305 // Couldn't detect the total RAM, don't restrict highmem processes. 1306 return parallel 1307 } else if c.totalRAM <= 16*1024*1024*1024 { 1308 // Less than 16GB of ram, restrict to 1 highmem processes 1309 return 1 1310 } else if c.totalRAM <= 32*1024*1024*1024 { 1311 // Less than 32GB of ram, restrict to 2 highmem processes 1312 return 2 1313 } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel { 1314 // If less than 8GB total RAM per process, reduce the number of highmem processes 1315 return p 1316 } 1317 // No restriction on highmem processes 1318 return parallel 1319} 1320 1321func (c *configImpl) TotalRAM() uint64 { 1322 return c.totalRAM 1323} 1324 1325// ForceUseGoma determines whether we should override Goma deprecation 1326// and use Goma for the current build or not. 1327func (c *configImpl) ForceUseGoma() bool { 1328 if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok { 1329 v = strings.TrimSpace(v) 1330 if v != "" && v != "false" { 1331 return true 1332 } 1333 } 1334 return false 1335} 1336 1337func (c *configImpl) UseGoma() bool { 1338 if v, ok := c.environ.Get("USE_GOMA"); ok { 1339 v = strings.TrimSpace(v) 1340 if v != "" && v != "false" { 1341 return true 1342 } 1343 } 1344 return false 1345} 1346 1347func (c *configImpl) StartGoma() bool { 1348 if !c.UseGoma() { 1349 return false 1350 } 1351 1352 if v, ok := c.environ.Get("NOSTART_GOMA"); ok { 1353 v = strings.TrimSpace(v) 1354 if v != "" && v != "false" { 1355 return false 1356 } 1357 } 1358 return true 1359} 1360 1361func (c *configImpl) canSupportRBE() bool { 1362 // Only supported on linux 1363 if runtime.GOOS != "linux" { 1364 return false 1365 } 1366 1367 // Do not use RBE with prod credentials in scenarios when stubby doesn't exist, since 1368 // its unlikely that we will be able to obtain necessary creds without stubby. 1369 authType, _ := c.rbeAuth() 1370 if !c.StubbyExists() && strings.Contains(authType, "use_google_prod_creds") { 1371 return false 1372 } 1373 if c.UseABFS() { 1374 return false 1375 } 1376 return true 1377} 1378 1379func (c *configImpl) UseABFS() bool { 1380 if c.ninjaCommand == NINJA_NINJAGO { 1381 return true 1382 } 1383 1384 if v, ok := c.environ.Get("NO_ABFS"); ok { 1385 v = strings.ToLower(strings.TrimSpace(v)) 1386 if v == "true" || v == "1" { 1387 return false 1388 } 1389 } 1390 1391 abfsBox := c.PrebuiltBuildTool("abfsbox") 1392 err := exec.Command(abfsBox, "hash", srcDirFileCheck).Run() 1393 return err == nil 1394} 1395 1396func (c *configImpl) sandboxPath(base, in string) string { 1397 if !c.UseABFS() { 1398 return in 1399 } 1400 1401 rel, err := filepath.Rel(base, in) 1402 if err != nil { 1403 return in 1404 } 1405 1406 return filepath.Join(abfsSrcDir, rel) 1407} 1408 1409func (c *configImpl) UseRBE() bool { 1410 // These alternate modes of running Soong do not use RBE / reclient. 1411 if c.JsonModuleGraph() { 1412 return false 1413 } 1414 1415 if !c.canSupportRBE() { 1416 return false 1417 } 1418 1419 if v, ok := c.Environment().Get("USE_RBE"); ok { 1420 v = strings.TrimSpace(v) 1421 if v != "" && v != "false" { 1422 return true 1423 } 1424 } 1425 return false 1426} 1427 1428func (c *configImpl) StartRBE() bool { 1429 if !c.UseRBE() { 1430 return false 1431 } 1432 1433 if v, ok := c.environ.Get("NOSTART_RBE"); ok { 1434 v = strings.TrimSpace(v) 1435 if v != "" && v != "false" { 1436 return false 1437 } 1438 } 1439 return true 1440} 1441 1442func (c *configImpl) rbeProxyLogsDir() string { 1443 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} { 1444 if v, ok := c.environ.Get(f); ok { 1445 return v 1446 } 1447 } 1448 return c.rbeTmpDir() 1449} 1450 1451func (c *configImpl) rbeDownloadTmpDir() string { 1452 for _, f := range []string{"RBE_download_tmp_dir", "FLAG_download_tmp_dir"} { 1453 if v, ok := c.environ.Get(f); ok { 1454 return v 1455 } 1456 } 1457 return c.rbeTmpDir() 1458} 1459 1460func (c *configImpl) rbeTmpDir() string { 1461 return filepath.Join(c.SoongOutDir(), "rbe") 1462} 1463 1464func (c *configImpl) rbeCacheDir() string { 1465 for _, f := range []string{"RBE_cache_dir", "FLAG_cache_dir"} { 1466 if v, ok := c.environ.Get(f); ok { 1467 return v 1468 } 1469 } 1470 return shared.JoinPath(c.SoongOutDir(), "rbe") 1471} 1472 1473func (c *configImpl) shouldCleanupRBELogsDir() bool { 1474 // Perform a log directory cleanup only when the log directory 1475 // is auto created by the build rather than user-specified. 1476 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} { 1477 if v, ok := c.environ.Get(f); ok { 1478 if v != c.rbeTmpDir() { 1479 return false 1480 } 1481 } 1482 } 1483 return true 1484} 1485 1486func (c *configImpl) rbeExecRoot() string { 1487 for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} { 1488 if v, ok := c.environ.Get(f); ok { 1489 return v 1490 } 1491 } 1492 wd, err := os.Getwd() 1493 if err != nil { 1494 return "" 1495 } 1496 return wd 1497} 1498 1499func (c *configImpl) rbeDir() string { 1500 if v, ok := c.environ.Get("RBE_DIR"); ok { 1501 return v 1502 } 1503 return "prebuilts/remoteexecution-client/live/" 1504} 1505 1506func (c *configImpl) rbeReproxy() string { 1507 for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} { 1508 if v, ok := c.environ.Get(f); ok { 1509 return v 1510 } 1511 } 1512 return filepath.Join(c.rbeDir(), "reproxy") 1513} 1514 1515func (c *configImpl) rbeAuth() (string, string) { 1516 credFlags := []string{ 1517 "use_application_default_credentials", 1518 "use_gce_credentials", 1519 "credential_file", 1520 "use_google_prod_creds", 1521 } 1522 for _, cf := range credFlags { 1523 for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} { 1524 if v, ok := c.environ.Get(f); ok { 1525 v = strings.TrimSpace(v) 1526 if v != "" && v != "false" && v != "0" { 1527 return "RBE_" + cf, v 1528 } 1529 } 1530 } 1531 } 1532 return "RBE_use_application_default_credentials", "true" 1533} 1534 1535func (c *configImpl) rbeSockAddr(dir string) (string, error) { 1536 // Absolute path socket addresses have a prefix of //. This should 1537 // be included in the length limit. 1538 maxNameLen := len(syscall.RawSockaddrUnix{}.Path) - 2 1539 base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix) 1540 1541 name := filepath.Join(dir, base) 1542 if len(name) < maxNameLen { 1543 return name, nil 1544 } 1545 1546 name = filepath.Join("/tmp", base) 1547 if len(name) < maxNameLen { 1548 return name, nil 1549 } 1550 1551 return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen) 1552} 1553 1554// IsGooglerEnvironment returns true if the current build is running 1555// on a Google developer machine and false otherwise. 1556func (c *configImpl) IsGooglerEnvironment() bool { 1557 cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG" 1558 if v, ok := c.environ.Get(cf); ok { 1559 return v == "googler" 1560 } 1561 return false 1562} 1563 1564// GoogleProdCredsExist determine whether credentials exist on the 1565// Googler machine to use remote execution. 1566func (c *configImpl) GoogleProdCredsExist() bool { 1567 if googleProdCredsExistCache { 1568 return googleProdCredsExistCache 1569 } 1570 if _, err := exec.Command("/usr/bin/gcertstatus", "-nocheck_ssh").Output(); err != nil { 1571 return false 1572 } 1573 googleProdCredsExistCache = true 1574 return true 1575} 1576 1577// UseRemoteBuild indicates whether to use a remote build acceleration system 1578// to speed up the build. 1579func (c *configImpl) UseRemoteBuild() bool { 1580 return c.UseGoma() || c.UseRBE() 1581} 1582 1583// StubbyExists checks whether the stubby binary exists on the machine running 1584// the build. 1585func (c *configImpl) StubbyExists() bool { 1586 if _, err := exec.LookPath("stubby"); err != nil { 1587 return false 1588 } 1589 return true 1590} 1591 1592// RemoteParallel controls how many remote jobs (i.e., commands which contain 1593// gomacc) are run in parallel. Note the parallelism of all other jobs is 1594// still limited by Parallel() 1595func (c *configImpl) RemoteParallel() int { 1596 if !c.UseRemoteBuild() { 1597 return 0 1598 } 1599 if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok { 1600 return i 1601 } 1602 return 500 1603} 1604 1605func (c *configImpl) SetKatiArgs(args []string) { 1606 c.katiArgs = args 1607} 1608 1609func (c *configImpl) SetNinjaArgs(args []string) { 1610 c.ninjaArgs = args 1611} 1612 1613func (c *configImpl) SetKatiSuffix(suffix string) { 1614 c.katiSuffix = suffix 1615} 1616 1617func (c *configImpl) LastKatiSuffixFile() string { 1618 return filepath.Join(c.OutDir(), "last_kati_suffix") 1619} 1620 1621func (c *configImpl) HasKatiSuffix() bool { 1622 return c.katiSuffix != "" 1623} 1624 1625func (c *configImpl) KatiEnvFile() string { 1626 return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh") 1627} 1628 1629func (c *configImpl) KatiBuildNinjaFile() string { 1630 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja") 1631} 1632 1633func (c *configImpl) KatiPackageNinjaFile() string { 1634 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja") 1635} 1636 1637func (c *configImpl) KatiSoongOnlyPackageNinjaFile() string { 1638 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiSoongOnlyPackageSuffix+".ninja") 1639} 1640 1641func (c *configImpl) SoongVarsFile() string { 1642 targetProduct, err := c.TargetProductOrErr() 1643 if err != nil { 1644 return filepath.Join(c.SoongOutDir(), "soong.variables") 1645 } else { 1646 return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+c.CoverageSuffix()+".variables") 1647 } 1648} 1649 1650func (c *configImpl) SoongExtraVarsFile() string { 1651 targetProduct, err := c.TargetProductOrErr() 1652 if err != nil { 1653 return filepath.Join(c.SoongOutDir(), "soong.extra.variables") 1654 } else { 1655 return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+c.CoverageSuffix()+".extra.variables") 1656 } 1657} 1658 1659func (c *configImpl) SoongNinjaFile() string { 1660 targetProduct, err := c.TargetProductOrErr() 1661 if err != nil { 1662 return filepath.Join(c.SoongOutDir(), "build.ninja") 1663 } else { 1664 return filepath.Join(c.SoongOutDir(), "build."+targetProduct+c.CoverageSuffix()+".ninja") 1665 } 1666} 1667 1668func (c *configImpl) CombinedNinjaFile() string { 1669 if c.katiSuffix == "" { 1670 return filepath.Join(c.OutDir(), "combined.ninja") 1671 } 1672 return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja") 1673} 1674 1675func (c *configImpl) SoongAndroidMk() string { 1676 return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+c.CoverageSuffix()+".mk") 1677} 1678 1679func (c *configImpl) SoongMakeVarsMk() string { 1680 return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+c.CoverageSuffix()+".mk") 1681} 1682 1683func (c *configImpl) SoongBuildMetrics() string { 1684 return filepath.Join(c.LogsDir(), "soong_build_metrics.pb") 1685} 1686 1687func (c *configImpl) ProductOut() string { 1688 return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice()) 1689} 1690 1691func (c *configImpl) DevicePreviousProductConfig() string { 1692 return filepath.Join(c.ProductOut(), "previous_build_config.mk") 1693} 1694 1695func (c *configImpl) DevicePreviousUsePartialCompile() string { 1696 return filepath.Join(c.ProductOut(), "previous_use_partial_compile.txt") 1697} 1698 1699func (c *configImpl) KatiPackageMkDir() string { 1700 return filepath.Join(c.SoongOutDir(), "kati_packaging"+c.KatiSuffix()) 1701} 1702 1703func (c *configImpl) hostOutRoot() string { 1704 return filepath.Join(c.OutDir(), "host") 1705} 1706 1707func (c *configImpl) HostOut() string { 1708 return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag()) 1709} 1710 1711// This probably needs to be multi-valued, so not exporting it for now 1712func (c *configImpl) hostCrossOut() string { 1713 if runtime.GOOS == "linux" { 1714 return filepath.Join(c.hostOutRoot(), "windows-x86") 1715 } else { 1716 return "" 1717 } 1718} 1719 1720func (c *configImpl) HostPrebuiltTag() string { 1721 return c.PrebuiltOS() 1722} 1723 1724func (c *configImpl) KatiBin() string { 1725 binName := "ckati" 1726 if c.UseABFS() { 1727 binName = "ckati-wrap" 1728 } 1729 1730 return c.PrebuiltBuildTool(binName) 1731} 1732 1733func (c *configImpl) NinjaBin() string { 1734 binName := "ninja" 1735 if c.UseABFS() { 1736 binName = "ninjago" 1737 } 1738 return c.PrebuiltBuildTool(binName) 1739} 1740 1741func (c *configImpl) N2Bin() string { 1742 path := c.PrebuiltBuildTool("n2") 1743 // Use musl instead of glibc because glibc on the build server is old and has bugs 1744 return strings.ReplaceAll(path, "/linux-x86/", "/linux_musl-x86/") 1745} 1746 1747func (c *configImpl) SisoBin() string { 1748 path := c.PrebuiltBuildTool("siso") 1749 // Use musl instead of glibc because glibc on the build server is old and has bugs 1750 return strings.ReplaceAll(path, "/linux-x86/", "/linux_musl-x86/") 1751} 1752 1753func (c *configImpl) PrebuiltBuildTool(name string) string { 1754 if c.environ.IsEnvTrue("SANITIZE_BUILD_TOOL_PREBUILTS") { 1755 asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name) 1756 if _, err := os.Stat(asan); err == nil { 1757 return asan 1758 } 1759 } 1760 return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name) 1761} 1762 1763func (c *configImpl) SetBuildBrokenDupRules(val bool) { 1764 c.brokenDupRules = val 1765} 1766 1767func (c *configImpl) BuildBrokenDupRules() bool { 1768 return c.brokenDupRules 1769} 1770 1771func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) { 1772 c.brokenUsesNetwork = val 1773} 1774 1775func (c *configImpl) BuildBrokenUsesNetwork() bool { 1776 return c.brokenUsesNetwork 1777} 1778 1779func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) { 1780 c.brokenNinjaEnvVars = val 1781} 1782 1783func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string { 1784 return c.brokenNinjaEnvVars 1785} 1786 1787func (c *configImpl) SetBuildBrokenMissingOutputs(val bool) { 1788 c.brokenMissingOutputs = val 1789} 1790 1791func (c *configImpl) BuildBrokenMissingOutputs() bool { 1792 return c.brokenMissingOutputs 1793} 1794 1795func (c *configImpl) SetTargetDeviceDir(dir string) { 1796 c.targetDeviceDir = dir 1797} 1798 1799func (c *configImpl) TargetDeviceDir() string { 1800 return c.targetDeviceDir 1801} 1802 1803func (c *configImpl) BuildDateTime() string { 1804 return c.buildDateTime 1805} 1806 1807func (c *configImpl) MetricsUploaderApp() string { 1808 return c.metricsUploader 1809} 1810 1811// LogsDir returns the absolute path to the logs directory where build log and 1812// metrics files are located. By default, the logs directory is the out 1813// directory. If the argument dist is specified, the logs directory 1814// is <dist_dir>/logs. 1815func (c *configImpl) LogsDir() string { 1816 dir := c.OutDir() 1817 if c.Dist() { 1818 // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files 1819 dir = filepath.Join(c.RealDistDir(), "logs") 1820 } 1821 absDir, err := filepath.Abs(dir) 1822 if err != nil { 1823 fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error()) 1824 os.Exit(1) 1825 } 1826 return absDir 1827} 1828 1829// MkFileMetrics returns the file path for make-related metrics. 1830func (c *configImpl) MkMetrics() string { 1831 return filepath.Join(c.LogsDir(), "mk_metrics.pb") 1832} 1833 1834func (c *configImpl) SetEmptyNinjaFile(v bool) { 1835 c.emptyNinjaFile = v 1836} 1837 1838func (c *configImpl) EmptyNinjaFile() bool { 1839 return c.emptyNinjaFile 1840} 1841 1842func (c *configImpl) SkipMetricsUpload() bool { 1843 // b/362625275 - Metrics upload sometimes prevents abfs unmount 1844 if c.UseABFS() { 1845 return true 1846 } 1847 1848 return c.skipMetricsUpload 1849} 1850 1851func (c *configImpl) EnsureAllowlistIntegrity() bool { 1852 return c.ensureAllowlistIntegrity 1853} 1854 1855// Returns a Time object if one was passed via a command-line flag. 1856// Otherwise returns the passed default. 1857func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time { 1858 if c.buildStartedTime == 0 { 1859 return defaultTime 1860 } 1861 return time.UnixMilli(c.buildStartedTime) 1862} 1863 1864func GetMetricsUploader(topDir string, env *Environment) string { 1865 if p, ok := env.Get("METRICS_UPLOADER"); ok { 1866 metricsUploader := filepath.Join(topDir, p) 1867 if _, err := os.Stat(metricsUploader); err == nil { 1868 return metricsUploader 1869 } 1870 } 1871 1872 return "" 1873} 1874