1// Copyright 2014 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package blueprint 16 17import ( 18 "bytes" 19 "context" 20 "errors" 21 "fmt" 22 "io" 23 "io/ioutil" 24 "os" 25 "path/filepath" 26 "reflect" 27 "runtime" 28 "runtime/pprof" 29 "sort" 30 "strings" 31 "sync" 32 "sync/atomic" 33 "text/scanner" 34 "text/template" 35 36 "github.com/google/blueprint/parser" 37 "github.com/google/blueprint/pathtools" 38 "github.com/google/blueprint/proptools" 39) 40 41var ErrBuildActionsNotReady = errors.New("build actions are not ready") 42 43const maxErrors = 10 44const MockModuleListFile = "bplist" 45 46// A Context contains all the state needed to parse a set of Blueprints files 47// and generate a Ninja file. The process of generating a Ninja file proceeds 48// through a series of four phases. Each phase corresponds with a some methods 49// on the Context object 50// 51// Phase Methods 52// ------------ ------------------------------------------- 53// 1. Registration RegisterModuleType, RegisterSingletonType 54// 55// 2. Parse ParseBlueprintsFiles, Parse 56// 57// 3. Generate ResolveDependencies, PrepareBuildActions 58// 59// 4. Write WriteBuildFile 60// 61// The registration phase prepares the context to process Blueprints files 62// containing various types of modules. The parse phase reads in one or more 63// Blueprints files and validates their contents against the module types that 64// have been registered. The generate phase then analyzes the parsed Blueprints 65// contents to create an internal representation for the build actions that must 66// be performed. This phase also performs validation of the module dependencies 67// and property values defined in the parsed Blueprints files. Finally, the 68// write phase generates the Ninja manifest text based on the generated build 69// actions. 70type Context struct { 71 context.Context 72 73 // set at instantiation 74 moduleFactories map[string]ModuleFactory 75 nameInterface NameInterface 76 moduleGroups []*moduleGroup 77 moduleInfo map[Module]*moduleInfo 78 modulesSorted []*moduleInfo 79 preSingletonInfo []*singletonInfo 80 singletonInfo []*singletonInfo 81 mutatorInfo []*mutatorInfo 82 earlyMutatorInfo []*mutatorInfo 83 variantMutatorNames []string 84 85 depsModified uint32 // positive if a mutator modified the dependencies 86 87 dependenciesReady bool // set to true on a successful ResolveDependencies 88 buildActionsReady bool // set to true on a successful PrepareBuildActions 89 90 // set by SetIgnoreUnknownModuleTypes 91 ignoreUnknownModuleTypes bool 92 93 // set by SetAllowMissingDependencies 94 allowMissingDependencies bool 95 96 // set during PrepareBuildActions 97 pkgNames map[*packageContext]string 98 liveGlobals *liveTracker 99 globalVariables map[Variable]*ninjaString 100 globalPools map[Pool]*poolDef 101 globalRules map[Rule]*ruleDef 102 103 // set during PrepareBuildActions 104 ninjaBuildDir *ninjaString // The builddir special Ninja variable 105 requiredNinjaMajor int // For the ninja_required_version variable 106 requiredNinjaMinor int // For the ninja_required_version variable 107 requiredNinjaMicro int // For the ninja_required_version variable 108 109 subninjas []string 110 111 // set lazily by sortedModuleGroups 112 cachedSortedModuleGroups []*moduleGroup 113 114 globs map[string]GlobPath 115 globLock sync.Mutex 116 117 fs pathtools.FileSystem 118 moduleListFile string 119} 120 121// An Error describes a problem that was encountered that is related to a 122// particular location in a Blueprints file. 123type BlueprintError struct { 124 Err error // the error that occurred 125 Pos scanner.Position // the relevant Blueprints file location 126} 127 128// A ModuleError describes a problem that was encountered that is related to a 129// particular module in a Blueprints file 130type ModuleError struct { 131 BlueprintError 132 module *moduleInfo 133} 134 135// A PropertyError describes a problem that was encountered that is related to a 136// particular property in a Blueprints file 137type PropertyError struct { 138 ModuleError 139 property string 140} 141 142func (e *BlueprintError) Error() string { 143 return fmt.Sprintf("%s: %s", e.Pos, e.Err) 144} 145 146func (e *ModuleError) Error() string { 147 return fmt.Sprintf("%s: %s: %s", e.Pos, e.module, e.Err) 148} 149 150func (e *PropertyError) Error() string { 151 return fmt.Sprintf("%s: %s: %s: %s", e.Pos, e.module, e.property, e.Err) 152} 153 154type localBuildActions struct { 155 variables []*localVariable 156 rules []*localRule 157 buildDefs []*buildDef 158} 159 160type moduleGroup struct { 161 name string 162 ninjaName string 163 164 modules []*moduleInfo 165 166 namespace Namespace 167} 168 169type moduleInfo struct { 170 // set during Parse 171 typeName string 172 factory ModuleFactory 173 relBlueprintsFile string 174 pos scanner.Position 175 propertyPos map[string]scanner.Position 176 177 variantName string 178 variant variationMap 179 dependencyVariant variationMap 180 181 logicModule Module 182 group *moduleGroup 183 properties []interface{} 184 185 // set during ResolveDependencies 186 directDeps []depInfo 187 missingDeps []string 188 189 // set during updateDependencies 190 reverseDeps []*moduleInfo 191 forwardDeps []*moduleInfo 192 193 // used by parallelVisitAllBottomUp 194 waitingCount int 195 196 // set during each runMutator 197 splitModules []*moduleInfo 198 199 // set during PrepareBuildActions 200 actionDefs localBuildActions 201} 202 203type depInfo struct { 204 module *moduleInfo 205 tag DependencyTag 206} 207 208func (module *moduleInfo) Name() string { 209 return module.group.name 210} 211 212func (module *moduleInfo) String() string { 213 s := fmt.Sprintf("module %q", module.Name()) 214 if module.variantName != "" { 215 s += fmt.Sprintf(" variant %q", module.variantName) 216 } 217 return s 218} 219 220func (module *moduleInfo) namespace() Namespace { 221 return module.group.namespace 222} 223 224// A Variation is a way that a variant of a module differs from other variants of the same module. 225// For example, two variants of the same module might have Variation{"arch","arm"} and 226// Variation{"arch","arm64"} 227type Variation struct { 228 // Mutator is the axis on which this variation applies, i.e. "arch" or "link" 229 Mutator string 230 // Variation is the name of the variation on the axis, i.e. "arm" or "arm64" for arch, or 231 // "shared" or "static" for link. 232 Variation string 233} 234 235// A variationMap stores a map of Mutator to Variation to specify a variant of a module. 236type variationMap map[string]string 237 238func (vm variationMap) clone() variationMap { 239 newVm := make(variationMap) 240 for k, v := range vm { 241 newVm[k] = v 242 } 243 244 return newVm 245} 246 247// Compare this variationMap to another one. Returns true if the every entry in this map 248// is either the same in the other map or doesn't exist in the other map. 249func (vm variationMap) subset(other variationMap) bool { 250 for k, v1 := range vm { 251 if v2, ok := other[k]; ok && v1 != v2 { 252 return false 253 } 254 } 255 return true 256} 257 258func (vm variationMap) equal(other variationMap) bool { 259 return reflect.DeepEqual(vm, other) 260} 261 262type singletonInfo struct { 263 // set during RegisterSingletonType 264 factory SingletonFactory 265 singleton Singleton 266 name string 267 268 // set during PrepareBuildActions 269 actionDefs localBuildActions 270} 271 272type mutatorInfo struct { 273 // set during RegisterMutator 274 topDownMutator TopDownMutator 275 bottomUpMutator BottomUpMutator 276 name string 277 parallel bool 278} 279 280func newContext() *Context { 281 return &Context{ 282 Context: context.Background(), 283 moduleFactories: make(map[string]ModuleFactory), 284 nameInterface: NewSimpleNameInterface(), 285 moduleInfo: make(map[Module]*moduleInfo), 286 globs: make(map[string]GlobPath), 287 fs: pathtools.OsFs, 288 ninjaBuildDir: nil, 289 requiredNinjaMajor: 1, 290 requiredNinjaMinor: 7, 291 requiredNinjaMicro: 0, 292 } 293} 294 295// NewContext creates a new Context object. The created context initially has 296// no module or singleton factories registered, so the RegisterModuleFactory and 297// RegisterSingletonFactory methods must be called before it can do anything 298// useful. 299func NewContext() *Context { 300 ctx := newContext() 301 302 ctx.RegisterBottomUpMutator("blueprint_deps", blueprintDepsMutator) 303 304 return ctx 305} 306 307// A ModuleFactory function creates a new Module object. See the 308// Context.RegisterModuleType method for details about how a registered 309// ModuleFactory is used by a Context. 310type ModuleFactory func() (m Module, propertyStructs []interface{}) 311 312// RegisterModuleType associates a module type name (which can appear in a 313// Blueprints file) with a Module factory function. When the given module type 314// name is encountered in a Blueprints file during parsing, the Module factory 315// is invoked to instantiate a new Module object to handle the build action 316// generation for the module. If a Mutator splits a module into multiple variants, 317// the factory is invoked again to create a new Module for each variant. 318// 319// The module type names given here must be unique for the context. The factory 320// function should be a named function so that its package and name can be 321// included in the generated Ninja file for debugging purposes. 322// 323// The factory function returns two values. The first is the newly created 324// Module object. The second is a slice of pointers to that Module object's 325// properties structs. Each properties struct is examined when parsing a module 326// definition of this type in a Blueprints file. Exported fields of the 327// properties structs are automatically set to the property values specified in 328// the Blueprints file. The properties struct field names determine the name of 329// the Blueprints file properties that are used - the Blueprints property name 330// matches that of the properties struct field name with the first letter 331// converted to lower-case. 332// 333// The fields of the properties struct must be either []string, a string, or 334// bool. The Context will panic if a Module gets instantiated with a properties 335// struct containing a field that is not one these supported types. 336// 337// Any properties that appear in the Blueprints files that are not built-in 338// module properties (such as "name" and "deps") and do not have a corresponding 339// field in the returned module properties struct result in an error during the 340// Context's parse phase. 341// 342// As an example, the follow code: 343// 344// type myModule struct { 345// properties struct { 346// Foo string 347// Bar []string 348// } 349// } 350// 351// func NewMyModule() (blueprint.Module, []interface{}) { 352// module := new(myModule) 353// properties := &module.properties 354// return module, []interface{}{properties} 355// } 356// 357// func main() { 358// ctx := blueprint.NewContext() 359// ctx.RegisterModuleType("my_module", NewMyModule) 360// // ... 361// } 362// 363// would support parsing a module defined in a Blueprints file as follows: 364// 365// my_module { 366// name: "myName", 367// foo: "my foo string", 368// bar: ["my", "bar", "strings"], 369// } 370// 371// The factory function may be called from multiple goroutines. Any accesses 372// to global variables must be synchronized. 373func (c *Context) RegisterModuleType(name string, factory ModuleFactory) { 374 if _, present := c.moduleFactories[name]; present { 375 panic(errors.New("module type name is already registered")) 376 } 377 c.moduleFactories[name] = factory 378} 379 380// A SingletonFactory function creates a new Singleton object. See the 381// Context.RegisterSingletonType method for details about how a registered 382// SingletonFactory is used by a Context. 383type SingletonFactory func() Singleton 384 385// RegisterSingletonType registers a singleton type that will be invoked to 386// generate build actions. Each registered singleton type is instantiated and 387// and invoked exactly once as part of the generate phase. Each registered 388// singleton is invoked in registration order. 389// 390// The singleton type names given here must be unique for the context. The 391// factory function should be a named function so that its package and name can 392// be included in the generated Ninja file for debugging purposes. 393func (c *Context) RegisterSingletonType(name string, factory SingletonFactory) { 394 for _, s := range c.singletonInfo { 395 if s.name == name { 396 panic(errors.New("singleton name is already registered")) 397 } 398 } 399 400 c.singletonInfo = append(c.singletonInfo, &singletonInfo{ 401 factory: factory, 402 singleton: factory(), 403 name: name, 404 }) 405} 406 407// RegisterPreSingletonType registers a presingleton type that will be invoked to 408// generate build actions before any Blueprint files have been read. Each registered 409// presingleton type is instantiated and invoked exactly once at the beginning of the 410// parse phase. Each registered presingleton is invoked in registration order. 411// 412// The presingleton type names given here must be unique for the context. The 413// factory function should be a named function so that its package and name can 414// be included in the generated Ninja file for debugging purposes. 415func (c *Context) RegisterPreSingletonType(name string, factory SingletonFactory) { 416 for _, s := range c.preSingletonInfo { 417 if s.name == name { 418 panic(errors.New("presingleton name is already registered")) 419 } 420 } 421 422 c.preSingletonInfo = append(c.preSingletonInfo, &singletonInfo{ 423 factory: factory, 424 singleton: factory(), 425 name: name, 426 }) 427} 428 429func (c *Context) SetNameInterface(i NameInterface) { 430 c.nameInterface = i 431} 432 433func singletonPkgPath(singleton Singleton) string { 434 typ := reflect.TypeOf(singleton) 435 for typ.Kind() == reflect.Ptr { 436 typ = typ.Elem() 437 } 438 return typ.PkgPath() 439} 440 441func singletonTypeName(singleton Singleton) string { 442 typ := reflect.TypeOf(singleton) 443 for typ.Kind() == reflect.Ptr { 444 typ = typ.Elem() 445 } 446 return typ.PkgPath() + "." + typ.Name() 447} 448 449// RegisterTopDownMutator registers a mutator that will be invoked to propagate dependency info 450// top-down between Modules. Each registered mutator is invoked in registration order (mixing 451// TopDownMutators and BottomUpMutators) once per Module, and the invocation on any module will 452// have returned before it is in invoked on any of its dependencies. 453// 454// The mutator type names given here must be unique to all top down mutators in 455// the Context. 456// 457// Returns a MutatorHandle, on which Parallel can be called to set the mutator to visit modules in 458// parallel while maintaining ordering. 459func (c *Context) RegisterTopDownMutator(name string, mutator TopDownMutator) MutatorHandle { 460 for _, m := range c.mutatorInfo { 461 if m.name == name && m.topDownMutator != nil { 462 panic(fmt.Errorf("mutator name %s is already registered", name)) 463 } 464 } 465 466 info := &mutatorInfo{ 467 topDownMutator: mutator, 468 name: name, 469 } 470 471 c.mutatorInfo = append(c.mutatorInfo, info) 472 473 return info 474} 475 476// RegisterBottomUpMutator registers a mutator that will be invoked to split Modules into variants. 477// Each registered mutator is invoked in registration order (mixing TopDownMutators and 478// BottomUpMutators) once per Module, will not be invoked on a module until the invocations on all 479// of the modules dependencies have returned. 480// 481// The mutator type names given here must be unique to all bottom up or early 482// mutators in the Context. 483// 484// Returns a MutatorHandle, on which Parallel can be called to set the mutator to visit modules in 485// parallel while maintaining ordering. 486func (c *Context) RegisterBottomUpMutator(name string, mutator BottomUpMutator) MutatorHandle { 487 for _, m := range c.variantMutatorNames { 488 if m == name { 489 panic(fmt.Errorf("mutator name %s is already registered", name)) 490 } 491 } 492 493 info := &mutatorInfo{ 494 bottomUpMutator: mutator, 495 name: name, 496 } 497 c.mutatorInfo = append(c.mutatorInfo, info) 498 499 c.variantMutatorNames = append(c.variantMutatorNames, name) 500 501 return info 502} 503 504type MutatorHandle interface { 505 // Set the mutator to visit modules in parallel while maintaining ordering. Calling any 506 // method on the mutator context is thread-safe, but the mutator must handle synchronization 507 // for any modifications to global state or any modules outside the one it was invoked on. 508 Parallel() MutatorHandle 509} 510 511func (mutator *mutatorInfo) Parallel() MutatorHandle { 512 mutator.parallel = true 513 return mutator 514} 515 516// RegisterEarlyMutator registers a mutator that will be invoked to split 517// Modules into multiple variant Modules before any dependencies have been 518// created. Each registered mutator is invoked in registration order once 519// per Module (including each variant from previous early mutators). Module 520// order is unpredictable. 521// 522// In order for dependencies to be satisifed in a later pass, all dependencies 523// of a module either must have an identical variant or must have no variations. 524// 525// The mutator type names given here must be unique to all bottom up or early 526// mutators in the Context. 527// 528// Deprecated, use a BottomUpMutator instead. The only difference between 529// EarlyMutator and BottomUpMutator is that EarlyMutator runs before the 530// deprecated DynamicDependencies. 531func (c *Context) RegisterEarlyMutator(name string, mutator EarlyMutator) { 532 for _, m := range c.variantMutatorNames { 533 if m == name { 534 panic(fmt.Errorf("mutator name %s is already registered", name)) 535 } 536 } 537 538 c.earlyMutatorInfo = append(c.earlyMutatorInfo, &mutatorInfo{ 539 bottomUpMutator: func(mctx BottomUpMutatorContext) { 540 mutator(mctx) 541 }, 542 name: name, 543 }) 544 545 c.variantMutatorNames = append(c.variantMutatorNames, name) 546} 547 548// SetIgnoreUnknownModuleTypes sets the behavior of the context in the case 549// where it encounters an unknown module type while parsing Blueprints files. By 550// default, the context will report unknown module types as an error. If this 551// method is called with ignoreUnknownModuleTypes set to true then the context 552// will silently ignore unknown module types. 553// 554// This method should generally not be used. It exists to facilitate the 555// bootstrapping process. 556func (c *Context) SetIgnoreUnknownModuleTypes(ignoreUnknownModuleTypes bool) { 557 c.ignoreUnknownModuleTypes = ignoreUnknownModuleTypes 558} 559 560// SetAllowMissingDependencies changes the behavior of Blueprint to ignore 561// unresolved dependencies. If the module's GenerateBuildActions calls 562// ModuleContext.GetMissingDependencies Blueprint will not emit any errors 563// for missing dependencies. 564func (c *Context) SetAllowMissingDependencies(allowMissingDependencies bool) { 565 c.allowMissingDependencies = allowMissingDependencies 566} 567 568func (c *Context) SetModuleListFile(listFile string) { 569 c.moduleListFile = listFile 570} 571 572func (c *Context) ListModulePaths(baseDir string) (paths []string, err error) { 573 reader, err := c.fs.Open(c.moduleListFile) 574 if err != nil { 575 return nil, err 576 } 577 bytes, err := ioutil.ReadAll(reader) 578 if err != nil { 579 return nil, err 580 } 581 text := string(bytes) 582 583 text = strings.Trim(text, "\n") 584 lines := strings.Split(text, "\n") 585 for i := range lines { 586 lines[i] = filepath.Join(baseDir, lines[i]) 587 } 588 589 return lines, nil 590} 591 592// a fileParseContext tells the status of parsing a particular file 593type fileParseContext struct { 594 // name of file 595 fileName string 596 597 // scope to use when resolving variables 598 Scope *parser.Scope 599 600 // pointer to the one in the parent directory 601 parent *fileParseContext 602 603 // is closed once FileHandler has completed for this file 604 doneVisiting chan struct{} 605} 606 607// ParseBlueprintsFiles parses a set of Blueprints files starting with the file 608// at rootFile. When it encounters a Blueprints file with a set of subdirs 609// listed it recursively parses any Blueprints files found in those 610// subdirectories. 611// 612// If no errors are encountered while parsing the files, the list of paths on 613// which the future output will depend is returned. This list will include both 614// Blueprints file paths as well as directory paths for cases where wildcard 615// subdirs are found. 616func (c *Context) ParseBlueprintsFiles(rootFile string) (deps []string, errs []error) { 617 baseDir := filepath.Dir(rootFile) 618 pathsToParse, err := c.ListModulePaths(baseDir) 619 if err != nil { 620 return nil, []error{err} 621 } 622 return c.ParseFileList(baseDir, pathsToParse) 623} 624 625func (c *Context) ParseFileList(rootDir string, filePaths []string) (deps []string, 626 errs []error) { 627 628 if len(filePaths) < 1 { 629 return nil, []error{fmt.Errorf("no paths provided to parse")} 630 } 631 632 c.dependenciesReady = false 633 634 moduleCh := make(chan *moduleInfo) 635 errsCh := make(chan []error) 636 doneCh := make(chan struct{}) 637 var numErrs uint32 638 var numGoroutines int32 639 640 // handler must be reentrant 641 handleOneFile := func(file *parser.File) { 642 if atomic.LoadUint32(&numErrs) > maxErrors { 643 return 644 } 645 646 for _, def := range file.Defs { 647 var module *moduleInfo 648 var errs []error 649 switch def := def.(type) { 650 case *parser.Module: 651 module, errs = c.processModuleDef(def, file.Name) 652 case *parser.Assignment: 653 // Already handled via Scope object 654 default: 655 panic("unknown definition type") 656 } 657 658 if len(errs) > 0 { 659 atomic.AddUint32(&numErrs, uint32(len(errs))) 660 errsCh <- errs 661 } else if module != nil { 662 moduleCh <- module 663 } 664 } 665 } 666 667 atomic.AddInt32(&numGoroutines, 1) 668 go func() { 669 var errs []error 670 deps, errs = c.WalkBlueprintsFiles(rootDir, filePaths, handleOneFile) 671 if len(errs) > 0 { 672 errsCh <- errs 673 } 674 doneCh <- struct{}{} 675 }() 676 677loop: 678 for { 679 select { 680 case newErrs := <-errsCh: 681 errs = append(errs, newErrs...) 682 case module := <-moduleCh: 683 newErrs := c.addModule(module) 684 if len(newErrs) > 0 { 685 errs = append(errs, newErrs...) 686 } 687 case <-doneCh: 688 n := atomic.AddInt32(&numGoroutines, -1) 689 if n == 0 { 690 break loop 691 } 692 } 693 } 694 695 return deps, errs 696} 697 698type FileHandler func(*parser.File) 699 700// WalkBlueprintsFiles walks a set of Blueprints files starting with the given filepaths, 701// calling the given file handler on each 702// 703// When WalkBlueprintsFiles encounters a Blueprints file with a set of subdirs listed, 704// it recursively parses any Blueprints files found in those subdirectories. 705// 706// If any of the file paths is an ancestor directory of any other of file path, the ancestor 707// will be parsed and visited first. 708// 709// the file handler will be called from a goroutine, so it must be reentrant. 710// 711// If no errors are encountered while parsing the files, the list of paths on 712// which the future output will depend is returned. This list will include both 713// Blueprints file paths as well as directory paths for cases where wildcard 714// subdirs are found. 715// 716// visitor will be called asynchronously, and will only be called once visitor for each 717// ancestor directory has completed. 718// 719// WalkBlueprintsFiles will not return until all calls to visitor have returned. 720func (c *Context) WalkBlueprintsFiles(rootDir string, filePaths []string, 721 visitor FileHandler) (deps []string, errs []error) { 722 723 // make a mapping from ancestors to their descendants to facilitate parsing ancestors first 724 descendantsMap, err := findBlueprintDescendants(filePaths) 725 if err != nil { 726 panic(err.Error()) 727 } 728 blueprintsSet := make(map[string]bool) 729 730 // Channels to receive data back from openAndParse goroutines 731 blueprintsCh := make(chan fileParseContext) 732 errsCh := make(chan []error) 733 depsCh := make(chan string) 734 735 // Channel to notify main loop that a openAndParse goroutine has finished 736 doneParsingCh := make(chan fileParseContext) 737 738 // Number of outstanding goroutines to wait for 739 activeCount := 0 740 var pending []fileParseContext 741 tooManyErrors := false 742 743 // Limit concurrent calls to parseBlueprintFiles to 200 744 // Darwin has a default limit of 256 open files 745 maxActiveCount := 200 746 747 // count the number of pending calls to visitor() 748 visitorWaitGroup := sync.WaitGroup{} 749 750 startParseBlueprintsFile := func(blueprint fileParseContext) { 751 if blueprintsSet[blueprint.fileName] { 752 return 753 } 754 blueprintsSet[blueprint.fileName] = true 755 activeCount++ 756 deps = append(deps, blueprint.fileName) 757 visitorWaitGroup.Add(1) 758 go func() { 759 file, blueprints, deps, errs := c.openAndParse(blueprint.fileName, blueprint.Scope, rootDir, 760 &blueprint) 761 if len(errs) > 0 { 762 errsCh <- errs 763 } 764 for _, blueprint := range blueprints { 765 blueprintsCh <- blueprint 766 } 767 for _, dep := range deps { 768 depsCh <- dep 769 } 770 doneParsingCh <- blueprint 771 772 if blueprint.parent != nil && blueprint.parent.doneVisiting != nil { 773 // wait for visitor() of parent to complete 774 <-blueprint.parent.doneVisiting 775 } 776 777 if len(errs) == 0 { 778 // process this file 779 visitor(file) 780 } 781 if blueprint.doneVisiting != nil { 782 close(blueprint.doneVisiting) 783 } 784 visitorWaitGroup.Done() 785 }() 786 } 787 788 foundParseableBlueprint := func(blueprint fileParseContext) { 789 if activeCount >= maxActiveCount { 790 pending = append(pending, blueprint) 791 } else { 792 startParseBlueprintsFile(blueprint) 793 } 794 } 795 796 startParseDescendants := func(blueprint fileParseContext) { 797 descendants, hasDescendants := descendantsMap[blueprint.fileName] 798 if hasDescendants { 799 for _, descendant := range descendants { 800 foundParseableBlueprint(fileParseContext{descendant, parser.NewScope(blueprint.Scope), &blueprint, make(chan struct{})}) 801 } 802 } 803 } 804 805 // begin parsing any files that have no ancestors 806 startParseDescendants(fileParseContext{"", parser.NewScope(nil), nil, nil}) 807 808loop: 809 for { 810 if len(errs) > maxErrors { 811 tooManyErrors = true 812 } 813 814 select { 815 case newErrs := <-errsCh: 816 errs = append(errs, newErrs...) 817 case dep := <-depsCh: 818 deps = append(deps, dep) 819 case blueprint := <-blueprintsCh: 820 if tooManyErrors { 821 continue 822 } 823 foundParseableBlueprint(blueprint) 824 case blueprint := <-doneParsingCh: 825 activeCount-- 826 if !tooManyErrors { 827 startParseDescendants(blueprint) 828 } 829 if activeCount < maxActiveCount && len(pending) > 0 { 830 // start to process the next one from the queue 831 next := pending[len(pending)-1] 832 pending = pending[:len(pending)-1] 833 startParseBlueprintsFile(next) 834 } 835 if activeCount == 0 { 836 break loop 837 } 838 } 839 } 840 841 sort.Strings(deps) 842 843 // wait for every visitor() to complete 844 visitorWaitGroup.Wait() 845 846 return 847} 848 849// MockFileSystem causes the Context to replace all reads with accesses to the provided map of 850// filenames to contents stored as a byte slice. 851func (c *Context) MockFileSystem(files map[string][]byte) { 852 // look for a module list file 853 _, ok := files[MockModuleListFile] 854 if !ok { 855 // no module list file specified; find every file named Blueprints 856 pathsToParse := []string{} 857 for candidate := range files { 858 if filepath.Base(candidate) == "Blueprints" { 859 pathsToParse = append(pathsToParse, candidate) 860 } 861 } 862 if len(pathsToParse) < 1 { 863 panic(fmt.Sprintf("No Blueprints files found in mock filesystem: %v\n", files)) 864 } 865 // put the list of Blueprints files into a list file 866 files[MockModuleListFile] = []byte(strings.Join(pathsToParse, "\n")) 867 } 868 c.SetModuleListFile(MockModuleListFile) 869 870 // mock the filesystem 871 c.fs = pathtools.MockFs(files) 872} 873 874// openAndParse opens and parses a single Blueprints file, and returns the results 875func (c *Context) openAndParse(filename string, scope *parser.Scope, rootDir string, 876 parent *fileParseContext) (file *parser.File, 877 subBlueprints []fileParseContext, deps []string, errs []error) { 878 879 f, err := c.fs.Open(filename) 880 if err != nil { 881 // couldn't open the file; see if we can provide a clearer error than "could not open file" 882 stats, statErr := c.fs.Lstat(filename) 883 if statErr == nil { 884 isSymlink := stats.Mode()&os.ModeSymlink != 0 885 if isSymlink { 886 err = fmt.Errorf("could not open symlink %v : %v", filename, err) 887 target, readlinkErr := os.Readlink(filename) 888 if readlinkErr == nil { 889 _, targetStatsErr := c.fs.Lstat(target) 890 if targetStatsErr != nil { 891 err = fmt.Errorf("could not open symlink %v; its target (%v) cannot be opened", filename, target) 892 } 893 } 894 } else { 895 err = fmt.Errorf("%v exists but could not be opened: %v", filename, err) 896 } 897 } 898 return nil, nil, nil, []error{err} 899 } 900 901 func() { 902 defer func() { 903 err = f.Close() 904 if err != nil { 905 errs = append(errs, err) 906 } 907 }() 908 file, subBlueprints, errs = c.parseOne(rootDir, filename, f, scope, parent) 909 }() 910 911 if len(errs) > 0 { 912 return nil, nil, nil, errs 913 } 914 915 for _, b := range subBlueprints { 916 deps = append(deps, b.fileName) 917 } 918 919 return file, subBlueprints, deps, nil 920} 921 922// parseOne parses a single Blueprints file from the given reader, creating Module 923// objects for each of the module definitions encountered. If the Blueprints 924// file contains an assignment to the "subdirs" variable, then the 925// subdirectories listed are searched for Blueprints files returned in the 926// subBlueprints return value. If the Blueprints file contains an assignment 927// to the "build" variable, then the file listed are returned in the 928// subBlueprints return value. 929// 930// rootDir specifies the path to the root directory of the source tree, while 931// filename specifies the path to the Blueprints file. These paths are used for 932// error reporting and for determining the module's directory. 933func (c *Context) parseOne(rootDir, filename string, reader io.Reader, 934 scope *parser.Scope, parent *fileParseContext) (file *parser.File, subBlueprints []fileParseContext, errs []error) { 935 936 relBlueprintsFile, err := filepath.Rel(rootDir, filename) 937 if err != nil { 938 return nil, nil, []error{err} 939 } 940 941 scope.Remove("subdirs") 942 scope.Remove("optional_subdirs") 943 scope.Remove("build") 944 file, errs = parser.ParseAndEval(filename, reader, scope) 945 if len(errs) > 0 { 946 for i, err := range errs { 947 if parseErr, ok := err.(*parser.ParseError); ok { 948 err = &BlueprintError{ 949 Err: parseErr.Err, 950 Pos: parseErr.Pos, 951 } 952 errs[i] = err 953 } 954 } 955 956 // If there were any parse errors don't bother trying to interpret the 957 // result. 958 return nil, nil, errs 959 } 960 file.Name = relBlueprintsFile 961 962 build, buildPos, err := getLocalStringListFromScope(scope, "build") 963 if err != nil { 964 errs = append(errs, err) 965 } 966 for _, buildEntry := range build { 967 if strings.Contains(buildEntry, "/") { 968 errs = append(errs, &BlueprintError{ 969 Err: fmt.Errorf("illegal value %v. The '/' character is not permitted", buildEntry), 970 Pos: buildPos, 971 }) 972 } 973 } 974 975 subBlueprintsName, _, err := getStringFromScope(scope, "subname") 976 if err != nil { 977 errs = append(errs, err) 978 } 979 980 if subBlueprintsName == "" { 981 subBlueprintsName = "Blueprints" 982 } 983 984 var blueprints []string 985 986 newBlueprints, newErrs := c.findBuildBlueprints(filepath.Dir(filename), build, buildPos) 987 blueprints = append(blueprints, newBlueprints...) 988 errs = append(errs, newErrs...) 989 990 subBlueprintsAndScope := make([]fileParseContext, len(blueprints)) 991 for i, b := range blueprints { 992 subBlueprintsAndScope[i] = fileParseContext{b, parser.NewScope(scope), parent, make(chan struct{})} 993 } 994 return file, subBlueprintsAndScope, errs 995} 996 997func (c *Context) findBuildBlueprints(dir string, build []string, 998 buildPos scanner.Position) ([]string, []error) { 999 1000 var blueprints []string 1001 var errs []error 1002 1003 for _, file := range build { 1004 pattern := filepath.Join(dir, file) 1005 var matches []string 1006 var err error 1007 1008 matches, err = c.glob(pattern, nil) 1009 1010 if err != nil { 1011 errs = append(errs, &BlueprintError{ 1012 Err: fmt.Errorf("%q: %s", pattern, err.Error()), 1013 Pos: buildPos, 1014 }) 1015 continue 1016 } 1017 1018 if len(matches) == 0 { 1019 errs = append(errs, &BlueprintError{ 1020 Err: fmt.Errorf("%q: not found", pattern), 1021 Pos: buildPos, 1022 }) 1023 } 1024 1025 for _, foundBlueprints := range matches { 1026 if strings.HasSuffix(foundBlueprints, "/") { 1027 errs = append(errs, &BlueprintError{ 1028 Err: fmt.Errorf("%q: is a directory", foundBlueprints), 1029 Pos: buildPos, 1030 }) 1031 } 1032 blueprints = append(blueprints, foundBlueprints) 1033 } 1034 } 1035 1036 return blueprints, errs 1037} 1038 1039func (c *Context) findSubdirBlueprints(dir string, subdirs []string, subdirsPos scanner.Position, 1040 subBlueprintsName string, optional bool) ([]string, []error) { 1041 1042 var blueprints []string 1043 var errs []error 1044 1045 for _, subdir := range subdirs { 1046 pattern := filepath.Join(dir, subdir, subBlueprintsName) 1047 var matches []string 1048 var err error 1049 1050 matches, err = c.glob(pattern, nil) 1051 1052 if err != nil { 1053 errs = append(errs, &BlueprintError{ 1054 Err: fmt.Errorf("%q: %s", pattern, err.Error()), 1055 Pos: subdirsPos, 1056 }) 1057 continue 1058 } 1059 1060 if len(matches) == 0 && !optional { 1061 errs = append(errs, &BlueprintError{ 1062 Err: fmt.Errorf("%q: not found", pattern), 1063 Pos: subdirsPos, 1064 }) 1065 } 1066 1067 for _, subBlueprints := range matches { 1068 if strings.HasSuffix(subBlueprints, "/") { 1069 errs = append(errs, &BlueprintError{ 1070 Err: fmt.Errorf("%q: is a directory", subBlueprints), 1071 Pos: subdirsPos, 1072 }) 1073 } 1074 blueprints = append(blueprints, subBlueprints) 1075 } 1076 } 1077 1078 return blueprints, errs 1079} 1080 1081func getLocalStringListFromScope(scope *parser.Scope, v string) ([]string, scanner.Position, error) { 1082 if assignment, local := scope.Get(v); assignment == nil || !local { 1083 return nil, scanner.Position{}, nil 1084 } else { 1085 switch value := assignment.Value.Eval().(type) { 1086 case *parser.List: 1087 ret := make([]string, 0, len(value.Values)) 1088 1089 for _, listValue := range value.Values { 1090 s, ok := listValue.(*parser.String) 1091 if !ok { 1092 // The parser should not produce this. 1093 panic("non-string value found in list") 1094 } 1095 1096 ret = append(ret, s.Value) 1097 } 1098 1099 return ret, assignment.EqualsPos, nil 1100 case *parser.Bool, *parser.String: 1101 return nil, scanner.Position{}, &BlueprintError{ 1102 Err: fmt.Errorf("%q must be a list of strings", v), 1103 Pos: assignment.EqualsPos, 1104 } 1105 default: 1106 panic(fmt.Errorf("unknown value type: %d", assignment.Value.Type())) 1107 } 1108 } 1109} 1110 1111func getStringFromScope(scope *parser.Scope, v string) (string, scanner.Position, error) { 1112 if assignment, _ := scope.Get(v); assignment == nil { 1113 return "", scanner.Position{}, nil 1114 } else { 1115 switch value := assignment.Value.Eval().(type) { 1116 case *parser.String: 1117 return value.Value, assignment.EqualsPos, nil 1118 case *parser.Bool, *parser.List: 1119 return "", scanner.Position{}, &BlueprintError{ 1120 Err: fmt.Errorf("%q must be a string", v), 1121 Pos: assignment.EqualsPos, 1122 } 1123 default: 1124 panic(fmt.Errorf("unknown value type: %d", assignment.Value.Type())) 1125 } 1126 } 1127} 1128 1129// Clones a build logic module by calling the factory method for its module type, and then cloning 1130// property values. Any values stored in the module object that are not stored in properties 1131// structs will be lost. 1132func (c *Context) cloneLogicModule(origModule *moduleInfo) (Module, []interface{}) { 1133 newLogicModule, newProperties := origModule.factory() 1134 1135 if len(newProperties) != len(origModule.properties) { 1136 panic("mismatched properties array length in " + origModule.Name()) 1137 } 1138 1139 for i := range newProperties { 1140 dst := reflect.ValueOf(newProperties[i]).Elem() 1141 src := reflect.ValueOf(origModule.properties[i]).Elem() 1142 1143 proptools.CopyProperties(dst, src) 1144 } 1145 1146 return newLogicModule, newProperties 1147} 1148 1149func (c *Context) createVariations(origModule *moduleInfo, mutatorName string, 1150 variationNames []string) ([]*moduleInfo, []error) { 1151 1152 if len(variationNames) == 0 { 1153 panic(fmt.Errorf("mutator %q passed zero-length variation list for module %q", 1154 mutatorName, origModule.Name())) 1155 } 1156 1157 newModules := []*moduleInfo{} 1158 1159 var errs []error 1160 1161 for i, variationName := range variationNames { 1162 var newLogicModule Module 1163 var newProperties []interface{} 1164 1165 if i == 0 { 1166 // Reuse the existing module for the first new variant 1167 // This both saves creating a new module, and causes the insertion in c.moduleInfo below 1168 // with logicModule as the key to replace the original entry in c.moduleInfo 1169 newLogicModule, newProperties = origModule.logicModule, origModule.properties 1170 } else { 1171 newLogicModule, newProperties = c.cloneLogicModule(origModule) 1172 } 1173 1174 newVariant := origModule.variant.clone() 1175 newVariant[mutatorName] = variationName 1176 1177 m := *origModule 1178 newModule := &m 1179 newModule.directDeps = append([]depInfo{}, origModule.directDeps...) 1180 newModule.logicModule = newLogicModule 1181 newModule.variant = newVariant 1182 newModule.dependencyVariant = origModule.dependencyVariant.clone() 1183 newModule.properties = newProperties 1184 1185 if variationName != "" { 1186 if newModule.variantName == "" { 1187 newModule.variantName = variationName 1188 } else { 1189 newModule.variantName += "_" + variationName 1190 } 1191 } 1192 1193 newModules = append(newModules, newModule) 1194 1195 newErrs := c.convertDepsToVariation(newModule, mutatorName, variationName) 1196 if len(newErrs) > 0 { 1197 errs = append(errs, newErrs...) 1198 } 1199 } 1200 1201 // Mark original variant as invalid. Modules that depend on this module will still 1202 // depend on origModule, but we'll fix it when the mutator is called on them. 1203 origModule.logicModule = nil 1204 origModule.splitModules = newModules 1205 1206 atomic.AddUint32(&c.depsModified, 1) 1207 1208 return newModules, errs 1209} 1210 1211func (c *Context) convertDepsToVariation(module *moduleInfo, 1212 mutatorName, variationName string) (errs []error) { 1213 1214 for i, dep := range module.directDeps { 1215 if dep.module.logicModule == nil { 1216 var newDep *moduleInfo 1217 for _, m := range dep.module.splitModules { 1218 if m.variant[mutatorName] == variationName { 1219 newDep = m 1220 break 1221 } 1222 } 1223 if newDep == nil { 1224 errs = append(errs, &BlueprintError{ 1225 Err: fmt.Errorf("failed to find variation %q for module %q needed by %q", 1226 variationName, dep.module.Name(), module.Name()), 1227 Pos: module.pos, 1228 }) 1229 continue 1230 } 1231 module.directDeps[i].module = newDep 1232 } 1233 } 1234 1235 return errs 1236} 1237 1238func (c *Context) prettyPrintVariant(variant variationMap) string { 1239 names := make([]string, 0, len(variant)) 1240 for _, m := range c.variantMutatorNames { 1241 if v, ok := variant[m]; ok { 1242 names = append(names, m+":"+v) 1243 } 1244 } 1245 1246 return strings.Join(names, ", ") 1247} 1248 1249func (c *Context) newModule(factory ModuleFactory) *moduleInfo { 1250 logicModule, properties := factory() 1251 1252 module := &moduleInfo{ 1253 logicModule: logicModule, 1254 factory: factory, 1255 } 1256 1257 module.properties = properties 1258 1259 return module 1260} 1261 1262func (c *Context) processModuleDef(moduleDef *parser.Module, 1263 relBlueprintsFile string) (*moduleInfo, []error) { 1264 1265 factory, ok := c.moduleFactories[moduleDef.Type] 1266 if !ok { 1267 if c.ignoreUnknownModuleTypes { 1268 return nil, nil 1269 } 1270 1271 return nil, []error{ 1272 &BlueprintError{ 1273 Err: fmt.Errorf("unrecognized module type %q", moduleDef.Type), 1274 Pos: moduleDef.TypePos, 1275 }, 1276 } 1277 } 1278 1279 module := c.newModule(factory) 1280 module.typeName = moduleDef.Type 1281 1282 module.relBlueprintsFile = relBlueprintsFile 1283 1284 propertyMap, errs := unpackProperties(moduleDef.Properties, module.properties...) 1285 if len(errs) > 0 { 1286 return nil, errs 1287 } 1288 1289 module.pos = moduleDef.TypePos 1290 module.propertyPos = make(map[string]scanner.Position) 1291 for name, propertyDef := range propertyMap { 1292 module.propertyPos[name] = propertyDef.ColonPos 1293 } 1294 1295 return module, nil 1296} 1297 1298func (c *Context) addModule(module *moduleInfo) []error { 1299 name := module.logicModule.Name() 1300 if name == "" { 1301 return []error{ 1302 &BlueprintError{ 1303 Err: fmt.Errorf("property 'name' is missing from a module"), 1304 Pos: module.pos, 1305 }, 1306 } 1307 } 1308 c.moduleInfo[module.logicModule] = module 1309 1310 group := &moduleGroup{ 1311 name: name, 1312 modules: []*moduleInfo{module}, 1313 } 1314 module.group = group 1315 namespace, errs := c.nameInterface.NewModule( 1316 newNamespaceContext(module), 1317 ModuleGroup{moduleGroup: group}, 1318 module.logicModule) 1319 if len(errs) > 0 { 1320 for i := range errs { 1321 errs[i] = &BlueprintError{Err: errs[i], Pos: module.pos} 1322 } 1323 return errs 1324 } 1325 group.namespace = namespace 1326 1327 c.moduleGroups = append(c.moduleGroups, group) 1328 1329 return nil 1330} 1331 1332// ResolveDependencies checks that the dependencies specified by all of the 1333// modules defined in the parsed Blueprints files are valid. This means that 1334// the modules depended upon are defined and that no circular dependencies 1335// exist. 1336func (c *Context) ResolveDependencies(config interface{}) (deps []string, errs []error) { 1337 return c.resolveDependencies(c.Context, config) 1338} 1339 1340func (c *Context) resolveDependencies(ctx context.Context, config interface{}) (deps []string, errs []error) { 1341 pprof.Do(ctx, pprof.Labels("blueprint", "ResolveDependencies"), func(ctx context.Context) { 1342 c.liveGlobals = newLiveTracker(config) 1343 1344 deps, errs = c.generateSingletonBuildActions(config, c.preSingletonInfo, c.liveGlobals) 1345 if len(errs) > 0 { 1346 return 1347 } 1348 1349 errs = c.updateDependencies() 1350 if len(errs) > 0 { 1351 return 1352 } 1353 1354 var mutatorDeps []string 1355 mutatorDeps, errs = c.runMutators(ctx, config) 1356 if len(errs) > 0 { 1357 return 1358 } 1359 deps = append(deps, mutatorDeps...) 1360 1361 c.cloneModules() 1362 1363 c.dependenciesReady = true 1364 }) 1365 1366 if len(errs) > 0 { 1367 return nil, errs 1368 } 1369 1370 return deps, nil 1371} 1372 1373// Default dependencies handling. If the module implements the (deprecated) 1374// DynamicDependerModule interface then this set consists of the union of those 1375// module names returned by its DynamicDependencies method and those added by calling 1376// AddDependencies or AddVariationDependencies on DynamicDependencyModuleContext. 1377func blueprintDepsMutator(ctx BottomUpMutatorContext) { 1378 if dynamicDepender, ok := ctx.Module().(DynamicDependerModule); ok { 1379 func() { 1380 defer func() { 1381 if r := recover(); r != nil { 1382 ctx.error(newPanicErrorf(r, "DynamicDependencies for %s", ctx.moduleInfo())) 1383 } 1384 }() 1385 dynamicDeps := dynamicDepender.DynamicDependencies(ctx) 1386 1387 if ctx.Failed() { 1388 return 1389 } 1390 1391 ctx.AddDependency(ctx.Module(), nil, dynamicDeps...) 1392 }() 1393 } 1394} 1395 1396// findMatchingVariant searches the moduleGroup for a module with the same variant as module, 1397// and returns the matching module, or nil if one is not found. 1398func (c *Context) findMatchingVariant(module *moduleInfo, possible []*moduleInfo) *moduleInfo { 1399 if len(possible) == 1 { 1400 return possible[0] 1401 } else { 1402 for _, m := range possible { 1403 if m.variant.equal(module.dependencyVariant) { 1404 return m 1405 } 1406 } 1407 } 1408 1409 return nil 1410} 1411 1412func (c *Context) addDependency(module *moduleInfo, tag DependencyTag, depName string) []error { 1413 if _, ok := tag.(BaseDependencyTag); ok { 1414 panic("BaseDependencyTag is not allowed to be used directly!") 1415 } 1416 1417 if depName == module.Name() { 1418 return []error{&BlueprintError{ 1419 Err: fmt.Errorf("%q depends on itself", depName), 1420 Pos: module.pos, 1421 }} 1422 } 1423 1424 possibleDeps := c.modulesFromName(depName, module.namespace()) 1425 if possibleDeps == nil { 1426 return c.discoveredMissingDependencies(module, depName) 1427 } 1428 1429 if m := c.findMatchingVariant(module, possibleDeps); m != nil { 1430 module.directDeps = append(module.directDeps, depInfo{m, tag}) 1431 atomic.AddUint32(&c.depsModified, 1) 1432 return nil 1433 } 1434 1435 variants := make([]string, len(possibleDeps)) 1436 for i, mod := range possibleDeps { 1437 variants[i] = c.prettyPrintVariant(mod.variant) 1438 } 1439 sort.Strings(variants) 1440 1441 return []error{&BlueprintError{ 1442 Err: fmt.Errorf("dependency %q of %q missing variant:\n %s\navailable variants:\n %s", 1443 depName, module.Name(), 1444 c.prettyPrintVariant(module.dependencyVariant), 1445 strings.Join(variants, "\n ")), 1446 Pos: module.pos, 1447 }} 1448} 1449 1450func (c *Context) findReverseDependency(module *moduleInfo, destName string) (*moduleInfo, []error) { 1451 if destName == module.Name() { 1452 return nil, []error{&BlueprintError{ 1453 Err: fmt.Errorf("%q depends on itself", destName), 1454 Pos: module.pos, 1455 }} 1456 } 1457 1458 possibleDeps := c.modulesFromName(destName, module.namespace()) 1459 if possibleDeps == nil { 1460 return nil, []error{&BlueprintError{ 1461 Err: fmt.Errorf("%q has a reverse dependency on undefined module %q", 1462 module.Name(), destName), 1463 Pos: module.pos, 1464 }} 1465 } 1466 1467 if m := c.findMatchingVariant(module, possibleDeps); m != nil { 1468 return m, nil 1469 } 1470 1471 variants := make([]string, len(possibleDeps)) 1472 for i, mod := range possibleDeps { 1473 variants[i] = c.prettyPrintVariant(mod.variant) 1474 } 1475 sort.Strings(variants) 1476 1477 return nil, []error{&BlueprintError{ 1478 Err: fmt.Errorf("reverse dependency %q of %q missing variant:\n %s\navailable variants:\n %s", 1479 destName, module.Name(), 1480 c.prettyPrintVariant(module.dependencyVariant), 1481 strings.Join(variants, "\n ")), 1482 Pos: module.pos, 1483 }} 1484} 1485 1486func (c *Context) addVariationDependency(module *moduleInfo, variations []Variation, 1487 tag DependencyTag, depName string, far bool) []error { 1488 if _, ok := tag.(BaseDependencyTag); ok { 1489 panic("BaseDependencyTag is not allowed to be used directly!") 1490 } 1491 1492 possibleDeps := c.modulesFromName(depName, module.namespace()) 1493 if possibleDeps == nil { 1494 return c.discoveredMissingDependencies(module, depName) 1495 } 1496 1497 // We can't just append variant.Variant to module.dependencyVariants.variantName and 1498 // compare the strings because the result won't be in mutator registration order. 1499 // Create a new map instead, and then deep compare the maps. 1500 var newVariant variationMap 1501 if !far { 1502 newVariant = module.dependencyVariant.clone() 1503 } else { 1504 newVariant = make(variationMap) 1505 } 1506 for _, v := range variations { 1507 newVariant[v.Mutator] = v.Variation 1508 } 1509 1510 for _, m := range possibleDeps { 1511 var found bool 1512 if far { 1513 found = m.variant.subset(newVariant) 1514 } else { 1515 found = m.variant.equal(newVariant) 1516 } 1517 if found { 1518 if module == m { 1519 return []error{&BlueprintError{ 1520 Err: fmt.Errorf("%q depends on itself", depName), 1521 Pos: module.pos, 1522 }} 1523 } 1524 // AddVariationDependency allows adding a dependency on itself, but only if 1525 // that module is earlier in the module list than this one, since we always 1526 // run GenerateBuildActions in order for the variants of a module 1527 if m.group == module.group && beforeInModuleList(module, m, module.group.modules) { 1528 return []error{&BlueprintError{ 1529 Err: fmt.Errorf("%q depends on later version of itself", depName), 1530 Pos: module.pos, 1531 }} 1532 } 1533 module.directDeps = append(module.directDeps, depInfo{m, tag}) 1534 atomic.AddUint32(&c.depsModified, 1) 1535 return nil 1536 } 1537 } 1538 1539 variants := make([]string, len(possibleDeps)) 1540 for i, mod := range possibleDeps { 1541 variants[i] = c.prettyPrintVariant(mod.variant) 1542 } 1543 sort.Strings(variants) 1544 1545 return []error{&BlueprintError{ 1546 Err: fmt.Errorf("dependency %q of %q missing variant:\n %s\navailable variants:\n %s", 1547 depName, module.Name(), 1548 c.prettyPrintVariant(newVariant), 1549 strings.Join(variants, "\n ")), 1550 Pos: module.pos, 1551 }} 1552} 1553 1554func (c *Context) addInterVariantDependency(origModule *moduleInfo, tag DependencyTag, 1555 from, to Module) { 1556 if _, ok := tag.(BaseDependencyTag); ok { 1557 panic("BaseDependencyTag is not allowed to be used directly!") 1558 } 1559 1560 var fromInfo, toInfo *moduleInfo 1561 for _, m := range origModule.splitModules { 1562 if m.logicModule == from { 1563 fromInfo = m 1564 } 1565 if m.logicModule == to { 1566 toInfo = m 1567 if fromInfo != nil { 1568 panic(fmt.Errorf("%q depends on later version of itself", origModule.Name())) 1569 } 1570 } 1571 } 1572 1573 if fromInfo == nil || toInfo == nil { 1574 panic(fmt.Errorf("AddInterVariantDependency called for module %q on invalid variant", 1575 origModule.Name())) 1576 } 1577 1578 fromInfo.directDeps = append(fromInfo.directDeps, depInfo{toInfo, tag}) 1579 atomic.AddUint32(&c.depsModified, 1) 1580} 1581 1582// findBlueprintDescendants returns a map linking parent Blueprints files to child Blueprints files 1583// For example, if paths = []string{"a/b/c/Android.bp", "a/Blueprints"}, 1584// then descendants = {"":[]string{"a/Blueprints"}, "a/Blueprints":[]string{"a/b/c/Android.bp"}} 1585func findBlueprintDescendants(paths []string) (descendants map[string][]string, err error) { 1586 // make mapping from dir path to file path 1587 filesByDir := make(map[string]string, len(paths)) 1588 for _, path := range paths { 1589 dir := filepath.Dir(path) 1590 _, alreadyFound := filesByDir[dir] 1591 if alreadyFound { 1592 return nil, fmt.Errorf("Found two Blueprint files in directory %v : %v and %v", dir, filesByDir[dir], path) 1593 } 1594 filesByDir[dir] = path 1595 } 1596 1597 findAncestor := func(childFile string) (ancestor string) { 1598 prevAncestorDir := filepath.Dir(childFile) 1599 for { 1600 ancestorDir := filepath.Dir(prevAncestorDir) 1601 if ancestorDir == prevAncestorDir { 1602 // reached the root dir without any matches; assign this as a descendant of "" 1603 return "" 1604 } 1605 1606 ancestorFile, ancestorExists := filesByDir[ancestorDir] 1607 if ancestorExists { 1608 return ancestorFile 1609 } 1610 prevAncestorDir = ancestorDir 1611 } 1612 } 1613 // generate the descendants map 1614 descendants = make(map[string][]string, len(filesByDir)) 1615 for _, childFile := range filesByDir { 1616 ancestorFile := findAncestor(childFile) 1617 descendants[ancestorFile] = append(descendants[ancestorFile], childFile) 1618 } 1619 return descendants, nil 1620} 1621 1622type visitOrderer interface { 1623 // returns the number of modules that this module needs to wait for 1624 waitCount(module *moduleInfo) int 1625 // returns the list of modules that are waiting for this module 1626 propagate(module *moduleInfo) []*moduleInfo 1627 // visit modules in order 1628 visit(modules []*moduleInfo, visit func(*moduleInfo) bool) 1629} 1630 1631type unorderedVisitorImpl struct{} 1632 1633func (unorderedVisitorImpl) waitCount(module *moduleInfo) int { 1634 return 0 1635} 1636 1637func (unorderedVisitorImpl) propagate(module *moduleInfo) []*moduleInfo { 1638 return nil 1639} 1640 1641func (unorderedVisitorImpl) visit(modules []*moduleInfo, visit func(*moduleInfo) bool) { 1642 for _, module := range modules { 1643 if visit(module) { 1644 return 1645 } 1646 } 1647} 1648 1649type bottomUpVisitorImpl struct{} 1650 1651func (bottomUpVisitorImpl) waitCount(module *moduleInfo) int { 1652 return len(module.forwardDeps) 1653} 1654 1655func (bottomUpVisitorImpl) propagate(module *moduleInfo) []*moduleInfo { 1656 return module.reverseDeps 1657} 1658 1659func (bottomUpVisitorImpl) visit(modules []*moduleInfo, visit func(*moduleInfo) bool) { 1660 for _, module := range modules { 1661 if visit(module) { 1662 return 1663 } 1664 } 1665} 1666 1667type topDownVisitorImpl struct{} 1668 1669func (topDownVisitorImpl) waitCount(module *moduleInfo) int { 1670 return len(module.reverseDeps) 1671} 1672 1673func (topDownVisitorImpl) propagate(module *moduleInfo) []*moduleInfo { 1674 return module.forwardDeps 1675} 1676 1677func (topDownVisitorImpl) visit(modules []*moduleInfo, visit func(*moduleInfo) bool) { 1678 for i := 0; i < len(modules); i++ { 1679 module := modules[len(modules)-1-i] 1680 if visit(module) { 1681 return 1682 } 1683 } 1684} 1685 1686var ( 1687 bottomUpVisitor bottomUpVisitorImpl 1688 topDownVisitor topDownVisitorImpl 1689) 1690 1691// Calls visit on each module, guaranteeing that visit is not called on a module until visit on all 1692// of its dependencies has finished. 1693func (c *Context) parallelVisit(order visitOrderer, visit func(group *moduleInfo) bool) { 1694 doneCh := make(chan *moduleInfo) 1695 cancelCh := make(chan bool) 1696 count := 0 1697 cancel := false 1698 var backlog []*moduleInfo 1699 const limit = 1000 1700 1701 for _, module := range c.modulesSorted { 1702 module.waitingCount = order.waitCount(module) 1703 } 1704 1705 visitOne := func(module *moduleInfo) { 1706 if count < limit { 1707 count++ 1708 go func() { 1709 ret := visit(module) 1710 if ret { 1711 cancelCh <- true 1712 } 1713 doneCh <- module 1714 }() 1715 } else { 1716 backlog = append(backlog, module) 1717 } 1718 } 1719 1720 for _, module := range c.modulesSorted { 1721 if module.waitingCount == 0 { 1722 visitOne(module) 1723 } 1724 } 1725 1726 for count > 0 || len(backlog) > 0 { 1727 select { 1728 case <-cancelCh: 1729 cancel = true 1730 backlog = nil 1731 case doneModule := <-doneCh: 1732 count-- 1733 if !cancel { 1734 for count < limit && len(backlog) > 0 { 1735 toVisit := backlog[0] 1736 backlog = backlog[1:] 1737 visitOne(toVisit) 1738 } 1739 for _, module := range order.propagate(doneModule) { 1740 module.waitingCount-- 1741 if module.waitingCount == 0 { 1742 visitOne(module) 1743 } 1744 } 1745 } 1746 } 1747 } 1748} 1749 1750// updateDependencies recursively walks the module dependency graph and updates 1751// additional fields based on the dependencies. It builds a sorted list of modules 1752// such that dependencies of a module always appear first, and populates reverse 1753// dependency links and counts of total dependencies. It also reports errors when 1754// it encounters dependency cycles. This should called after resolveDependencies, 1755// as well as after any mutator pass has called addDependency 1756func (c *Context) updateDependencies() (errs []error) { 1757 visited := make(map[*moduleInfo]bool) // modules that were already checked 1758 checking := make(map[*moduleInfo]bool) // modules actively being checked 1759 1760 sorted := make([]*moduleInfo, 0, len(c.moduleInfo)) 1761 1762 var check func(group *moduleInfo) []*moduleInfo 1763 1764 cycleError := func(cycle []*moduleInfo) { 1765 // We are the "start" of the cycle, so we're responsible 1766 // for generating the errors. The cycle list is in 1767 // reverse order because all the 'check' calls append 1768 // their own module to the list. 1769 errs = append(errs, &BlueprintError{ 1770 Err: fmt.Errorf("encountered dependency cycle:"), 1771 Pos: cycle[len(cycle)-1].pos, 1772 }) 1773 1774 // Iterate backwards through the cycle list. 1775 curModule := cycle[0] 1776 for i := len(cycle) - 1; i >= 0; i-- { 1777 nextModule := cycle[i] 1778 errs = append(errs, &BlueprintError{ 1779 Err: fmt.Errorf(" %q depends on %q", 1780 curModule.Name(), 1781 nextModule.Name()), 1782 Pos: curModule.pos, 1783 }) 1784 curModule = nextModule 1785 } 1786 } 1787 1788 check = func(module *moduleInfo) []*moduleInfo { 1789 visited[module] = true 1790 checking[module] = true 1791 defer delete(checking, module) 1792 1793 deps := make(map[*moduleInfo]bool) 1794 1795 // Add an implicit dependency ordering on all earlier modules in the same module group 1796 for _, dep := range module.group.modules { 1797 if dep == module { 1798 break 1799 } 1800 deps[dep] = true 1801 } 1802 1803 for _, dep := range module.directDeps { 1804 deps[dep.module] = true 1805 } 1806 1807 module.reverseDeps = []*moduleInfo{} 1808 module.forwardDeps = []*moduleInfo{} 1809 1810 for dep := range deps { 1811 if checking[dep] { 1812 // This is a cycle. 1813 return []*moduleInfo{dep, module} 1814 } 1815 1816 if !visited[dep] { 1817 cycle := check(dep) 1818 if cycle != nil { 1819 if cycle[0] == module { 1820 // We are the "start" of the cycle, so we're responsible 1821 // for generating the errors. The cycle list is in 1822 // reverse order because all the 'check' calls append 1823 // their own module to the list. 1824 cycleError(cycle) 1825 1826 // We can continue processing this module's children to 1827 // find more cycles. Since all the modules that were 1828 // part of the found cycle were marked as visited we 1829 // won't run into that cycle again. 1830 } else { 1831 // We're not the "start" of the cycle, so we just append 1832 // our module to the list and return it. 1833 return append(cycle, module) 1834 } 1835 } 1836 } 1837 1838 module.forwardDeps = append(module.forwardDeps, dep) 1839 dep.reverseDeps = append(dep.reverseDeps, module) 1840 } 1841 1842 sorted = append(sorted, module) 1843 1844 return nil 1845 } 1846 1847 for _, module := range c.moduleInfo { 1848 if !visited[module] { 1849 cycle := check(module) 1850 if cycle != nil { 1851 if cycle[len(cycle)-1] != module { 1852 panic("inconceivable!") 1853 } 1854 cycleError(cycle) 1855 } 1856 } 1857 } 1858 1859 c.modulesSorted = sorted 1860 1861 return 1862} 1863 1864// PrepareBuildActions generates an internal representation of all the build 1865// actions that need to be performed. This process involves invoking the 1866// GenerateBuildActions method on each of the Module objects created during the 1867// parse phase and then on each of the registered Singleton objects. 1868// 1869// If the ResolveDependencies method has not already been called it is called 1870// automatically by this method. 1871// 1872// The config argument is made available to all of the Module and Singleton 1873// objects via the Config method on the ModuleContext and SingletonContext 1874// objects passed to GenerateBuildActions. It is also passed to the functions 1875// specified via PoolFunc, RuleFunc, and VariableFunc so that they can compute 1876// config-specific values. 1877// 1878// The returned deps is a list of the ninja files dependencies that were added 1879// by the modules and singletons via the ModuleContext.AddNinjaFileDeps(), 1880// SingletonContext.AddNinjaFileDeps(), and PackageContext.AddNinjaFileDeps() 1881// methods. 1882func (c *Context) PrepareBuildActions(config interface{}) (deps []string, errs []error) { 1883 pprof.Do(c.Context, pprof.Labels("blueprint", "PrepareBuildActions"), func(ctx context.Context) { 1884 c.buildActionsReady = false 1885 1886 if !c.dependenciesReady { 1887 var extraDeps []string 1888 extraDeps, errs = c.resolveDependencies(ctx, config) 1889 if len(errs) > 0 { 1890 return 1891 } 1892 deps = append(deps, extraDeps...) 1893 } 1894 1895 var depsModules []string 1896 depsModules, errs = c.generateModuleBuildActions(config, c.liveGlobals) 1897 if len(errs) > 0 { 1898 return 1899 } 1900 1901 var depsSingletons []string 1902 depsSingletons, errs = c.generateSingletonBuildActions(config, c.singletonInfo, c.liveGlobals) 1903 if len(errs) > 0 { 1904 return 1905 } 1906 1907 deps = append(deps, depsModules...) 1908 deps = append(deps, depsSingletons...) 1909 1910 if c.ninjaBuildDir != nil { 1911 err := c.liveGlobals.addNinjaStringDeps(c.ninjaBuildDir) 1912 if err != nil { 1913 errs = []error{err} 1914 return 1915 } 1916 } 1917 1918 pkgNames, depsPackages := c.makeUniquePackageNames(c.liveGlobals) 1919 1920 deps = append(deps, depsPackages...) 1921 1922 // This will panic if it finds a problem since it's a programming error. 1923 c.checkForVariableReferenceCycles(c.liveGlobals.variables, pkgNames) 1924 1925 c.pkgNames = pkgNames 1926 c.globalVariables = c.liveGlobals.variables 1927 c.globalPools = c.liveGlobals.pools 1928 c.globalRules = c.liveGlobals.rules 1929 1930 c.buildActionsReady = true 1931 }) 1932 1933 if len(errs) > 0 { 1934 return nil, errs 1935 } 1936 1937 return deps, nil 1938} 1939 1940func (c *Context) runMutators(ctx context.Context, config interface{}) (deps []string, errs []error) { 1941 var mutators []*mutatorInfo 1942 1943 pprof.Do(ctx, pprof.Labels("blueprint", "runMutators"), func(ctx context.Context) { 1944 mutators = append(mutators, c.earlyMutatorInfo...) 1945 mutators = append(mutators, c.mutatorInfo...) 1946 1947 for _, mutator := range mutators { 1948 pprof.Do(ctx, pprof.Labels("mutator", mutator.name), func(context.Context) { 1949 var newDeps []string 1950 if mutator.topDownMutator != nil { 1951 newDeps, errs = c.runMutator(config, mutator, topDownMutator) 1952 } else if mutator.bottomUpMutator != nil { 1953 newDeps, errs = c.runMutator(config, mutator, bottomUpMutator) 1954 } else { 1955 panic("no mutator set on " + mutator.name) 1956 } 1957 if len(errs) > 0 { 1958 return 1959 } 1960 deps = append(deps, newDeps...) 1961 }) 1962 if len(errs) > 0 { 1963 return 1964 } 1965 } 1966 }) 1967 1968 if len(errs) > 0 { 1969 return nil, errs 1970 } 1971 1972 return deps, nil 1973} 1974 1975type mutatorDirection interface { 1976 run(mutator *mutatorInfo, ctx *mutatorContext) 1977 orderer() visitOrderer 1978 fmt.Stringer 1979} 1980 1981type bottomUpMutatorImpl struct{} 1982 1983func (bottomUpMutatorImpl) run(mutator *mutatorInfo, ctx *mutatorContext) { 1984 mutator.bottomUpMutator(ctx) 1985} 1986 1987func (bottomUpMutatorImpl) orderer() visitOrderer { 1988 return bottomUpVisitor 1989} 1990 1991func (bottomUpMutatorImpl) String() string { 1992 return "bottom up mutator" 1993} 1994 1995type topDownMutatorImpl struct{} 1996 1997func (topDownMutatorImpl) run(mutator *mutatorInfo, ctx *mutatorContext) { 1998 mutator.topDownMutator(ctx) 1999} 2000 2001func (topDownMutatorImpl) orderer() visitOrderer { 2002 return topDownVisitor 2003} 2004 2005func (topDownMutatorImpl) String() string { 2006 return "top down mutator" 2007} 2008 2009var ( 2010 topDownMutator topDownMutatorImpl 2011 bottomUpMutator bottomUpMutatorImpl 2012) 2013 2014type reverseDep struct { 2015 module *moduleInfo 2016 dep depInfo 2017} 2018 2019func (c *Context) runMutator(config interface{}, mutator *mutatorInfo, 2020 direction mutatorDirection) (deps []string, errs []error) { 2021 2022 newModuleInfo := make(map[Module]*moduleInfo) 2023 for k, v := range c.moduleInfo { 2024 newModuleInfo[k] = v 2025 } 2026 2027 type globalStateChange struct { 2028 reverse []reverseDep 2029 rename []rename 2030 replace []replace 2031 newModules []*moduleInfo 2032 deps []string 2033 } 2034 2035 reverseDeps := make(map[*moduleInfo][]depInfo) 2036 var rename []rename 2037 var replace []replace 2038 var newModules []*moduleInfo 2039 2040 errsCh := make(chan []error) 2041 globalStateCh := make(chan globalStateChange) 2042 newVariationsCh := make(chan []*moduleInfo) 2043 done := make(chan bool) 2044 2045 c.depsModified = 0 2046 2047 visit := func(module *moduleInfo) bool { 2048 if module.splitModules != nil { 2049 panic("split module found in sorted module list") 2050 } 2051 2052 mctx := &mutatorContext{ 2053 baseModuleContext: baseModuleContext{ 2054 context: c, 2055 config: config, 2056 module: module, 2057 }, 2058 name: mutator.name, 2059 } 2060 2061 func() { 2062 defer func() { 2063 if r := recover(); r != nil { 2064 in := fmt.Sprintf("%s %q for %s", direction, mutator.name, module) 2065 if err, ok := r.(panicError); ok { 2066 err.addIn(in) 2067 mctx.error(err) 2068 } else { 2069 mctx.error(newPanicErrorf(r, in)) 2070 } 2071 } 2072 }() 2073 direction.run(mutator, mctx) 2074 }() 2075 2076 if len(mctx.errs) > 0 { 2077 errsCh <- mctx.errs 2078 return true 2079 } 2080 2081 if len(mctx.newVariations) > 0 { 2082 newVariationsCh <- mctx.newVariations 2083 } 2084 2085 if len(mctx.reverseDeps) > 0 || len(mctx.replace) > 0 || len(mctx.rename) > 0 || len(mctx.newModules) > 0 { 2086 globalStateCh <- globalStateChange{ 2087 reverse: mctx.reverseDeps, 2088 replace: mctx.replace, 2089 rename: mctx.rename, 2090 newModules: mctx.newModules, 2091 deps: mctx.ninjaFileDeps, 2092 } 2093 } 2094 2095 return false 2096 } 2097 2098 // Process errs and reverseDeps in a single goroutine 2099 go func() { 2100 for { 2101 select { 2102 case newErrs := <-errsCh: 2103 errs = append(errs, newErrs...) 2104 case globalStateChange := <-globalStateCh: 2105 for _, r := range globalStateChange.reverse { 2106 reverseDeps[r.module] = append(reverseDeps[r.module], r.dep) 2107 } 2108 replace = append(replace, globalStateChange.replace...) 2109 rename = append(rename, globalStateChange.rename...) 2110 newModules = append(newModules, globalStateChange.newModules...) 2111 deps = append(deps, globalStateChange.deps...) 2112 case newVariations := <-newVariationsCh: 2113 for _, m := range newVariations { 2114 newModuleInfo[m.logicModule] = m 2115 } 2116 case <-done: 2117 return 2118 } 2119 } 2120 }() 2121 2122 if mutator.parallel { 2123 c.parallelVisit(direction.orderer(), visit) 2124 } else { 2125 direction.orderer().visit(c.modulesSorted, visit) 2126 } 2127 2128 done <- true 2129 2130 if len(errs) > 0 { 2131 return nil, errs 2132 } 2133 2134 c.moduleInfo = newModuleInfo 2135 2136 for _, group := range c.moduleGroups { 2137 for i := 0; i < len(group.modules); i++ { 2138 module := group.modules[i] 2139 2140 // Update module group to contain newly split variants 2141 if module.splitModules != nil { 2142 group.modules, i = spliceModules(group.modules, i, module.splitModules) 2143 } 2144 2145 // Fix up any remaining dependencies on modules that were split into variants 2146 // by replacing them with the first variant 2147 for j, dep := range module.directDeps { 2148 if dep.module.logicModule == nil { 2149 module.directDeps[j].module = dep.module.splitModules[0] 2150 } 2151 } 2152 } 2153 } 2154 2155 for module, deps := range reverseDeps { 2156 sort.Sort(depSorter(deps)) 2157 module.directDeps = append(module.directDeps, deps...) 2158 c.depsModified++ 2159 } 2160 2161 for _, module := range newModules { 2162 errs = c.addModule(module) 2163 if len(errs) > 0 { 2164 return nil, errs 2165 } 2166 atomic.AddUint32(&c.depsModified, 1) 2167 } 2168 2169 errs = c.handleRenames(rename) 2170 if len(errs) > 0 { 2171 return nil, errs 2172 } 2173 2174 errs = c.handleReplacements(replace) 2175 if len(errs) > 0 { 2176 return nil, errs 2177 } 2178 2179 if c.depsModified > 0 { 2180 errs = c.updateDependencies() 2181 if len(errs) > 0 { 2182 return nil, errs 2183 } 2184 } 2185 2186 return deps, errs 2187} 2188 2189// Replaces every build logic module with a clone of itself. Prevents introducing problems where 2190// a mutator sets a non-property member variable on a module, which works until a later mutator 2191// creates variants of that module. 2192func (c *Context) cloneModules() { 2193 type update struct { 2194 orig Module 2195 clone *moduleInfo 2196 } 2197 ch := make(chan update) 2198 doneCh := make(chan bool) 2199 go func() { 2200 c.parallelVisit(unorderedVisitorImpl{}, func(m *moduleInfo) bool { 2201 origLogicModule := m.logicModule 2202 m.logicModule, m.properties = c.cloneLogicModule(m) 2203 ch <- update{origLogicModule, m} 2204 return false 2205 }) 2206 doneCh <- true 2207 }() 2208 2209 done := false 2210 for !done { 2211 select { 2212 case <-doneCh: 2213 done = true 2214 case update := <-ch: 2215 delete(c.moduleInfo, update.orig) 2216 c.moduleInfo[update.clone.logicModule] = update.clone 2217 } 2218 } 2219} 2220 2221// Removes modules[i] from the list and inserts newModules... where it was located, returning 2222// the new slice and the index of the last inserted element 2223func spliceModules(modules []*moduleInfo, i int, newModules []*moduleInfo) ([]*moduleInfo, int) { 2224 spliceSize := len(newModules) 2225 newLen := len(modules) + spliceSize - 1 2226 var dest []*moduleInfo 2227 if cap(modules) >= len(modules)-1+len(newModules) { 2228 // We can fit the splice in the existing capacity, do everything in place 2229 dest = modules[:newLen] 2230 } else { 2231 dest = make([]*moduleInfo, newLen) 2232 copy(dest, modules[:i]) 2233 } 2234 2235 // Move the end of the slice over by spliceSize-1 2236 copy(dest[i+spliceSize:], modules[i+1:]) 2237 2238 // Copy the new modules into the slice 2239 copy(dest[i:], newModules) 2240 2241 return dest, i + spliceSize - 1 2242} 2243 2244func (c *Context) generateModuleBuildActions(config interface{}, 2245 liveGlobals *liveTracker) ([]string, []error) { 2246 2247 var deps []string 2248 var errs []error 2249 2250 cancelCh := make(chan struct{}) 2251 errsCh := make(chan []error) 2252 depsCh := make(chan []string) 2253 2254 go func() { 2255 for { 2256 select { 2257 case <-cancelCh: 2258 close(cancelCh) 2259 return 2260 case newErrs := <-errsCh: 2261 errs = append(errs, newErrs...) 2262 case newDeps := <-depsCh: 2263 deps = append(deps, newDeps...) 2264 2265 } 2266 } 2267 }() 2268 2269 c.parallelVisit(bottomUpVisitor, func(module *moduleInfo) bool { 2270 2271 uniqueName := c.nameInterface.UniqueName(newNamespaceContext(module), module.group.name) 2272 sanitizedName := toNinjaName(uniqueName) 2273 2274 prefix := moduleNamespacePrefix(sanitizedName + "_" + module.variantName) 2275 2276 // The parent scope of the moduleContext's local scope gets overridden to be that of the 2277 // calling Go package on a per-call basis. Since the initial parent scope doesn't matter we 2278 // just set it to nil. 2279 scope := newLocalScope(nil, prefix) 2280 2281 mctx := &moduleContext{ 2282 baseModuleContext: baseModuleContext{ 2283 context: c, 2284 config: config, 2285 module: module, 2286 }, 2287 scope: scope, 2288 handledMissingDeps: module.missingDeps == nil, 2289 } 2290 2291 func() { 2292 defer func() { 2293 if r := recover(); r != nil { 2294 in := fmt.Sprintf("GenerateBuildActions for %s", module) 2295 if err, ok := r.(panicError); ok { 2296 err.addIn(in) 2297 mctx.error(err) 2298 } else { 2299 mctx.error(newPanicErrorf(r, in)) 2300 } 2301 } 2302 }() 2303 mctx.module.logicModule.GenerateBuildActions(mctx) 2304 }() 2305 2306 if len(mctx.errs) > 0 { 2307 errsCh <- mctx.errs 2308 return true 2309 } 2310 2311 if module.missingDeps != nil && !mctx.handledMissingDeps { 2312 var errs []error 2313 for _, depName := range module.missingDeps { 2314 errs = append(errs, c.missingDependencyError(module, depName)) 2315 } 2316 errsCh <- errs 2317 return true 2318 } 2319 2320 depsCh <- mctx.ninjaFileDeps 2321 2322 newErrs := c.processLocalBuildActions(&module.actionDefs, 2323 &mctx.actionDefs, liveGlobals) 2324 if len(newErrs) > 0 { 2325 errsCh <- newErrs 2326 return true 2327 } 2328 return false 2329 }) 2330 2331 cancelCh <- struct{}{} 2332 <-cancelCh 2333 2334 return deps, errs 2335} 2336 2337func (c *Context) generateSingletonBuildActions(config interface{}, 2338 singletons []*singletonInfo, liveGlobals *liveTracker) ([]string, []error) { 2339 2340 var deps []string 2341 var errs []error 2342 2343 for _, info := range singletons { 2344 // The parent scope of the singletonContext's local scope gets overridden to be that of the 2345 // calling Go package on a per-call basis. Since the initial parent scope doesn't matter we 2346 // just set it to nil. 2347 scope := newLocalScope(nil, singletonNamespacePrefix(info.name)) 2348 2349 sctx := &singletonContext{ 2350 name: info.name, 2351 context: c, 2352 config: config, 2353 scope: scope, 2354 globals: liveGlobals, 2355 } 2356 2357 func() { 2358 defer func() { 2359 if r := recover(); r != nil { 2360 in := fmt.Sprintf("GenerateBuildActions for singleton %s", info.name) 2361 if err, ok := r.(panicError); ok { 2362 err.addIn(in) 2363 sctx.error(err) 2364 } else { 2365 sctx.error(newPanicErrorf(r, in)) 2366 } 2367 } 2368 }() 2369 info.singleton.GenerateBuildActions(sctx) 2370 }() 2371 2372 if len(sctx.errs) > 0 { 2373 errs = append(errs, sctx.errs...) 2374 if len(errs) > maxErrors { 2375 break 2376 } 2377 continue 2378 } 2379 2380 deps = append(deps, sctx.ninjaFileDeps...) 2381 2382 newErrs := c.processLocalBuildActions(&info.actionDefs, 2383 &sctx.actionDefs, liveGlobals) 2384 errs = append(errs, newErrs...) 2385 if len(errs) > maxErrors { 2386 break 2387 } 2388 } 2389 2390 return deps, errs 2391} 2392 2393func (c *Context) processLocalBuildActions(out, in *localBuildActions, 2394 liveGlobals *liveTracker) []error { 2395 2396 var errs []error 2397 2398 // First we go through and add everything referenced by the module's 2399 // buildDefs to the live globals set. This will end up adding the live 2400 // locals to the set as well, but we'll take them out after. 2401 for _, def := range in.buildDefs { 2402 err := liveGlobals.AddBuildDefDeps(def) 2403 if err != nil { 2404 errs = append(errs, err) 2405 } 2406 } 2407 2408 if len(errs) > 0 { 2409 return errs 2410 } 2411 2412 out.buildDefs = append(out.buildDefs, in.buildDefs...) 2413 2414 // We use the now-incorrect set of live "globals" to determine which local 2415 // definitions are live. As we go through copying those live locals to the 2416 // moduleGroup we remove them from the live globals set. 2417 for _, v := range in.variables { 2418 isLive := liveGlobals.RemoveVariableIfLive(v) 2419 if isLive { 2420 out.variables = append(out.variables, v) 2421 } 2422 } 2423 2424 for _, r := range in.rules { 2425 isLive := liveGlobals.RemoveRuleIfLive(r) 2426 if isLive { 2427 out.rules = append(out.rules, r) 2428 } 2429 } 2430 2431 return nil 2432} 2433 2434func (c *Context) walkDeps(topModule *moduleInfo, allowDuplicates bool, 2435 visitDown func(depInfo, *moduleInfo) bool, visitUp func(depInfo, *moduleInfo)) { 2436 2437 visited := make(map[*moduleInfo]bool) 2438 var visiting *moduleInfo 2439 2440 defer func() { 2441 if r := recover(); r != nil { 2442 panic(newPanicErrorf(r, "WalkDeps(%s, %s, %s) for dependency %s", 2443 topModule, funcName(visitDown), funcName(visitUp), visiting)) 2444 } 2445 }() 2446 2447 var walk func(module *moduleInfo) 2448 walk = func(module *moduleInfo) { 2449 for _, dep := range module.directDeps { 2450 if allowDuplicates || !visited[dep.module] { 2451 visiting = dep.module 2452 recurse := true 2453 if visitDown != nil { 2454 recurse = visitDown(dep, module) 2455 } 2456 if recurse && !visited[dep.module] { 2457 walk(dep.module) 2458 } 2459 visited[dep.module] = true 2460 if visitUp != nil { 2461 visitUp(dep, module) 2462 } 2463 } 2464 } 2465 } 2466 2467 walk(topModule) 2468} 2469 2470type replace struct { 2471 from, to *moduleInfo 2472} 2473 2474type rename struct { 2475 group *moduleGroup 2476 name string 2477} 2478 2479func (c *Context) moduleMatchingVariant(module *moduleInfo, name string) *moduleInfo { 2480 targets := c.modulesFromName(name, module.namespace()) 2481 2482 if targets == nil { 2483 return nil 2484 } 2485 2486 for _, m := range targets { 2487 if module.variantName == m.variantName { 2488 return m 2489 } 2490 } 2491 2492 return nil 2493} 2494 2495func (c *Context) handleRenames(renames []rename) []error { 2496 var errs []error 2497 for _, rename := range renames { 2498 group, name := rename.group, rename.name 2499 if name == group.name || len(group.modules) < 1 { 2500 continue 2501 } 2502 2503 errs = append(errs, c.nameInterface.Rename(group.name, rename.name, group.namespace)...) 2504 } 2505 2506 return errs 2507} 2508 2509func (c *Context) handleReplacements(replacements []replace) []error { 2510 var errs []error 2511 for _, replace := range replacements { 2512 for _, m := range replace.from.reverseDeps { 2513 for i, d := range m.directDeps { 2514 if d.module == replace.from { 2515 m.directDeps[i].module = replace.to 2516 } 2517 } 2518 } 2519 2520 atomic.AddUint32(&c.depsModified, 1) 2521 } 2522 2523 return errs 2524} 2525 2526func (c *Context) discoveredMissingDependencies(module *moduleInfo, depName string) (errs []error) { 2527 if c.allowMissingDependencies { 2528 module.missingDeps = append(module.missingDeps, depName) 2529 return nil 2530 } 2531 return []error{c.missingDependencyError(module, depName)} 2532} 2533 2534func (c *Context) missingDependencyError(module *moduleInfo, depName string) (errs error) { 2535 err := c.nameInterface.MissingDependencyError(module.Name(), module.namespace(), depName) 2536 2537 return &BlueprintError{ 2538 Err: err, 2539 Pos: module.pos, 2540 } 2541} 2542 2543func (c *Context) modulesFromName(name string, namespace Namespace) []*moduleInfo { 2544 group, exists := c.nameInterface.ModuleFromName(name, namespace) 2545 if exists { 2546 return group.modules 2547 } 2548 return nil 2549} 2550 2551func (c *Context) sortedModuleGroups() []*moduleGroup { 2552 if c.cachedSortedModuleGroups == nil { 2553 unwrap := func(wrappers []ModuleGroup) []*moduleGroup { 2554 result := make([]*moduleGroup, 0, len(wrappers)) 2555 for _, group := range wrappers { 2556 result = append(result, group.moduleGroup) 2557 } 2558 return result 2559 } 2560 2561 c.cachedSortedModuleGroups = unwrap(c.nameInterface.AllModules()) 2562 } 2563 2564 return c.cachedSortedModuleGroups 2565} 2566 2567func (c *Context) visitAllModules(visit func(Module)) { 2568 var module *moduleInfo 2569 2570 defer func() { 2571 if r := recover(); r != nil { 2572 panic(newPanicErrorf(r, "VisitAllModules(%s) for %s", 2573 funcName(visit), module)) 2574 } 2575 }() 2576 2577 for _, moduleGroup := range c.sortedModuleGroups() { 2578 for _, module = range moduleGroup.modules { 2579 visit(module.logicModule) 2580 } 2581 } 2582} 2583 2584func (c *Context) visitAllModulesIf(pred func(Module) bool, 2585 visit func(Module)) { 2586 2587 var module *moduleInfo 2588 2589 defer func() { 2590 if r := recover(); r != nil { 2591 panic(newPanicErrorf(r, "VisitAllModulesIf(%s, %s) for %s", 2592 funcName(pred), funcName(visit), module)) 2593 } 2594 }() 2595 2596 for _, moduleGroup := range c.sortedModuleGroups() { 2597 for _, module := range moduleGroup.modules { 2598 if pred(module.logicModule) { 2599 visit(module.logicModule) 2600 } 2601 } 2602 } 2603} 2604 2605func (c *Context) visitAllModuleVariants(module *moduleInfo, 2606 visit func(Module)) { 2607 2608 var variant *moduleInfo 2609 2610 defer func() { 2611 if r := recover(); r != nil { 2612 panic(newPanicErrorf(r, "VisitAllModuleVariants(%s, %s) for %s", 2613 module, funcName(visit), variant)) 2614 } 2615 }() 2616 2617 for _, variant = range module.group.modules { 2618 visit(variant.logicModule) 2619 } 2620} 2621 2622func (c *Context) requireNinjaVersion(major, minor, micro int) { 2623 if major != 1 { 2624 panic("ninja version with major version != 1 not supported") 2625 } 2626 if c.requiredNinjaMinor < minor { 2627 c.requiredNinjaMinor = minor 2628 c.requiredNinjaMicro = micro 2629 } 2630 if c.requiredNinjaMinor == minor && c.requiredNinjaMicro < micro { 2631 c.requiredNinjaMicro = micro 2632 } 2633} 2634 2635func (c *Context) setNinjaBuildDir(value *ninjaString) { 2636 if c.ninjaBuildDir == nil { 2637 c.ninjaBuildDir = value 2638 } 2639} 2640 2641func (c *Context) makeUniquePackageNames( 2642 liveGlobals *liveTracker) (map[*packageContext]string, []string) { 2643 2644 pkgs := make(map[string]*packageContext) 2645 pkgNames := make(map[*packageContext]string) 2646 longPkgNames := make(map[*packageContext]bool) 2647 2648 processPackage := func(pctx *packageContext) { 2649 if pctx == nil { 2650 // This is a built-in rule and has no package. 2651 return 2652 } 2653 if _, ok := pkgNames[pctx]; ok { 2654 // We've already processed this package. 2655 return 2656 } 2657 2658 otherPkg, present := pkgs[pctx.shortName] 2659 if present { 2660 // Short name collision. Both this package and the one that's 2661 // already there need to use their full names. We leave the short 2662 // name in pkgNames for now so future collisions still get caught. 2663 longPkgNames[pctx] = true 2664 longPkgNames[otherPkg] = true 2665 } else { 2666 // No collision so far. Tentatively set the package's name to be 2667 // its short name. 2668 pkgNames[pctx] = pctx.shortName 2669 pkgs[pctx.shortName] = pctx 2670 } 2671 } 2672 2673 // We try to give all packages their short name, but when we get collisions 2674 // we need to use the full unique package name. 2675 for v, _ := range liveGlobals.variables { 2676 processPackage(v.packageContext()) 2677 } 2678 for p, _ := range liveGlobals.pools { 2679 processPackage(p.packageContext()) 2680 } 2681 for r, _ := range liveGlobals.rules { 2682 processPackage(r.packageContext()) 2683 } 2684 2685 // Add the packages that had collisions using their full unique names. This 2686 // will overwrite any short names that were added in the previous step. 2687 for pctx := range longPkgNames { 2688 pkgNames[pctx] = pctx.fullName 2689 } 2690 2691 // Create deps list from calls to PackageContext.AddNinjaFileDeps 2692 deps := []string{} 2693 for _, pkg := range pkgs { 2694 deps = append(deps, pkg.ninjaFileDeps...) 2695 } 2696 2697 return pkgNames, deps 2698} 2699 2700func (c *Context) checkForVariableReferenceCycles( 2701 variables map[Variable]*ninjaString, pkgNames map[*packageContext]string) { 2702 2703 visited := make(map[Variable]bool) // variables that were already checked 2704 checking := make(map[Variable]bool) // variables actively being checked 2705 2706 var check func(v Variable) []Variable 2707 2708 check = func(v Variable) []Variable { 2709 visited[v] = true 2710 checking[v] = true 2711 defer delete(checking, v) 2712 2713 value := variables[v] 2714 for _, dep := range value.variables { 2715 if checking[dep] { 2716 // This is a cycle. 2717 return []Variable{dep, v} 2718 } 2719 2720 if !visited[dep] { 2721 cycle := check(dep) 2722 if cycle != nil { 2723 if cycle[0] == v { 2724 // We are the "start" of the cycle, so we're responsible 2725 // for generating the errors. The cycle list is in 2726 // reverse order because all the 'check' calls append 2727 // their own module to the list. 2728 msgs := []string{"detected variable reference cycle:"} 2729 2730 // Iterate backwards through the cycle list. 2731 curName := v.fullName(pkgNames) 2732 curValue := value.Value(pkgNames) 2733 for i := len(cycle) - 1; i >= 0; i-- { 2734 next := cycle[i] 2735 nextName := next.fullName(pkgNames) 2736 nextValue := variables[next].Value(pkgNames) 2737 2738 msgs = append(msgs, fmt.Sprintf( 2739 " %q depends on %q", curName, nextName)) 2740 msgs = append(msgs, fmt.Sprintf( 2741 " [%s = %s]", curName, curValue)) 2742 2743 curName = nextName 2744 curValue = nextValue 2745 } 2746 2747 // Variable reference cycles are a programming error, 2748 // not the fault of the Blueprint file authors. 2749 panic(strings.Join(msgs, "\n")) 2750 } else { 2751 // We're not the "start" of the cycle, so we just append 2752 // our module to the list and return it. 2753 return append(cycle, v) 2754 } 2755 } 2756 } 2757 } 2758 2759 return nil 2760 } 2761 2762 for v := range variables { 2763 if !visited[v] { 2764 cycle := check(v) 2765 if cycle != nil { 2766 panic("inconceivable!") 2767 } 2768 } 2769 } 2770} 2771 2772// AllTargets returns a map all the build target names to the rule used to build 2773// them. This is the same information that is output by running 'ninja -t 2774// targets all'. If this is called before PrepareBuildActions successfully 2775// completes then ErrbuildActionsNotReady is returned. 2776func (c *Context) AllTargets() (map[string]string, error) { 2777 if !c.buildActionsReady { 2778 return nil, ErrBuildActionsNotReady 2779 } 2780 2781 targets := map[string]string{} 2782 2783 // Collect all the module build targets. 2784 for _, module := range c.moduleInfo { 2785 for _, buildDef := range module.actionDefs.buildDefs { 2786 ruleName := buildDef.Rule.fullName(c.pkgNames) 2787 for _, output := range append(buildDef.Outputs, buildDef.ImplicitOutputs...) { 2788 outputValue, err := output.Eval(c.globalVariables) 2789 if err != nil { 2790 return nil, err 2791 } 2792 targets[outputValue] = ruleName 2793 } 2794 } 2795 } 2796 2797 // Collect all the singleton build targets. 2798 for _, info := range c.singletonInfo { 2799 for _, buildDef := range info.actionDefs.buildDefs { 2800 ruleName := buildDef.Rule.fullName(c.pkgNames) 2801 for _, output := range append(buildDef.Outputs, buildDef.ImplicitOutputs...) { 2802 outputValue, err := output.Eval(c.globalVariables) 2803 if err != nil { 2804 return nil, err 2805 } 2806 targets[outputValue] = ruleName 2807 } 2808 } 2809 } 2810 2811 return targets, nil 2812} 2813 2814func (c *Context) NinjaBuildDir() (string, error) { 2815 if c.ninjaBuildDir != nil { 2816 return c.ninjaBuildDir.Eval(c.globalVariables) 2817 } else { 2818 return "", nil 2819 } 2820} 2821 2822// ModuleTypePropertyStructs returns a mapping from module type name to a list of pointers to 2823// property structs returned by the factory for that module type. 2824func (c *Context) ModuleTypePropertyStructs() map[string][]interface{} { 2825 ret := make(map[string][]interface{}) 2826 for moduleType, factory := range c.moduleFactories { 2827 _, ret[moduleType] = factory() 2828 } 2829 2830 return ret 2831} 2832 2833func (c *Context) ModuleTypeFactories() map[string]ModuleFactory { 2834 ret := make(map[string]ModuleFactory) 2835 for k, v := range c.moduleFactories { 2836 ret[k] = v 2837 } 2838 return ret 2839} 2840 2841func (c *Context) ModuleName(logicModule Module) string { 2842 module := c.moduleInfo[logicModule] 2843 return module.Name() 2844} 2845 2846func (c *Context) ModulePath(logicModule Module) string { 2847 module := c.moduleInfo[logicModule] 2848 return module.relBlueprintsFile 2849} 2850 2851func (c *Context) ModuleDir(logicModule Module) string { 2852 return filepath.Dir(c.ModulePath(logicModule)) 2853} 2854 2855func (c *Context) ModuleSubDir(logicModule Module) string { 2856 module := c.moduleInfo[logicModule] 2857 return module.variantName 2858} 2859 2860func (c *Context) ModuleType(logicModule Module) string { 2861 module := c.moduleInfo[logicModule] 2862 return module.typeName 2863} 2864 2865func (c *Context) BlueprintFile(logicModule Module) string { 2866 module := c.moduleInfo[logicModule] 2867 return module.relBlueprintsFile 2868} 2869 2870func (c *Context) ModuleErrorf(logicModule Module, format string, 2871 args ...interface{}) error { 2872 2873 module := c.moduleInfo[logicModule] 2874 return &BlueprintError{ 2875 Err: fmt.Errorf(format, args...), 2876 Pos: module.pos, 2877 } 2878} 2879 2880func (c *Context) VisitAllModules(visit func(Module)) { 2881 c.visitAllModules(visit) 2882} 2883 2884func (c *Context) VisitAllModulesIf(pred func(Module) bool, 2885 visit func(Module)) { 2886 2887 c.visitAllModulesIf(pred, visit) 2888} 2889 2890func (c *Context) VisitDirectDeps(module Module, visit func(Module)) { 2891 topModule := c.moduleInfo[module] 2892 2893 var visiting *moduleInfo 2894 2895 defer func() { 2896 if r := recover(); r != nil { 2897 panic(newPanicErrorf(r, "VisitDirectDeps(%s, %s) for dependency %s", 2898 topModule, funcName(visit), visiting)) 2899 } 2900 }() 2901 2902 for _, dep := range topModule.directDeps { 2903 visiting = dep.module 2904 visit(dep.module.logicModule) 2905 } 2906} 2907 2908func (c *Context) VisitDirectDepsIf(module Module, pred func(Module) bool, visit func(Module)) { 2909 topModule := c.moduleInfo[module] 2910 2911 var visiting *moduleInfo 2912 2913 defer func() { 2914 if r := recover(); r != nil { 2915 panic(newPanicErrorf(r, "VisitDirectDepsIf(%s, %s, %s) for dependency %s", 2916 topModule, funcName(pred), funcName(visit), visiting)) 2917 } 2918 }() 2919 2920 for _, dep := range topModule.directDeps { 2921 visiting = dep.module 2922 if pred(dep.module.logicModule) { 2923 visit(dep.module.logicModule) 2924 } 2925 } 2926} 2927 2928func (c *Context) VisitDepsDepthFirst(module Module, visit func(Module)) { 2929 topModule := c.moduleInfo[module] 2930 2931 var visiting *moduleInfo 2932 2933 defer func() { 2934 if r := recover(); r != nil { 2935 panic(newPanicErrorf(r, "VisitDepsDepthFirst(%s, %s) for dependency %s", 2936 topModule, funcName(visit), visiting)) 2937 } 2938 }() 2939 2940 c.walkDeps(topModule, false, nil, func(dep depInfo, parent *moduleInfo) { 2941 visiting = dep.module 2942 visit(dep.module.logicModule) 2943 }) 2944} 2945 2946func (c *Context) VisitDepsDepthFirstIf(module Module, pred func(Module) bool, visit func(Module)) { 2947 topModule := c.moduleInfo[module] 2948 2949 var visiting *moduleInfo 2950 2951 defer func() { 2952 if r := recover(); r != nil { 2953 panic(newPanicErrorf(r, "VisitDepsDepthFirstIf(%s, %s, %s) for dependency %s", 2954 topModule, funcName(pred), funcName(visit), visiting)) 2955 } 2956 }() 2957 2958 c.walkDeps(topModule, false, nil, func(dep depInfo, parent *moduleInfo) { 2959 if pred(dep.module.logicModule) { 2960 visiting = dep.module 2961 visit(dep.module.logicModule) 2962 } 2963 }) 2964} 2965 2966func (c *Context) PrimaryModule(module Module) Module { 2967 return c.moduleInfo[module].group.modules[0].logicModule 2968} 2969 2970func (c *Context) FinalModule(module Module) Module { 2971 modules := c.moduleInfo[module].group.modules 2972 return modules[len(modules)-1].logicModule 2973} 2974 2975func (c *Context) VisitAllModuleVariants(module Module, 2976 visit func(Module)) { 2977 2978 c.visitAllModuleVariants(c.moduleInfo[module], visit) 2979} 2980 2981// Singletons returns a list of all registered Singletons. 2982func (c *Context) Singletons() []Singleton { 2983 var ret []Singleton 2984 for _, s := range c.singletonInfo { 2985 ret = append(ret, s.singleton) 2986 } 2987 return ret 2988} 2989 2990// SingletonName returns the name that the given singleton was registered with. 2991func (c *Context) SingletonName(singleton Singleton) string { 2992 for _, s := range c.singletonInfo { 2993 if s.singleton == singleton { 2994 return s.name 2995 } 2996 } 2997 return "" 2998} 2999 3000// WriteBuildFile writes the Ninja manifeset text for the generated build 3001// actions to w. If this is called before PrepareBuildActions successfully 3002// completes then ErrBuildActionsNotReady is returned. 3003func (c *Context) WriteBuildFile(w io.Writer) error { 3004 var err error 3005 pprof.Do(c.Context, pprof.Labels("blueprint", "WriteBuildFile"), func(ctx context.Context) { 3006 if !c.buildActionsReady { 3007 err = ErrBuildActionsNotReady 3008 return 3009 } 3010 3011 nw := newNinjaWriter(w) 3012 3013 err = c.writeBuildFileHeader(nw) 3014 if err != nil { 3015 return 3016 } 3017 3018 err = c.writeNinjaRequiredVersion(nw) 3019 if err != nil { 3020 return 3021 } 3022 3023 err = c.writeSubninjas(nw) 3024 if err != nil { 3025 return 3026 } 3027 3028 // TODO: Group the globals by package. 3029 3030 err = c.writeGlobalVariables(nw) 3031 if err != nil { 3032 return 3033 } 3034 3035 err = c.writeGlobalPools(nw) 3036 if err != nil { 3037 return 3038 } 3039 3040 err = c.writeBuildDir(nw) 3041 if err != nil { 3042 return 3043 } 3044 3045 err = c.writeGlobalRules(nw) 3046 if err != nil { 3047 return 3048 } 3049 3050 err = c.writeAllModuleActions(nw) 3051 if err != nil { 3052 return 3053 } 3054 3055 err = c.writeAllSingletonActions(nw) 3056 if err != nil { 3057 return 3058 } 3059 }) 3060 3061 if err != nil { 3062 return err 3063 } 3064 3065 return nil 3066} 3067 3068type pkgAssociation struct { 3069 PkgName string 3070 PkgPath string 3071} 3072 3073type pkgAssociationSorter struct { 3074 pkgs []pkgAssociation 3075} 3076 3077func (s *pkgAssociationSorter) Len() int { 3078 return len(s.pkgs) 3079} 3080 3081func (s *pkgAssociationSorter) Less(i, j int) bool { 3082 iName := s.pkgs[i].PkgName 3083 jName := s.pkgs[j].PkgName 3084 return iName < jName 3085} 3086 3087func (s *pkgAssociationSorter) Swap(i, j int) { 3088 s.pkgs[i], s.pkgs[j] = s.pkgs[j], s.pkgs[i] 3089} 3090 3091func (c *Context) writeBuildFileHeader(nw *ninjaWriter) error { 3092 headerTemplate := template.New("fileHeader") 3093 _, err := headerTemplate.Parse(fileHeaderTemplate) 3094 if err != nil { 3095 // This is a programming error. 3096 panic(err) 3097 } 3098 3099 var pkgs []pkgAssociation 3100 maxNameLen := 0 3101 for pkg, name := range c.pkgNames { 3102 pkgs = append(pkgs, pkgAssociation{ 3103 PkgName: name, 3104 PkgPath: pkg.pkgPath, 3105 }) 3106 if len(name) > maxNameLen { 3107 maxNameLen = len(name) 3108 } 3109 } 3110 3111 for i := range pkgs { 3112 pkgs[i].PkgName += strings.Repeat(" ", maxNameLen-len(pkgs[i].PkgName)) 3113 } 3114 3115 sort.Sort(&pkgAssociationSorter{pkgs}) 3116 3117 params := map[string]interface{}{ 3118 "Pkgs": pkgs, 3119 } 3120 3121 buf := bytes.NewBuffer(nil) 3122 err = headerTemplate.Execute(buf, params) 3123 if err != nil { 3124 return err 3125 } 3126 3127 return nw.Comment(buf.String()) 3128} 3129 3130func (c *Context) writeNinjaRequiredVersion(nw *ninjaWriter) error { 3131 value := fmt.Sprintf("%d.%d.%d", c.requiredNinjaMajor, c.requiredNinjaMinor, 3132 c.requiredNinjaMicro) 3133 3134 err := nw.Assign("ninja_required_version", value) 3135 if err != nil { 3136 return err 3137 } 3138 3139 return nw.BlankLine() 3140} 3141 3142func (c *Context) writeSubninjas(nw *ninjaWriter) error { 3143 for _, subninja := range c.subninjas { 3144 err := nw.Subninja(subninja) 3145 if err != nil { 3146 return err 3147 } 3148 } 3149 return nw.BlankLine() 3150} 3151 3152func (c *Context) writeBuildDir(nw *ninjaWriter) error { 3153 if c.ninjaBuildDir != nil { 3154 err := nw.Assign("builddir", c.ninjaBuildDir.Value(c.pkgNames)) 3155 if err != nil { 3156 return err 3157 } 3158 3159 err = nw.BlankLine() 3160 if err != nil { 3161 return err 3162 } 3163 } 3164 return nil 3165} 3166 3167type globalEntity interface { 3168 fullName(pkgNames map[*packageContext]string) string 3169} 3170 3171type globalEntitySorter struct { 3172 pkgNames map[*packageContext]string 3173 entities []globalEntity 3174} 3175 3176func (s *globalEntitySorter) Len() int { 3177 return len(s.entities) 3178} 3179 3180func (s *globalEntitySorter) Less(i, j int) bool { 3181 iName := s.entities[i].fullName(s.pkgNames) 3182 jName := s.entities[j].fullName(s.pkgNames) 3183 return iName < jName 3184} 3185 3186func (s *globalEntitySorter) Swap(i, j int) { 3187 s.entities[i], s.entities[j] = s.entities[j], s.entities[i] 3188} 3189 3190func (c *Context) writeGlobalVariables(nw *ninjaWriter) error { 3191 visited := make(map[Variable]bool) 3192 3193 var walk func(v Variable) error 3194 walk = func(v Variable) error { 3195 visited[v] = true 3196 3197 // First visit variables on which this variable depends. 3198 value := c.globalVariables[v] 3199 for _, dep := range value.variables { 3200 if !visited[dep] { 3201 err := walk(dep) 3202 if err != nil { 3203 return err 3204 } 3205 } 3206 } 3207 3208 err := nw.Assign(v.fullName(c.pkgNames), value.Value(c.pkgNames)) 3209 if err != nil { 3210 return err 3211 } 3212 3213 err = nw.BlankLine() 3214 if err != nil { 3215 return err 3216 } 3217 3218 return nil 3219 } 3220 3221 globalVariables := make([]globalEntity, 0, len(c.globalVariables)) 3222 for variable := range c.globalVariables { 3223 globalVariables = append(globalVariables, variable) 3224 } 3225 3226 sort.Sort(&globalEntitySorter{c.pkgNames, globalVariables}) 3227 3228 for _, entity := range globalVariables { 3229 v := entity.(Variable) 3230 if !visited[v] { 3231 err := walk(v) 3232 if err != nil { 3233 return nil 3234 } 3235 } 3236 } 3237 3238 return nil 3239} 3240 3241func (c *Context) writeGlobalPools(nw *ninjaWriter) error { 3242 globalPools := make([]globalEntity, 0, len(c.globalPools)) 3243 for pool := range c.globalPools { 3244 globalPools = append(globalPools, pool) 3245 } 3246 3247 sort.Sort(&globalEntitySorter{c.pkgNames, globalPools}) 3248 3249 for _, entity := range globalPools { 3250 pool := entity.(Pool) 3251 name := pool.fullName(c.pkgNames) 3252 def := c.globalPools[pool] 3253 err := def.WriteTo(nw, name) 3254 if err != nil { 3255 return err 3256 } 3257 3258 err = nw.BlankLine() 3259 if err != nil { 3260 return err 3261 } 3262 } 3263 3264 return nil 3265} 3266 3267func (c *Context) writeGlobalRules(nw *ninjaWriter) error { 3268 globalRules := make([]globalEntity, 0, len(c.globalRules)) 3269 for rule := range c.globalRules { 3270 globalRules = append(globalRules, rule) 3271 } 3272 3273 sort.Sort(&globalEntitySorter{c.pkgNames, globalRules}) 3274 3275 for _, entity := range globalRules { 3276 rule := entity.(Rule) 3277 name := rule.fullName(c.pkgNames) 3278 def := c.globalRules[rule] 3279 err := def.WriteTo(nw, name, c.pkgNames) 3280 if err != nil { 3281 return err 3282 } 3283 3284 err = nw.BlankLine() 3285 if err != nil { 3286 return err 3287 } 3288 } 3289 3290 return nil 3291} 3292 3293type depSorter []depInfo 3294 3295func (s depSorter) Len() int { 3296 return len(s) 3297} 3298 3299func (s depSorter) Less(i, j int) bool { 3300 iName := s[i].module.Name() 3301 jName := s[j].module.Name() 3302 if iName == jName { 3303 iName = s[i].module.variantName 3304 jName = s[j].module.variantName 3305 } 3306 return iName < jName 3307} 3308 3309func (s depSorter) Swap(i, j int) { 3310 s[i], s[j] = s[j], s[i] 3311} 3312 3313type moduleSorter struct { 3314 modules []*moduleInfo 3315 nameInterface NameInterface 3316} 3317 3318func (s moduleSorter) Len() int { 3319 return len(s.modules) 3320} 3321 3322func (s moduleSorter) Less(i, j int) bool { 3323 iMod := s.modules[i] 3324 jMod := s.modules[j] 3325 iName := s.nameInterface.UniqueName(newNamespaceContext(iMod), iMod.group.name) 3326 jName := s.nameInterface.UniqueName(newNamespaceContext(jMod), jMod.group.name) 3327 if iName == jName { 3328 iName = s.modules[i].variantName 3329 jName = s.modules[j].variantName 3330 } 3331 3332 if iName == jName { 3333 panic(fmt.Sprintf("duplicate module name: %s: %#v and %#v\n", iName, iMod, jMod)) 3334 } 3335 return iName < jName 3336} 3337 3338func (s moduleSorter) Swap(i, j int) { 3339 s.modules[i], s.modules[j] = s.modules[j], s.modules[i] 3340} 3341 3342func (c *Context) writeAllModuleActions(nw *ninjaWriter) error { 3343 headerTemplate := template.New("moduleHeader") 3344 _, err := headerTemplate.Parse(moduleHeaderTemplate) 3345 if err != nil { 3346 // This is a programming error. 3347 panic(err) 3348 } 3349 3350 modules := make([]*moduleInfo, 0, len(c.moduleInfo)) 3351 for _, module := range c.moduleInfo { 3352 modules = append(modules, module) 3353 } 3354 sort.Sort(moduleSorter{modules, c.nameInterface}) 3355 3356 buf := bytes.NewBuffer(nil) 3357 3358 for _, module := range modules { 3359 if len(module.actionDefs.variables)+len(module.actionDefs.rules)+len(module.actionDefs.buildDefs) == 0 { 3360 continue 3361 } 3362 3363 buf.Reset() 3364 3365 // In order to make the bootstrap build manifest independent of the 3366 // build dir we need to output the Blueprints file locations in the 3367 // comments as paths relative to the source directory. 3368 relPos := module.pos 3369 relPos.Filename = module.relBlueprintsFile 3370 3371 // Get the name and location of the factory function for the module. 3372 factoryFunc := runtime.FuncForPC(reflect.ValueOf(module.factory).Pointer()) 3373 factoryName := factoryFunc.Name() 3374 3375 infoMap := map[string]interface{}{ 3376 "name": module.Name(), 3377 "typeName": module.typeName, 3378 "goFactory": factoryName, 3379 "pos": relPos, 3380 "variant": module.variantName, 3381 } 3382 err = headerTemplate.Execute(buf, infoMap) 3383 if err != nil { 3384 return err 3385 } 3386 3387 err = nw.Comment(buf.String()) 3388 if err != nil { 3389 return err 3390 } 3391 3392 err = nw.BlankLine() 3393 if err != nil { 3394 return err 3395 } 3396 3397 err = c.writeLocalBuildActions(nw, &module.actionDefs) 3398 if err != nil { 3399 return err 3400 } 3401 3402 err = nw.BlankLine() 3403 if err != nil { 3404 return err 3405 } 3406 } 3407 3408 return nil 3409} 3410 3411func (c *Context) writeAllSingletonActions(nw *ninjaWriter) error { 3412 headerTemplate := template.New("singletonHeader") 3413 _, err := headerTemplate.Parse(singletonHeaderTemplate) 3414 if err != nil { 3415 // This is a programming error. 3416 panic(err) 3417 } 3418 3419 buf := bytes.NewBuffer(nil) 3420 3421 for _, info := range c.singletonInfo { 3422 if len(info.actionDefs.variables)+len(info.actionDefs.rules)+len(info.actionDefs.buildDefs) == 0 { 3423 continue 3424 } 3425 3426 // Get the name of the factory function for the module. 3427 factory := info.factory 3428 factoryFunc := runtime.FuncForPC(reflect.ValueOf(factory).Pointer()) 3429 factoryName := factoryFunc.Name() 3430 3431 buf.Reset() 3432 infoMap := map[string]interface{}{ 3433 "name": info.name, 3434 "goFactory": factoryName, 3435 } 3436 err = headerTemplate.Execute(buf, infoMap) 3437 if err != nil { 3438 return err 3439 } 3440 3441 err = nw.Comment(buf.String()) 3442 if err != nil { 3443 return err 3444 } 3445 3446 err = nw.BlankLine() 3447 if err != nil { 3448 return err 3449 } 3450 3451 err = c.writeLocalBuildActions(nw, &info.actionDefs) 3452 if err != nil { 3453 return err 3454 } 3455 3456 err = nw.BlankLine() 3457 if err != nil { 3458 return err 3459 } 3460 } 3461 3462 return nil 3463} 3464 3465func (c *Context) writeLocalBuildActions(nw *ninjaWriter, 3466 defs *localBuildActions) error { 3467 3468 // Write the local variable assignments. 3469 for _, v := range defs.variables { 3470 // A localVariable doesn't need the package names or config to 3471 // determine its name or value. 3472 name := v.fullName(nil) 3473 value, err := v.value(nil) 3474 if err != nil { 3475 panic(err) 3476 } 3477 err = nw.Assign(name, value.Value(c.pkgNames)) 3478 if err != nil { 3479 return err 3480 } 3481 } 3482 3483 if len(defs.variables) > 0 { 3484 err := nw.BlankLine() 3485 if err != nil { 3486 return err 3487 } 3488 } 3489 3490 // Write the local rules. 3491 for _, r := range defs.rules { 3492 // A localRule doesn't need the package names or config to determine 3493 // its name or definition. 3494 name := r.fullName(nil) 3495 def, err := r.def(nil) 3496 if err != nil { 3497 panic(err) 3498 } 3499 3500 err = def.WriteTo(nw, name, c.pkgNames) 3501 if err != nil { 3502 return err 3503 } 3504 3505 err = nw.BlankLine() 3506 if err != nil { 3507 return err 3508 } 3509 } 3510 3511 // Write the build definitions. 3512 for _, buildDef := range defs.buildDefs { 3513 err := buildDef.WriteTo(nw, c.pkgNames) 3514 if err != nil { 3515 return err 3516 } 3517 3518 if len(buildDef.Args) > 0 { 3519 err = nw.BlankLine() 3520 if err != nil { 3521 return err 3522 } 3523 } 3524 } 3525 3526 return nil 3527} 3528 3529func beforeInModuleList(a, b *moduleInfo, list []*moduleInfo) bool { 3530 found := false 3531 if a == b { 3532 return false 3533 } 3534 for _, l := range list { 3535 if l == a { 3536 found = true 3537 } else if l == b { 3538 return found 3539 } 3540 } 3541 3542 missing := a 3543 if found { 3544 missing = b 3545 } 3546 panic(fmt.Errorf("element %v not found in list %v", missing, list)) 3547} 3548 3549type panicError struct { 3550 panic interface{} 3551 stack []byte 3552 in string 3553} 3554 3555func newPanicErrorf(panic interface{}, in string, a ...interface{}) error { 3556 buf := make([]byte, 4096) 3557 count := runtime.Stack(buf, false) 3558 return panicError{ 3559 panic: panic, 3560 in: fmt.Sprintf(in, a...), 3561 stack: buf[:count], 3562 } 3563} 3564 3565func (p panicError) Error() string { 3566 return fmt.Sprintf("panic in %s\n%s\n%s\n", p.in, p.panic, p.stack) 3567} 3568 3569func (p *panicError) addIn(in string) { 3570 p.in += " in " + in 3571} 3572 3573func funcName(f interface{}) string { 3574 return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() 3575} 3576 3577var fileHeaderTemplate = `****************************************************************************** 3578*** This file is generated and should not be edited *** 3579****************************************************************************** 3580{{if .Pkgs}} 3581This file contains variables, rules, and pools with name prefixes indicating 3582they were generated by the following Go packages: 3583{{range .Pkgs}} 3584 {{.PkgName}} [from Go package {{.PkgPath}}]{{end}}{{end}} 3585 3586` 3587 3588var moduleHeaderTemplate = `# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 3589Module: {{.name}} 3590Variant: {{.variant}} 3591Type: {{.typeName}} 3592Factory: {{.goFactory}} 3593Defined: {{.pos}} 3594` 3595 3596var singletonHeaderTemplate = `# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 3597Singleton: {{.name}} 3598Factory: {{.goFactory}} 3599` 3600