• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2014 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package blueprint
16
17import (
18	"bytes"
19	"context"
20	"errors"
21	"fmt"
22	"io"
23	"io/ioutil"
24	"os"
25	"path/filepath"
26	"reflect"
27	"runtime"
28	"runtime/pprof"
29	"sort"
30	"strings"
31	"sync"
32	"sync/atomic"
33	"text/scanner"
34	"text/template"
35
36	"github.com/google/blueprint/parser"
37	"github.com/google/blueprint/pathtools"
38	"github.com/google/blueprint/proptools"
39)
40
41var ErrBuildActionsNotReady = errors.New("build actions are not ready")
42
43const maxErrors = 10
44const MockModuleListFile = "bplist"
45
46// A Context contains all the state needed to parse a set of Blueprints files
47// and generate a Ninja file.  The process of generating a Ninja file proceeds
48// through a series of four phases.  Each phase corresponds with a some methods
49// on the Context object
50//
51//         Phase                            Methods
52//      ------------      -------------------------------------------
53//   1. Registration         RegisterModuleType, RegisterSingletonType
54//
55//   2. Parse                    ParseBlueprintsFiles, Parse
56//
57//   3. Generate            ResolveDependencies, PrepareBuildActions
58//
59//   4. Write                           WriteBuildFile
60//
61// The registration phase prepares the context to process Blueprints files
62// containing various types of modules.  The parse phase reads in one or more
63// Blueprints files and validates their contents against the module types that
64// have been registered.  The generate phase then analyzes the parsed Blueprints
65// contents to create an internal representation for the build actions that must
66// be performed.  This phase also performs validation of the module dependencies
67// and property values defined in the parsed Blueprints files.  Finally, the
68// write phase generates the Ninja manifest text based on the generated build
69// actions.
70type Context struct {
71	context.Context
72
73	// set at instantiation
74	moduleFactories     map[string]ModuleFactory
75	nameInterface       NameInterface
76	moduleGroups        []*moduleGroup
77	moduleInfo          map[Module]*moduleInfo
78	modulesSorted       []*moduleInfo
79	preSingletonInfo    []*singletonInfo
80	singletonInfo       []*singletonInfo
81	mutatorInfo         []*mutatorInfo
82	earlyMutatorInfo    []*mutatorInfo
83	variantMutatorNames []string
84
85	depsModified uint32 // positive if a mutator modified the dependencies
86
87	dependenciesReady bool // set to true on a successful ResolveDependencies
88	buildActionsReady bool // set to true on a successful PrepareBuildActions
89
90	// set by SetIgnoreUnknownModuleTypes
91	ignoreUnknownModuleTypes bool
92
93	// set by SetAllowMissingDependencies
94	allowMissingDependencies bool
95
96	// set during PrepareBuildActions
97	pkgNames        map[*packageContext]string
98	liveGlobals     *liveTracker
99	globalVariables map[Variable]*ninjaString
100	globalPools     map[Pool]*poolDef
101	globalRules     map[Rule]*ruleDef
102
103	// set during PrepareBuildActions
104	ninjaBuildDir      *ninjaString // The builddir special Ninja variable
105	requiredNinjaMajor int          // For the ninja_required_version variable
106	requiredNinjaMinor int          // For the ninja_required_version variable
107	requiredNinjaMicro int          // For the ninja_required_version variable
108
109	subninjas []string
110
111	// set lazily by sortedModuleGroups
112	cachedSortedModuleGroups []*moduleGroup
113
114	globs    map[string]GlobPath
115	globLock sync.Mutex
116
117	fs             pathtools.FileSystem
118	moduleListFile string
119}
120
121// An Error describes a problem that was encountered that is related to a
122// particular location in a Blueprints file.
123type BlueprintError struct {
124	Err error            // the error that occurred
125	Pos scanner.Position // the relevant Blueprints file location
126}
127
128// A ModuleError describes a problem that was encountered that is related to a
129// particular module in a Blueprints file
130type ModuleError struct {
131	BlueprintError
132	module *moduleInfo
133}
134
135// A PropertyError describes a problem that was encountered that is related to a
136// particular property in a Blueprints file
137type PropertyError struct {
138	ModuleError
139	property string
140}
141
142func (e *BlueprintError) Error() string {
143	return fmt.Sprintf("%s: %s", e.Pos, e.Err)
144}
145
146func (e *ModuleError) Error() string {
147	return fmt.Sprintf("%s: %s: %s", e.Pos, e.module, e.Err)
148}
149
150func (e *PropertyError) Error() string {
151	return fmt.Sprintf("%s: %s: %s: %s", e.Pos, e.module, e.property, e.Err)
152}
153
154type localBuildActions struct {
155	variables []*localVariable
156	rules     []*localRule
157	buildDefs []*buildDef
158}
159
160type moduleGroup struct {
161	name      string
162	ninjaName string
163
164	modules []*moduleInfo
165
166	namespace Namespace
167}
168
169type moduleInfo struct {
170	// set during Parse
171	typeName          string
172	factory           ModuleFactory
173	relBlueprintsFile string
174	pos               scanner.Position
175	propertyPos       map[string]scanner.Position
176
177	variantName       string
178	variant           variationMap
179	dependencyVariant variationMap
180
181	logicModule Module
182	group       *moduleGroup
183	properties  []interface{}
184
185	// set during ResolveDependencies
186	directDeps  []depInfo
187	missingDeps []string
188
189	// set during updateDependencies
190	reverseDeps []*moduleInfo
191	forwardDeps []*moduleInfo
192
193	// used by parallelVisitAllBottomUp
194	waitingCount int
195
196	// set during each runMutator
197	splitModules []*moduleInfo
198
199	// set during PrepareBuildActions
200	actionDefs localBuildActions
201}
202
203type depInfo struct {
204	module *moduleInfo
205	tag    DependencyTag
206}
207
208func (module *moduleInfo) Name() string {
209	return module.group.name
210}
211
212func (module *moduleInfo) String() string {
213	s := fmt.Sprintf("module %q", module.Name())
214	if module.variantName != "" {
215		s += fmt.Sprintf(" variant %q", module.variantName)
216	}
217	return s
218}
219
220func (module *moduleInfo) namespace() Namespace {
221	return module.group.namespace
222}
223
224// A Variation is a way that a variant of a module differs from other variants of the same module.
225// For example, two variants of the same module might have Variation{"arch","arm"} and
226// Variation{"arch","arm64"}
227type Variation struct {
228	// Mutator is the axis on which this variation applies, i.e. "arch" or "link"
229	Mutator string
230	// Variation is the name of the variation on the axis, i.e. "arm" or "arm64" for arch, or
231	// "shared" or "static" for link.
232	Variation string
233}
234
235// A variationMap stores a map of Mutator to Variation to specify a variant of a module.
236type variationMap map[string]string
237
238func (vm variationMap) clone() variationMap {
239	newVm := make(variationMap)
240	for k, v := range vm {
241		newVm[k] = v
242	}
243
244	return newVm
245}
246
247// Compare this variationMap to another one.  Returns true if the every entry in this map
248// is either the same in the other map or doesn't exist in the other map.
249func (vm variationMap) subset(other variationMap) bool {
250	for k, v1 := range vm {
251		if v2, ok := other[k]; ok && v1 != v2 {
252			return false
253		}
254	}
255	return true
256}
257
258func (vm variationMap) equal(other variationMap) bool {
259	return reflect.DeepEqual(vm, other)
260}
261
262type singletonInfo struct {
263	// set during RegisterSingletonType
264	factory   SingletonFactory
265	singleton Singleton
266	name      string
267
268	// set during PrepareBuildActions
269	actionDefs localBuildActions
270}
271
272type mutatorInfo struct {
273	// set during RegisterMutator
274	topDownMutator  TopDownMutator
275	bottomUpMutator BottomUpMutator
276	name            string
277	parallel        bool
278}
279
280func newContext() *Context {
281	return &Context{
282		Context:            context.Background(),
283		moduleFactories:    make(map[string]ModuleFactory),
284		nameInterface:      NewSimpleNameInterface(),
285		moduleInfo:         make(map[Module]*moduleInfo),
286		globs:              make(map[string]GlobPath),
287		fs:                 pathtools.OsFs,
288		ninjaBuildDir:      nil,
289		requiredNinjaMajor: 1,
290		requiredNinjaMinor: 7,
291		requiredNinjaMicro: 0,
292	}
293}
294
295// NewContext creates a new Context object.  The created context initially has
296// no module or singleton factories registered, so the RegisterModuleFactory and
297// RegisterSingletonFactory methods must be called before it can do anything
298// useful.
299func NewContext() *Context {
300	ctx := newContext()
301
302	ctx.RegisterBottomUpMutator("blueprint_deps", blueprintDepsMutator)
303
304	return ctx
305}
306
307// A ModuleFactory function creates a new Module object.  See the
308// Context.RegisterModuleType method for details about how a registered
309// ModuleFactory is used by a Context.
310type ModuleFactory func() (m Module, propertyStructs []interface{})
311
312// RegisterModuleType associates a module type name (which can appear in a
313// Blueprints file) with a Module factory function.  When the given module type
314// name is encountered in a Blueprints file during parsing, the Module factory
315// is invoked to instantiate a new Module object to handle the build action
316// generation for the module.  If a Mutator splits a module into multiple variants,
317// the factory is invoked again to create a new Module for each variant.
318//
319// The module type names given here must be unique for the context.  The factory
320// function should be a named function so that its package and name can be
321// included in the generated Ninja file for debugging purposes.
322//
323// The factory function returns two values.  The first is the newly created
324// Module object.  The second is a slice of pointers to that Module object's
325// properties structs.  Each properties struct is examined when parsing a module
326// definition of this type in a Blueprints file.  Exported fields of the
327// properties structs are automatically set to the property values specified in
328// the Blueprints file.  The properties struct field names determine the name of
329// the Blueprints file properties that are used - the Blueprints property name
330// matches that of the properties struct field name with the first letter
331// converted to lower-case.
332//
333// The fields of the properties struct must be either []string, a string, or
334// bool. The Context will panic if a Module gets instantiated with a properties
335// struct containing a field that is not one these supported types.
336//
337// Any properties that appear in the Blueprints files that are not built-in
338// module properties (such as "name" and "deps") and do not have a corresponding
339// field in the returned module properties struct result in an error during the
340// Context's parse phase.
341//
342// As an example, the follow code:
343//
344//   type myModule struct {
345//       properties struct {
346//           Foo string
347//           Bar []string
348//       }
349//   }
350//
351//   func NewMyModule() (blueprint.Module, []interface{}) {
352//       module := new(myModule)
353//       properties := &module.properties
354//       return module, []interface{}{properties}
355//   }
356//
357//   func main() {
358//       ctx := blueprint.NewContext()
359//       ctx.RegisterModuleType("my_module", NewMyModule)
360//       // ...
361//   }
362//
363// would support parsing a module defined in a Blueprints file as follows:
364//
365//   my_module {
366//       name: "myName",
367//       foo:  "my foo string",
368//       bar:  ["my", "bar", "strings"],
369//   }
370//
371// The factory function may be called from multiple goroutines.  Any accesses
372// to global variables must be synchronized.
373func (c *Context) RegisterModuleType(name string, factory ModuleFactory) {
374	if _, present := c.moduleFactories[name]; present {
375		panic(errors.New("module type name is already registered"))
376	}
377	c.moduleFactories[name] = factory
378}
379
380// A SingletonFactory function creates a new Singleton object.  See the
381// Context.RegisterSingletonType method for details about how a registered
382// SingletonFactory is used by a Context.
383type SingletonFactory func() Singleton
384
385// RegisterSingletonType registers a singleton type that will be invoked to
386// generate build actions.  Each registered singleton type is instantiated and
387// and invoked exactly once as part of the generate phase.  Each registered
388// singleton is invoked in registration order.
389//
390// The singleton type names given here must be unique for the context.  The
391// factory function should be a named function so that its package and name can
392// be included in the generated Ninja file for debugging purposes.
393func (c *Context) RegisterSingletonType(name string, factory SingletonFactory) {
394	for _, s := range c.singletonInfo {
395		if s.name == name {
396			panic(errors.New("singleton name is already registered"))
397		}
398	}
399
400	c.singletonInfo = append(c.singletonInfo, &singletonInfo{
401		factory:   factory,
402		singleton: factory(),
403		name:      name,
404	})
405}
406
407// RegisterPreSingletonType registers a presingleton type that will be invoked to
408// generate build actions before any Blueprint files have been read.  Each registered
409// presingleton type is instantiated and invoked exactly once at the beginning of the
410// parse phase.  Each registered presingleton is invoked in registration order.
411//
412// The presingleton type names given here must be unique for the context.  The
413// factory function should be a named function so that its package and name can
414// be included in the generated Ninja file for debugging purposes.
415func (c *Context) RegisterPreSingletonType(name string, factory SingletonFactory) {
416	for _, s := range c.preSingletonInfo {
417		if s.name == name {
418			panic(errors.New("presingleton name is already registered"))
419		}
420	}
421
422	c.preSingletonInfo = append(c.preSingletonInfo, &singletonInfo{
423		factory:   factory,
424		singleton: factory(),
425		name:      name,
426	})
427}
428
429func (c *Context) SetNameInterface(i NameInterface) {
430	c.nameInterface = i
431}
432
433func singletonPkgPath(singleton Singleton) string {
434	typ := reflect.TypeOf(singleton)
435	for typ.Kind() == reflect.Ptr {
436		typ = typ.Elem()
437	}
438	return typ.PkgPath()
439}
440
441func singletonTypeName(singleton Singleton) string {
442	typ := reflect.TypeOf(singleton)
443	for typ.Kind() == reflect.Ptr {
444		typ = typ.Elem()
445	}
446	return typ.PkgPath() + "." + typ.Name()
447}
448
449// RegisterTopDownMutator registers a mutator that will be invoked to propagate dependency info
450// top-down between Modules.  Each registered mutator is invoked in registration order (mixing
451// TopDownMutators and BottomUpMutators) once per Module, and the invocation on any module will
452// have returned before it is in invoked on any of its dependencies.
453//
454// The mutator type names given here must be unique to all top down mutators in
455// the Context.
456//
457// Returns a MutatorHandle, on which Parallel can be called to set the mutator to visit modules in
458// parallel while maintaining ordering.
459func (c *Context) RegisterTopDownMutator(name string, mutator TopDownMutator) MutatorHandle {
460	for _, m := range c.mutatorInfo {
461		if m.name == name && m.topDownMutator != nil {
462			panic(fmt.Errorf("mutator name %s is already registered", name))
463		}
464	}
465
466	info := &mutatorInfo{
467		topDownMutator: mutator,
468		name:           name,
469	}
470
471	c.mutatorInfo = append(c.mutatorInfo, info)
472
473	return info
474}
475
476// RegisterBottomUpMutator registers a mutator that will be invoked to split Modules into variants.
477// Each registered mutator is invoked in registration order (mixing TopDownMutators and
478// BottomUpMutators) once per Module, will not be invoked on a module until the invocations on all
479// of the modules dependencies have returned.
480//
481// The mutator type names given here must be unique to all bottom up or early
482// mutators in the Context.
483//
484// Returns a MutatorHandle, on which Parallel can be called to set the mutator to visit modules in
485// parallel while maintaining ordering.
486func (c *Context) RegisterBottomUpMutator(name string, mutator BottomUpMutator) MutatorHandle {
487	for _, m := range c.variantMutatorNames {
488		if m == name {
489			panic(fmt.Errorf("mutator name %s is already registered", name))
490		}
491	}
492
493	info := &mutatorInfo{
494		bottomUpMutator: mutator,
495		name:            name,
496	}
497	c.mutatorInfo = append(c.mutatorInfo, info)
498
499	c.variantMutatorNames = append(c.variantMutatorNames, name)
500
501	return info
502}
503
504type MutatorHandle interface {
505	// Set the mutator to visit modules in parallel while maintaining ordering.  Calling any
506	// method on the mutator context is thread-safe, but the mutator must handle synchronization
507	// for any modifications to global state or any modules outside the one it was invoked on.
508	Parallel() MutatorHandle
509}
510
511func (mutator *mutatorInfo) Parallel() MutatorHandle {
512	mutator.parallel = true
513	return mutator
514}
515
516// RegisterEarlyMutator registers a mutator that will be invoked to split
517// Modules into multiple variant Modules before any dependencies have been
518// created.  Each registered mutator is invoked in registration order once
519// per Module (including each variant from previous early mutators).  Module
520// order is unpredictable.
521//
522// In order for dependencies to be satisifed in a later pass, all dependencies
523// of a module either must have an identical variant or must have no variations.
524//
525// The mutator type names given here must be unique to all bottom up or early
526// mutators in the Context.
527//
528// Deprecated, use a BottomUpMutator instead.  The only difference between
529// EarlyMutator and BottomUpMutator is that EarlyMutator runs before the
530// deprecated DynamicDependencies.
531func (c *Context) RegisterEarlyMutator(name string, mutator EarlyMutator) {
532	for _, m := range c.variantMutatorNames {
533		if m == name {
534			panic(fmt.Errorf("mutator name %s is already registered", name))
535		}
536	}
537
538	c.earlyMutatorInfo = append(c.earlyMutatorInfo, &mutatorInfo{
539		bottomUpMutator: func(mctx BottomUpMutatorContext) {
540			mutator(mctx)
541		},
542		name: name,
543	})
544
545	c.variantMutatorNames = append(c.variantMutatorNames, name)
546}
547
548// SetIgnoreUnknownModuleTypes sets the behavior of the context in the case
549// where it encounters an unknown module type while parsing Blueprints files. By
550// default, the context will report unknown module types as an error.  If this
551// method is called with ignoreUnknownModuleTypes set to true then the context
552// will silently ignore unknown module types.
553//
554// This method should generally not be used.  It exists to facilitate the
555// bootstrapping process.
556func (c *Context) SetIgnoreUnknownModuleTypes(ignoreUnknownModuleTypes bool) {
557	c.ignoreUnknownModuleTypes = ignoreUnknownModuleTypes
558}
559
560// SetAllowMissingDependencies changes the behavior of Blueprint to ignore
561// unresolved dependencies.  If the module's GenerateBuildActions calls
562// ModuleContext.GetMissingDependencies Blueprint will not emit any errors
563// for missing dependencies.
564func (c *Context) SetAllowMissingDependencies(allowMissingDependencies bool) {
565	c.allowMissingDependencies = allowMissingDependencies
566}
567
568func (c *Context) SetModuleListFile(listFile string) {
569	c.moduleListFile = listFile
570}
571
572func (c *Context) ListModulePaths(baseDir string) (paths []string, err error) {
573	reader, err := c.fs.Open(c.moduleListFile)
574	if err != nil {
575		return nil, err
576	}
577	bytes, err := ioutil.ReadAll(reader)
578	if err != nil {
579		return nil, err
580	}
581	text := string(bytes)
582
583	text = strings.Trim(text, "\n")
584	lines := strings.Split(text, "\n")
585	for i := range lines {
586		lines[i] = filepath.Join(baseDir, lines[i])
587	}
588
589	return lines, nil
590}
591
592// a fileParseContext tells the status of parsing a particular file
593type fileParseContext struct {
594	// name of file
595	fileName string
596
597	// scope to use when resolving variables
598	Scope *parser.Scope
599
600	// pointer to the one in the parent directory
601	parent *fileParseContext
602
603	// is closed once FileHandler has completed for this file
604	doneVisiting chan struct{}
605}
606
607// ParseBlueprintsFiles parses a set of Blueprints files starting with the file
608// at rootFile.  When it encounters a Blueprints file with a set of subdirs
609// listed it recursively parses any Blueprints files found in those
610// subdirectories.
611//
612// If no errors are encountered while parsing the files, the list of paths on
613// which the future output will depend is returned.  This list will include both
614// Blueprints file paths as well as directory paths for cases where wildcard
615// subdirs are found.
616func (c *Context) ParseBlueprintsFiles(rootFile string) (deps []string, errs []error) {
617	baseDir := filepath.Dir(rootFile)
618	pathsToParse, err := c.ListModulePaths(baseDir)
619	if err != nil {
620		return nil, []error{err}
621	}
622	return c.ParseFileList(baseDir, pathsToParse)
623}
624
625func (c *Context) ParseFileList(rootDir string, filePaths []string) (deps []string,
626	errs []error) {
627
628	if len(filePaths) < 1 {
629		return nil, []error{fmt.Errorf("no paths provided to parse")}
630	}
631
632	c.dependenciesReady = false
633
634	moduleCh := make(chan *moduleInfo)
635	errsCh := make(chan []error)
636	doneCh := make(chan struct{})
637	var numErrs uint32
638	var numGoroutines int32
639
640	// handler must be reentrant
641	handleOneFile := func(file *parser.File) {
642		if atomic.LoadUint32(&numErrs) > maxErrors {
643			return
644		}
645
646		for _, def := range file.Defs {
647			var module *moduleInfo
648			var errs []error
649			switch def := def.(type) {
650			case *parser.Module:
651				module, errs = c.processModuleDef(def, file.Name)
652			case *parser.Assignment:
653				// Already handled via Scope object
654			default:
655				panic("unknown definition type")
656			}
657
658			if len(errs) > 0 {
659				atomic.AddUint32(&numErrs, uint32(len(errs)))
660				errsCh <- errs
661			} else if module != nil {
662				moduleCh <- module
663			}
664		}
665	}
666
667	atomic.AddInt32(&numGoroutines, 1)
668	go func() {
669		var errs []error
670		deps, errs = c.WalkBlueprintsFiles(rootDir, filePaths, handleOneFile)
671		if len(errs) > 0 {
672			errsCh <- errs
673		}
674		doneCh <- struct{}{}
675	}()
676
677loop:
678	for {
679		select {
680		case newErrs := <-errsCh:
681			errs = append(errs, newErrs...)
682		case module := <-moduleCh:
683			newErrs := c.addModule(module)
684			if len(newErrs) > 0 {
685				errs = append(errs, newErrs...)
686			}
687		case <-doneCh:
688			n := atomic.AddInt32(&numGoroutines, -1)
689			if n == 0 {
690				break loop
691			}
692		}
693	}
694
695	return deps, errs
696}
697
698type FileHandler func(*parser.File)
699
700// WalkBlueprintsFiles walks a set of Blueprints files starting with the given filepaths,
701// calling the given file handler on each
702//
703// When WalkBlueprintsFiles encounters a Blueprints file with a set of subdirs listed,
704// it recursively parses any Blueprints files found in those subdirectories.
705//
706// If any of the file paths is an ancestor directory of any other of file path, the ancestor
707// will be parsed and visited first.
708//
709// the file handler will be called from a goroutine, so it must be reentrant.
710//
711// If no errors are encountered while parsing the files, the list of paths on
712// which the future output will depend is returned.  This list will include both
713// Blueprints file paths as well as directory paths for cases where wildcard
714// subdirs are found.
715//
716// visitor will be called asynchronously, and will only be called once visitor for each
717// ancestor directory has completed.
718//
719// WalkBlueprintsFiles will not return until all calls to visitor have returned.
720func (c *Context) WalkBlueprintsFiles(rootDir string, filePaths []string,
721	visitor FileHandler) (deps []string, errs []error) {
722
723	// make a mapping from ancestors to their descendants to facilitate parsing ancestors first
724	descendantsMap, err := findBlueprintDescendants(filePaths)
725	if err != nil {
726		panic(err.Error())
727	}
728	blueprintsSet := make(map[string]bool)
729
730	// Channels to receive data back from openAndParse goroutines
731	blueprintsCh := make(chan fileParseContext)
732	errsCh := make(chan []error)
733	depsCh := make(chan string)
734
735	// Channel to notify main loop that a openAndParse goroutine has finished
736	doneParsingCh := make(chan fileParseContext)
737
738	// Number of outstanding goroutines to wait for
739	activeCount := 0
740	var pending []fileParseContext
741	tooManyErrors := false
742
743	// Limit concurrent calls to parseBlueprintFiles to 200
744	// Darwin has a default limit of 256 open files
745	maxActiveCount := 200
746
747	// count the number of pending calls to visitor()
748	visitorWaitGroup := sync.WaitGroup{}
749
750	startParseBlueprintsFile := func(blueprint fileParseContext) {
751		if blueprintsSet[blueprint.fileName] {
752			return
753		}
754		blueprintsSet[blueprint.fileName] = true
755		activeCount++
756		deps = append(deps, blueprint.fileName)
757		visitorWaitGroup.Add(1)
758		go func() {
759			file, blueprints, deps, errs := c.openAndParse(blueprint.fileName, blueprint.Scope, rootDir,
760				&blueprint)
761			if len(errs) > 0 {
762				errsCh <- errs
763			}
764			for _, blueprint := range blueprints {
765				blueprintsCh <- blueprint
766			}
767			for _, dep := range deps {
768				depsCh <- dep
769			}
770			doneParsingCh <- blueprint
771
772			if blueprint.parent != nil && blueprint.parent.doneVisiting != nil {
773				// wait for visitor() of parent to complete
774				<-blueprint.parent.doneVisiting
775			}
776
777			if len(errs) == 0 {
778				// process this file
779				visitor(file)
780			}
781			if blueprint.doneVisiting != nil {
782				close(blueprint.doneVisiting)
783			}
784			visitorWaitGroup.Done()
785		}()
786	}
787
788	foundParseableBlueprint := func(blueprint fileParseContext) {
789		if activeCount >= maxActiveCount {
790			pending = append(pending, blueprint)
791		} else {
792			startParseBlueprintsFile(blueprint)
793		}
794	}
795
796	startParseDescendants := func(blueprint fileParseContext) {
797		descendants, hasDescendants := descendantsMap[blueprint.fileName]
798		if hasDescendants {
799			for _, descendant := range descendants {
800				foundParseableBlueprint(fileParseContext{descendant, parser.NewScope(blueprint.Scope), &blueprint, make(chan struct{})})
801			}
802		}
803	}
804
805	// begin parsing any files that have no ancestors
806	startParseDescendants(fileParseContext{"", parser.NewScope(nil), nil, nil})
807
808loop:
809	for {
810		if len(errs) > maxErrors {
811			tooManyErrors = true
812		}
813
814		select {
815		case newErrs := <-errsCh:
816			errs = append(errs, newErrs...)
817		case dep := <-depsCh:
818			deps = append(deps, dep)
819		case blueprint := <-blueprintsCh:
820			if tooManyErrors {
821				continue
822			}
823			foundParseableBlueprint(blueprint)
824		case blueprint := <-doneParsingCh:
825			activeCount--
826			if !tooManyErrors {
827				startParseDescendants(blueprint)
828			}
829			if activeCount < maxActiveCount && len(pending) > 0 {
830				// start to process the next one from the queue
831				next := pending[len(pending)-1]
832				pending = pending[:len(pending)-1]
833				startParseBlueprintsFile(next)
834			}
835			if activeCount == 0 {
836				break loop
837			}
838		}
839	}
840
841	sort.Strings(deps)
842
843	// wait for every visitor() to complete
844	visitorWaitGroup.Wait()
845
846	return
847}
848
849// MockFileSystem causes the Context to replace all reads with accesses to the provided map of
850// filenames to contents stored as a byte slice.
851func (c *Context) MockFileSystem(files map[string][]byte) {
852	// look for a module list file
853	_, ok := files[MockModuleListFile]
854	if !ok {
855		// no module list file specified; find every file named Blueprints
856		pathsToParse := []string{}
857		for candidate := range files {
858			if filepath.Base(candidate) == "Blueprints" {
859				pathsToParse = append(pathsToParse, candidate)
860			}
861		}
862		if len(pathsToParse) < 1 {
863			panic(fmt.Sprintf("No Blueprints files found in mock filesystem: %v\n", files))
864		}
865		// put the list of Blueprints files into a list file
866		files[MockModuleListFile] = []byte(strings.Join(pathsToParse, "\n"))
867	}
868	c.SetModuleListFile(MockModuleListFile)
869
870	// mock the filesystem
871	c.fs = pathtools.MockFs(files)
872}
873
874// openAndParse opens and parses a single Blueprints file, and returns the results
875func (c *Context) openAndParse(filename string, scope *parser.Scope, rootDir string,
876	parent *fileParseContext) (file *parser.File,
877	subBlueprints []fileParseContext, deps []string, errs []error) {
878
879	f, err := c.fs.Open(filename)
880	if err != nil {
881		// couldn't open the file; see if we can provide a clearer error than "could not open file"
882		stats, statErr := c.fs.Lstat(filename)
883		if statErr == nil {
884			isSymlink := stats.Mode()&os.ModeSymlink != 0
885			if isSymlink {
886				err = fmt.Errorf("could not open symlink %v : %v", filename, err)
887				target, readlinkErr := os.Readlink(filename)
888				if readlinkErr == nil {
889					_, targetStatsErr := c.fs.Lstat(target)
890					if targetStatsErr != nil {
891						err = fmt.Errorf("could not open symlink %v; its target (%v) cannot be opened", filename, target)
892					}
893				}
894			} else {
895				err = fmt.Errorf("%v exists but could not be opened: %v", filename, err)
896			}
897		}
898		return nil, nil, nil, []error{err}
899	}
900
901	func() {
902		defer func() {
903			err = f.Close()
904			if err != nil {
905				errs = append(errs, err)
906			}
907		}()
908		file, subBlueprints, errs = c.parseOne(rootDir, filename, f, scope, parent)
909	}()
910
911	if len(errs) > 0 {
912		return nil, nil, nil, errs
913	}
914
915	for _, b := range subBlueprints {
916		deps = append(deps, b.fileName)
917	}
918
919	return file, subBlueprints, deps, nil
920}
921
922// parseOne parses a single Blueprints file from the given reader, creating Module
923// objects for each of the module definitions encountered.  If the Blueprints
924// file contains an assignment to the "subdirs" variable, then the
925// subdirectories listed are searched for Blueprints files returned in the
926// subBlueprints return value.  If the Blueprints file contains an assignment
927// to the "build" variable, then the file listed are returned in the
928// subBlueprints return value.
929//
930// rootDir specifies the path to the root directory of the source tree, while
931// filename specifies the path to the Blueprints file.  These paths are used for
932// error reporting and for determining the module's directory.
933func (c *Context) parseOne(rootDir, filename string, reader io.Reader,
934	scope *parser.Scope, parent *fileParseContext) (file *parser.File, subBlueprints []fileParseContext, errs []error) {
935
936	relBlueprintsFile, err := filepath.Rel(rootDir, filename)
937	if err != nil {
938		return nil, nil, []error{err}
939	}
940
941	scope.Remove("subdirs")
942	scope.Remove("optional_subdirs")
943	scope.Remove("build")
944	file, errs = parser.ParseAndEval(filename, reader, scope)
945	if len(errs) > 0 {
946		for i, err := range errs {
947			if parseErr, ok := err.(*parser.ParseError); ok {
948				err = &BlueprintError{
949					Err: parseErr.Err,
950					Pos: parseErr.Pos,
951				}
952				errs[i] = err
953			}
954		}
955
956		// If there were any parse errors don't bother trying to interpret the
957		// result.
958		return nil, nil, errs
959	}
960	file.Name = relBlueprintsFile
961
962	build, buildPos, err := getLocalStringListFromScope(scope, "build")
963	if err != nil {
964		errs = append(errs, err)
965	}
966	for _, buildEntry := range build {
967		if strings.Contains(buildEntry, "/") {
968			errs = append(errs, &BlueprintError{
969				Err: fmt.Errorf("illegal value %v. The '/' character is not permitted", buildEntry),
970				Pos: buildPos,
971			})
972		}
973	}
974
975	subBlueprintsName, _, err := getStringFromScope(scope, "subname")
976	if err != nil {
977		errs = append(errs, err)
978	}
979
980	if subBlueprintsName == "" {
981		subBlueprintsName = "Blueprints"
982	}
983
984	var blueprints []string
985
986	newBlueprints, newErrs := c.findBuildBlueprints(filepath.Dir(filename), build, buildPos)
987	blueprints = append(blueprints, newBlueprints...)
988	errs = append(errs, newErrs...)
989
990	subBlueprintsAndScope := make([]fileParseContext, len(blueprints))
991	for i, b := range blueprints {
992		subBlueprintsAndScope[i] = fileParseContext{b, parser.NewScope(scope), parent, make(chan struct{})}
993	}
994	return file, subBlueprintsAndScope, errs
995}
996
997func (c *Context) findBuildBlueprints(dir string, build []string,
998	buildPos scanner.Position) ([]string, []error) {
999
1000	var blueprints []string
1001	var errs []error
1002
1003	for _, file := range build {
1004		pattern := filepath.Join(dir, file)
1005		var matches []string
1006		var err error
1007
1008		matches, err = c.glob(pattern, nil)
1009
1010		if err != nil {
1011			errs = append(errs, &BlueprintError{
1012				Err: fmt.Errorf("%q: %s", pattern, err.Error()),
1013				Pos: buildPos,
1014			})
1015			continue
1016		}
1017
1018		if len(matches) == 0 {
1019			errs = append(errs, &BlueprintError{
1020				Err: fmt.Errorf("%q: not found", pattern),
1021				Pos: buildPos,
1022			})
1023		}
1024
1025		for _, foundBlueprints := range matches {
1026			if strings.HasSuffix(foundBlueprints, "/") {
1027				errs = append(errs, &BlueprintError{
1028					Err: fmt.Errorf("%q: is a directory", foundBlueprints),
1029					Pos: buildPos,
1030				})
1031			}
1032			blueprints = append(blueprints, foundBlueprints)
1033		}
1034	}
1035
1036	return blueprints, errs
1037}
1038
1039func (c *Context) findSubdirBlueprints(dir string, subdirs []string, subdirsPos scanner.Position,
1040	subBlueprintsName string, optional bool) ([]string, []error) {
1041
1042	var blueprints []string
1043	var errs []error
1044
1045	for _, subdir := range subdirs {
1046		pattern := filepath.Join(dir, subdir, subBlueprintsName)
1047		var matches []string
1048		var err error
1049
1050		matches, err = c.glob(pattern, nil)
1051
1052		if err != nil {
1053			errs = append(errs, &BlueprintError{
1054				Err: fmt.Errorf("%q: %s", pattern, err.Error()),
1055				Pos: subdirsPos,
1056			})
1057			continue
1058		}
1059
1060		if len(matches) == 0 && !optional {
1061			errs = append(errs, &BlueprintError{
1062				Err: fmt.Errorf("%q: not found", pattern),
1063				Pos: subdirsPos,
1064			})
1065		}
1066
1067		for _, subBlueprints := range matches {
1068			if strings.HasSuffix(subBlueprints, "/") {
1069				errs = append(errs, &BlueprintError{
1070					Err: fmt.Errorf("%q: is a directory", subBlueprints),
1071					Pos: subdirsPos,
1072				})
1073			}
1074			blueprints = append(blueprints, subBlueprints)
1075		}
1076	}
1077
1078	return blueprints, errs
1079}
1080
1081func getLocalStringListFromScope(scope *parser.Scope, v string) ([]string, scanner.Position, error) {
1082	if assignment, local := scope.Get(v); assignment == nil || !local {
1083		return nil, scanner.Position{}, nil
1084	} else {
1085		switch value := assignment.Value.Eval().(type) {
1086		case *parser.List:
1087			ret := make([]string, 0, len(value.Values))
1088
1089			for _, listValue := range value.Values {
1090				s, ok := listValue.(*parser.String)
1091				if !ok {
1092					// The parser should not produce this.
1093					panic("non-string value found in list")
1094				}
1095
1096				ret = append(ret, s.Value)
1097			}
1098
1099			return ret, assignment.EqualsPos, nil
1100		case *parser.Bool, *parser.String:
1101			return nil, scanner.Position{}, &BlueprintError{
1102				Err: fmt.Errorf("%q must be a list of strings", v),
1103				Pos: assignment.EqualsPos,
1104			}
1105		default:
1106			panic(fmt.Errorf("unknown value type: %d", assignment.Value.Type()))
1107		}
1108	}
1109}
1110
1111func getStringFromScope(scope *parser.Scope, v string) (string, scanner.Position, error) {
1112	if assignment, _ := scope.Get(v); assignment == nil {
1113		return "", scanner.Position{}, nil
1114	} else {
1115		switch value := assignment.Value.Eval().(type) {
1116		case *parser.String:
1117			return value.Value, assignment.EqualsPos, nil
1118		case *parser.Bool, *parser.List:
1119			return "", scanner.Position{}, &BlueprintError{
1120				Err: fmt.Errorf("%q must be a string", v),
1121				Pos: assignment.EqualsPos,
1122			}
1123		default:
1124			panic(fmt.Errorf("unknown value type: %d", assignment.Value.Type()))
1125		}
1126	}
1127}
1128
1129// Clones a build logic module by calling the factory method for its module type, and then cloning
1130// property values.  Any values stored in the module object that are not stored in properties
1131// structs will be lost.
1132func (c *Context) cloneLogicModule(origModule *moduleInfo) (Module, []interface{}) {
1133	newLogicModule, newProperties := origModule.factory()
1134
1135	if len(newProperties) != len(origModule.properties) {
1136		panic("mismatched properties array length in " + origModule.Name())
1137	}
1138
1139	for i := range newProperties {
1140		dst := reflect.ValueOf(newProperties[i]).Elem()
1141		src := reflect.ValueOf(origModule.properties[i]).Elem()
1142
1143		proptools.CopyProperties(dst, src)
1144	}
1145
1146	return newLogicModule, newProperties
1147}
1148
1149func (c *Context) createVariations(origModule *moduleInfo, mutatorName string,
1150	variationNames []string) ([]*moduleInfo, []error) {
1151
1152	if len(variationNames) == 0 {
1153		panic(fmt.Errorf("mutator %q passed zero-length variation list for module %q",
1154			mutatorName, origModule.Name()))
1155	}
1156
1157	newModules := []*moduleInfo{}
1158
1159	var errs []error
1160
1161	for i, variationName := range variationNames {
1162		var newLogicModule Module
1163		var newProperties []interface{}
1164
1165		if i == 0 {
1166			// Reuse the existing module for the first new variant
1167			// This both saves creating a new module, and causes the insertion in c.moduleInfo below
1168			// with logicModule as the key to replace the original entry in c.moduleInfo
1169			newLogicModule, newProperties = origModule.logicModule, origModule.properties
1170		} else {
1171			newLogicModule, newProperties = c.cloneLogicModule(origModule)
1172		}
1173
1174		newVariant := origModule.variant.clone()
1175		newVariant[mutatorName] = variationName
1176
1177		m := *origModule
1178		newModule := &m
1179		newModule.directDeps = append([]depInfo{}, origModule.directDeps...)
1180		newModule.logicModule = newLogicModule
1181		newModule.variant = newVariant
1182		newModule.dependencyVariant = origModule.dependencyVariant.clone()
1183		newModule.properties = newProperties
1184
1185		if variationName != "" {
1186			if newModule.variantName == "" {
1187				newModule.variantName = variationName
1188			} else {
1189				newModule.variantName += "_" + variationName
1190			}
1191		}
1192
1193		newModules = append(newModules, newModule)
1194
1195		newErrs := c.convertDepsToVariation(newModule, mutatorName, variationName)
1196		if len(newErrs) > 0 {
1197			errs = append(errs, newErrs...)
1198		}
1199	}
1200
1201	// Mark original variant as invalid.  Modules that depend on this module will still
1202	// depend on origModule, but we'll fix it when the mutator is called on them.
1203	origModule.logicModule = nil
1204	origModule.splitModules = newModules
1205
1206	atomic.AddUint32(&c.depsModified, 1)
1207
1208	return newModules, errs
1209}
1210
1211func (c *Context) convertDepsToVariation(module *moduleInfo,
1212	mutatorName, variationName string) (errs []error) {
1213
1214	for i, dep := range module.directDeps {
1215		if dep.module.logicModule == nil {
1216			var newDep *moduleInfo
1217			for _, m := range dep.module.splitModules {
1218				if m.variant[mutatorName] == variationName {
1219					newDep = m
1220					break
1221				}
1222			}
1223			if newDep == nil {
1224				errs = append(errs, &BlueprintError{
1225					Err: fmt.Errorf("failed to find variation %q for module %q needed by %q",
1226						variationName, dep.module.Name(), module.Name()),
1227					Pos: module.pos,
1228				})
1229				continue
1230			}
1231			module.directDeps[i].module = newDep
1232		}
1233	}
1234
1235	return errs
1236}
1237
1238func (c *Context) prettyPrintVariant(variant variationMap) string {
1239	names := make([]string, 0, len(variant))
1240	for _, m := range c.variantMutatorNames {
1241		if v, ok := variant[m]; ok {
1242			names = append(names, m+":"+v)
1243		}
1244	}
1245
1246	return strings.Join(names, ", ")
1247}
1248
1249func (c *Context) newModule(factory ModuleFactory) *moduleInfo {
1250	logicModule, properties := factory()
1251
1252	module := &moduleInfo{
1253		logicModule: logicModule,
1254		factory:     factory,
1255	}
1256
1257	module.properties = properties
1258
1259	return module
1260}
1261
1262func (c *Context) processModuleDef(moduleDef *parser.Module,
1263	relBlueprintsFile string) (*moduleInfo, []error) {
1264
1265	factory, ok := c.moduleFactories[moduleDef.Type]
1266	if !ok {
1267		if c.ignoreUnknownModuleTypes {
1268			return nil, nil
1269		}
1270
1271		return nil, []error{
1272			&BlueprintError{
1273				Err: fmt.Errorf("unrecognized module type %q", moduleDef.Type),
1274				Pos: moduleDef.TypePos,
1275			},
1276		}
1277	}
1278
1279	module := c.newModule(factory)
1280	module.typeName = moduleDef.Type
1281
1282	module.relBlueprintsFile = relBlueprintsFile
1283
1284	propertyMap, errs := unpackProperties(moduleDef.Properties, module.properties...)
1285	if len(errs) > 0 {
1286		return nil, errs
1287	}
1288
1289	module.pos = moduleDef.TypePos
1290	module.propertyPos = make(map[string]scanner.Position)
1291	for name, propertyDef := range propertyMap {
1292		module.propertyPos[name] = propertyDef.ColonPos
1293	}
1294
1295	return module, nil
1296}
1297
1298func (c *Context) addModule(module *moduleInfo) []error {
1299	name := module.logicModule.Name()
1300	if name == "" {
1301		return []error{
1302			&BlueprintError{
1303				Err: fmt.Errorf("property 'name' is missing from a module"),
1304				Pos: module.pos,
1305			},
1306		}
1307	}
1308	c.moduleInfo[module.logicModule] = module
1309
1310	group := &moduleGroup{
1311		name:    name,
1312		modules: []*moduleInfo{module},
1313	}
1314	module.group = group
1315	namespace, errs := c.nameInterface.NewModule(
1316		newNamespaceContext(module),
1317		ModuleGroup{moduleGroup: group},
1318		module.logicModule)
1319	if len(errs) > 0 {
1320		for i := range errs {
1321			errs[i] = &BlueprintError{Err: errs[i], Pos: module.pos}
1322		}
1323		return errs
1324	}
1325	group.namespace = namespace
1326
1327	c.moduleGroups = append(c.moduleGroups, group)
1328
1329	return nil
1330}
1331
1332// ResolveDependencies checks that the dependencies specified by all of the
1333// modules defined in the parsed Blueprints files are valid.  This means that
1334// the modules depended upon are defined and that no circular dependencies
1335// exist.
1336func (c *Context) ResolveDependencies(config interface{}) (deps []string, errs []error) {
1337	return c.resolveDependencies(c.Context, config)
1338}
1339
1340func (c *Context) resolveDependencies(ctx context.Context, config interface{}) (deps []string, errs []error) {
1341	pprof.Do(ctx, pprof.Labels("blueprint", "ResolveDependencies"), func(ctx context.Context) {
1342		c.liveGlobals = newLiveTracker(config)
1343
1344		deps, errs = c.generateSingletonBuildActions(config, c.preSingletonInfo, c.liveGlobals)
1345		if len(errs) > 0 {
1346			return
1347		}
1348
1349		errs = c.updateDependencies()
1350		if len(errs) > 0 {
1351			return
1352		}
1353
1354		var mutatorDeps []string
1355		mutatorDeps, errs = c.runMutators(ctx, config)
1356		if len(errs) > 0 {
1357			return
1358		}
1359		deps = append(deps, mutatorDeps...)
1360
1361		c.cloneModules()
1362
1363		c.dependenciesReady = true
1364	})
1365
1366	if len(errs) > 0 {
1367		return nil, errs
1368	}
1369
1370	return deps, nil
1371}
1372
1373// Default dependencies handling.  If the module implements the (deprecated)
1374// DynamicDependerModule interface then this set consists of the union of those
1375// module names returned by its DynamicDependencies method and those added by calling
1376// AddDependencies or AddVariationDependencies on DynamicDependencyModuleContext.
1377func blueprintDepsMutator(ctx BottomUpMutatorContext) {
1378	if dynamicDepender, ok := ctx.Module().(DynamicDependerModule); ok {
1379		func() {
1380			defer func() {
1381				if r := recover(); r != nil {
1382					ctx.error(newPanicErrorf(r, "DynamicDependencies for %s", ctx.moduleInfo()))
1383				}
1384			}()
1385			dynamicDeps := dynamicDepender.DynamicDependencies(ctx)
1386
1387			if ctx.Failed() {
1388				return
1389			}
1390
1391			ctx.AddDependency(ctx.Module(), nil, dynamicDeps...)
1392		}()
1393	}
1394}
1395
1396// findMatchingVariant searches the moduleGroup for a module with the same variant as module,
1397// and returns the matching module, or nil if one is not found.
1398func (c *Context) findMatchingVariant(module *moduleInfo, possible []*moduleInfo) *moduleInfo {
1399	if len(possible) == 1 {
1400		return possible[0]
1401	} else {
1402		for _, m := range possible {
1403			if m.variant.equal(module.dependencyVariant) {
1404				return m
1405			}
1406		}
1407	}
1408
1409	return nil
1410}
1411
1412func (c *Context) addDependency(module *moduleInfo, tag DependencyTag, depName string) []error {
1413	if _, ok := tag.(BaseDependencyTag); ok {
1414		panic("BaseDependencyTag is not allowed to be used directly!")
1415	}
1416
1417	if depName == module.Name() {
1418		return []error{&BlueprintError{
1419			Err: fmt.Errorf("%q depends on itself", depName),
1420			Pos: module.pos,
1421		}}
1422	}
1423
1424	possibleDeps := c.modulesFromName(depName, module.namespace())
1425	if possibleDeps == nil {
1426		return c.discoveredMissingDependencies(module, depName)
1427	}
1428
1429	if m := c.findMatchingVariant(module, possibleDeps); m != nil {
1430		module.directDeps = append(module.directDeps, depInfo{m, tag})
1431		atomic.AddUint32(&c.depsModified, 1)
1432		return nil
1433	}
1434
1435	variants := make([]string, len(possibleDeps))
1436	for i, mod := range possibleDeps {
1437		variants[i] = c.prettyPrintVariant(mod.variant)
1438	}
1439	sort.Strings(variants)
1440
1441	return []error{&BlueprintError{
1442		Err: fmt.Errorf("dependency %q of %q missing variant:\n  %s\navailable variants:\n  %s",
1443			depName, module.Name(),
1444			c.prettyPrintVariant(module.dependencyVariant),
1445			strings.Join(variants, "\n  ")),
1446		Pos: module.pos,
1447	}}
1448}
1449
1450func (c *Context) findReverseDependency(module *moduleInfo, destName string) (*moduleInfo, []error) {
1451	if destName == module.Name() {
1452		return nil, []error{&BlueprintError{
1453			Err: fmt.Errorf("%q depends on itself", destName),
1454			Pos: module.pos,
1455		}}
1456	}
1457
1458	possibleDeps := c.modulesFromName(destName, module.namespace())
1459	if possibleDeps == nil {
1460		return nil, []error{&BlueprintError{
1461			Err: fmt.Errorf("%q has a reverse dependency on undefined module %q",
1462				module.Name(), destName),
1463			Pos: module.pos,
1464		}}
1465	}
1466
1467	if m := c.findMatchingVariant(module, possibleDeps); m != nil {
1468		return m, nil
1469	}
1470
1471	variants := make([]string, len(possibleDeps))
1472	for i, mod := range possibleDeps {
1473		variants[i] = c.prettyPrintVariant(mod.variant)
1474	}
1475	sort.Strings(variants)
1476
1477	return nil, []error{&BlueprintError{
1478		Err: fmt.Errorf("reverse dependency %q of %q missing variant:\n  %s\navailable variants:\n  %s",
1479			destName, module.Name(),
1480			c.prettyPrintVariant(module.dependencyVariant),
1481			strings.Join(variants, "\n  ")),
1482		Pos: module.pos,
1483	}}
1484}
1485
1486func (c *Context) addVariationDependency(module *moduleInfo, variations []Variation,
1487	tag DependencyTag, depName string, far bool) []error {
1488	if _, ok := tag.(BaseDependencyTag); ok {
1489		panic("BaseDependencyTag is not allowed to be used directly!")
1490	}
1491
1492	possibleDeps := c.modulesFromName(depName, module.namespace())
1493	if possibleDeps == nil {
1494		return c.discoveredMissingDependencies(module, depName)
1495	}
1496
1497	// We can't just append variant.Variant to module.dependencyVariants.variantName and
1498	// compare the strings because the result won't be in mutator registration order.
1499	// Create a new map instead, and then deep compare the maps.
1500	var newVariant variationMap
1501	if !far {
1502		newVariant = module.dependencyVariant.clone()
1503	} else {
1504		newVariant = make(variationMap)
1505	}
1506	for _, v := range variations {
1507		newVariant[v.Mutator] = v.Variation
1508	}
1509
1510	for _, m := range possibleDeps {
1511		var found bool
1512		if far {
1513			found = m.variant.subset(newVariant)
1514		} else {
1515			found = m.variant.equal(newVariant)
1516		}
1517		if found {
1518			if module == m {
1519				return []error{&BlueprintError{
1520					Err: fmt.Errorf("%q depends on itself", depName),
1521					Pos: module.pos,
1522				}}
1523			}
1524			// AddVariationDependency allows adding a dependency on itself, but only if
1525			// that module is earlier in the module list than this one, since we always
1526			// run GenerateBuildActions in order for the variants of a module
1527			if m.group == module.group && beforeInModuleList(module, m, module.group.modules) {
1528				return []error{&BlueprintError{
1529					Err: fmt.Errorf("%q depends on later version of itself", depName),
1530					Pos: module.pos,
1531				}}
1532			}
1533			module.directDeps = append(module.directDeps, depInfo{m, tag})
1534			atomic.AddUint32(&c.depsModified, 1)
1535			return nil
1536		}
1537	}
1538
1539	variants := make([]string, len(possibleDeps))
1540	for i, mod := range possibleDeps {
1541		variants[i] = c.prettyPrintVariant(mod.variant)
1542	}
1543	sort.Strings(variants)
1544
1545	return []error{&BlueprintError{
1546		Err: fmt.Errorf("dependency %q of %q missing variant:\n  %s\navailable variants:\n  %s",
1547			depName, module.Name(),
1548			c.prettyPrintVariant(newVariant),
1549			strings.Join(variants, "\n  ")),
1550		Pos: module.pos,
1551	}}
1552}
1553
1554func (c *Context) addInterVariantDependency(origModule *moduleInfo, tag DependencyTag,
1555	from, to Module) {
1556	if _, ok := tag.(BaseDependencyTag); ok {
1557		panic("BaseDependencyTag is not allowed to be used directly!")
1558	}
1559
1560	var fromInfo, toInfo *moduleInfo
1561	for _, m := range origModule.splitModules {
1562		if m.logicModule == from {
1563			fromInfo = m
1564		}
1565		if m.logicModule == to {
1566			toInfo = m
1567			if fromInfo != nil {
1568				panic(fmt.Errorf("%q depends on later version of itself", origModule.Name()))
1569			}
1570		}
1571	}
1572
1573	if fromInfo == nil || toInfo == nil {
1574		panic(fmt.Errorf("AddInterVariantDependency called for module %q on invalid variant",
1575			origModule.Name()))
1576	}
1577
1578	fromInfo.directDeps = append(fromInfo.directDeps, depInfo{toInfo, tag})
1579	atomic.AddUint32(&c.depsModified, 1)
1580}
1581
1582// findBlueprintDescendants returns a map linking parent Blueprints files to child Blueprints files
1583// For example, if paths = []string{"a/b/c/Android.bp", "a/Blueprints"},
1584// then descendants = {"":[]string{"a/Blueprints"}, "a/Blueprints":[]string{"a/b/c/Android.bp"}}
1585func findBlueprintDescendants(paths []string) (descendants map[string][]string, err error) {
1586	// make mapping from dir path to file path
1587	filesByDir := make(map[string]string, len(paths))
1588	for _, path := range paths {
1589		dir := filepath.Dir(path)
1590		_, alreadyFound := filesByDir[dir]
1591		if alreadyFound {
1592			return nil, fmt.Errorf("Found two Blueprint files in directory %v : %v and %v", dir, filesByDir[dir], path)
1593		}
1594		filesByDir[dir] = path
1595	}
1596
1597	findAncestor := func(childFile string) (ancestor string) {
1598		prevAncestorDir := filepath.Dir(childFile)
1599		for {
1600			ancestorDir := filepath.Dir(prevAncestorDir)
1601			if ancestorDir == prevAncestorDir {
1602				// reached the root dir without any matches; assign this as a descendant of ""
1603				return ""
1604			}
1605
1606			ancestorFile, ancestorExists := filesByDir[ancestorDir]
1607			if ancestorExists {
1608				return ancestorFile
1609			}
1610			prevAncestorDir = ancestorDir
1611		}
1612	}
1613	// generate the descendants map
1614	descendants = make(map[string][]string, len(filesByDir))
1615	for _, childFile := range filesByDir {
1616		ancestorFile := findAncestor(childFile)
1617		descendants[ancestorFile] = append(descendants[ancestorFile], childFile)
1618	}
1619	return descendants, nil
1620}
1621
1622type visitOrderer interface {
1623	// returns the number of modules that this module needs to wait for
1624	waitCount(module *moduleInfo) int
1625	// returns the list of modules that are waiting for this module
1626	propagate(module *moduleInfo) []*moduleInfo
1627	// visit modules in order
1628	visit(modules []*moduleInfo, visit func(*moduleInfo) bool)
1629}
1630
1631type unorderedVisitorImpl struct{}
1632
1633func (unorderedVisitorImpl) waitCount(module *moduleInfo) int {
1634	return 0
1635}
1636
1637func (unorderedVisitorImpl) propagate(module *moduleInfo) []*moduleInfo {
1638	return nil
1639}
1640
1641func (unorderedVisitorImpl) visit(modules []*moduleInfo, visit func(*moduleInfo) bool) {
1642	for _, module := range modules {
1643		if visit(module) {
1644			return
1645		}
1646	}
1647}
1648
1649type bottomUpVisitorImpl struct{}
1650
1651func (bottomUpVisitorImpl) waitCount(module *moduleInfo) int {
1652	return len(module.forwardDeps)
1653}
1654
1655func (bottomUpVisitorImpl) propagate(module *moduleInfo) []*moduleInfo {
1656	return module.reverseDeps
1657}
1658
1659func (bottomUpVisitorImpl) visit(modules []*moduleInfo, visit func(*moduleInfo) bool) {
1660	for _, module := range modules {
1661		if visit(module) {
1662			return
1663		}
1664	}
1665}
1666
1667type topDownVisitorImpl struct{}
1668
1669func (topDownVisitorImpl) waitCount(module *moduleInfo) int {
1670	return len(module.reverseDeps)
1671}
1672
1673func (topDownVisitorImpl) propagate(module *moduleInfo) []*moduleInfo {
1674	return module.forwardDeps
1675}
1676
1677func (topDownVisitorImpl) visit(modules []*moduleInfo, visit func(*moduleInfo) bool) {
1678	for i := 0; i < len(modules); i++ {
1679		module := modules[len(modules)-1-i]
1680		if visit(module) {
1681			return
1682		}
1683	}
1684}
1685
1686var (
1687	bottomUpVisitor bottomUpVisitorImpl
1688	topDownVisitor  topDownVisitorImpl
1689)
1690
1691// Calls visit on each module, guaranteeing that visit is not called on a module until visit on all
1692// of its dependencies has finished.
1693func (c *Context) parallelVisit(order visitOrderer, visit func(group *moduleInfo) bool) {
1694	doneCh := make(chan *moduleInfo)
1695	cancelCh := make(chan bool)
1696	count := 0
1697	cancel := false
1698	var backlog []*moduleInfo
1699	const limit = 1000
1700
1701	for _, module := range c.modulesSorted {
1702		module.waitingCount = order.waitCount(module)
1703	}
1704
1705	visitOne := func(module *moduleInfo) {
1706		if count < limit {
1707			count++
1708			go func() {
1709				ret := visit(module)
1710				if ret {
1711					cancelCh <- true
1712				}
1713				doneCh <- module
1714			}()
1715		} else {
1716			backlog = append(backlog, module)
1717		}
1718	}
1719
1720	for _, module := range c.modulesSorted {
1721		if module.waitingCount == 0 {
1722			visitOne(module)
1723		}
1724	}
1725
1726	for count > 0 || len(backlog) > 0 {
1727		select {
1728		case <-cancelCh:
1729			cancel = true
1730			backlog = nil
1731		case doneModule := <-doneCh:
1732			count--
1733			if !cancel {
1734				for count < limit && len(backlog) > 0 {
1735					toVisit := backlog[0]
1736					backlog = backlog[1:]
1737					visitOne(toVisit)
1738				}
1739				for _, module := range order.propagate(doneModule) {
1740					module.waitingCount--
1741					if module.waitingCount == 0 {
1742						visitOne(module)
1743					}
1744				}
1745			}
1746		}
1747	}
1748}
1749
1750// updateDependencies recursively walks the module dependency graph and updates
1751// additional fields based on the dependencies.  It builds a sorted list of modules
1752// such that dependencies of a module always appear first, and populates reverse
1753// dependency links and counts of total dependencies.  It also reports errors when
1754// it encounters dependency cycles.  This should called after resolveDependencies,
1755// as well as after any mutator pass has called addDependency
1756func (c *Context) updateDependencies() (errs []error) {
1757	visited := make(map[*moduleInfo]bool)  // modules that were already checked
1758	checking := make(map[*moduleInfo]bool) // modules actively being checked
1759
1760	sorted := make([]*moduleInfo, 0, len(c.moduleInfo))
1761
1762	var check func(group *moduleInfo) []*moduleInfo
1763
1764	cycleError := func(cycle []*moduleInfo) {
1765		// We are the "start" of the cycle, so we're responsible
1766		// for generating the errors.  The cycle list is in
1767		// reverse order because all the 'check' calls append
1768		// their own module to the list.
1769		errs = append(errs, &BlueprintError{
1770			Err: fmt.Errorf("encountered dependency cycle:"),
1771			Pos: cycle[len(cycle)-1].pos,
1772		})
1773
1774		// Iterate backwards through the cycle list.
1775		curModule := cycle[0]
1776		for i := len(cycle) - 1; i >= 0; i-- {
1777			nextModule := cycle[i]
1778			errs = append(errs, &BlueprintError{
1779				Err: fmt.Errorf("    %q depends on %q",
1780					curModule.Name(),
1781					nextModule.Name()),
1782				Pos: curModule.pos,
1783			})
1784			curModule = nextModule
1785		}
1786	}
1787
1788	check = func(module *moduleInfo) []*moduleInfo {
1789		visited[module] = true
1790		checking[module] = true
1791		defer delete(checking, module)
1792
1793		deps := make(map[*moduleInfo]bool)
1794
1795		// Add an implicit dependency ordering on all earlier modules in the same module group
1796		for _, dep := range module.group.modules {
1797			if dep == module {
1798				break
1799			}
1800			deps[dep] = true
1801		}
1802
1803		for _, dep := range module.directDeps {
1804			deps[dep.module] = true
1805		}
1806
1807		module.reverseDeps = []*moduleInfo{}
1808		module.forwardDeps = []*moduleInfo{}
1809
1810		for dep := range deps {
1811			if checking[dep] {
1812				// This is a cycle.
1813				return []*moduleInfo{dep, module}
1814			}
1815
1816			if !visited[dep] {
1817				cycle := check(dep)
1818				if cycle != nil {
1819					if cycle[0] == module {
1820						// We are the "start" of the cycle, so we're responsible
1821						// for generating the errors.  The cycle list is in
1822						// reverse order because all the 'check' calls append
1823						// their own module to the list.
1824						cycleError(cycle)
1825
1826						// We can continue processing this module's children to
1827						// find more cycles.  Since all the modules that were
1828						// part of the found cycle were marked as visited we
1829						// won't run into that cycle again.
1830					} else {
1831						// We're not the "start" of the cycle, so we just append
1832						// our module to the list and return it.
1833						return append(cycle, module)
1834					}
1835				}
1836			}
1837
1838			module.forwardDeps = append(module.forwardDeps, dep)
1839			dep.reverseDeps = append(dep.reverseDeps, module)
1840		}
1841
1842		sorted = append(sorted, module)
1843
1844		return nil
1845	}
1846
1847	for _, module := range c.moduleInfo {
1848		if !visited[module] {
1849			cycle := check(module)
1850			if cycle != nil {
1851				if cycle[len(cycle)-1] != module {
1852					panic("inconceivable!")
1853				}
1854				cycleError(cycle)
1855			}
1856		}
1857	}
1858
1859	c.modulesSorted = sorted
1860
1861	return
1862}
1863
1864// PrepareBuildActions generates an internal representation of all the build
1865// actions that need to be performed.  This process involves invoking the
1866// GenerateBuildActions method on each of the Module objects created during the
1867// parse phase and then on each of the registered Singleton objects.
1868//
1869// If the ResolveDependencies method has not already been called it is called
1870// automatically by this method.
1871//
1872// The config argument is made available to all of the Module and Singleton
1873// objects via the Config method on the ModuleContext and SingletonContext
1874// objects passed to GenerateBuildActions.  It is also passed to the functions
1875// specified via PoolFunc, RuleFunc, and VariableFunc so that they can compute
1876// config-specific values.
1877//
1878// The returned deps is a list of the ninja files dependencies that were added
1879// by the modules and singletons via the ModuleContext.AddNinjaFileDeps(),
1880// SingletonContext.AddNinjaFileDeps(), and PackageContext.AddNinjaFileDeps()
1881// methods.
1882func (c *Context) PrepareBuildActions(config interface{}) (deps []string, errs []error) {
1883	pprof.Do(c.Context, pprof.Labels("blueprint", "PrepareBuildActions"), func(ctx context.Context) {
1884		c.buildActionsReady = false
1885
1886		if !c.dependenciesReady {
1887			var extraDeps []string
1888			extraDeps, errs = c.resolveDependencies(ctx, config)
1889			if len(errs) > 0 {
1890				return
1891			}
1892			deps = append(deps, extraDeps...)
1893		}
1894
1895		var depsModules []string
1896		depsModules, errs = c.generateModuleBuildActions(config, c.liveGlobals)
1897		if len(errs) > 0 {
1898			return
1899		}
1900
1901		var depsSingletons []string
1902		depsSingletons, errs = c.generateSingletonBuildActions(config, c.singletonInfo, c.liveGlobals)
1903		if len(errs) > 0 {
1904			return
1905		}
1906
1907		deps = append(deps, depsModules...)
1908		deps = append(deps, depsSingletons...)
1909
1910		if c.ninjaBuildDir != nil {
1911			err := c.liveGlobals.addNinjaStringDeps(c.ninjaBuildDir)
1912			if err != nil {
1913				errs = []error{err}
1914				return
1915			}
1916		}
1917
1918		pkgNames, depsPackages := c.makeUniquePackageNames(c.liveGlobals)
1919
1920		deps = append(deps, depsPackages...)
1921
1922		// This will panic if it finds a problem since it's a programming error.
1923		c.checkForVariableReferenceCycles(c.liveGlobals.variables, pkgNames)
1924
1925		c.pkgNames = pkgNames
1926		c.globalVariables = c.liveGlobals.variables
1927		c.globalPools = c.liveGlobals.pools
1928		c.globalRules = c.liveGlobals.rules
1929
1930		c.buildActionsReady = true
1931	})
1932
1933	if len(errs) > 0 {
1934		return nil, errs
1935	}
1936
1937	return deps, nil
1938}
1939
1940func (c *Context) runMutators(ctx context.Context, config interface{}) (deps []string, errs []error) {
1941	var mutators []*mutatorInfo
1942
1943	pprof.Do(ctx, pprof.Labels("blueprint", "runMutators"), func(ctx context.Context) {
1944		mutators = append(mutators, c.earlyMutatorInfo...)
1945		mutators = append(mutators, c.mutatorInfo...)
1946
1947		for _, mutator := range mutators {
1948			pprof.Do(ctx, pprof.Labels("mutator", mutator.name), func(context.Context) {
1949				var newDeps []string
1950				if mutator.topDownMutator != nil {
1951					newDeps, errs = c.runMutator(config, mutator, topDownMutator)
1952				} else if mutator.bottomUpMutator != nil {
1953					newDeps, errs = c.runMutator(config, mutator, bottomUpMutator)
1954				} else {
1955					panic("no mutator set on " + mutator.name)
1956				}
1957				if len(errs) > 0 {
1958					return
1959				}
1960				deps = append(deps, newDeps...)
1961			})
1962			if len(errs) > 0 {
1963				return
1964			}
1965		}
1966	})
1967
1968	if len(errs) > 0 {
1969		return nil, errs
1970	}
1971
1972	return deps, nil
1973}
1974
1975type mutatorDirection interface {
1976	run(mutator *mutatorInfo, ctx *mutatorContext)
1977	orderer() visitOrderer
1978	fmt.Stringer
1979}
1980
1981type bottomUpMutatorImpl struct{}
1982
1983func (bottomUpMutatorImpl) run(mutator *mutatorInfo, ctx *mutatorContext) {
1984	mutator.bottomUpMutator(ctx)
1985}
1986
1987func (bottomUpMutatorImpl) orderer() visitOrderer {
1988	return bottomUpVisitor
1989}
1990
1991func (bottomUpMutatorImpl) String() string {
1992	return "bottom up mutator"
1993}
1994
1995type topDownMutatorImpl struct{}
1996
1997func (topDownMutatorImpl) run(mutator *mutatorInfo, ctx *mutatorContext) {
1998	mutator.topDownMutator(ctx)
1999}
2000
2001func (topDownMutatorImpl) orderer() visitOrderer {
2002	return topDownVisitor
2003}
2004
2005func (topDownMutatorImpl) String() string {
2006	return "top down mutator"
2007}
2008
2009var (
2010	topDownMutator  topDownMutatorImpl
2011	bottomUpMutator bottomUpMutatorImpl
2012)
2013
2014type reverseDep struct {
2015	module *moduleInfo
2016	dep    depInfo
2017}
2018
2019func (c *Context) runMutator(config interface{}, mutator *mutatorInfo,
2020	direction mutatorDirection) (deps []string, errs []error) {
2021
2022	newModuleInfo := make(map[Module]*moduleInfo)
2023	for k, v := range c.moduleInfo {
2024		newModuleInfo[k] = v
2025	}
2026
2027	type globalStateChange struct {
2028		reverse    []reverseDep
2029		rename     []rename
2030		replace    []replace
2031		newModules []*moduleInfo
2032		deps       []string
2033	}
2034
2035	reverseDeps := make(map[*moduleInfo][]depInfo)
2036	var rename []rename
2037	var replace []replace
2038	var newModules []*moduleInfo
2039
2040	errsCh := make(chan []error)
2041	globalStateCh := make(chan globalStateChange)
2042	newVariationsCh := make(chan []*moduleInfo)
2043	done := make(chan bool)
2044
2045	c.depsModified = 0
2046
2047	visit := func(module *moduleInfo) bool {
2048		if module.splitModules != nil {
2049			panic("split module found in sorted module list")
2050		}
2051
2052		mctx := &mutatorContext{
2053			baseModuleContext: baseModuleContext{
2054				context: c,
2055				config:  config,
2056				module:  module,
2057			},
2058			name: mutator.name,
2059		}
2060
2061		func() {
2062			defer func() {
2063				if r := recover(); r != nil {
2064					in := fmt.Sprintf("%s %q for %s", direction, mutator.name, module)
2065					if err, ok := r.(panicError); ok {
2066						err.addIn(in)
2067						mctx.error(err)
2068					} else {
2069						mctx.error(newPanicErrorf(r, in))
2070					}
2071				}
2072			}()
2073			direction.run(mutator, mctx)
2074		}()
2075
2076		if len(mctx.errs) > 0 {
2077			errsCh <- mctx.errs
2078			return true
2079		}
2080
2081		if len(mctx.newVariations) > 0 {
2082			newVariationsCh <- mctx.newVariations
2083		}
2084
2085		if len(mctx.reverseDeps) > 0 || len(mctx.replace) > 0 || len(mctx.rename) > 0 || len(mctx.newModules) > 0 {
2086			globalStateCh <- globalStateChange{
2087				reverse:    mctx.reverseDeps,
2088				replace:    mctx.replace,
2089				rename:     mctx.rename,
2090				newModules: mctx.newModules,
2091				deps:       mctx.ninjaFileDeps,
2092			}
2093		}
2094
2095		return false
2096	}
2097
2098	// Process errs and reverseDeps in a single goroutine
2099	go func() {
2100		for {
2101			select {
2102			case newErrs := <-errsCh:
2103				errs = append(errs, newErrs...)
2104			case globalStateChange := <-globalStateCh:
2105				for _, r := range globalStateChange.reverse {
2106					reverseDeps[r.module] = append(reverseDeps[r.module], r.dep)
2107				}
2108				replace = append(replace, globalStateChange.replace...)
2109				rename = append(rename, globalStateChange.rename...)
2110				newModules = append(newModules, globalStateChange.newModules...)
2111				deps = append(deps, globalStateChange.deps...)
2112			case newVariations := <-newVariationsCh:
2113				for _, m := range newVariations {
2114					newModuleInfo[m.logicModule] = m
2115				}
2116			case <-done:
2117				return
2118			}
2119		}
2120	}()
2121
2122	if mutator.parallel {
2123		c.parallelVisit(direction.orderer(), visit)
2124	} else {
2125		direction.orderer().visit(c.modulesSorted, visit)
2126	}
2127
2128	done <- true
2129
2130	if len(errs) > 0 {
2131		return nil, errs
2132	}
2133
2134	c.moduleInfo = newModuleInfo
2135
2136	for _, group := range c.moduleGroups {
2137		for i := 0; i < len(group.modules); i++ {
2138			module := group.modules[i]
2139
2140			// Update module group to contain newly split variants
2141			if module.splitModules != nil {
2142				group.modules, i = spliceModules(group.modules, i, module.splitModules)
2143			}
2144
2145			// Fix up any remaining dependencies on modules that were split into variants
2146			// by replacing them with the first variant
2147			for j, dep := range module.directDeps {
2148				if dep.module.logicModule == nil {
2149					module.directDeps[j].module = dep.module.splitModules[0]
2150				}
2151			}
2152		}
2153	}
2154
2155	for module, deps := range reverseDeps {
2156		sort.Sort(depSorter(deps))
2157		module.directDeps = append(module.directDeps, deps...)
2158		c.depsModified++
2159	}
2160
2161	for _, module := range newModules {
2162		errs = c.addModule(module)
2163		if len(errs) > 0 {
2164			return nil, errs
2165		}
2166		atomic.AddUint32(&c.depsModified, 1)
2167	}
2168
2169	errs = c.handleRenames(rename)
2170	if len(errs) > 0 {
2171		return nil, errs
2172	}
2173
2174	errs = c.handleReplacements(replace)
2175	if len(errs) > 0 {
2176		return nil, errs
2177	}
2178
2179	if c.depsModified > 0 {
2180		errs = c.updateDependencies()
2181		if len(errs) > 0 {
2182			return nil, errs
2183		}
2184	}
2185
2186	return deps, errs
2187}
2188
2189// Replaces every build logic module with a clone of itself.  Prevents introducing problems where
2190// a mutator sets a non-property member variable on a module, which works until a later mutator
2191// creates variants of that module.
2192func (c *Context) cloneModules() {
2193	type update struct {
2194		orig  Module
2195		clone *moduleInfo
2196	}
2197	ch := make(chan update)
2198	doneCh := make(chan bool)
2199	go func() {
2200		c.parallelVisit(unorderedVisitorImpl{}, func(m *moduleInfo) bool {
2201			origLogicModule := m.logicModule
2202			m.logicModule, m.properties = c.cloneLogicModule(m)
2203			ch <- update{origLogicModule, m}
2204			return false
2205		})
2206		doneCh <- true
2207	}()
2208
2209	done := false
2210	for !done {
2211		select {
2212		case <-doneCh:
2213			done = true
2214		case update := <-ch:
2215			delete(c.moduleInfo, update.orig)
2216			c.moduleInfo[update.clone.logicModule] = update.clone
2217		}
2218	}
2219}
2220
2221// Removes modules[i] from the list and inserts newModules... where it was located, returning
2222// the new slice and the index of the last inserted element
2223func spliceModules(modules []*moduleInfo, i int, newModules []*moduleInfo) ([]*moduleInfo, int) {
2224	spliceSize := len(newModules)
2225	newLen := len(modules) + spliceSize - 1
2226	var dest []*moduleInfo
2227	if cap(modules) >= len(modules)-1+len(newModules) {
2228		// We can fit the splice in the existing capacity, do everything in place
2229		dest = modules[:newLen]
2230	} else {
2231		dest = make([]*moduleInfo, newLen)
2232		copy(dest, modules[:i])
2233	}
2234
2235	// Move the end of the slice over by spliceSize-1
2236	copy(dest[i+spliceSize:], modules[i+1:])
2237
2238	// Copy the new modules into the slice
2239	copy(dest[i:], newModules)
2240
2241	return dest, i + spliceSize - 1
2242}
2243
2244func (c *Context) generateModuleBuildActions(config interface{},
2245	liveGlobals *liveTracker) ([]string, []error) {
2246
2247	var deps []string
2248	var errs []error
2249
2250	cancelCh := make(chan struct{})
2251	errsCh := make(chan []error)
2252	depsCh := make(chan []string)
2253
2254	go func() {
2255		for {
2256			select {
2257			case <-cancelCh:
2258				close(cancelCh)
2259				return
2260			case newErrs := <-errsCh:
2261				errs = append(errs, newErrs...)
2262			case newDeps := <-depsCh:
2263				deps = append(deps, newDeps...)
2264
2265			}
2266		}
2267	}()
2268
2269	c.parallelVisit(bottomUpVisitor, func(module *moduleInfo) bool {
2270
2271		uniqueName := c.nameInterface.UniqueName(newNamespaceContext(module), module.group.name)
2272		sanitizedName := toNinjaName(uniqueName)
2273
2274		prefix := moduleNamespacePrefix(sanitizedName + "_" + module.variantName)
2275
2276		// The parent scope of the moduleContext's local scope gets overridden to be that of the
2277		// calling Go package on a per-call basis.  Since the initial parent scope doesn't matter we
2278		// just set it to nil.
2279		scope := newLocalScope(nil, prefix)
2280
2281		mctx := &moduleContext{
2282			baseModuleContext: baseModuleContext{
2283				context: c,
2284				config:  config,
2285				module:  module,
2286			},
2287			scope:              scope,
2288			handledMissingDeps: module.missingDeps == nil,
2289		}
2290
2291		func() {
2292			defer func() {
2293				if r := recover(); r != nil {
2294					in := fmt.Sprintf("GenerateBuildActions for %s", module)
2295					if err, ok := r.(panicError); ok {
2296						err.addIn(in)
2297						mctx.error(err)
2298					} else {
2299						mctx.error(newPanicErrorf(r, in))
2300					}
2301				}
2302			}()
2303			mctx.module.logicModule.GenerateBuildActions(mctx)
2304		}()
2305
2306		if len(mctx.errs) > 0 {
2307			errsCh <- mctx.errs
2308			return true
2309		}
2310
2311		if module.missingDeps != nil && !mctx.handledMissingDeps {
2312			var errs []error
2313			for _, depName := range module.missingDeps {
2314				errs = append(errs, c.missingDependencyError(module, depName))
2315			}
2316			errsCh <- errs
2317			return true
2318		}
2319
2320		depsCh <- mctx.ninjaFileDeps
2321
2322		newErrs := c.processLocalBuildActions(&module.actionDefs,
2323			&mctx.actionDefs, liveGlobals)
2324		if len(newErrs) > 0 {
2325			errsCh <- newErrs
2326			return true
2327		}
2328		return false
2329	})
2330
2331	cancelCh <- struct{}{}
2332	<-cancelCh
2333
2334	return deps, errs
2335}
2336
2337func (c *Context) generateSingletonBuildActions(config interface{},
2338	singletons []*singletonInfo, liveGlobals *liveTracker) ([]string, []error) {
2339
2340	var deps []string
2341	var errs []error
2342
2343	for _, info := range singletons {
2344		// The parent scope of the singletonContext's local scope gets overridden to be that of the
2345		// calling Go package on a per-call basis.  Since the initial parent scope doesn't matter we
2346		// just set it to nil.
2347		scope := newLocalScope(nil, singletonNamespacePrefix(info.name))
2348
2349		sctx := &singletonContext{
2350			name:    info.name,
2351			context: c,
2352			config:  config,
2353			scope:   scope,
2354			globals: liveGlobals,
2355		}
2356
2357		func() {
2358			defer func() {
2359				if r := recover(); r != nil {
2360					in := fmt.Sprintf("GenerateBuildActions for singleton %s", info.name)
2361					if err, ok := r.(panicError); ok {
2362						err.addIn(in)
2363						sctx.error(err)
2364					} else {
2365						sctx.error(newPanicErrorf(r, in))
2366					}
2367				}
2368			}()
2369			info.singleton.GenerateBuildActions(sctx)
2370		}()
2371
2372		if len(sctx.errs) > 0 {
2373			errs = append(errs, sctx.errs...)
2374			if len(errs) > maxErrors {
2375				break
2376			}
2377			continue
2378		}
2379
2380		deps = append(deps, sctx.ninjaFileDeps...)
2381
2382		newErrs := c.processLocalBuildActions(&info.actionDefs,
2383			&sctx.actionDefs, liveGlobals)
2384		errs = append(errs, newErrs...)
2385		if len(errs) > maxErrors {
2386			break
2387		}
2388	}
2389
2390	return deps, errs
2391}
2392
2393func (c *Context) processLocalBuildActions(out, in *localBuildActions,
2394	liveGlobals *liveTracker) []error {
2395
2396	var errs []error
2397
2398	// First we go through and add everything referenced by the module's
2399	// buildDefs to the live globals set.  This will end up adding the live
2400	// locals to the set as well, but we'll take them out after.
2401	for _, def := range in.buildDefs {
2402		err := liveGlobals.AddBuildDefDeps(def)
2403		if err != nil {
2404			errs = append(errs, err)
2405		}
2406	}
2407
2408	if len(errs) > 0 {
2409		return errs
2410	}
2411
2412	out.buildDefs = append(out.buildDefs, in.buildDefs...)
2413
2414	// We use the now-incorrect set of live "globals" to determine which local
2415	// definitions are live.  As we go through copying those live locals to the
2416	// moduleGroup we remove them from the live globals set.
2417	for _, v := range in.variables {
2418		isLive := liveGlobals.RemoveVariableIfLive(v)
2419		if isLive {
2420			out.variables = append(out.variables, v)
2421		}
2422	}
2423
2424	for _, r := range in.rules {
2425		isLive := liveGlobals.RemoveRuleIfLive(r)
2426		if isLive {
2427			out.rules = append(out.rules, r)
2428		}
2429	}
2430
2431	return nil
2432}
2433
2434func (c *Context) walkDeps(topModule *moduleInfo, allowDuplicates bool,
2435	visitDown func(depInfo, *moduleInfo) bool, visitUp func(depInfo, *moduleInfo)) {
2436
2437	visited := make(map[*moduleInfo]bool)
2438	var visiting *moduleInfo
2439
2440	defer func() {
2441		if r := recover(); r != nil {
2442			panic(newPanicErrorf(r, "WalkDeps(%s, %s, %s) for dependency %s",
2443				topModule, funcName(visitDown), funcName(visitUp), visiting))
2444		}
2445	}()
2446
2447	var walk func(module *moduleInfo)
2448	walk = func(module *moduleInfo) {
2449		for _, dep := range module.directDeps {
2450			if allowDuplicates || !visited[dep.module] {
2451				visiting = dep.module
2452				recurse := true
2453				if visitDown != nil {
2454					recurse = visitDown(dep, module)
2455				}
2456				if recurse && !visited[dep.module] {
2457					walk(dep.module)
2458				}
2459				visited[dep.module] = true
2460				if visitUp != nil {
2461					visitUp(dep, module)
2462				}
2463			}
2464		}
2465	}
2466
2467	walk(topModule)
2468}
2469
2470type replace struct {
2471	from, to *moduleInfo
2472}
2473
2474type rename struct {
2475	group *moduleGroup
2476	name  string
2477}
2478
2479func (c *Context) moduleMatchingVariant(module *moduleInfo, name string) *moduleInfo {
2480	targets := c.modulesFromName(name, module.namespace())
2481
2482	if targets == nil {
2483		return nil
2484	}
2485
2486	for _, m := range targets {
2487		if module.variantName == m.variantName {
2488			return m
2489		}
2490	}
2491
2492	return nil
2493}
2494
2495func (c *Context) handleRenames(renames []rename) []error {
2496	var errs []error
2497	for _, rename := range renames {
2498		group, name := rename.group, rename.name
2499		if name == group.name || len(group.modules) < 1 {
2500			continue
2501		}
2502
2503		errs = append(errs, c.nameInterface.Rename(group.name, rename.name, group.namespace)...)
2504	}
2505
2506	return errs
2507}
2508
2509func (c *Context) handleReplacements(replacements []replace) []error {
2510	var errs []error
2511	for _, replace := range replacements {
2512		for _, m := range replace.from.reverseDeps {
2513			for i, d := range m.directDeps {
2514				if d.module == replace.from {
2515					m.directDeps[i].module = replace.to
2516				}
2517			}
2518		}
2519
2520		atomic.AddUint32(&c.depsModified, 1)
2521	}
2522
2523	return errs
2524}
2525
2526func (c *Context) discoveredMissingDependencies(module *moduleInfo, depName string) (errs []error) {
2527	if c.allowMissingDependencies {
2528		module.missingDeps = append(module.missingDeps, depName)
2529		return nil
2530	}
2531	return []error{c.missingDependencyError(module, depName)}
2532}
2533
2534func (c *Context) missingDependencyError(module *moduleInfo, depName string) (errs error) {
2535	err := c.nameInterface.MissingDependencyError(module.Name(), module.namespace(), depName)
2536
2537	return &BlueprintError{
2538		Err: err,
2539		Pos: module.pos,
2540	}
2541}
2542
2543func (c *Context) modulesFromName(name string, namespace Namespace) []*moduleInfo {
2544	group, exists := c.nameInterface.ModuleFromName(name, namespace)
2545	if exists {
2546		return group.modules
2547	}
2548	return nil
2549}
2550
2551func (c *Context) sortedModuleGroups() []*moduleGroup {
2552	if c.cachedSortedModuleGroups == nil {
2553		unwrap := func(wrappers []ModuleGroup) []*moduleGroup {
2554			result := make([]*moduleGroup, 0, len(wrappers))
2555			for _, group := range wrappers {
2556				result = append(result, group.moduleGroup)
2557			}
2558			return result
2559		}
2560
2561		c.cachedSortedModuleGroups = unwrap(c.nameInterface.AllModules())
2562	}
2563
2564	return c.cachedSortedModuleGroups
2565}
2566
2567func (c *Context) visitAllModules(visit func(Module)) {
2568	var module *moduleInfo
2569
2570	defer func() {
2571		if r := recover(); r != nil {
2572			panic(newPanicErrorf(r, "VisitAllModules(%s) for %s",
2573				funcName(visit), module))
2574		}
2575	}()
2576
2577	for _, moduleGroup := range c.sortedModuleGroups() {
2578		for _, module = range moduleGroup.modules {
2579			visit(module.logicModule)
2580		}
2581	}
2582}
2583
2584func (c *Context) visitAllModulesIf(pred func(Module) bool,
2585	visit func(Module)) {
2586
2587	var module *moduleInfo
2588
2589	defer func() {
2590		if r := recover(); r != nil {
2591			panic(newPanicErrorf(r, "VisitAllModulesIf(%s, %s) for %s",
2592				funcName(pred), funcName(visit), module))
2593		}
2594	}()
2595
2596	for _, moduleGroup := range c.sortedModuleGroups() {
2597		for _, module := range moduleGroup.modules {
2598			if pred(module.logicModule) {
2599				visit(module.logicModule)
2600			}
2601		}
2602	}
2603}
2604
2605func (c *Context) visitAllModuleVariants(module *moduleInfo,
2606	visit func(Module)) {
2607
2608	var variant *moduleInfo
2609
2610	defer func() {
2611		if r := recover(); r != nil {
2612			panic(newPanicErrorf(r, "VisitAllModuleVariants(%s, %s) for %s",
2613				module, funcName(visit), variant))
2614		}
2615	}()
2616
2617	for _, variant = range module.group.modules {
2618		visit(variant.logicModule)
2619	}
2620}
2621
2622func (c *Context) requireNinjaVersion(major, minor, micro int) {
2623	if major != 1 {
2624		panic("ninja version with major version != 1 not supported")
2625	}
2626	if c.requiredNinjaMinor < minor {
2627		c.requiredNinjaMinor = minor
2628		c.requiredNinjaMicro = micro
2629	}
2630	if c.requiredNinjaMinor == minor && c.requiredNinjaMicro < micro {
2631		c.requiredNinjaMicro = micro
2632	}
2633}
2634
2635func (c *Context) setNinjaBuildDir(value *ninjaString) {
2636	if c.ninjaBuildDir == nil {
2637		c.ninjaBuildDir = value
2638	}
2639}
2640
2641func (c *Context) makeUniquePackageNames(
2642	liveGlobals *liveTracker) (map[*packageContext]string, []string) {
2643
2644	pkgs := make(map[string]*packageContext)
2645	pkgNames := make(map[*packageContext]string)
2646	longPkgNames := make(map[*packageContext]bool)
2647
2648	processPackage := func(pctx *packageContext) {
2649		if pctx == nil {
2650			// This is a built-in rule and has no package.
2651			return
2652		}
2653		if _, ok := pkgNames[pctx]; ok {
2654			// We've already processed this package.
2655			return
2656		}
2657
2658		otherPkg, present := pkgs[pctx.shortName]
2659		if present {
2660			// Short name collision.  Both this package and the one that's
2661			// already there need to use their full names.  We leave the short
2662			// name in pkgNames for now so future collisions still get caught.
2663			longPkgNames[pctx] = true
2664			longPkgNames[otherPkg] = true
2665		} else {
2666			// No collision so far.  Tentatively set the package's name to be
2667			// its short name.
2668			pkgNames[pctx] = pctx.shortName
2669			pkgs[pctx.shortName] = pctx
2670		}
2671	}
2672
2673	// We try to give all packages their short name, but when we get collisions
2674	// we need to use the full unique package name.
2675	for v, _ := range liveGlobals.variables {
2676		processPackage(v.packageContext())
2677	}
2678	for p, _ := range liveGlobals.pools {
2679		processPackage(p.packageContext())
2680	}
2681	for r, _ := range liveGlobals.rules {
2682		processPackage(r.packageContext())
2683	}
2684
2685	// Add the packages that had collisions using their full unique names.  This
2686	// will overwrite any short names that were added in the previous step.
2687	for pctx := range longPkgNames {
2688		pkgNames[pctx] = pctx.fullName
2689	}
2690
2691	// Create deps list from calls to PackageContext.AddNinjaFileDeps
2692	deps := []string{}
2693	for _, pkg := range pkgs {
2694		deps = append(deps, pkg.ninjaFileDeps...)
2695	}
2696
2697	return pkgNames, deps
2698}
2699
2700func (c *Context) checkForVariableReferenceCycles(
2701	variables map[Variable]*ninjaString, pkgNames map[*packageContext]string) {
2702
2703	visited := make(map[Variable]bool)  // variables that were already checked
2704	checking := make(map[Variable]bool) // variables actively being checked
2705
2706	var check func(v Variable) []Variable
2707
2708	check = func(v Variable) []Variable {
2709		visited[v] = true
2710		checking[v] = true
2711		defer delete(checking, v)
2712
2713		value := variables[v]
2714		for _, dep := range value.variables {
2715			if checking[dep] {
2716				// This is a cycle.
2717				return []Variable{dep, v}
2718			}
2719
2720			if !visited[dep] {
2721				cycle := check(dep)
2722				if cycle != nil {
2723					if cycle[0] == v {
2724						// We are the "start" of the cycle, so we're responsible
2725						// for generating the errors.  The cycle list is in
2726						// reverse order because all the 'check' calls append
2727						// their own module to the list.
2728						msgs := []string{"detected variable reference cycle:"}
2729
2730						// Iterate backwards through the cycle list.
2731						curName := v.fullName(pkgNames)
2732						curValue := value.Value(pkgNames)
2733						for i := len(cycle) - 1; i >= 0; i-- {
2734							next := cycle[i]
2735							nextName := next.fullName(pkgNames)
2736							nextValue := variables[next].Value(pkgNames)
2737
2738							msgs = append(msgs, fmt.Sprintf(
2739								"    %q depends on %q", curName, nextName))
2740							msgs = append(msgs, fmt.Sprintf(
2741								"    [%s = %s]", curName, curValue))
2742
2743							curName = nextName
2744							curValue = nextValue
2745						}
2746
2747						// Variable reference cycles are a programming error,
2748						// not the fault of the Blueprint file authors.
2749						panic(strings.Join(msgs, "\n"))
2750					} else {
2751						// We're not the "start" of the cycle, so we just append
2752						// our module to the list and return it.
2753						return append(cycle, v)
2754					}
2755				}
2756			}
2757		}
2758
2759		return nil
2760	}
2761
2762	for v := range variables {
2763		if !visited[v] {
2764			cycle := check(v)
2765			if cycle != nil {
2766				panic("inconceivable!")
2767			}
2768		}
2769	}
2770}
2771
2772// AllTargets returns a map all the build target names to the rule used to build
2773// them.  This is the same information that is output by running 'ninja -t
2774// targets all'.  If this is called before PrepareBuildActions successfully
2775// completes then ErrbuildActionsNotReady is returned.
2776func (c *Context) AllTargets() (map[string]string, error) {
2777	if !c.buildActionsReady {
2778		return nil, ErrBuildActionsNotReady
2779	}
2780
2781	targets := map[string]string{}
2782
2783	// Collect all the module build targets.
2784	for _, module := range c.moduleInfo {
2785		for _, buildDef := range module.actionDefs.buildDefs {
2786			ruleName := buildDef.Rule.fullName(c.pkgNames)
2787			for _, output := range append(buildDef.Outputs, buildDef.ImplicitOutputs...) {
2788				outputValue, err := output.Eval(c.globalVariables)
2789				if err != nil {
2790					return nil, err
2791				}
2792				targets[outputValue] = ruleName
2793			}
2794		}
2795	}
2796
2797	// Collect all the singleton build targets.
2798	for _, info := range c.singletonInfo {
2799		for _, buildDef := range info.actionDefs.buildDefs {
2800			ruleName := buildDef.Rule.fullName(c.pkgNames)
2801			for _, output := range append(buildDef.Outputs, buildDef.ImplicitOutputs...) {
2802				outputValue, err := output.Eval(c.globalVariables)
2803				if err != nil {
2804					return nil, err
2805				}
2806				targets[outputValue] = ruleName
2807			}
2808		}
2809	}
2810
2811	return targets, nil
2812}
2813
2814func (c *Context) NinjaBuildDir() (string, error) {
2815	if c.ninjaBuildDir != nil {
2816		return c.ninjaBuildDir.Eval(c.globalVariables)
2817	} else {
2818		return "", nil
2819	}
2820}
2821
2822// ModuleTypePropertyStructs returns a mapping from module type name to a list of pointers to
2823// property structs returned by the factory for that module type.
2824func (c *Context) ModuleTypePropertyStructs() map[string][]interface{} {
2825	ret := make(map[string][]interface{})
2826	for moduleType, factory := range c.moduleFactories {
2827		_, ret[moduleType] = factory()
2828	}
2829
2830	return ret
2831}
2832
2833func (c *Context) ModuleTypeFactories() map[string]ModuleFactory {
2834	ret := make(map[string]ModuleFactory)
2835	for k, v := range c.moduleFactories {
2836		ret[k] = v
2837	}
2838	return ret
2839}
2840
2841func (c *Context) ModuleName(logicModule Module) string {
2842	module := c.moduleInfo[logicModule]
2843	return module.Name()
2844}
2845
2846func (c *Context) ModulePath(logicModule Module) string {
2847	module := c.moduleInfo[logicModule]
2848	return module.relBlueprintsFile
2849}
2850
2851func (c *Context) ModuleDir(logicModule Module) string {
2852	return filepath.Dir(c.ModulePath(logicModule))
2853}
2854
2855func (c *Context) ModuleSubDir(logicModule Module) string {
2856	module := c.moduleInfo[logicModule]
2857	return module.variantName
2858}
2859
2860func (c *Context) ModuleType(logicModule Module) string {
2861	module := c.moduleInfo[logicModule]
2862	return module.typeName
2863}
2864
2865func (c *Context) BlueprintFile(logicModule Module) string {
2866	module := c.moduleInfo[logicModule]
2867	return module.relBlueprintsFile
2868}
2869
2870func (c *Context) ModuleErrorf(logicModule Module, format string,
2871	args ...interface{}) error {
2872
2873	module := c.moduleInfo[logicModule]
2874	return &BlueprintError{
2875		Err: fmt.Errorf(format, args...),
2876		Pos: module.pos,
2877	}
2878}
2879
2880func (c *Context) VisitAllModules(visit func(Module)) {
2881	c.visitAllModules(visit)
2882}
2883
2884func (c *Context) VisitAllModulesIf(pred func(Module) bool,
2885	visit func(Module)) {
2886
2887	c.visitAllModulesIf(pred, visit)
2888}
2889
2890func (c *Context) VisitDirectDeps(module Module, visit func(Module)) {
2891	topModule := c.moduleInfo[module]
2892
2893	var visiting *moduleInfo
2894
2895	defer func() {
2896		if r := recover(); r != nil {
2897			panic(newPanicErrorf(r, "VisitDirectDeps(%s, %s) for dependency %s",
2898				topModule, funcName(visit), visiting))
2899		}
2900	}()
2901
2902	for _, dep := range topModule.directDeps {
2903		visiting = dep.module
2904		visit(dep.module.logicModule)
2905	}
2906}
2907
2908func (c *Context) VisitDirectDepsIf(module Module, pred func(Module) bool, visit func(Module)) {
2909	topModule := c.moduleInfo[module]
2910
2911	var visiting *moduleInfo
2912
2913	defer func() {
2914		if r := recover(); r != nil {
2915			panic(newPanicErrorf(r, "VisitDirectDepsIf(%s, %s, %s) for dependency %s",
2916				topModule, funcName(pred), funcName(visit), visiting))
2917		}
2918	}()
2919
2920	for _, dep := range topModule.directDeps {
2921		visiting = dep.module
2922		if pred(dep.module.logicModule) {
2923			visit(dep.module.logicModule)
2924		}
2925	}
2926}
2927
2928func (c *Context) VisitDepsDepthFirst(module Module, visit func(Module)) {
2929	topModule := c.moduleInfo[module]
2930
2931	var visiting *moduleInfo
2932
2933	defer func() {
2934		if r := recover(); r != nil {
2935			panic(newPanicErrorf(r, "VisitDepsDepthFirst(%s, %s) for dependency %s",
2936				topModule, funcName(visit), visiting))
2937		}
2938	}()
2939
2940	c.walkDeps(topModule, false, nil, func(dep depInfo, parent *moduleInfo) {
2941		visiting = dep.module
2942		visit(dep.module.logicModule)
2943	})
2944}
2945
2946func (c *Context) VisitDepsDepthFirstIf(module Module, pred func(Module) bool, visit func(Module)) {
2947	topModule := c.moduleInfo[module]
2948
2949	var visiting *moduleInfo
2950
2951	defer func() {
2952		if r := recover(); r != nil {
2953			panic(newPanicErrorf(r, "VisitDepsDepthFirstIf(%s, %s, %s) for dependency %s",
2954				topModule, funcName(pred), funcName(visit), visiting))
2955		}
2956	}()
2957
2958	c.walkDeps(topModule, false, nil, func(dep depInfo, parent *moduleInfo) {
2959		if pred(dep.module.logicModule) {
2960			visiting = dep.module
2961			visit(dep.module.logicModule)
2962		}
2963	})
2964}
2965
2966func (c *Context) PrimaryModule(module Module) Module {
2967	return c.moduleInfo[module].group.modules[0].logicModule
2968}
2969
2970func (c *Context) FinalModule(module Module) Module {
2971	modules := c.moduleInfo[module].group.modules
2972	return modules[len(modules)-1].logicModule
2973}
2974
2975func (c *Context) VisitAllModuleVariants(module Module,
2976	visit func(Module)) {
2977
2978	c.visitAllModuleVariants(c.moduleInfo[module], visit)
2979}
2980
2981// Singletons returns a list of all registered Singletons.
2982func (c *Context) Singletons() []Singleton {
2983	var ret []Singleton
2984	for _, s := range c.singletonInfo {
2985		ret = append(ret, s.singleton)
2986	}
2987	return ret
2988}
2989
2990// SingletonName returns the name that the given singleton was registered with.
2991func (c *Context) SingletonName(singleton Singleton) string {
2992	for _, s := range c.singletonInfo {
2993		if s.singleton == singleton {
2994			return s.name
2995		}
2996	}
2997	return ""
2998}
2999
3000// WriteBuildFile writes the Ninja manifeset text for the generated build
3001// actions to w.  If this is called before PrepareBuildActions successfully
3002// completes then ErrBuildActionsNotReady is returned.
3003func (c *Context) WriteBuildFile(w io.Writer) error {
3004	var err error
3005	pprof.Do(c.Context, pprof.Labels("blueprint", "WriteBuildFile"), func(ctx context.Context) {
3006		if !c.buildActionsReady {
3007			err = ErrBuildActionsNotReady
3008			return
3009		}
3010
3011		nw := newNinjaWriter(w)
3012
3013		err = c.writeBuildFileHeader(nw)
3014		if err != nil {
3015			return
3016		}
3017
3018		err = c.writeNinjaRequiredVersion(nw)
3019		if err != nil {
3020			return
3021		}
3022
3023		err = c.writeSubninjas(nw)
3024		if err != nil {
3025			return
3026		}
3027
3028		// TODO: Group the globals by package.
3029
3030		err = c.writeGlobalVariables(nw)
3031		if err != nil {
3032			return
3033		}
3034
3035		err = c.writeGlobalPools(nw)
3036		if err != nil {
3037			return
3038		}
3039
3040		err = c.writeBuildDir(nw)
3041		if err != nil {
3042			return
3043		}
3044
3045		err = c.writeGlobalRules(nw)
3046		if err != nil {
3047			return
3048		}
3049
3050		err = c.writeAllModuleActions(nw)
3051		if err != nil {
3052			return
3053		}
3054
3055		err = c.writeAllSingletonActions(nw)
3056		if err != nil {
3057			return
3058		}
3059	})
3060
3061	if err != nil {
3062		return err
3063	}
3064
3065	return nil
3066}
3067
3068type pkgAssociation struct {
3069	PkgName string
3070	PkgPath string
3071}
3072
3073type pkgAssociationSorter struct {
3074	pkgs []pkgAssociation
3075}
3076
3077func (s *pkgAssociationSorter) Len() int {
3078	return len(s.pkgs)
3079}
3080
3081func (s *pkgAssociationSorter) Less(i, j int) bool {
3082	iName := s.pkgs[i].PkgName
3083	jName := s.pkgs[j].PkgName
3084	return iName < jName
3085}
3086
3087func (s *pkgAssociationSorter) Swap(i, j int) {
3088	s.pkgs[i], s.pkgs[j] = s.pkgs[j], s.pkgs[i]
3089}
3090
3091func (c *Context) writeBuildFileHeader(nw *ninjaWriter) error {
3092	headerTemplate := template.New("fileHeader")
3093	_, err := headerTemplate.Parse(fileHeaderTemplate)
3094	if err != nil {
3095		// This is a programming error.
3096		panic(err)
3097	}
3098
3099	var pkgs []pkgAssociation
3100	maxNameLen := 0
3101	for pkg, name := range c.pkgNames {
3102		pkgs = append(pkgs, pkgAssociation{
3103			PkgName: name,
3104			PkgPath: pkg.pkgPath,
3105		})
3106		if len(name) > maxNameLen {
3107			maxNameLen = len(name)
3108		}
3109	}
3110
3111	for i := range pkgs {
3112		pkgs[i].PkgName += strings.Repeat(" ", maxNameLen-len(pkgs[i].PkgName))
3113	}
3114
3115	sort.Sort(&pkgAssociationSorter{pkgs})
3116
3117	params := map[string]interface{}{
3118		"Pkgs": pkgs,
3119	}
3120
3121	buf := bytes.NewBuffer(nil)
3122	err = headerTemplate.Execute(buf, params)
3123	if err != nil {
3124		return err
3125	}
3126
3127	return nw.Comment(buf.String())
3128}
3129
3130func (c *Context) writeNinjaRequiredVersion(nw *ninjaWriter) error {
3131	value := fmt.Sprintf("%d.%d.%d", c.requiredNinjaMajor, c.requiredNinjaMinor,
3132		c.requiredNinjaMicro)
3133
3134	err := nw.Assign("ninja_required_version", value)
3135	if err != nil {
3136		return err
3137	}
3138
3139	return nw.BlankLine()
3140}
3141
3142func (c *Context) writeSubninjas(nw *ninjaWriter) error {
3143	for _, subninja := range c.subninjas {
3144		err := nw.Subninja(subninja)
3145		if err != nil {
3146			return err
3147		}
3148	}
3149	return nw.BlankLine()
3150}
3151
3152func (c *Context) writeBuildDir(nw *ninjaWriter) error {
3153	if c.ninjaBuildDir != nil {
3154		err := nw.Assign("builddir", c.ninjaBuildDir.Value(c.pkgNames))
3155		if err != nil {
3156			return err
3157		}
3158
3159		err = nw.BlankLine()
3160		if err != nil {
3161			return err
3162		}
3163	}
3164	return nil
3165}
3166
3167type globalEntity interface {
3168	fullName(pkgNames map[*packageContext]string) string
3169}
3170
3171type globalEntitySorter struct {
3172	pkgNames map[*packageContext]string
3173	entities []globalEntity
3174}
3175
3176func (s *globalEntitySorter) Len() int {
3177	return len(s.entities)
3178}
3179
3180func (s *globalEntitySorter) Less(i, j int) bool {
3181	iName := s.entities[i].fullName(s.pkgNames)
3182	jName := s.entities[j].fullName(s.pkgNames)
3183	return iName < jName
3184}
3185
3186func (s *globalEntitySorter) Swap(i, j int) {
3187	s.entities[i], s.entities[j] = s.entities[j], s.entities[i]
3188}
3189
3190func (c *Context) writeGlobalVariables(nw *ninjaWriter) error {
3191	visited := make(map[Variable]bool)
3192
3193	var walk func(v Variable) error
3194	walk = func(v Variable) error {
3195		visited[v] = true
3196
3197		// First visit variables on which this variable depends.
3198		value := c.globalVariables[v]
3199		for _, dep := range value.variables {
3200			if !visited[dep] {
3201				err := walk(dep)
3202				if err != nil {
3203					return err
3204				}
3205			}
3206		}
3207
3208		err := nw.Assign(v.fullName(c.pkgNames), value.Value(c.pkgNames))
3209		if err != nil {
3210			return err
3211		}
3212
3213		err = nw.BlankLine()
3214		if err != nil {
3215			return err
3216		}
3217
3218		return nil
3219	}
3220
3221	globalVariables := make([]globalEntity, 0, len(c.globalVariables))
3222	for variable := range c.globalVariables {
3223		globalVariables = append(globalVariables, variable)
3224	}
3225
3226	sort.Sort(&globalEntitySorter{c.pkgNames, globalVariables})
3227
3228	for _, entity := range globalVariables {
3229		v := entity.(Variable)
3230		if !visited[v] {
3231			err := walk(v)
3232			if err != nil {
3233				return nil
3234			}
3235		}
3236	}
3237
3238	return nil
3239}
3240
3241func (c *Context) writeGlobalPools(nw *ninjaWriter) error {
3242	globalPools := make([]globalEntity, 0, len(c.globalPools))
3243	for pool := range c.globalPools {
3244		globalPools = append(globalPools, pool)
3245	}
3246
3247	sort.Sort(&globalEntitySorter{c.pkgNames, globalPools})
3248
3249	for _, entity := range globalPools {
3250		pool := entity.(Pool)
3251		name := pool.fullName(c.pkgNames)
3252		def := c.globalPools[pool]
3253		err := def.WriteTo(nw, name)
3254		if err != nil {
3255			return err
3256		}
3257
3258		err = nw.BlankLine()
3259		if err != nil {
3260			return err
3261		}
3262	}
3263
3264	return nil
3265}
3266
3267func (c *Context) writeGlobalRules(nw *ninjaWriter) error {
3268	globalRules := make([]globalEntity, 0, len(c.globalRules))
3269	for rule := range c.globalRules {
3270		globalRules = append(globalRules, rule)
3271	}
3272
3273	sort.Sort(&globalEntitySorter{c.pkgNames, globalRules})
3274
3275	for _, entity := range globalRules {
3276		rule := entity.(Rule)
3277		name := rule.fullName(c.pkgNames)
3278		def := c.globalRules[rule]
3279		err := def.WriteTo(nw, name, c.pkgNames)
3280		if err != nil {
3281			return err
3282		}
3283
3284		err = nw.BlankLine()
3285		if err != nil {
3286			return err
3287		}
3288	}
3289
3290	return nil
3291}
3292
3293type depSorter []depInfo
3294
3295func (s depSorter) Len() int {
3296	return len(s)
3297}
3298
3299func (s depSorter) Less(i, j int) bool {
3300	iName := s[i].module.Name()
3301	jName := s[j].module.Name()
3302	if iName == jName {
3303		iName = s[i].module.variantName
3304		jName = s[j].module.variantName
3305	}
3306	return iName < jName
3307}
3308
3309func (s depSorter) Swap(i, j int) {
3310	s[i], s[j] = s[j], s[i]
3311}
3312
3313type moduleSorter struct {
3314	modules       []*moduleInfo
3315	nameInterface NameInterface
3316}
3317
3318func (s moduleSorter) Len() int {
3319	return len(s.modules)
3320}
3321
3322func (s moduleSorter) Less(i, j int) bool {
3323	iMod := s.modules[i]
3324	jMod := s.modules[j]
3325	iName := s.nameInterface.UniqueName(newNamespaceContext(iMod), iMod.group.name)
3326	jName := s.nameInterface.UniqueName(newNamespaceContext(jMod), jMod.group.name)
3327	if iName == jName {
3328		iName = s.modules[i].variantName
3329		jName = s.modules[j].variantName
3330	}
3331
3332	if iName == jName {
3333		panic(fmt.Sprintf("duplicate module name: %s: %#v and %#v\n", iName, iMod, jMod))
3334	}
3335	return iName < jName
3336}
3337
3338func (s moduleSorter) Swap(i, j int) {
3339	s.modules[i], s.modules[j] = s.modules[j], s.modules[i]
3340}
3341
3342func (c *Context) writeAllModuleActions(nw *ninjaWriter) error {
3343	headerTemplate := template.New("moduleHeader")
3344	_, err := headerTemplate.Parse(moduleHeaderTemplate)
3345	if err != nil {
3346		// This is a programming error.
3347		panic(err)
3348	}
3349
3350	modules := make([]*moduleInfo, 0, len(c.moduleInfo))
3351	for _, module := range c.moduleInfo {
3352		modules = append(modules, module)
3353	}
3354	sort.Sort(moduleSorter{modules, c.nameInterface})
3355
3356	buf := bytes.NewBuffer(nil)
3357
3358	for _, module := range modules {
3359		if len(module.actionDefs.variables)+len(module.actionDefs.rules)+len(module.actionDefs.buildDefs) == 0 {
3360			continue
3361		}
3362
3363		buf.Reset()
3364
3365		// In order to make the bootstrap build manifest independent of the
3366		// build dir we need to output the Blueprints file locations in the
3367		// comments as paths relative to the source directory.
3368		relPos := module.pos
3369		relPos.Filename = module.relBlueprintsFile
3370
3371		// Get the name and location of the factory function for the module.
3372		factoryFunc := runtime.FuncForPC(reflect.ValueOf(module.factory).Pointer())
3373		factoryName := factoryFunc.Name()
3374
3375		infoMap := map[string]interface{}{
3376			"name":      module.Name(),
3377			"typeName":  module.typeName,
3378			"goFactory": factoryName,
3379			"pos":       relPos,
3380			"variant":   module.variantName,
3381		}
3382		err = headerTemplate.Execute(buf, infoMap)
3383		if err != nil {
3384			return err
3385		}
3386
3387		err = nw.Comment(buf.String())
3388		if err != nil {
3389			return err
3390		}
3391
3392		err = nw.BlankLine()
3393		if err != nil {
3394			return err
3395		}
3396
3397		err = c.writeLocalBuildActions(nw, &module.actionDefs)
3398		if err != nil {
3399			return err
3400		}
3401
3402		err = nw.BlankLine()
3403		if err != nil {
3404			return err
3405		}
3406	}
3407
3408	return nil
3409}
3410
3411func (c *Context) writeAllSingletonActions(nw *ninjaWriter) error {
3412	headerTemplate := template.New("singletonHeader")
3413	_, err := headerTemplate.Parse(singletonHeaderTemplate)
3414	if err != nil {
3415		// This is a programming error.
3416		panic(err)
3417	}
3418
3419	buf := bytes.NewBuffer(nil)
3420
3421	for _, info := range c.singletonInfo {
3422		if len(info.actionDefs.variables)+len(info.actionDefs.rules)+len(info.actionDefs.buildDefs) == 0 {
3423			continue
3424		}
3425
3426		// Get the name of the factory function for the module.
3427		factory := info.factory
3428		factoryFunc := runtime.FuncForPC(reflect.ValueOf(factory).Pointer())
3429		factoryName := factoryFunc.Name()
3430
3431		buf.Reset()
3432		infoMap := map[string]interface{}{
3433			"name":      info.name,
3434			"goFactory": factoryName,
3435		}
3436		err = headerTemplate.Execute(buf, infoMap)
3437		if err != nil {
3438			return err
3439		}
3440
3441		err = nw.Comment(buf.String())
3442		if err != nil {
3443			return err
3444		}
3445
3446		err = nw.BlankLine()
3447		if err != nil {
3448			return err
3449		}
3450
3451		err = c.writeLocalBuildActions(nw, &info.actionDefs)
3452		if err != nil {
3453			return err
3454		}
3455
3456		err = nw.BlankLine()
3457		if err != nil {
3458			return err
3459		}
3460	}
3461
3462	return nil
3463}
3464
3465func (c *Context) writeLocalBuildActions(nw *ninjaWriter,
3466	defs *localBuildActions) error {
3467
3468	// Write the local variable assignments.
3469	for _, v := range defs.variables {
3470		// A localVariable doesn't need the package names or config to
3471		// determine its name or value.
3472		name := v.fullName(nil)
3473		value, err := v.value(nil)
3474		if err != nil {
3475			panic(err)
3476		}
3477		err = nw.Assign(name, value.Value(c.pkgNames))
3478		if err != nil {
3479			return err
3480		}
3481	}
3482
3483	if len(defs.variables) > 0 {
3484		err := nw.BlankLine()
3485		if err != nil {
3486			return err
3487		}
3488	}
3489
3490	// Write the local rules.
3491	for _, r := range defs.rules {
3492		// A localRule doesn't need the package names or config to determine
3493		// its name or definition.
3494		name := r.fullName(nil)
3495		def, err := r.def(nil)
3496		if err != nil {
3497			panic(err)
3498		}
3499
3500		err = def.WriteTo(nw, name, c.pkgNames)
3501		if err != nil {
3502			return err
3503		}
3504
3505		err = nw.BlankLine()
3506		if err != nil {
3507			return err
3508		}
3509	}
3510
3511	// Write the build definitions.
3512	for _, buildDef := range defs.buildDefs {
3513		err := buildDef.WriteTo(nw, c.pkgNames)
3514		if err != nil {
3515			return err
3516		}
3517
3518		if len(buildDef.Args) > 0 {
3519			err = nw.BlankLine()
3520			if err != nil {
3521				return err
3522			}
3523		}
3524	}
3525
3526	return nil
3527}
3528
3529func beforeInModuleList(a, b *moduleInfo, list []*moduleInfo) bool {
3530	found := false
3531	if a == b {
3532		return false
3533	}
3534	for _, l := range list {
3535		if l == a {
3536			found = true
3537		} else if l == b {
3538			return found
3539		}
3540	}
3541
3542	missing := a
3543	if found {
3544		missing = b
3545	}
3546	panic(fmt.Errorf("element %v not found in list %v", missing, list))
3547}
3548
3549type panicError struct {
3550	panic interface{}
3551	stack []byte
3552	in    string
3553}
3554
3555func newPanicErrorf(panic interface{}, in string, a ...interface{}) error {
3556	buf := make([]byte, 4096)
3557	count := runtime.Stack(buf, false)
3558	return panicError{
3559		panic: panic,
3560		in:    fmt.Sprintf(in, a...),
3561		stack: buf[:count],
3562	}
3563}
3564
3565func (p panicError) Error() string {
3566	return fmt.Sprintf("panic in %s\n%s\n%s\n", p.in, p.panic, p.stack)
3567}
3568
3569func (p *panicError) addIn(in string) {
3570	p.in += " in " + in
3571}
3572
3573func funcName(f interface{}) string {
3574	return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
3575}
3576
3577var fileHeaderTemplate = `******************************************************************************
3578***            This file is generated and should not be edited             ***
3579******************************************************************************
3580{{if .Pkgs}}
3581This file contains variables, rules, and pools with name prefixes indicating
3582they were generated by the following Go packages:
3583{{range .Pkgs}}
3584    {{.PkgName}} [from Go package {{.PkgPath}}]{{end}}{{end}}
3585
3586`
3587
3588var moduleHeaderTemplate = `# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
3589Module:  {{.name}}
3590Variant: {{.variant}}
3591Type:    {{.typeName}}
3592Factory: {{.goFactory}}
3593Defined: {{.pos}}
3594`
3595
3596var singletonHeaderTemplate = `# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
3597Singleton: {{.name}}
3598Factory:   {{.goFactory}}
3599`
3600