1// Copyright 2021 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15// Convert makefile containing device configuration to Starlark file 16// The conversion can handle the following constructs in a makefile: 17// - comments 18// - simple variable assignments 19// - $(call init-product,<file>) 20// - $(call inherit-product-if-exists 21// - if directives 22// 23// All other constructs are carried over to the output starlark file as comments. 24package mk2rbc 25 26import ( 27 "bytes" 28 "fmt" 29 "io" 30 "io/fs" 31 "io/ioutil" 32 "os" 33 "path/filepath" 34 "regexp" 35 "sort" 36 "strconv" 37 "strings" 38 "text/scanner" 39 40 mkparser "android/soong/androidmk/parser" 41) 42 43const ( 44 annotationCommentPrefix = "RBC#" 45 baseUri = "//build/make/core:product_config.rbc" 46 // The name of the struct exported by the product_config.rbc 47 // that contains the functions and variables available to 48 // product configuration Starlark files. 49 baseName = "rblf" 50 51 soongNsPrefix = "SOONG_CONFIG_" 52 53 // And here are the functions and variables: 54 cfnGetCfg = baseName + ".cfg" 55 cfnMain = baseName + ".product_configuration" 56 cfnBoardMain = baseName + ".board_configuration" 57 cfnPrintVars = baseName + ".printvars" 58 cfnInherit = baseName + ".inherit" 59 cfnSetListDefault = baseName + ".setdefault" 60) 61 62const ( 63 soongConfigAppend = "soong_config_append" 64 soongConfigAssign = "soong_config_set" 65) 66 67var knownFunctions = map[string]interface { 68 parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr 69}{ 70 "abspath": &simpleCallParser{name: baseName + ".abspath", returnType: starlarkTypeString}, 71 "add-product-dex-preopt-module-config": &simpleCallParser{name: baseName + ".add_product_dex_preopt_module_config", returnType: starlarkTypeString, addHandle: true}, 72 "add_soong_config_namespace": &simpleCallParser{name: baseName + ".soong_config_namespace", returnType: starlarkTypeVoid, addGlobals: true}, 73 "add_soong_config_var_value": &simpleCallParser{name: baseName + ".soong_config_set", returnType: starlarkTypeVoid, addGlobals: true}, 74 soongConfigAssign: &simpleCallParser{name: baseName + ".soong_config_set", returnType: starlarkTypeVoid, addGlobals: true}, 75 soongConfigAppend: &simpleCallParser{name: baseName + ".soong_config_append", returnType: starlarkTypeVoid, addGlobals: true}, 76 "soong_config_get": &simpleCallParser{name: baseName + ".soong_config_get", returnType: starlarkTypeString, addGlobals: true}, 77 "add-to-product-copy-files-if-exists": &simpleCallParser{name: baseName + ".copy_if_exists", returnType: starlarkTypeList}, 78 "addprefix": &simpleCallParser{name: baseName + ".addprefix", returnType: starlarkTypeList}, 79 "addsuffix": &simpleCallParser{name: baseName + ".addsuffix", returnType: starlarkTypeList}, 80 "and": &andOrParser{isAnd: true}, 81 "clear-var-list": &simpleCallParser{name: baseName + ".clear_var_list", returnType: starlarkTypeVoid, addGlobals: true, addHandle: true}, 82 "copy-files": &simpleCallParser{name: baseName + ".copy_files", returnType: starlarkTypeList}, 83 "dir": &simpleCallParser{name: baseName + ".dir", returnType: starlarkTypeString}, 84 "dist-for-goals": &simpleCallParser{name: baseName + ".mkdist_for_goals", returnType: starlarkTypeVoid, addGlobals: true}, 85 "enforce-product-packages-exist": &simpleCallParser{name: baseName + ".enforce_product_packages_exist", returnType: starlarkTypeVoid, addHandle: true}, 86 "error": &makeControlFuncParser{name: baseName + ".mkerror"}, 87 "findstring": &simpleCallParser{name: baseName + ".findstring", returnType: starlarkTypeInt}, 88 "find-copy-subdir-files": &simpleCallParser{name: baseName + ".find_and_copy", returnType: starlarkTypeList}, 89 "filter": &simpleCallParser{name: baseName + ".filter", returnType: starlarkTypeList}, 90 "filter-out": &simpleCallParser{name: baseName + ".filter_out", returnType: starlarkTypeList}, 91 "firstword": &simpleCallParser{name: baseName + ".first_word", returnType: starlarkTypeString}, 92 "foreach": &foreachCallParser{}, 93 "if": &ifCallParser{}, 94 "info": &makeControlFuncParser{name: baseName + ".mkinfo"}, 95 "is-board-platform": &simpleCallParser{name: baseName + ".board_platform_is", returnType: starlarkTypeBool, addGlobals: true}, 96 "is-board-platform2": &simpleCallParser{name: baseName + ".board_platform_is", returnType: starlarkTypeBool, addGlobals: true}, 97 "is-board-platform-in-list": &simpleCallParser{name: baseName + ".board_platform_in", returnType: starlarkTypeBool, addGlobals: true}, 98 "is-board-platform-in-list2": &simpleCallParser{name: baseName + ".board_platform_in", returnType: starlarkTypeBool, addGlobals: true}, 99 "is-product-in-list": &isProductInListCallParser{}, 100 "is-vendor-board-platform": &isVendorBoardPlatformCallParser{}, 101 "is-vendor-board-qcom": &isVendorBoardQcomCallParser{}, 102 "lastword": &simpleCallParser{name: baseName + ".last_word", returnType: starlarkTypeString}, 103 "notdir": &simpleCallParser{name: baseName + ".notdir", returnType: starlarkTypeString}, 104 "math_max": &mathMaxOrMinCallParser{function: "max"}, 105 "math_min": &mathMaxOrMinCallParser{function: "min"}, 106 "math_gt_or_eq": &mathComparisonCallParser{op: ">="}, 107 "math_gt": &mathComparisonCallParser{op: ">"}, 108 "math_lt": &mathComparisonCallParser{op: "<"}, 109 "my-dir": &myDirCallParser{}, 110 "or": &andOrParser{isAnd: false}, 111 "patsubst": &substCallParser{fname: "patsubst"}, 112 "product-copy-files-by-pattern": &simpleCallParser{name: baseName + ".product_copy_files_by_pattern", returnType: starlarkTypeList}, 113 "require-artifacts-in-path": &simpleCallParser{name: baseName + ".require_artifacts_in_path", returnType: starlarkTypeVoid, addHandle: true}, 114 "require-artifacts-in-path-relaxed": &simpleCallParser{name: baseName + ".require_artifacts_in_path_relaxed", returnType: starlarkTypeVoid, addHandle: true}, 115 // TODO(asmundak): remove it once all calls are removed from configuration makefiles. see b/183161002 116 "shell": &shellCallParser{}, 117 "sort": &simpleCallParser{name: baseName + ".mksort", returnType: starlarkTypeList}, 118 "strip": &simpleCallParser{name: baseName + ".mkstrip", returnType: starlarkTypeString}, 119 "subst": &substCallParser{fname: "subst"}, 120 "to-lower": &lowerUpperParser{isUpper: false}, 121 "to-upper": &lowerUpperParser{isUpper: true}, 122 "warning": &makeControlFuncParser{name: baseName + ".mkwarning"}, 123 "word": &wordCallParser{}, 124 "words": &wordsCallParser{}, 125 "wildcard": &simpleCallParser{name: baseName + ".expand_wildcard", returnType: starlarkTypeList}, 126} 127 128// The same as knownFunctions, but returns a []starlarkNode instead of a starlarkExpr 129var knownNodeFunctions = map[string]interface { 130 parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode 131}{ 132 "eval": &evalNodeParser{}, 133 "if": &ifCallNodeParser{}, 134 "inherit-product": &inheritProductCallParser{loadAlways: true}, 135 "inherit-product-if-exists": &inheritProductCallParser{loadAlways: false}, 136 "foreach": &foreachCallNodeParser{}, 137} 138 139// These look like variables, but are actually functions, and would give 140// undefined variable errors if we converted them as variables. Instead, 141// emit an error instead of converting them. 142var unsupportedFunctions = map[string]bool{ 143 "local-generated-sources-dir": true, 144 "local-intermediates-dir": true, 145} 146 147// These are functions that we don't implement conversions for, but 148// we allow seeing their definitions in the product config files. 149var ignoredDefines = map[string]bool{ 150 "find-word-in-list": true, // internal macro 151 "get-vendor-board-platforms": true, // internal macro, used by is-board-platform, etc. 152 "is-android-codename": true, // unused by product config 153 "is-android-codename-in-list": true, // unused by product config 154 "is-chipset-in-board-platform": true, // unused by product config 155 "is-chipset-prefix-in-board-platform": true, // unused by product config 156 "is-not-board-platform": true, // defined but never used 157 "is-platform-sdk-version-at-least": true, // unused by product config 158 "match-prefix": true, // internal macro 159 "match-word": true, // internal macro 160 "match-word-in-list": true, // internal macro 161 "tb-modules": true, // defined in hardware/amlogic/tb_modules/tb_detect.mk, unused 162} 163 164var identifierFullMatchRegex = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$") 165 166// Conversion request parameters 167type Request struct { 168 MkFile string // file to convert 169 Reader io.Reader // if set, read input from this stream instead 170 OutputSuffix string // generated Starlark files suffix 171 OutputDir string // if set, root of the output hierarchy 172 ErrorLogger ErrorLogger 173 TracedVariables []string // trace assignment to these variables 174 TraceCalls bool 175 SourceFS fs.FS 176 MakefileFinder MakefileFinder 177} 178 179// ErrorLogger prints errors and gathers error statistics. 180// Its NewError function is called on every error encountered during the conversion. 181type ErrorLogger interface { 182 NewError(el ErrorLocation, node mkparser.Node, text string, args ...interface{}) 183} 184 185type ErrorLocation struct { 186 MkFile string 187 MkLine int 188} 189 190func (el ErrorLocation) String() string { 191 return fmt.Sprintf("%s:%d", el.MkFile, el.MkLine) 192} 193 194// Derives module name for a given file. It is base name 195// (file name without suffix), with some characters replaced to make it a Starlark identifier 196func moduleNameForFile(mkFile string) string { 197 base := strings.TrimSuffix(filepath.Base(mkFile), filepath.Ext(mkFile)) 198 // TODO(asmundak): what else can be in the product file names? 199 return strings.NewReplacer("-", "_", ".", "_").Replace(base) 200 201} 202 203func cloneMakeString(mkString *mkparser.MakeString) *mkparser.MakeString { 204 r := &mkparser.MakeString{StringPos: mkString.StringPos} 205 r.Strings = append(r.Strings, mkString.Strings...) 206 r.Variables = append(r.Variables, mkString.Variables...) 207 return r 208} 209 210func isMakeControlFunc(s string) bool { 211 return s == "error" || s == "warning" || s == "info" 212} 213 214// varAssignmentScope points to the last assignment for each variable 215// in the current block. It is used during the parsing to chain 216// the assignments to a variable together. 217type varAssignmentScope struct { 218 outer *varAssignmentScope 219 vars map[string]bool 220} 221 222// Starlark output generation context 223type generationContext struct { 224 buf strings.Builder 225 starScript *StarlarkScript 226 indentLevel int 227 inAssignment bool 228 tracedCount int 229 varAssignments *varAssignmentScope 230} 231 232func NewGenerateContext(ss *StarlarkScript) *generationContext { 233 return &generationContext{ 234 starScript: ss, 235 varAssignments: &varAssignmentScope{ 236 outer: nil, 237 vars: make(map[string]bool), 238 }, 239 } 240} 241 242func (gctx *generationContext) pushVariableAssignments() { 243 va := &varAssignmentScope{ 244 outer: gctx.varAssignments, 245 vars: make(map[string]bool), 246 } 247 gctx.varAssignments = va 248} 249 250func (gctx *generationContext) popVariableAssignments() { 251 gctx.varAssignments = gctx.varAssignments.outer 252} 253 254func (gctx *generationContext) hasBeenAssigned(v variable) bool { 255 for va := gctx.varAssignments; va != nil; va = va.outer { 256 if _, ok := va.vars[v.name()]; ok { 257 return true 258 } 259 } 260 return false 261} 262 263func (gctx *generationContext) setHasBeenAssigned(v variable) { 264 gctx.varAssignments.vars[v.name()] = true 265} 266 267// emit returns generated script 268func (gctx *generationContext) emit() string { 269 ss := gctx.starScript 270 271 // The emitted code has the following layout: 272 // <initial comments> 273 // preamble, i.e., 274 // load statement for the runtime support 275 // load statement for each unique submodule pulled in by this one 276 // def init(g, handle): 277 // cfg = rblf.cfg(handle) 278 // <statements> 279 // <warning if conversion was not clean> 280 281 iNode := len(ss.nodes) 282 for i, node := range ss.nodes { 283 if _, ok := node.(*commentNode); !ok { 284 iNode = i 285 break 286 } 287 node.emit(gctx) 288 } 289 290 gctx.emitPreamble() 291 292 gctx.newLine() 293 // The arguments passed to the init function are the global dictionary 294 // ('g') and the product configuration dictionary ('cfg') 295 gctx.write("def init(g, handle):") 296 gctx.indentLevel++ 297 if gctx.starScript.traceCalls { 298 gctx.newLine() 299 gctx.writef(`print(">%s")`, gctx.starScript.mkFile) 300 } 301 gctx.newLine() 302 gctx.writef("cfg = %s(handle)", cfnGetCfg) 303 for _, node := range ss.nodes[iNode:] { 304 node.emit(gctx) 305 } 306 307 if gctx.starScript.traceCalls { 308 gctx.newLine() 309 gctx.writef(`print("<%s")`, gctx.starScript.mkFile) 310 } 311 gctx.indentLevel-- 312 gctx.write("\n") 313 return gctx.buf.String() 314} 315 316func (gctx *generationContext) emitPreamble() { 317 gctx.newLine() 318 gctx.writef("load(%q, %q)", baseUri, baseName) 319 // Emit exactly one load statement for each URI. 320 loadedSubConfigs := make(map[string]string) 321 for _, mi := range gctx.starScript.inherited { 322 uri := mi.path 323 if m, ok := loadedSubConfigs[uri]; ok { 324 // No need to emit load statement, but fix module name. 325 mi.moduleLocalName = m 326 continue 327 } 328 if mi.optional || mi.missing { 329 uri += "|init" 330 } 331 gctx.newLine() 332 gctx.writef("load(%q, %s = \"init\")", uri, mi.entryName()) 333 loadedSubConfigs[uri] = mi.moduleLocalName 334 } 335 gctx.write("\n") 336} 337 338func (gctx *generationContext) emitPass() { 339 gctx.newLine() 340 gctx.write("pass") 341} 342 343func (gctx *generationContext) write(ss ...string) { 344 for _, s := range ss { 345 gctx.buf.WriteString(s) 346 } 347} 348 349func (gctx *generationContext) writef(format string, args ...interface{}) { 350 gctx.write(fmt.Sprintf(format, args...)) 351} 352 353func (gctx *generationContext) newLine() { 354 if gctx.buf.Len() == 0 { 355 return 356 } 357 gctx.write("\n") 358 gctx.writef("%*s", 2*gctx.indentLevel, "") 359} 360 361func (gctx *generationContext) emitConversionError(el ErrorLocation, message string) { 362 gctx.writef(`rblf.mk2rbc_error("%s", %q)`, el, message) 363} 364 365func (gctx *generationContext) emitLoadCheck(im inheritedModule) { 366 if !im.needsLoadCheck() { 367 return 368 } 369 gctx.newLine() 370 gctx.writef("if not %s:", im.entryName()) 371 gctx.indentLevel++ 372 gctx.newLine() 373 gctx.write(`rblf.mkerror("`, gctx.starScript.mkFile, `", "Cannot find %s" % (`) 374 im.pathExpr().emit(gctx) 375 gctx.write("))") 376 gctx.indentLevel-- 377} 378 379type knownVariable struct { 380 name string 381 class varClass 382 valueType starlarkType 383} 384 385type knownVariables map[string]knownVariable 386 387func (pcv knownVariables) NewVariable(name string, varClass varClass, valueType starlarkType) { 388 v, exists := pcv[name] 389 if !exists { 390 pcv[name] = knownVariable{name, varClass, valueType} 391 return 392 } 393 // Conflict resolution: 394 // * config class trumps everything 395 // * any type trumps unknown type 396 match := varClass == v.class 397 if !match { 398 if varClass == VarClassConfig { 399 v.class = VarClassConfig 400 match = true 401 } else if v.class == VarClassConfig { 402 match = true 403 } 404 } 405 if valueType != v.valueType { 406 if valueType != starlarkTypeUnknown { 407 if v.valueType == starlarkTypeUnknown { 408 v.valueType = valueType 409 } else { 410 match = false 411 } 412 } 413 } 414 if !match { 415 fmt.Fprintf(os.Stderr, "cannot redefine %s as %v/%v (already defined as %v/%v)\n", 416 name, varClass, valueType, v.class, v.valueType) 417 } 418} 419 420// All known product variables. 421var KnownVariables = make(knownVariables) 422 423func init() { 424 for _, kv := range []string{ 425 // Kernel-related variables that we know are lists. 426 "BOARD_VENDOR_KERNEL_MODULES", 427 "BOARD_VENDOR_RAMDISK_KERNEL_MODULES", 428 "BOARD_VENDOR_RAMDISK_KERNEL_MODULES_LOAD", 429 "BOARD_RECOVERY_KERNEL_MODULES", 430 // Other variables we knwo are lists 431 "ART_APEX_JARS", 432 } { 433 KnownVariables.NewVariable(kv, VarClassSoong, starlarkTypeList) 434 } 435} 436 437// Information about the generated Starlark script. 438type StarlarkScript struct { 439 mkFile string 440 moduleName string 441 mkPos scanner.Position 442 nodes []starlarkNode 443 inherited []*moduleInfo 444 hasErrors bool 445 traceCalls bool // print enter/exit each init function 446 sourceFS fs.FS 447 makefileFinder MakefileFinder 448 nodeLocator func(pos mkparser.Pos) int 449} 450 451// parseContext holds the script we are generating and all the ephemeral data 452// needed during the parsing. 453type parseContext struct { 454 script *StarlarkScript 455 nodes []mkparser.Node // Makefile as parsed by mkparser 456 currentNodeIndex int // Node in it we are processing 457 ifNestLevel int 458 moduleNameCount map[string]int // count of imported modules with given basename 459 fatalError error 460 outputSuffix string 461 errorLogger ErrorLogger 462 tracedVariables map[string]bool // variables to be traced in the generated script 463 variables map[string]variable 464 outputDir string 465 dependentModules map[string]*moduleInfo 466 soongNamespaces map[string]map[string]bool 467 includeTops []string 468 typeHints map[string]starlarkType 469 atTopOfMakefile bool 470} 471 472func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext { 473 predefined := []struct{ name, value string }{ 474 {"SRC_TARGET_DIR", filepath.Join("build", "make", "target")}, 475 {"LOCAL_PATH", filepath.Dir(ss.mkFile)}, 476 {"MAKEFILE_LIST", ss.mkFile}, 477 {"TOPDIR", ""}, // TOPDIR is just set to an empty string in cleanbuild.mk and core.mk 478 // TODO(asmundak): maybe read it from build/make/core/envsetup.mk? 479 {"TARGET_COPY_OUT_SYSTEM", "system"}, 480 {"TARGET_COPY_OUT_SYSTEM_OTHER", "system_other"}, 481 {"TARGET_COPY_OUT_DATA", "data"}, 482 {"TARGET_COPY_OUT_ASAN", filepath.Join("data", "asan")}, 483 {"TARGET_COPY_OUT_OEM", "oem"}, 484 {"TARGET_COPY_OUT_RAMDISK", "ramdisk"}, 485 {"TARGET_COPY_OUT_DEBUG_RAMDISK", "debug_ramdisk"}, 486 {"TARGET_COPY_OUT_VENDOR_DEBUG_RAMDISK", "vendor_debug_ramdisk"}, 487 {"TARGET_COPY_OUT_TEST_HARNESS_RAMDISK", "test_harness_ramdisk"}, 488 {"TARGET_COPY_OUT_ROOT", "root"}, 489 {"TARGET_COPY_OUT_RECOVERY", "recovery"}, 490 {"TARGET_COPY_OUT_VENDOR_RAMDISK", "vendor_ramdisk"}, 491 // TODO(asmundak): to process internal config files, we need the following variables: 492 // TARGET_VENDOR 493 // target_base_product 494 // 495 496 // the following utility variables are set in build/make/common/core.mk: 497 {"empty", ""}, 498 {"space", " "}, 499 {"comma", ","}, 500 {"newline", "\n"}, 501 {"pound", "#"}, 502 {"backslash", "\\"}, 503 } 504 ctx := &parseContext{ 505 script: ss, 506 nodes: nodes, 507 currentNodeIndex: 0, 508 ifNestLevel: 0, 509 moduleNameCount: make(map[string]int), 510 variables: make(map[string]variable), 511 dependentModules: make(map[string]*moduleInfo), 512 soongNamespaces: make(map[string]map[string]bool), 513 includeTops: []string{}, 514 typeHints: make(map[string]starlarkType), 515 atTopOfMakefile: true, 516 } 517 for _, item := range predefined { 518 ctx.variables[item.name] = &predefinedVariable{ 519 baseVariable: baseVariable{nam: item.name, typ: starlarkTypeString}, 520 value: &stringLiteralExpr{item.value}, 521 } 522 } 523 524 return ctx 525} 526 527func (ctx *parseContext) hasNodes() bool { 528 return ctx.currentNodeIndex < len(ctx.nodes) 529} 530 531func (ctx *parseContext) getNode() mkparser.Node { 532 if !ctx.hasNodes() { 533 return nil 534 } 535 node := ctx.nodes[ctx.currentNodeIndex] 536 ctx.currentNodeIndex++ 537 return node 538} 539 540func (ctx *parseContext) backNode() { 541 if ctx.currentNodeIndex <= 0 { 542 panic("Cannot back off") 543 } 544 ctx.currentNodeIndex-- 545} 546 547func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode { 548 // Handle only simple variables 549 if !a.Name.Const() || a.Target != nil { 550 return []starlarkNode{ctx.newBadNode(a, "Only simple variables are handled")} 551 } 552 name := a.Name.Strings[0] 553 // The `override` directive 554 // override FOO := 555 // is parsed as an assignment to a variable named `override FOO`. 556 // There are very few places where `override` is used, just flag it. 557 if strings.HasPrefix(name, "override ") { 558 return []starlarkNode{ctx.newBadNode(a, "cannot handle override directive")} 559 } 560 if name == ".KATI_READONLY" { 561 // Skip assignments to .KATI_READONLY. If it was in the output file, it 562 // would be an error because it would be sorted before the definition of 563 // the variable it's trying to make readonly. 564 return []starlarkNode{} 565 } 566 567 // Soong configuration 568 if strings.HasPrefix(name, soongNsPrefix) { 569 return ctx.handleSoongNsAssignment(strings.TrimPrefix(name, soongNsPrefix), a) 570 } 571 lhs := ctx.addVariable(name) 572 if lhs == nil { 573 return []starlarkNode{ctx.newBadNode(a, "unknown variable %s", name)} 574 } 575 _, isTraced := ctx.tracedVariables[lhs.name()] 576 asgn := &assignmentNode{lhs: lhs, mkValue: a.Value, isTraced: isTraced, location: ctx.errorLocation(a)} 577 if lhs.valueType() == starlarkTypeUnknown { 578 // Try to divine variable type from the RHS 579 asgn.value = ctx.parseMakeString(a, a.Value) 580 inferred_type := asgn.value.typ() 581 if inferred_type != starlarkTypeUnknown { 582 lhs.setValueType(inferred_type) 583 } 584 } 585 if lhs.valueType() == starlarkTypeList { 586 xConcat, xBad := ctx.buildConcatExpr(a) 587 if xBad != nil { 588 asgn.value = xBad 589 } else { 590 switch len(xConcat.items) { 591 case 0: 592 asgn.value = &listExpr{} 593 case 1: 594 asgn.value = xConcat.items[0] 595 default: 596 asgn.value = xConcat 597 } 598 } 599 } else { 600 asgn.value = ctx.parseMakeString(a, a.Value) 601 } 602 603 if asgn.lhs.valueType() == starlarkTypeString && 604 asgn.value.typ() != starlarkTypeUnknown && 605 asgn.value.typ() != starlarkTypeString { 606 asgn.value = &toStringExpr{expr: asgn.value} 607 } 608 609 switch a.Type { 610 case "=", ":=": 611 asgn.flavor = asgnSet 612 case "+=": 613 asgn.flavor = asgnAppend 614 case "?=": 615 asgn.flavor = asgnMaybeSet 616 default: 617 panic(fmt.Errorf("unexpected assignment type %s", a.Type)) 618 } 619 620 return []starlarkNode{asgn} 621} 622 623func (ctx *parseContext) handleSoongNsAssignment(name string, asgn *mkparser.Assignment) []starlarkNode { 624 val := ctx.parseMakeString(asgn, asgn.Value) 625 if xBad, ok := val.(*badExpr); ok { 626 return []starlarkNode{&exprNode{expr: xBad}} 627 } 628 629 // Unfortunately, Soong namespaces can be set up by directly setting corresponding Make 630 // variables instead of via add_soong_config_namespace + add_soong_config_var_value. 631 // Try to divine the call from the assignment as follows: 632 if name == "NAMESPACES" { 633 // Upon seeng 634 // SOONG_CONFIG_NAMESPACES += foo 635 // remember that there is a namespace `foo` and act as we saw 636 // $(call add_soong_config_namespace,foo) 637 s, ok := maybeString(val) 638 if !ok { 639 return []starlarkNode{ctx.newBadNode(asgn, "cannot handle variables in SOONG_CONFIG_NAMESPACES assignment, please use add_soong_config_namespace instead")} 640 } 641 result := make([]starlarkNode, 0) 642 for _, ns := range strings.Fields(s) { 643 ctx.addSoongNamespace(ns) 644 result = append(result, &exprNode{&callExpr{ 645 name: baseName + ".soong_config_namespace", 646 args: []starlarkExpr{&globalsExpr{}, &stringLiteralExpr{ns}}, 647 returnType: starlarkTypeVoid, 648 }}) 649 } 650 return result 651 } else { 652 // Upon seeing 653 // SOONG_CONFIG_x_y = v 654 // find a namespace called `x` and act as if we encountered 655 // $(call soong_config_set,x,y,v) 656 // or check that `x_y` is a namespace, and then add the RHS of this assignment as variables in 657 // it. 658 // Emit an error in the ambiguous situation (namespaces `foo_bar` with a variable `baz` 659 // and `foo` with a variable `bar_baz`. 660 namespaceName := "" 661 if ctx.hasSoongNamespace(name) { 662 namespaceName = name 663 } 664 var varName string 665 for pos, ch := range name { 666 if !(ch == '_' && ctx.hasSoongNamespace(name[0:pos])) { 667 continue 668 } 669 if namespaceName != "" { 670 return []starlarkNode{ctx.newBadNode(asgn, "ambiguous soong namespace (may be either `%s` or `%s`)", namespaceName, name[0:pos])} 671 } 672 namespaceName = name[0:pos] 673 varName = name[pos+1:] 674 } 675 if namespaceName == "" { 676 return []starlarkNode{ctx.newBadNode(asgn, "cannot figure out Soong namespace, please use add_soong_config_var_value macro instead")} 677 } 678 if varName == "" { 679 // Remember variables in this namespace 680 s, ok := maybeString(val) 681 if !ok { 682 return []starlarkNode{ctx.newBadNode(asgn, "cannot handle variables in SOONG_CONFIG_ assignment, please use add_soong_config_var_value instead")} 683 } 684 ctx.updateSoongNamespace(asgn.Type != "+=", namespaceName, strings.Fields(s)) 685 return []starlarkNode{} 686 } 687 688 // Finally, handle assignment to a namespace variable 689 if !ctx.hasNamespaceVar(namespaceName, varName) { 690 return []starlarkNode{ctx.newBadNode(asgn, "no %s variable in %s namespace, please use add_soong_config_var_value instead", varName, namespaceName)} 691 } 692 fname := baseName + "." + soongConfigAssign 693 if asgn.Type == "+=" { 694 fname = baseName + "." + soongConfigAppend 695 } 696 return []starlarkNode{&exprNode{&callExpr{ 697 name: fname, 698 args: []starlarkExpr{&globalsExpr{}, &stringLiteralExpr{namespaceName}, &stringLiteralExpr{varName}, val}, 699 returnType: starlarkTypeVoid, 700 }}} 701 } 702} 703 704func (ctx *parseContext) buildConcatExpr(a *mkparser.Assignment) (*concatExpr, *badExpr) { 705 xConcat := &concatExpr{} 706 var xItemList *listExpr 707 addToItemList := func(x ...starlarkExpr) { 708 if xItemList == nil { 709 xItemList = &listExpr{[]starlarkExpr{}} 710 } 711 xItemList.items = append(xItemList.items, x...) 712 } 713 finishItemList := func() { 714 if xItemList != nil { 715 xConcat.items = append(xConcat.items, xItemList) 716 xItemList = nil 717 } 718 } 719 720 items := a.Value.Words() 721 for _, item := range items { 722 // A function call in RHS is supposed to return a list, all other item 723 // expressions return individual elements. 724 switch x := ctx.parseMakeString(a, item).(type) { 725 case *badExpr: 726 return nil, x 727 case *stringLiteralExpr: 728 addToItemList(maybeConvertToStringList(x).(*listExpr).items...) 729 default: 730 switch x.typ() { 731 case starlarkTypeList: 732 finishItemList() 733 xConcat.items = append(xConcat.items, x) 734 case starlarkTypeString: 735 finishItemList() 736 xConcat.items = append(xConcat.items, &callExpr{ 737 object: x, 738 name: "split", 739 args: nil, 740 returnType: starlarkTypeList, 741 }) 742 default: 743 addToItemList(x) 744 } 745 } 746 } 747 if xItemList != nil { 748 xConcat.items = append(xConcat.items, xItemList) 749 } 750 return xConcat, nil 751} 752 753func (ctx *parseContext) newDependentModule(path string, optional bool) *moduleInfo { 754 modulePath := ctx.loadedModulePath(path) 755 if mi, ok := ctx.dependentModules[modulePath]; ok { 756 mi.optional = mi.optional && optional 757 return mi 758 } 759 moduleName := moduleNameForFile(path) 760 moduleLocalName := "_" + moduleName 761 n, found := ctx.moduleNameCount[moduleName] 762 if found { 763 moduleLocalName += fmt.Sprintf("%d", n) 764 } 765 ctx.moduleNameCount[moduleName] = n + 1 766 _, err := fs.Stat(ctx.script.sourceFS, path) 767 mi := &moduleInfo{ 768 path: modulePath, 769 originalPath: path, 770 moduleLocalName: moduleLocalName, 771 optional: optional, 772 missing: err != nil, 773 } 774 ctx.dependentModules[modulePath] = mi 775 ctx.script.inherited = append(ctx.script.inherited, mi) 776 return mi 777} 778 779func (ctx *parseContext) handleSubConfig( 780 v mkparser.Node, pathExpr starlarkExpr, loadAlways bool, processModule func(inheritedModule) starlarkNode) []starlarkNode { 781 782 // Allow seeing $(sort $(wildcard realPathExpr)) or $(wildcard realPathExpr) 783 // because those are functionally the same as not having the sort/wildcard calls. 784 if ce, ok := pathExpr.(*callExpr); ok && ce.name == "rblf.mksort" && len(ce.args) == 1 { 785 if ce2, ok2 := ce.args[0].(*callExpr); ok2 && ce2.name == "rblf.expand_wildcard" && len(ce2.args) == 1 { 786 pathExpr = ce2.args[0] 787 } 788 } else if ce2, ok2 := pathExpr.(*callExpr); ok2 && ce2.name == "rblf.expand_wildcard" && len(ce2.args) == 1 { 789 pathExpr = ce2.args[0] 790 } 791 792 // In a simple case, the name of a module to inherit/include is known statically. 793 if path, ok := maybeString(pathExpr); ok { 794 // Note that even if this directive loads a module unconditionally, a module may be 795 // absent without causing any harm if this directive is inside an if/else block. 796 moduleShouldExist := loadAlways && ctx.ifNestLevel == 0 797 if strings.Contains(path, "*") { 798 if paths, err := fs.Glob(ctx.script.sourceFS, path); err == nil { 799 sort.Strings(paths) 800 result := make([]starlarkNode, 0) 801 for _, p := range paths { 802 mi := ctx.newDependentModule(p, !moduleShouldExist) 803 result = append(result, processModule(inheritedStaticModule{mi, loadAlways})) 804 } 805 return result 806 } else { 807 return []starlarkNode{ctx.newBadNode(v, "cannot glob wildcard argument")} 808 } 809 } else { 810 mi := ctx.newDependentModule(path, !moduleShouldExist) 811 return []starlarkNode{processModule(inheritedStaticModule{mi, loadAlways})} 812 } 813 } 814 815 // If module path references variables (e.g., $(v1)/foo/$(v2)/device-config.mk), find all the paths in the 816 // source tree that may be a match and the corresponding variable values. For instance, if the source tree 817 // contains vendor1/foo/abc/dev.mk and vendor2/foo/def/dev.mk, the first one will be inherited when 818 // (v1, v2) == ('vendor1', 'abc'), and the second one when (v1, v2) == ('vendor2', 'def'). 819 // We then emit the code that loads all of them, e.g.: 820 // load("//vendor1/foo/abc:dev.rbc", _dev1_init="init") 821 // load("//vendor2/foo/def/dev.rbc", _dev2_init="init") 822 // And then inherit it as follows: 823 // _e = { 824 // "vendor1/foo/abc/dev.mk": ("vendor1/foo/abc/dev", _dev1_init), 825 // "vendor2/foo/def/dev.mk": ("vendor2/foo/def/dev", _dev_init2) }.get("%s/foo/%s/dev.mk" % (v1, v2)) 826 // if _e: 827 // rblf.inherit(handle, _e[0], _e[1]) 828 // 829 var matchingPaths []string 830 var needsWarning = false 831 if interpolate, ok := pathExpr.(*interpolateExpr); ok { 832 pathPattern := []string{interpolate.chunks[0]} 833 for _, chunk := range interpolate.chunks[1:] { 834 if chunk != "" { 835 pathPattern = append(pathPattern, chunk) 836 } 837 } 838 if len(pathPattern) == 1 { 839 pathPattern = append(pathPattern, "") 840 } 841 matchingPaths = ctx.findMatchingPaths(pathPattern) 842 needsWarning = pathPattern[0] == "" && len(ctx.includeTops) == 0 843 } else if len(ctx.includeTops) > 0 { 844 matchingPaths = append(matchingPaths, ctx.findMatchingPaths([]string{"", ""})...) 845 } else { 846 return []starlarkNode{ctx.newBadNode(v, "inherit-product/include argument is too complex")} 847 } 848 849 // Safeguard against $(call inherit-product,$(PRODUCT_PATH)) 850 const maxMatchingFiles = 150 851 if len(matchingPaths) > maxMatchingFiles { 852 return []starlarkNode{ctx.newBadNode(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)} 853 } 854 855 res := inheritedDynamicModule{pathExpr, []*moduleInfo{}, loadAlways, ctx.errorLocation(v), needsWarning} 856 for _, p := range matchingPaths { 857 // A product configuration files discovered dynamically may attempt to inherit 858 // from another one which does not exist in this source tree. Prevent load errors 859 // by always loading the dynamic files as optional. 860 res.candidateModules = append(res.candidateModules, ctx.newDependentModule(p, true)) 861 } 862 return []starlarkNode{processModule(res)} 863} 864 865func (ctx *parseContext) findMatchingPaths(pattern []string) []string { 866 files := ctx.script.makefileFinder.Find(".") 867 if len(pattern) == 0 { 868 return files 869 } 870 871 // Create regular expression from the pattern 872 regexString := "^" + regexp.QuoteMeta(pattern[0]) 873 for _, s := range pattern[1:] { 874 regexString += ".*" + regexp.QuoteMeta(s) 875 } 876 regexString += "$" 877 rex := regexp.MustCompile(regexString) 878 879 includeTopRegexString := "" 880 if len(ctx.includeTops) > 0 { 881 for i, top := range ctx.includeTops { 882 if i > 0 { 883 includeTopRegexString += "|" 884 } 885 includeTopRegexString += "^" + regexp.QuoteMeta(top) 886 } 887 } else { 888 includeTopRegexString = ".*" 889 } 890 891 includeTopRegex := regexp.MustCompile(includeTopRegexString) 892 893 // Now match 894 var res []string 895 for _, p := range files { 896 if rex.MatchString(p) && includeTopRegex.MatchString(p) { 897 res = append(res, p) 898 } 899 } 900 return res 901} 902 903type inheritProductCallParser struct { 904 loadAlways bool 905} 906 907func (p *inheritProductCallParser) parse(ctx *parseContext, v mkparser.Node, args *mkparser.MakeString) []starlarkNode { 908 args.TrimLeftSpaces() 909 args.TrimRightSpaces() 910 pathExpr := ctx.parseMakeString(v, args) 911 if _, ok := pathExpr.(*badExpr); ok { 912 return []starlarkNode{ctx.newBadNode(v, "Unable to parse argument to inherit")} 913 } 914 return ctx.handleSubConfig(v, pathExpr, p.loadAlways, func(im inheritedModule) starlarkNode { 915 return &inheritNode{im, p.loadAlways} 916 }) 917} 918 919func (ctx *parseContext) handleInclude(v *mkparser.Directive) []starlarkNode { 920 loadAlways := v.Name[0] != '-' 921 return ctx.handleSubConfig(v, ctx.parseMakeString(v, v.Args), loadAlways, func(im inheritedModule) starlarkNode { 922 return &includeNode{im, loadAlways} 923 }) 924} 925 926func (ctx *parseContext) handleVariable(v *mkparser.Variable) []starlarkNode { 927 // Handle: 928 // $(call inherit-product,...) 929 // $(call inherit-product-if-exists,...) 930 // $(info xxx) 931 // $(warning xxx) 932 // $(error xxx) 933 // $(call other-custom-functions,...) 934 935 if name, args, ok := ctx.maybeParseFunctionCall(v, v.Name); ok { 936 if kf, ok := knownNodeFunctions[name]; ok { 937 return kf.parse(ctx, v, args) 938 } 939 } 940 941 return []starlarkNode{&exprNode{expr: ctx.parseReference(v, v.Name)}} 942} 943 944func (ctx *parseContext) maybeHandleDefine(directive *mkparser.Directive) starlarkNode { 945 macro_name := strings.Fields(directive.Args.Strings[0])[0] 946 // Ignore the macros that we handle 947 _, ignored := ignoredDefines[macro_name] 948 _, known := knownFunctions[macro_name] 949 if !ignored && !known { 950 return ctx.newBadNode(directive, "define is not supported: %s", macro_name) 951 } 952 return nil 953} 954 955func (ctx *parseContext) handleIfBlock(ifDirective *mkparser.Directive) starlarkNode { 956 ssSwitch := &switchNode{ 957 ssCases: []*switchCase{ctx.processBranch(ifDirective)}, 958 } 959 for ctx.hasNodes() && ctx.fatalError == nil { 960 node := ctx.getNode() 961 switch x := node.(type) { 962 case *mkparser.Directive: 963 switch x.Name { 964 case "else", "elifdef", "elifndef", "elifeq", "elifneq": 965 ssSwitch.ssCases = append(ssSwitch.ssCases, ctx.processBranch(x)) 966 case "endif": 967 return ssSwitch 968 default: 969 return ctx.newBadNode(node, "unexpected directive %s", x.Name) 970 } 971 default: 972 return ctx.newBadNode(ifDirective, "unexpected statement") 973 } 974 } 975 if ctx.fatalError == nil { 976 ctx.fatalError = fmt.Errorf("no matching endif for %s", ifDirective.Dump()) 977 } 978 return ctx.newBadNode(ifDirective, "no matching endif for %s", ifDirective.Dump()) 979} 980 981// processBranch processes a single branch (if/elseif/else) until the next directive 982// on the same level. 983func (ctx *parseContext) processBranch(check *mkparser.Directive) *switchCase { 984 block := &switchCase{gate: ctx.parseCondition(check)} 985 defer func() { 986 ctx.ifNestLevel-- 987 }() 988 ctx.ifNestLevel++ 989 990 for ctx.hasNodes() { 991 node := ctx.getNode() 992 if d, ok := node.(*mkparser.Directive); ok { 993 switch d.Name { 994 case "else", "elifdef", "elifndef", "elifeq", "elifneq", "endif": 995 ctx.backNode() 996 return block 997 } 998 } 999 block.nodes = append(block.nodes, ctx.handleSimpleStatement(node)...) 1000 } 1001 ctx.fatalError = fmt.Errorf("no matching endif for %s", check.Dump()) 1002 return block 1003} 1004 1005func (ctx *parseContext) parseCondition(check *mkparser.Directive) starlarkNode { 1006 switch check.Name { 1007 case "ifdef", "ifndef", "elifdef", "elifndef": 1008 if !check.Args.Const() { 1009 return ctx.newBadNode(check, "ifdef variable ref too complex: %s", check.Args.Dump()) 1010 } 1011 v := NewVariableRefExpr(ctx.addVariable(check.Args.Strings[0])) 1012 if strings.HasSuffix(check.Name, "ndef") { 1013 v = ¬Expr{v} 1014 } 1015 return &ifNode{ 1016 isElif: strings.HasPrefix(check.Name, "elif"), 1017 expr: v, 1018 } 1019 case "ifeq", "ifneq", "elifeq", "elifneq": 1020 return &ifNode{ 1021 isElif: strings.HasPrefix(check.Name, "elif"), 1022 expr: ctx.parseCompare(check), 1023 } 1024 case "else": 1025 return &elseNode{} 1026 default: 1027 panic(fmt.Errorf("%s: unknown directive: %s", ctx.script.mkFile, check.Dump())) 1028 } 1029} 1030 1031func (ctx *parseContext) newBadExpr(node mkparser.Node, text string, args ...interface{}) starlarkExpr { 1032 if ctx.errorLogger != nil { 1033 ctx.errorLogger.NewError(ctx.errorLocation(node), node, text, args...) 1034 } 1035 ctx.script.hasErrors = true 1036 return &badExpr{errorLocation: ctx.errorLocation(node), message: fmt.Sprintf(text, args...)} 1037} 1038 1039// records that the given node failed to be converted and includes an explanatory message 1040func (ctx *parseContext) newBadNode(failedNode mkparser.Node, message string, args ...interface{}) starlarkNode { 1041 return &exprNode{ctx.newBadExpr(failedNode, message, args...)} 1042} 1043 1044func (ctx *parseContext) parseCompare(cond *mkparser.Directive) starlarkExpr { 1045 // Strip outer parentheses 1046 mkArg := cloneMakeString(cond.Args) 1047 mkArg.Strings[0] = strings.TrimLeft(mkArg.Strings[0], "( ") 1048 n := len(mkArg.Strings) 1049 mkArg.Strings[n-1] = strings.TrimRight(mkArg.Strings[n-1], ") ") 1050 args := mkArg.Split(",") 1051 // TODO(asmundak): handle the case where the arguments are in quotes and space-separated 1052 if len(args) != 2 { 1053 return ctx.newBadExpr(cond, "ifeq/ifneq len(args) != 2 %s", cond.Dump()) 1054 } 1055 args[0].TrimRightSpaces() 1056 args[1].TrimLeftSpaces() 1057 1058 isEq := !strings.HasSuffix(cond.Name, "neq") 1059 xLeft := ctx.parseMakeString(cond, args[0]) 1060 xRight := ctx.parseMakeString(cond, args[1]) 1061 if bad, ok := xLeft.(*badExpr); ok { 1062 return bad 1063 } 1064 if bad, ok := xRight.(*badExpr); ok { 1065 return bad 1066 } 1067 1068 if expr, ok := ctx.parseCompareSpecialCases(cond, xLeft, xRight); ok { 1069 return expr 1070 } 1071 1072 var stringOperand string 1073 var otherOperand starlarkExpr 1074 if s, ok := maybeString(xLeft); ok { 1075 stringOperand = s 1076 otherOperand = xRight 1077 } else if s, ok := maybeString(xRight); ok { 1078 stringOperand = s 1079 otherOperand = xLeft 1080 } 1081 1082 // If we've identified one of the operands as being a string literal, check 1083 // for some special cases we can do to simplify the resulting expression. 1084 if otherOperand != nil { 1085 if stringOperand == "" { 1086 if isEq { 1087 return negateExpr(otherOperand) 1088 } else { 1089 return otherOperand 1090 } 1091 } 1092 if stringOperand == "true" && otherOperand.typ() == starlarkTypeBool { 1093 if !isEq { 1094 return negateExpr(otherOperand) 1095 } else { 1096 return otherOperand 1097 } 1098 } 1099 if otherOperand.typ() == starlarkTypeList { 1100 fields := strings.Fields(stringOperand) 1101 elements := make([]starlarkExpr, len(fields)) 1102 for i, s := range fields { 1103 elements[i] = &stringLiteralExpr{literal: s} 1104 } 1105 return &eqExpr{ 1106 left: otherOperand, 1107 right: &listExpr{elements}, 1108 isEq: isEq, 1109 } 1110 } 1111 if intOperand, err := strconv.Atoi(strings.TrimSpace(stringOperand)); err == nil && otherOperand.typ() == starlarkTypeInt { 1112 return &eqExpr{ 1113 left: otherOperand, 1114 right: &intLiteralExpr{literal: intOperand}, 1115 isEq: isEq, 1116 } 1117 } 1118 } 1119 1120 return &eqExpr{left: xLeft, right: xRight, isEq: isEq} 1121} 1122 1123// Given an if statement's directive and the left/right starlarkExprs, 1124// check if the starlarkExprs are one of a few hardcoded special cases 1125// that can be converted to a simpler equality expression than simply comparing 1126// the two. 1127func (ctx *parseContext) parseCompareSpecialCases(directive *mkparser.Directive, left starlarkExpr, 1128 right starlarkExpr) (starlarkExpr, bool) { 1129 isEq := !strings.HasSuffix(directive.Name, "neq") 1130 1131 // All the special cases require a call on one side and a 1132 // string literal/variable on the other. Turn the left/right variables into 1133 // call/value variables, and return false if that's not possible. 1134 var value starlarkExpr = nil 1135 call, ok := left.(*callExpr) 1136 if ok { 1137 switch right.(type) { 1138 case *stringLiteralExpr, *variableRefExpr: 1139 value = right 1140 } 1141 } else { 1142 call, _ = right.(*callExpr) 1143 switch left.(type) { 1144 case *stringLiteralExpr, *variableRefExpr: 1145 value = left 1146 } 1147 } 1148 1149 if call == nil || value == nil { 1150 return nil, false 1151 } 1152 1153 switch call.name { 1154 case baseName + ".filter": 1155 return ctx.parseCompareFilterFuncResult(directive, call, value, isEq) 1156 case baseName + ".findstring": 1157 return ctx.parseCheckFindstringFuncResult(directive, call, value, !isEq), true 1158 case baseName + ".strip": 1159 return ctx.parseCompareStripFuncResult(directive, call, value, !isEq), true 1160 } 1161 return nil, false 1162} 1163 1164func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive, 1165 filterFuncCall *callExpr, xValue starlarkExpr, negate bool) (starlarkExpr, bool) { 1166 // We handle: 1167 // * ifeq/ifneq (,$(filter v1 v2 ..., EXPR) becomes if EXPR not in/in ["v1", "v2", ...] 1168 // * ifeq/ifneq (,$(filter EXPR, v1 v2 ...) becomes if EXPR not in/in ["v1", "v2", ...] 1169 if x, ok := xValue.(*stringLiteralExpr); !ok || x.literal != "" { 1170 return nil, false 1171 } 1172 xPattern := filterFuncCall.args[0] 1173 xText := filterFuncCall.args[1] 1174 var xInList *stringLiteralExpr 1175 var expr starlarkExpr 1176 var ok bool 1177 if xInList, ok = xPattern.(*stringLiteralExpr); ok && !strings.ContainsRune(xInList.literal, '%') && xText.typ() == starlarkTypeList { 1178 expr = xText 1179 } else if xInList, ok = xText.(*stringLiteralExpr); ok { 1180 expr = xPattern 1181 } else { 1182 return nil, false 1183 } 1184 slExpr := newStringListExpr(strings.Fields(xInList.literal)) 1185 // Generate simpler code for the common cases: 1186 if expr.typ() == starlarkTypeList { 1187 if len(slExpr.items) == 1 { 1188 // Checking that a string belongs to list 1189 return &inExpr{isNot: negate, list: expr, expr: slExpr.items[0]}, true 1190 } else { 1191 return nil, false 1192 } 1193 } else if len(slExpr.items) == 1 { 1194 return &eqExpr{left: expr, right: slExpr.items[0], isEq: !negate}, true 1195 } else { 1196 return &inExpr{isNot: negate, list: newStringListExpr(strings.Fields(xInList.literal)), expr: expr}, true 1197 } 1198} 1199 1200func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive, 1201 xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr { 1202 if isEmptyString(xValue) { 1203 return &eqExpr{ 1204 left: &callExpr{ 1205 object: xCall.args[1], 1206 name: "find", 1207 args: []starlarkExpr{xCall.args[0]}, 1208 returnType: starlarkTypeInt, 1209 }, 1210 right: &intLiteralExpr{-1}, 1211 isEq: !negate, 1212 } 1213 } else if s, ok := maybeString(xValue); ok { 1214 if s2, ok := maybeString(xCall.args[0]); ok && s == s2 { 1215 return &eqExpr{ 1216 left: &callExpr{ 1217 object: xCall.args[1], 1218 name: "find", 1219 args: []starlarkExpr{xCall.args[0]}, 1220 returnType: starlarkTypeInt, 1221 }, 1222 right: &intLiteralExpr{-1}, 1223 isEq: negate, 1224 } 1225 } 1226 } 1227 return ctx.newBadExpr(directive, "$(findstring) can only be compared to nothing or its first argument") 1228} 1229 1230func (ctx *parseContext) parseCompareStripFuncResult(directive *mkparser.Directive, 1231 xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr { 1232 if _, ok := xValue.(*stringLiteralExpr); !ok { 1233 return ctx.newBadExpr(directive, "strip result can be compared only to string: %s", xValue) 1234 } 1235 return &eqExpr{ 1236 left: &callExpr{ 1237 name: "strip", 1238 args: xCall.args, 1239 returnType: starlarkTypeString, 1240 }, 1241 right: xValue, isEq: !negate} 1242} 1243 1244func (ctx *parseContext) maybeParseFunctionCall(node mkparser.Node, ref *mkparser.MakeString) (name string, args *mkparser.MakeString, ok bool) { 1245 ref.TrimLeftSpaces() 1246 ref.TrimRightSpaces() 1247 1248 words := ref.SplitN(" ", 2) 1249 if !words[0].Const() { 1250 return "", nil, false 1251 } 1252 1253 name = words[0].Dump() 1254 args = mkparser.SimpleMakeString("", words[0].Pos()) 1255 if len(words) >= 2 { 1256 args = words[1] 1257 } 1258 args.TrimLeftSpaces() 1259 if name == "call" { 1260 words = args.SplitN(",", 2) 1261 if words[0].Empty() || !words[0].Const() { 1262 return "", nil, false 1263 } 1264 name = words[0].Dump() 1265 if len(words) < 2 { 1266 args = mkparser.SimpleMakeString("", words[0].Pos()) 1267 } else { 1268 args = words[1] 1269 } 1270 } 1271 ok = true 1272 return 1273} 1274 1275// parses $(...), returning an expression 1276func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeString) starlarkExpr { 1277 ref.TrimLeftSpaces() 1278 ref.TrimRightSpaces() 1279 refDump := ref.Dump() 1280 1281 // Handle only the case where the first (or only) word is constant 1282 words := ref.SplitN(" ", 2) 1283 if !words[0].Const() { 1284 if len(words) == 1 { 1285 expr := ctx.parseMakeString(node, ref) 1286 return &callExpr{ 1287 object: &identifierExpr{"cfg"}, 1288 name: "get", 1289 args: []starlarkExpr{ 1290 expr, 1291 &callExpr{ 1292 object: &identifierExpr{"g"}, 1293 name: "get", 1294 args: []starlarkExpr{ 1295 expr, 1296 &stringLiteralExpr{literal: ""}, 1297 }, 1298 returnType: starlarkTypeUnknown, 1299 }, 1300 }, 1301 returnType: starlarkTypeUnknown, 1302 } 1303 } else { 1304 return ctx.newBadExpr(node, "reference is too complex: %s", refDump) 1305 } 1306 } 1307 1308 if name, _, ok := ctx.maybeParseFunctionCall(node, ref); ok { 1309 if _, unsupported := unsupportedFunctions[name]; unsupported { 1310 return ctx.newBadExpr(node, "%s is not supported", refDump) 1311 } 1312 } 1313 1314 // If it is a single word, it can be a simple variable 1315 // reference or a function call 1316 if len(words) == 1 && !isMakeControlFunc(refDump) && refDump != "shell" && refDump != "eval" { 1317 if strings.HasPrefix(refDump, soongNsPrefix) { 1318 // TODO (asmundak): if we find many, maybe handle them. 1319 return ctx.newBadExpr(node, "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: %s", refDump) 1320 } 1321 // Handle substitution references: https://www.gnu.org/software/make/manual/html_node/Substitution-Refs.html 1322 if strings.Contains(refDump, ":") { 1323 parts := strings.SplitN(refDump, ":", 2) 1324 substParts := strings.SplitN(parts[1], "=", 2) 1325 if len(substParts) < 2 || strings.Count(substParts[0], "%") > 1 { 1326 return ctx.newBadExpr(node, "Invalid substitution reference") 1327 } 1328 if !strings.Contains(substParts[0], "%") { 1329 if strings.Contains(substParts[1], "%") { 1330 return ctx.newBadExpr(node, "A substitution reference must have a %% in the \"before\" part of the substitution if it has one in the \"after\" part.") 1331 } 1332 substParts[0] = "%" + substParts[0] 1333 substParts[1] = "%" + substParts[1] 1334 } 1335 v := ctx.addVariable(parts[0]) 1336 if v == nil { 1337 return ctx.newBadExpr(node, "unknown variable %s", refDump) 1338 } 1339 return &callExpr{ 1340 name: baseName + ".mkpatsubst", 1341 returnType: starlarkTypeString, 1342 args: []starlarkExpr{ 1343 &stringLiteralExpr{literal: substParts[0]}, 1344 &stringLiteralExpr{literal: substParts[1]}, 1345 NewVariableRefExpr(v), 1346 }, 1347 } 1348 } 1349 if v := ctx.addVariable(refDump); v != nil { 1350 return NewVariableRefExpr(v) 1351 } 1352 return ctx.newBadExpr(node, "unknown variable %s", refDump) 1353 } 1354 1355 if name, args, ok := ctx.maybeParseFunctionCall(node, ref); ok { 1356 if kf, found := knownFunctions[name]; found { 1357 return kf.parse(ctx, node, args) 1358 } else { 1359 return ctx.newBadExpr(node, "cannot handle invoking %s", name) 1360 } 1361 } 1362 return ctx.newBadExpr(node, "cannot handle %s", refDump) 1363} 1364 1365type simpleCallParser struct { 1366 name string 1367 returnType starlarkType 1368 addGlobals bool 1369 addHandle bool 1370} 1371 1372func (p *simpleCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1373 expr := &callExpr{name: p.name, returnType: p.returnType} 1374 if p.addGlobals { 1375 expr.args = append(expr.args, &globalsExpr{}) 1376 } 1377 if p.addHandle { 1378 expr.args = append(expr.args, &identifierExpr{name: "handle"}) 1379 } 1380 for _, arg := range args.Split(",") { 1381 arg.TrimLeftSpaces() 1382 arg.TrimRightSpaces() 1383 x := ctx.parseMakeString(node, arg) 1384 if xBad, ok := x.(*badExpr); ok { 1385 return xBad 1386 } 1387 expr.args = append(expr.args, x) 1388 } 1389 return expr 1390} 1391 1392type makeControlFuncParser struct { 1393 name string 1394} 1395 1396func (p *makeControlFuncParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1397 // Make control functions need special treatment as everything 1398 // after the name is a single text argument 1399 x := ctx.parseMakeString(node, args) 1400 if xBad, ok := x.(*badExpr); ok { 1401 return xBad 1402 } 1403 return &callExpr{ 1404 name: p.name, 1405 args: []starlarkExpr{ 1406 &stringLiteralExpr{ctx.script.mkFile}, 1407 x, 1408 }, 1409 returnType: starlarkTypeUnknown, 1410 } 1411} 1412 1413type shellCallParser struct{} 1414 1415func (p *shellCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1416 // Shell functions need special treatment as everything 1417 // after the name is a single text argument 1418 x := ctx.parseMakeString(node, args) 1419 if xBad, ok := x.(*badExpr); ok { 1420 return xBad 1421 } 1422 return &callExpr{ 1423 name: baseName + ".shell", 1424 args: []starlarkExpr{x}, 1425 returnType: starlarkTypeUnknown, 1426 } 1427} 1428 1429type myDirCallParser struct{} 1430 1431func (p *myDirCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1432 if !args.Empty() { 1433 return ctx.newBadExpr(node, "my-dir function cannot have any arguments passed to it.") 1434 } 1435 return &stringLiteralExpr{literal: filepath.Dir(ctx.script.mkFile)} 1436} 1437 1438type andOrParser struct { 1439 isAnd bool 1440} 1441 1442func (p *andOrParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1443 if args.Empty() { 1444 return ctx.newBadExpr(node, "and/or function must have at least 1 argument") 1445 } 1446 op := "or" 1447 if p.isAnd { 1448 op = "and" 1449 } 1450 1451 argsParsed := make([]starlarkExpr, 0) 1452 1453 for _, arg := range args.Split(",") { 1454 arg.TrimLeftSpaces() 1455 arg.TrimRightSpaces() 1456 x := ctx.parseMakeString(node, arg) 1457 if xBad, ok := x.(*badExpr); ok { 1458 return xBad 1459 } 1460 argsParsed = append(argsParsed, x) 1461 } 1462 typ := starlarkTypeUnknown 1463 for _, arg := range argsParsed { 1464 if typ != arg.typ() && arg.typ() != starlarkTypeUnknown && typ != starlarkTypeUnknown { 1465 return ctx.newBadExpr(node, "Expected all arguments to $(or) or $(and) to have the same type, found %q and %q", typ.String(), arg.typ().String()) 1466 } 1467 if arg.typ() != starlarkTypeUnknown { 1468 typ = arg.typ() 1469 } 1470 } 1471 result := argsParsed[0] 1472 for _, arg := range argsParsed[1:] { 1473 result = &binaryOpExpr{ 1474 left: result, 1475 right: arg, 1476 op: op, 1477 returnType: typ, 1478 } 1479 } 1480 return result 1481} 1482 1483type isProductInListCallParser struct{} 1484 1485func (p *isProductInListCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1486 if args.Empty() { 1487 return ctx.newBadExpr(node, "is-product-in-list requires an argument") 1488 } 1489 return &inExpr{ 1490 expr: NewVariableRefExpr(ctx.addVariable("TARGET_PRODUCT")), 1491 list: maybeConvertToStringList(ctx.parseMakeString(node, args)), 1492 isNot: false, 1493 } 1494} 1495 1496type isVendorBoardPlatformCallParser struct{} 1497 1498func (p *isVendorBoardPlatformCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1499 if args.Empty() || !identifierFullMatchRegex.MatchString(args.Dump()) { 1500 return ctx.newBadExpr(node, "cannot handle non-constant argument to is-vendor-board-platform") 1501 } 1502 return &inExpr{ 1503 expr: NewVariableRefExpr(ctx.addVariable("TARGET_BOARD_PLATFORM")), 1504 list: NewVariableRefExpr(ctx.addVariable(args.Dump() + "_BOARD_PLATFORMS")), 1505 isNot: false, 1506 } 1507} 1508 1509type isVendorBoardQcomCallParser struct{} 1510 1511func (p *isVendorBoardQcomCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1512 if !args.Empty() { 1513 return ctx.newBadExpr(node, "is-vendor-board-qcom does not accept any arguments") 1514 } 1515 return &inExpr{ 1516 expr: NewVariableRefExpr(ctx.addVariable("TARGET_BOARD_PLATFORM")), 1517 list: NewVariableRefExpr(ctx.addVariable("QCOM_BOARD_PLATFORMS")), 1518 isNot: false, 1519 } 1520} 1521 1522type substCallParser struct { 1523 fname string 1524} 1525 1526func (p *substCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1527 words := args.Split(",") 1528 if len(words) != 3 { 1529 return ctx.newBadExpr(node, "%s function should have 3 arguments", p.fname) 1530 } 1531 from := ctx.parseMakeString(node, words[0]) 1532 if xBad, ok := from.(*badExpr); ok { 1533 return xBad 1534 } 1535 to := ctx.parseMakeString(node, words[1]) 1536 if xBad, ok := to.(*badExpr); ok { 1537 return xBad 1538 } 1539 words[2].TrimLeftSpaces() 1540 words[2].TrimRightSpaces() 1541 obj := ctx.parseMakeString(node, words[2]) 1542 typ := obj.typ() 1543 if typ == starlarkTypeString && p.fname == "subst" { 1544 // Optimization: if it's $(subst from, to, string), emit string.replace(from, to) 1545 return &callExpr{ 1546 object: obj, 1547 name: "replace", 1548 args: []starlarkExpr{from, to}, 1549 returnType: typ, 1550 } 1551 } 1552 return &callExpr{ 1553 name: baseName + ".mk" + p.fname, 1554 args: []starlarkExpr{from, to, obj}, 1555 returnType: obj.typ(), 1556 } 1557} 1558 1559type ifCallParser struct{} 1560 1561func (p *ifCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1562 words := args.Split(",") 1563 if len(words) != 2 && len(words) != 3 { 1564 return ctx.newBadExpr(node, "if function should have 2 or 3 arguments, found "+strconv.Itoa(len(words))) 1565 } 1566 condition := ctx.parseMakeString(node, words[0]) 1567 ifTrue := ctx.parseMakeString(node, words[1]) 1568 var ifFalse starlarkExpr 1569 if len(words) == 3 { 1570 ifFalse = ctx.parseMakeString(node, words[2]) 1571 } else { 1572 switch ifTrue.typ() { 1573 case starlarkTypeList: 1574 ifFalse = &listExpr{items: []starlarkExpr{}} 1575 case starlarkTypeInt: 1576 ifFalse = &intLiteralExpr{literal: 0} 1577 case starlarkTypeBool: 1578 ifFalse = &boolLiteralExpr{literal: false} 1579 default: 1580 ifFalse = &stringLiteralExpr{literal: ""} 1581 } 1582 } 1583 return &ifExpr{ 1584 condition, 1585 ifTrue, 1586 ifFalse, 1587 } 1588} 1589 1590type ifCallNodeParser struct{} 1591 1592func (p *ifCallNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode { 1593 words := args.Split(",") 1594 if len(words) != 2 && len(words) != 3 { 1595 return []starlarkNode{ctx.newBadNode(node, "if function should have 2 or 3 arguments, found "+strconv.Itoa(len(words)))} 1596 } 1597 1598 ifn := &ifNode{expr: ctx.parseMakeString(node, words[0])} 1599 cases := []*switchCase{ 1600 { 1601 gate: ifn, 1602 nodes: ctx.parseNodeMakeString(node, words[1]), 1603 }, 1604 } 1605 if len(words) == 3 { 1606 cases = append(cases, &switchCase{ 1607 gate: &elseNode{}, 1608 nodes: ctx.parseNodeMakeString(node, words[2]), 1609 }) 1610 } 1611 if len(cases) == 2 { 1612 if len(cases[1].nodes) == 0 { 1613 // Remove else branch if it has no contents 1614 cases = cases[:1] 1615 } else if len(cases[0].nodes) == 0 { 1616 // If the if branch has no contents but the else does, 1617 // move them to the if and negate its condition 1618 ifn.expr = negateExpr(ifn.expr) 1619 cases[0].nodes = cases[1].nodes 1620 cases = cases[:1] 1621 } 1622 } 1623 1624 return []starlarkNode{&switchNode{ssCases: cases}} 1625} 1626 1627type foreachCallParser struct{} 1628 1629func (p *foreachCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1630 words := args.Split(",") 1631 if len(words) != 3 { 1632 return ctx.newBadExpr(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words))) 1633 } 1634 if !words[0].Const() || words[0].Empty() || !identifierFullMatchRegex.MatchString(words[0].Strings[0]) { 1635 return ctx.newBadExpr(node, "first argument to foreach function must be a simple string identifier") 1636 } 1637 loopVarName := words[0].Strings[0] 1638 list := ctx.parseMakeString(node, words[1]) 1639 action := ctx.parseMakeString(node, words[2]).transform(func(expr starlarkExpr) starlarkExpr { 1640 if varRefExpr, ok := expr.(*variableRefExpr); ok && varRefExpr.ref.name() == loopVarName { 1641 return &identifierExpr{loopVarName} 1642 } 1643 return nil 1644 }) 1645 1646 if list.typ() != starlarkTypeList { 1647 list = &callExpr{ 1648 name: baseName + ".words", 1649 returnType: starlarkTypeList, 1650 args: []starlarkExpr{list}, 1651 } 1652 } 1653 1654 var result starlarkExpr = &foreachExpr{ 1655 varName: loopVarName, 1656 list: list, 1657 action: action, 1658 } 1659 1660 if action.typ() == starlarkTypeList { 1661 result = &callExpr{ 1662 name: baseName + ".flatten_2d_list", 1663 args: []starlarkExpr{result}, 1664 returnType: starlarkTypeList, 1665 } 1666 } 1667 1668 return result 1669} 1670 1671func transformNode(node starlarkNode, transformer func(expr starlarkExpr) starlarkExpr) { 1672 switch a := node.(type) { 1673 case *ifNode: 1674 a.expr = a.expr.transform(transformer) 1675 case *switchCase: 1676 transformNode(a.gate, transformer) 1677 for _, n := range a.nodes { 1678 transformNode(n, transformer) 1679 } 1680 case *switchNode: 1681 for _, n := range a.ssCases { 1682 transformNode(n, transformer) 1683 } 1684 case *exprNode: 1685 a.expr = a.expr.transform(transformer) 1686 case *assignmentNode: 1687 a.value = a.value.transform(transformer) 1688 case *foreachNode: 1689 a.list = a.list.transform(transformer) 1690 for _, n := range a.actions { 1691 transformNode(n, transformer) 1692 } 1693 case *inheritNode: 1694 if b, ok := a.module.(inheritedDynamicModule); ok { 1695 b.path = b.path.transform(transformer) 1696 a.module = b 1697 } 1698 case *includeNode: 1699 if b, ok := a.module.(inheritedDynamicModule); ok { 1700 b.path = b.path.transform(transformer) 1701 a.module = b 1702 } 1703 } 1704} 1705 1706type foreachCallNodeParser struct{} 1707 1708func (p *foreachCallNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode { 1709 words := args.Split(",") 1710 if len(words) != 3 { 1711 return []starlarkNode{ctx.newBadNode(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words)))} 1712 } 1713 if !words[0].Const() || words[0].Empty() || !identifierFullMatchRegex.MatchString(words[0].Strings[0]) { 1714 return []starlarkNode{ctx.newBadNode(node, "first argument to foreach function must be a simple string identifier")} 1715 } 1716 1717 loopVarName := words[0].Strings[0] 1718 1719 list := ctx.parseMakeString(node, words[1]) 1720 if list.typ() != starlarkTypeList { 1721 list = &callExpr{ 1722 name: baseName + ".words", 1723 returnType: starlarkTypeList, 1724 args: []starlarkExpr{list}, 1725 } 1726 } 1727 1728 actions := ctx.parseNodeMakeString(node, words[2]) 1729 // TODO(colefaust): Replace transforming code with something more elegant 1730 for _, action := range actions { 1731 transformNode(action, func(expr starlarkExpr) starlarkExpr { 1732 if varRefExpr, ok := expr.(*variableRefExpr); ok && varRefExpr.ref.name() == loopVarName { 1733 return &identifierExpr{loopVarName} 1734 } 1735 return nil 1736 }) 1737 } 1738 1739 return []starlarkNode{&foreachNode{ 1740 varName: loopVarName, 1741 list: list, 1742 actions: actions, 1743 }} 1744} 1745 1746type wordCallParser struct{} 1747 1748func (p *wordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1749 words := args.Split(",") 1750 if len(words) != 2 { 1751 return ctx.newBadExpr(node, "word function should have 2 arguments") 1752 } 1753 var index = 0 1754 if words[0].Const() { 1755 if i, err := strconv.Atoi(strings.TrimSpace(words[0].Strings[0])); err == nil { 1756 index = i 1757 } 1758 } 1759 if index < 1 { 1760 return ctx.newBadExpr(node, "word index should be constant positive integer") 1761 } 1762 words[1].TrimLeftSpaces() 1763 words[1].TrimRightSpaces() 1764 array := ctx.parseMakeString(node, words[1]) 1765 if bad, ok := array.(*badExpr); ok { 1766 return bad 1767 } 1768 if array.typ() != starlarkTypeList { 1769 array = &callExpr{ 1770 name: baseName + ".words", 1771 args: []starlarkExpr{array}, 1772 returnType: starlarkTypeList, 1773 } 1774 } 1775 return &indexExpr{array, &intLiteralExpr{index - 1}} 1776} 1777 1778type wordsCallParser struct{} 1779 1780func (p *wordsCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1781 args.TrimLeftSpaces() 1782 args.TrimRightSpaces() 1783 array := ctx.parseMakeString(node, args) 1784 if bad, ok := array.(*badExpr); ok { 1785 return bad 1786 } 1787 if array.typ() != starlarkTypeList { 1788 array = &callExpr{ 1789 name: baseName + ".words", 1790 args: []starlarkExpr{array}, 1791 returnType: starlarkTypeList, 1792 } 1793 } 1794 return &callExpr{ 1795 name: "len", 1796 args: []starlarkExpr{array}, 1797 returnType: starlarkTypeInt, 1798 } 1799} 1800 1801func parseIntegerArguments(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString, expectedArgs int) ([]starlarkExpr, error) { 1802 parsedArgs := make([]starlarkExpr, 0) 1803 for _, arg := range args.Split(",") { 1804 expr := ctx.parseMakeString(node, arg) 1805 if expr.typ() == starlarkTypeList { 1806 return nil, fmt.Errorf("argument to math argument has type list, which cannot be converted to int") 1807 } 1808 if s, ok := maybeString(expr); ok { 1809 intVal, err := strconv.Atoi(strings.TrimSpace(s)) 1810 if err != nil { 1811 return nil, err 1812 } 1813 expr = &intLiteralExpr{literal: intVal} 1814 } else if expr.typ() != starlarkTypeInt { 1815 expr = &callExpr{ 1816 name: "int", 1817 args: []starlarkExpr{expr}, 1818 returnType: starlarkTypeInt, 1819 } 1820 } 1821 parsedArgs = append(parsedArgs, expr) 1822 } 1823 if len(parsedArgs) != expectedArgs { 1824 return nil, fmt.Errorf("function should have %d arguments", expectedArgs) 1825 } 1826 return parsedArgs, nil 1827} 1828 1829type mathComparisonCallParser struct { 1830 op string 1831} 1832 1833func (p *mathComparisonCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1834 parsedArgs, err := parseIntegerArguments(ctx, node, args, 2) 1835 if err != nil { 1836 return ctx.newBadExpr(node, err.Error()) 1837 } 1838 return &binaryOpExpr{ 1839 left: parsedArgs[0], 1840 right: parsedArgs[1], 1841 op: p.op, 1842 returnType: starlarkTypeBool, 1843 } 1844} 1845 1846type mathMaxOrMinCallParser struct { 1847 function string 1848} 1849 1850func (p *mathMaxOrMinCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1851 parsedArgs, err := parseIntegerArguments(ctx, node, args, 2) 1852 if err != nil { 1853 return ctx.newBadExpr(node, err.Error()) 1854 } 1855 return &callExpr{ 1856 object: nil, 1857 name: p.function, 1858 args: parsedArgs, 1859 returnType: starlarkTypeInt, 1860 } 1861} 1862 1863type evalNodeParser struct{} 1864 1865func (p *evalNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode { 1866 parser := mkparser.NewParser("Eval expression", strings.NewReader(args.Dump())) 1867 nodes, errs := parser.Parse() 1868 if errs != nil { 1869 return []starlarkNode{ctx.newBadNode(node, "Unable to parse eval statement")} 1870 } 1871 1872 if len(nodes) == 0 { 1873 return []starlarkNode{} 1874 } else if len(nodes) == 1 { 1875 // Replace the nodeLocator with one that just returns the location of 1876 // the $(eval) node. Otherwise, statements inside an $(eval) will show as 1877 // being on line 1 of the file, because they're on line 1 of 1878 // strings.NewReader(args.Dump()) 1879 oldNodeLocator := ctx.script.nodeLocator 1880 ctx.script.nodeLocator = func(pos mkparser.Pos) int { 1881 return oldNodeLocator(node.Pos()) 1882 } 1883 defer func() { 1884 ctx.script.nodeLocator = oldNodeLocator 1885 }() 1886 1887 switch n := nodes[0].(type) { 1888 case *mkparser.Assignment: 1889 if n.Name.Const() { 1890 return ctx.handleAssignment(n) 1891 } 1892 case *mkparser.Comment: 1893 return []starlarkNode{&commentNode{strings.TrimSpace("#" + n.Comment)}} 1894 case *mkparser.Directive: 1895 if n.Name == "include" || n.Name == "-include" { 1896 return ctx.handleInclude(n) 1897 } 1898 case *mkparser.Variable: 1899 // Technically inherit-product(-if-exists) don't need to be put inside 1900 // an eval, but some makefiles do it, presumably because they copy+pasted 1901 // from a $(eval include ...) 1902 if name, _, ok := ctx.maybeParseFunctionCall(n, n.Name); ok { 1903 if name == "inherit-product" || name == "inherit-product-if-exists" { 1904 return ctx.handleVariable(n) 1905 } 1906 } 1907 } 1908 } 1909 1910 return []starlarkNode{ctx.newBadNode(node, "Eval expression too complex; only assignments, comments, includes, and inherit-products are supported")} 1911} 1912 1913type lowerUpperParser struct { 1914 isUpper bool 1915} 1916 1917func (p *lowerUpperParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr { 1918 fn := "lower" 1919 if p.isUpper { 1920 fn = "upper" 1921 } 1922 arg := ctx.parseMakeString(node, args) 1923 1924 return &callExpr{ 1925 object: arg, 1926 name: fn, 1927 returnType: starlarkTypeString, 1928 } 1929} 1930 1931func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr { 1932 if mk.Const() { 1933 return &stringLiteralExpr{mk.Dump()} 1934 } 1935 if mkRef, ok := mk.SingleVariable(); ok { 1936 return ctx.parseReference(node, mkRef) 1937 } 1938 // If we reached here, it's neither string literal nor a simple variable, 1939 // we need a full-blown interpolation node that will generate 1940 // "a%b%c" % (X, Y) for a$(X)b$(Y)c 1941 parts := make([]starlarkExpr, len(mk.Variables)+len(mk.Strings)) 1942 for i := 0; i < len(parts); i++ { 1943 if i%2 == 0 { 1944 parts[i] = &stringLiteralExpr{literal: mk.Strings[i/2]} 1945 } else { 1946 parts[i] = ctx.parseReference(node, mk.Variables[i/2].Name) 1947 if x, ok := parts[i].(*badExpr); ok { 1948 return x 1949 } 1950 } 1951 } 1952 return NewInterpolateExpr(parts) 1953} 1954 1955func (ctx *parseContext) parseNodeMakeString(node mkparser.Node, mk *mkparser.MakeString) []starlarkNode { 1956 // Discard any constant values in the make string, as they would be top level 1957 // string literals and do nothing. 1958 result := make([]starlarkNode, 0, len(mk.Variables)) 1959 for i := range mk.Variables { 1960 result = append(result, ctx.handleVariable(&mk.Variables[i])...) 1961 } 1962 return result 1963} 1964 1965// Handles the statements whose treatment is the same in all contexts: comment, 1966// assignment, variable (which is a macro call in reality) and all constructs that 1967// do not handle in any context ('define directive and any unrecognized stuff). 1968func (ctx *parseContext) handleSimpleStatement(node mkparser.Node) []starlarkNode { 1969 var result []starlarkNode 1970 switch x := node.(type) { 1971 case *mkparser.Comment: 1972 if n, handled := ctx.maybeHandleAnnotation(x); handled && n != nil { 1973 result = []starlarkNode{n} 1974 } else if !handled { 1975 result = []starlarkNode{&commentNode{strings.TrimSpace("#" + x.Comment)}} 1976 } 1977 case *mkparser.Assignment: 1978 result = ctx.handleAssignment(x) 1979 case *mkparser.Variable: 1980 result = ctx.handleVariable(x) 1981 case *mkparser.Directive: 1982 switch x.Name { 1983 case "define": 1984 if res := ctx.maybeHandleDefine(x); res != nil { 1985 result = []starlarkNode{res} 1986 } 1987 case "include", "-include": 1988 result = ctx.handleInclude(x) 1989 case "ifeq", "ifneq", "ifdef", "ifndef": 1990 result = []starlarkNode{ctx.handleIfBlock(x)} 1991 default: 1992 result = []starlarkNode{ctx.newBadNode(x, "unexpected directive %s", x.Name)} 1993 } 1994 default: 1995 result = []starlarkNode{ctx.newBadNode(x, "unsupported line %s", strings.ReplaceAll(x.Dump(), "\n", "\n#"))} 1996 } 1997 1998 // Clear the includeTops after each non-comment statement 1999 // so that include annotations placed on certain statements don't apply 2000 // globally for the rest of the makefile was well. 2001 if _, wasComment := node.(*mkparser.Comment); !wasComment { 2002 ctx.atTopOfMakefile = false 2003 ctx.includeTops = []string{} 2004 } 2005 2006 if result == nil { 2007 result = []starlarkNode{} 2008 } 2009 2010 return result 2011} 2012 2013// The types allowed in a type_hint 2014var typeHintMap = map[string]starlarkType{ 2015 "string": starlarkTypeString, 2016 "list": starlarkTypeList, 2017} 2018 2019// Processes annotation. An annotation is a comment that starts with #RBC# and provides 2020// a conversion hint -- say, where to look for the dynamically calculated inherit/include 2021// paths. Returns true if the comment was a successfully-handled annotation. 2022func (ctx *parseContext) maybeHandleAnnotation(cnode *mkparser.Comment) (starlarkNode, bool) { 2023 maybeTrim := func(s, prefix string) (string, bool) { 2024 if strings.HasPrefix(s, prefix) { 2025 return strings.TrimSpace(strings.TrimPrefix(s, prefix)), true 2026 } 2027 return s, false 2028 } 2029 annotation, ok := maybeTrim(cnode.Comment, annotationCommentPrefix) 2030 if !ok { 2031 return nil, false 2032 } 2033 if p, ok := maybeTrim(annotation, "include_top"); ok { 2034 // Don't allow duplicate include tops, because then we will generate 2035 // invalid starlark code. (duplicate keys in the _entry dictionary) 2036 for _, top := range ctx.includeTops { 2037 if top == p { 2038 return nil, true 2039 } 2040 } 2041 ctx.includeTops = append(ctx.includeTops, p) 2042 return nil, true 2043 } else if p, ok := maybeTrim(annotation, "type_hint"); ok { 2044 // Type hints must come at the beginning the file, to avoid confusion 2045 // if a type hint was specified later and thus only takes effect for half 2046 // of the file. 2047 if !ctx.atTopOfMakefile { 2048 return ctx.newBadNode(cnode, "type_hint annotations must come before the first Makefile statement"), true 2049 } 2050 2051 parts := strings.Fields(p) 2052 if len(parts) <= 1 { 2053 return ctx.newBadNode(cnode, "Invalid type_hint annotation: %s. Must be a variable type followed by a list of variables of that type", p), true 2054 } 2055 2056 var varType starlarkType 2057 if varType, ok = typeHintMap[parts[0]]; !ok { 2058 varType = starlarkTypeUnknown 2059 } 2060 if varType == starlarkTypeUnknown { 2061 return ctx.newBadNode(cnode, "Invalid type_hint annotation. Only list/string types are accepted, found %s", parts[0]), true 2062 } 2063 2064 for _, name := range parts[1:] { 2065 // Don't allow duplicate type hints 2066 if _, ok := ctx.typeHints[name]; ok { 2067 return ctx.newBadNode(cnode, "Duplicate type hint for variable %s", name), true 2068 } 2069 ctx.typeHints[name] = varType 2070 } 2071 return nil, true 2072 } 2073 return ctx.newBadNode(cnode, "unsupported annotation %s", cnode.Comment), true 2074} 2075 2076func (ctx *parseContext) loadedModulePath(path string) string { 2077 // During the transition to Roboleaf some of the product configuration files 2078 // will be converted and checked in while the others will be generated on the fly 2079 // and run. The runner (rbcrun application) accommodates this by allowing three 2080 // different ways to specify the loaded file location: 2081 // 1) load(":<file>",...) loads <file> from the same directory 2082 // 2) load("//path/relative/to/source/root:<file>", ...) loads <file> source tree 2083 // 3) load("/absolute/path/to/<file> absolute path 2084 // If the file being generated and the file it wants to load are in the same directory, 2085 // generate option 1. 2086 // Otherwise, if output directory is not specified, generate 2) 2087 // Finally, if output directory has been specified and the file being generated and 2088 // the file it wants to load from are in the different directories, generate 2) or 3): 2089 // * if the file being loaded exists in the source tree, generate 2) 2090 // * otherwise, generate 3) 2091 // Finally, figure out the loaded module path and name and create a node for it 2092 loadedModuleDir := filepath.Dir(path) 2093 base := filepath.Base(path) 2094 loadedModuleName := strings.TrimSuffix(base, filepath.Ext(base)) + ctx.outputSuffix 2095 if loadedModuleDir == filepath.Dir(ctx.script.mkFile) { 2096 return ":" + loadedModuleName 2097 } 2098 if ctx.outputDir == "" { 2099 return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName) 2100 } 2101 if _, err := os.Stat(filepath.Join(loadedModuleDir, loadedModuleName)); err == nil { 2102 return fmt.Sprintf("//%s:%s", loadedModuleDir, loadedModuleName) 2103 } 2104 return filepath.Join(ctx.outputDir, loadedModuleDir, loadedModuleName) 2105} 2106 2107func (ctx *parseContext) addSoongNamespace(ns string) { 2108 if _, ok := ctx.soongNamespaces[ns]; ok { 2109 return 2110 } 2111 ctx.soongNamespaces[ns] = make(map[string]bool) 2112} 2113 2114func (ctx *parseContext) hasSoongNamespace(name string) bool { 2115 _, ok := ctx.soongNamespaces[name] 2116 return ok 2117} 2118 2119func (ctx *parseContext) updateSoongNamespace(replace bool, namespaceName string, varNames []string) { 2120 ctx.addSoongNamespace(namespaceName) 2121 vars := ctx.soongNamespaces[namespaceName] 2122 if replace { 2123 vars = make(map[string]bool) 2124 ctx.soongNamespaces[namespaceName] = vars 2125 } 2126 for _, v := range varNames { 2127 vars[v] = true 2128 } 2129} 2130 2131func (ctx *parseContext) hasNamespaceVar(namespaceName string, varName string) bool { 2132 vars, ok := ctx.soongNamespaces[namespaceName] 2133 if ok { 2134 _, ok = vars[varName] 2135 } 2136 return ok 2137} 2138 2139func (ctx *parseContext) errorLocation(node mkparser.Node) ErrorLocation { 2140 return ErrorLocation{ctx.script.mkFile, ctx.script.nodeLocator(node.Pos())} 2141} 2142 2143func (ss *StarlarkScript) String() string { 2144 return NewGenerateContext(ss).emit() 2145} 2146 2147func (ss *StarlarkScript) SubConfigFiles() []string { 2148 2149 var subs []string 2150 for _, src := range ss.inherited { 2151 subs = append(subs, src.originalPath) 2152 } 2153 return subs 2154} 2155 2156func (ss *StarlarkScript) HasErrors() bool { 2157 return ss.hasErrors 2158} 2159 2160// Convert reads and parses a makefile. If successful, parsed tree 2161// is returned and then can be passed to String() to get the generated 2162// Starlark file. 2163func Convert(req Request) (*StarlarkScript, error) { 2164 reader := req.Reader 2165 if reader == nil { 2166 mkContents, err := ioutil.ReadFile(req.MkFile) 2167 if err != nil { 2168 return nil, err 2169 } 2170 reader = bytes.NewBuffer(mkContents) 2171 } 2172 parser := mkparser.NewParser(req.MkFile, reader) 2173 nodes, errs := parser.Parse() 2174 if len(errs) > 0 { 2175 for _, e := range errs { 2176 fmt.Fprintln(os.Stderr, "ERROR:", e) 2177 } 2178 return nil, fmt.Errorf("bad makefile %s", req.MkFile) 2179 } 2180 starScript := &StarlarkScript{ 2181 moduleName: moduleNameForFile(req.MkFile), 2182 mkFile: req.MkFile, 2183 traceCalls: req.TraceCalls, 2184 sourceFS: req.SourceFS, 2185 makefileFinder: req.MakefileFinder, 2186 nodeLocator: func(pos mkparser.Pos) int { return parser.Unpack(pos).Line }, 2187 nodes: make([]starlarkNode, 0), 2188 } 2189 ctx := newParseContext(starScript, nodes) 2190 ctx.outputSuffix = req.OutputSuffix 2191 ctx.outputDir = req.OutputDir 2192 ctx.errorLogger = req.ErrorLogger 2193 if len(req.TracedVariables) > 0 { 2194 ctx.tracedVariables = make(map[string]bool) 2195 for _, v := range req.TracedVariables { 2196 ctx.tracedVariables[v] = true 2197 } 2198 } 2199 for ctx.hasNodes() && ctx.fatalError == nil { 2200 starScript.nodes = append(starScript.nodes, ctx.handleSimpleStatement(ctx.getNode())...) 2201 } 2202 if ctx.fatalError != nil { 2203 return nil, ctx.fatalError 2204 } 2205 return starScript, nil 2206} 2207 2208func Launcher(mainModuleUri, inputVariablesUri, mainModuleName string) string { 2209 var buf bytes.Buffer 2210 fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName) 2211 fmt.Fprintf(&buf, "load(%q, input_variables_init = \"init\")\n", inputVariablesUri) 2212 fmt.Fprintf(&buf, "load(%q, \"init\")\n", mainModuleUri) 2213 fmt.Fprintf(&buf, "%s(%s(%q, init, input_variables_init))\n", cfnPrintVars, cfnMain, mainModuleName) 2214 return buf.String() 2215} 2216 2217func BoardLauncher(mainModuleUri string, inputVariablesUri string) string { 2218 var buf bytes.Buffer 2219 fmt.Fprintf(&buf, "load(%q, %q)\n", baseUri, baseName) 2220 fmt.Fprintf(&buf, "load(%q, \"init\")\n", mainModuleUri) 2221 fmt.Fprintf(&buf, "load(%q, input_variables_init = \"init\")\n", inputVariablesUri) 2222 fmt.Fprintf(&buf, "%s(%s(init, input_variables_init))\n", cfnPrintVars, cfnBoardMain) 2223 return buf.String() 2224} 2225 2226func MakePath2ModuleName(mkPath string) string { 2227 return strings.TrimSuffix(mkPath, filepath.Ext(mkPath)) 2228} 2229