1## @file 2# The engine for building files 3# 4# Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR> 5# This program and the accompanying materials 6# are licensed and made available under the terms and conditions of the BSD License 7# which accompanies this distribution. The full text of the license may be found at 8# http://opensource.org/licenses/bsd-license.php 9# 10# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, 11# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. 12# 13 14## 15# Import Modules 16# 17import Common.LongFilePathOs as os 18import re 19import copy 20import string 21from Common.LongFilePathSupport import OpenLongFilePath as open 22 23from Common.GlobalData import * 24from Common.BuildToolError import * 25from Common.Misc import tdict, PathClass 26from Common.String import NormPath 27from Common.DataType import * 28 29import Common.EdkLogger as EdkLogger 30 31## Convert file type to file list macro name 32# 33# @param FileType The name of file type 34# 35# @retval string The name of macro 36# 37def FileListMacro(FileType): 38 return "%sS" % FileType.replace("-", "_").upper() 39 40## Convert file type to list file macro name 41# 42# @param FileType The name of file type 43# 44# @retval string The name of macro 45# 46def ListFileMacro(FileType): 47 return "%s_LIST" % FileListMacro(FileType) 48 49class TargetDescBlock(object): 50 _Cache_ = {} # {TargetFile : TargetDescBlock object} 51 52 # Factory method 53 def __new__(Class, Inputs, Outputs, Commands, Dependencies): 54 if Outputs[0] in Class._Cache_: 55 Tdb = Class._Cache_[Outputs[0]] 56 for File in Inputs: 57 Tdb.AddInput(File) 58 else: 59 Tdb = super(TargetDescBlock, Class).__new__(Class) 60 Tdb._Init(Inputs, Outputs, Commands, Dependencies) 61 #Class._Cache_[Outputs[0]] = Tdb 62 return Tdb 63 64 def _Init(self, Inputs, Outputs, Commands, Dependencies): 65 self.Inputs = Inputs 66 self.Outputs = Outputs 67 self.Commands = Commands 68 self.Dependencies = Dependencies 69 if self.Outputs: 70 self.Target = self.Outputs[0] 71 else: 72 self.Target = None 73 74 def __str__(self): 75 return self.Target.Path 76 77 def __hash__(self): 78 return hash(self.Target.Path) 79 80 def __eq__(self, Other): 81 if type(Other) == type(self): 82 return Other.Target.Path == self.Target.Path 83 else: 84 return str(Other) == self.Target.Path 85 86 def AddInput(self, Input): 87 if Input not in self.Inputs: 88 self.Inputs.append(Input) 89 90 def IsMultipleInput(self): 91 return len(self.Inputs) > 1 92 93 @staticmethod 94 def Renew(): 95 TargetDescBlock._Cache_ = {} 96 97## Class for one build rule 98# 99# This represents a build rule which can give out corresponding command list for 100# building the given source file(s). The result can be used for generating the 101# target for makefile. 102# 103class FileBuildRule: 104 INC_LIST_MACRO = "INC_LIST" 105 INC_MACRO = "INC" 106 107 ## constructor 108 # 109 # @param Input The dictionary represeting input file(s) for a rule 110 # @param Output The list represeting output file(s) for a rule 111 # @param Command The list containing commands to generate the output from input 112 # 113 def __init__(self, Type, Input, Output, Command, ExtraDependency=None): 114 # The Input should not be empty 115 if not Input: 116 Input = [] 117 if not Output: 118 Output = [] 119 if not Command: 120 Command = [] 121 122 self.FileListMacro = FileListMacro(Type) 123 self.ListFileMacro = ListFileMacro(Type) 124 self.IncListFileMacro = self.INC_LIST_MACRO 125 126 self.SourceFileType = Type 127 # source files listed not in "*" or "?" pattern format 128 if not ExtraDependency: 129 self.ExtraSourceFileList = [] 130 else: 131 self.ExtraSourceFileList = ExtraDependency 132 133 # 134 # Search macros used in command lines for <FILE_TYPE>_LIST and INC_LIST. 135 # If found, generate a file to keep the input files used to get over the 136 # limitation of command line length 137 # 138 self.MacroList = [] 139 self.CommandList = [] 140 for CmdLine in Command: 141 self.MacroList.extend(gMacroRefPattern.findall(CmdLine)) 142 # replace path separator with native one 143 self.CommandList.append(CmdLine) 144 145 # Indicate what should be generated 146 if self.FileListMacro in self.MacroList: 147 self.GenFileListMacro = True 148 else: 149 self.GenFileListMacro = False 150 151 if self.ListFileMacro in self.MacroList: 152 self.GenListFile = True 153 self.GenFileListMacro = True 154 else: 155 self.GenListFile = False 156 157 if self.INC_LIST_MACRO in self.MacroList: 158 self.GenIncListFile = True 159 else: 160 self.GenIncListFile = False 161 162 # Check input files 163 self.IsMultipleInput = False 164 self.SourceFileExtList = [] 165 for File in Input: 166 Base, Ext = os.path.splitext(File) 167 if Base.find("*") >= 0: 168 # There's "*" in the file name 169 self.IsMultipleInput = True 170 self.GenFileListMacro = True 171 elif Base.find("?") < 0: 172 # There's no "*" and "?" in file name 173 self.ExtraSourceFileList.append(File) 174 continue 175 if Ext not in self.SourceFileExtList: 176 self.SourceFileExtList.append(Ext) 177 178 # Check output files 179 self.DestFileList = [] 180 for File in Output: 181 self.DestFileList.append(File) 182 183 # All build targets generated by this rule for a module 184 self.BuildTargets = {} 185 186 ## str() function support 187 # 188 # @retval string 189 # 190 def __str__(self): 191 SourceString = "" 192 SourceString += " %s %s %s" % (self.SourceFileType, " ".join(self.SourceFileExtList), self.ExtraSourceFileList) 193 DestString = ", ".join(self.DestFileList) 194 CommandString = "\n\t".join(self.CommandList) 195 return "%s : %s\n\t%s" % (DestString, SourceString, CommandString) 196 197 ## Check if given file extension is supported by this rule 198 # 199 # @param FileExt The extension of a file 200 # 201 # @retval True If the extension is supported 202 # @retval False If the extension is not supported 203 # 204 def IsSupported(self, FileExt): 205 return FileExt in self.SourceFileExtList 206 207 def Instantiate(self, Macros={}): 208 NewRuleObject = copy.copy(self) 209 NewRuleObject.BuildTargets = {} 210 NewRuleObject.DestFileList = [] 211 for File in self.DestFileList: 212 NewRuleObject.DestFileList.append(PathClass(NormPath(File, Macros))) 213 return NewRuleObject 214 215 ## Apply the rule to given source file(s) 216 # 217 # @param SourceFile One file or a list of files to be built 218 # @param RelativeToDir The relative path of the source file 219 # @param PathSeparator Path separator 220 # 221 # @retval tuple (Source file in full path, List of individual sourcefiles, Destionation file, List of build commands) 222 # 223 def Apply(self, SourceFile, BuildRuleOrder=None): 224 if not self.CommandList or not self.DestFileList: 225 return None 226 227 # source file 228 if self.IsMultipleInput: 229 SrcFileName = "" 230 SrcFileBase = "" 231 SrcFileExt = "" 232 SrcFileDir = "" 233 SrcPath = "" 234 # SourceFile must be a list 235 SrcFile = "$(%s)" % self.FileListMacro 236 else: 237 SrcFileName, SrcFileBase, SrcFileExt = SourceFile.Name, SourceFile.BaseName, SourceFile.Ext 238 if SourceFile.Root: 239 SrcFileDir = SourceFile.SubDir 240 if SrcFileDir == "": 241 SrcFileDir = "." 242 else: 243 SrcFileDir = "." 244 SrcFile = SourceFile.Path 245 SrcPath = SourceFile.Dir 246 247 # destination file (the first one) 248 if self.DestFileList: 249 DestFile = self.DestFileList[0].Path 250 DestPath = self.DestFileList[0].Dir 251 DestFileName = self.DestFileList[0].Name 252 DestFileBase, DestFileExt = self.DestFileList[0].BaseName, self.DestFileList[0].Ext 253 else: 254 DestFile = "" 255 DestPath = "" 256 DestFileName = "" 257 DestFileBase = "" 258 DestFileExt = "" 259 260 BuildRulePlaceholderDict = { 261 # source file 262 "src" : SrcFile, 263 "s_path" : SrcPath, 264 "s_dir" : SrcFileDir, 265 "s_name" : SrcFileName, 266 "s_base" : SrcFileBase, 267 "s_ext" : SrcFileExt, 268 # destination file 269 "dst" : DestFile, 270 "d_path" : DestPath, 271 "d_name" : DestFileName, 272 "d_base" : DestFileBase, 273 "d_ext" : DestFileExt, 274 } 275 276 DstFile = [] 277 for File in self.DestFileList: 278 File = string.Template(str(File)).safe_substitute(BuildRulePlaceholderDict) 279 File = string.Template(str(File)).safe_substitute(BuildRulePlaceholderDict) 280 DstFile.append(PathClass(File, IsBinary=True)) 281 282 if DstFile[0] in self.BuildTargets: 283 TargetDesc = self.BuildTargets[DstFile[0]] 284 if BuildRuleOrder and SourceFile.Ext in BuildRuleOrder: 285 Index = BuildRuleOrder.index(SourceFile.Ext) 286 for Input in TargetDesc.Inputs: 287 if Input.Ext not in BuildRuleOrder or BuildRuleOrder.index(Input.Ext) > Index: 288 # 289 # Command line should be regenerated since some macros are different 290 # 291 CommandList = self._BuildCommand(BuildRulePlaceholderDict) 292 TargetDesc._Init([SourceFile], DstFile, CommandList, self.ExtraSourceFileList) 293 break 294 else: 295 TargetDesc.AddInput(SourceFile) 296 else: 297 CommandList = self._BuildCommand(BuildRulePlaceholderDict) 298 TargetDesc = TargetDescBlock([SourceFile], DstFile, CommandList, self.ExtraSourceFileList) 299 TargetDesc.ListFileMacro = self.ListFileMacro 300 TargetDesc.FileListMacro = self.FileListMacro 301 TargetDesc.IncListFileMacro = self.IncListFileMacro 302 TargetDesc.GenFileListMacro = self.GenFileListMacro 303 TargetDesc.GenListFile = self.GenListFile 304 TargetDesc.GenIncListFile = self.GenIncListFile 305 self.BuildTargets[DstFile[0]] = TargetDesc 306 return TargetDesc 307 308 def _BuildCommand(self, Macros): 309 CommandList = [] 310 for CommandString in self.CommandList: 311 CommandString = string.Template(CommandString).safe_substitute(Macros) 312 CommandString = string.Template(CommandString).safe_substitute(Macros) 313 CommandList.append(CommandString) 314 return CommandList 315 316## Class for build rules 317# 318# BuildRule class parses rules defined in a file or passed by caller, and converts 319# the rule into FileBuildRule object. 320# 321class BuildRule: 322 _SectionHeader = "SECTIONHEADER" 323 _Section = "SECTION" 324 _SubSectionHeader = "SUBSECTIONHEADER" 325 _SubSection = "SUBSECTION" 326 _InputFile = "INPUTFILE" 327 _OutputFile = "OUTPUTFILE" 328 _ExtraDependency = "EXTRADEPENDENCY" 329 _Command = "COMMAND" 330 _UnknownSection = "UNKNOWNSECTION" 331 332 _SubSectionList = [_InputFile, _OutputFile, _Command] 333 334 _PATH_SEP = "(+)" 335 _FileTypePattern = re.compile("^[_a-zA-Z][_\-0-9a-zA-Z]*$") 336 _BinaryFileRule = FileBuildRule(TAB_DEFAULT_BINARY_FILE, [], [os.path.join("$(OUTPUT_DIR)", "${s_name}")], 337 ["$(CP) ${src} ${dst}"], []) 338 339 ## Constructor 340 # 341 # @param File The file containing build rules in a well defined format 342 # @param Content The string list of build rules in a well defined format 343 # @param LineIndex The line number from which the parsing will begin 344 # @param SupportedFamily The list of supported tool chain families 345 # 346 def __init__(self, File=None, Content=None, LineIndex=0, SupportedFamily=["MSFT", "INTEL", "GCC", "RVCT"]): 347 self.RuleFile = File 348 # Read build rules from file if it's not none 349 if File != None: 350 try: 351 self.RuleContent = open(File, 'r').readlines() 352 except: 353 EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File) 354 elif Content != None: 355 self.RuleContent = Content 356 else: 357 EdkLogger.error("build", PARAMETER_MISSING, ExtraData="No rule file or string given") 358 359 self.SupportedToolChainFamilyList = SupportedFamily 360 self.RuleDatabase = tdict(True, 4) # {FileExt, ModuleType, Arch, Family : FileBuildRule object} 361 self.Ext2FileType = {} # {ext : file-type} 362 self.FileTypeList = set() 363 364 self._LineIndex = LineIndex 365 self._State = "" 366 self._RuleInfo = tdict(True, 2) # {toolchain family : {"InputFile": {}, "OutputFile" : [], "Command" : []}} 367 self._FileType = '' 368 self._BuildTypeList = [] 369 self._ArchList = [] 370 self._FamilyList = [] 371 self._TotalToolChainFamilySet = set() 372 self._RuleObjectList = [] # FileBuildRule object list 373 self._FileVersion = "" 374 375 self.Parse() 376 377 # some intrinsic rules 378 self.RuleDatabase[TAB_DEFAULT_BINARY_FILE, "COMMON", "COMMON", "COMMON"] = self._BinaryFileRule 379 self.FileTypeList.add(TAB_DEFAULT_BINARY_FILE) 380 381 ## Parse the build rule strings 382 def Parse(self): 383 self._State = self._Section 384 for Index in range(self._LineIndex, len(self.RuleContent)): 385 # Clean up the line and replace path separator with native one 386 Line = self.RuleContent[Index].strip().replace(self._PATH_SEP, os.path.sep) 387 self.RuleContent[Index] = Line 388 389 # find the build_rule_version 390 if Line and Line[0] == "#" and Line.find(TAB_BUILD_RULE_VERSION) <> -1: 391 if Line.find("=") <> -1 and Line.find("=") < (len(Line) - 1) and (Line[(Line.find("=") + 1):]).split(): 392 self._FileVersion = (Line[(Line.find("=") + 1):]).split()[0] 393 # skip empty or comment line 394 if Line == "" or Line[0] == "#": 395 continue 396 397 # find out section header, enclosed by [] 398 if Line[0] == '[' and Line[-1] == ']': 399 # merge last section information into rule database 400 self.EndOfSection() 401 self._State = self._SectionHeader 402 # find out sub-section header, enclosed by <> 403 elif Line[0] == '<' and Line[-1] == '>': 404 if self._State != self._UnknownSection: 405 self._State = self._SubSectionHeader 406 407 # call section handler to parse each (sub)section 408 self._StateHandler[self._State](self, Index) 409 # merge last section information into rule database 410 self.EndOfSection() 411 412 ## Parse definitions under a section 413 # 414 # @param LineIndex The line index of build rule text 415 # 416 def ParseSection(self, LineIndex): 417 pass 418 419 ## Parse definitions under a subsection 420 # 421 # @param LineIndex The line index of build rule text 422 # 423 def ParseSubSection(self, LineIndex): 424 # currenly nothing here 425 pass 426 427 ## Placeholder for not supported sections 428 # 429 # @param LineIndex The line index of build rule text 430 # 431 def SkipSection(self, LineIndex): 432 pass 433 434 ## Merge section information just got into rule database 435 def EndOfSection(self): 436 Database = self.RuleDatabase 437 # if there's specific toochain family, 'COMMON' doesn't make sense any more 438 if len(self._TotalToolChainFamilySet) > 1 and 'COMMON' in self._TotalToolChainFamilySet: 439 self._TotalToolChainFamilySet.remove('COMMON') 440 for Family in self._TotalToolChainFamilySet: 441 Input = self._RuleInfo[Family, self._InputFile] 442 Output = self._RuleInfo[Family, self._OutputFile] 443 Command = self._RuleInfo[Family, self._Command] 444 ExtraDependency = self._RuleInfo[Family, self._ExtraDependency] 445 446 BuildRule = FileBuildRule(self._FileType, Input, Output, Command, ExtraDependency) 447 for BuildType in self._BuildTypeList: 448 for Arch in self._ArchList: 449 Database[self._FileType, BuildType, Arch, Family] = BuildRule 450 for FileExt in BuildRule.SourceFileExtList: 451 self.Ext2FileType[FileExt] = self._FileType 452 453 ## Parse section header 454 # 455 # @param LineIndex The line index of build rule text 456 # 457 def ParseSectionHeader(self, LineIndex): 458 self._RuleInfo = tdict(True, 2) 459 self._BuildTypeList = [] 460 self._ArchList = [] 461 self._FamilyList = [] 462 self._TotalToolChainFamilySet = set() 463 FileType = '' 464 RuleNameList = self.RuleContent[LineIndex][1:-1].split(',') 465 for RuleName in RuleNameList: 466 Arch = 'COMMON' 467 BuildType = 'COMMON' 468 TokenList = [Token.strip().upper() for Token in RuleName.split('.')] 469 # old format: Build.File-Type 470 if TokenList[0] == "BUILD": 471 if len(TokenList) == 1: 472 EdkLogger.error("build", FORMAT_INVALID, "Invalid rule section", 473 File=self.RuleFile, Line=LineIndex + 1, 474 ExtraData=self.RuleContent[LineIndex]) 475 476 FileType = TokenList[1] 477 if FileType == '': 478 EdkLogger.error("build", FORMAT_INVALID, "No file type given", 479 File=self.RuleFile, Line=LineIndex + 1, 480 ExtraData=self.RuleContent[LineIndex]) 481 if self._FileTypePattern.match(FileType) == None: 482 EdkLogger.error("build", FORMAT_INVALID, File=self.RuleFile, Line=LineIndex + 1, 483 ExtraData="Only character, number (non-first character), '_' and '-' are allowed in file type") 484 # new format: File-Type.Build-Type.Arch 485 else: 486 if FileType == '': 487 FileType = TokenList[0] 488 elif FileType != TokenList[0]: 489 EdkLogger.error("build", FORMAT_INVALID, 490 "Different file types are not allowed in the same rule section", 491 File=self.RuleFile, Line=LineIndex + 1, 492 ExtraData=self.RuleContent[LineIndex]) 493 if len(TokenList) > 1: 494 BuildType = TokenList[1] 495 if len(TokenList) > 2: 496 Arch = TokenList[2] 497 if BuildType not in self._BuildTypeList: 498 self._BuildTypeList.append(BuildType) 499 if Arch not in self._ArchList: 500 self._ArchList.append(Arch) 501 502 if 'COMMON' in self._BuildTypeList and len(self._BuildTypeList) > 1: 503 EdkLogger.error("build", FORMAT_INVALID, 504 "Specific build types must not be mixed with common one", 505 File=self.RuleFile, Line=LineIndex + 1, 506 ExtraData=self.RuleContent[LineIndex]) 507 if 'COMMON' in self._ArchList and len(self._ArchList) > 1: 508 EdkLogger.error("build", FORMAT_INVALID, 509 "Specific ARCH must not be mixed with common one", 510 File=self.RuleFile, Line=LineIndex + 1, 511 ExtraData=self.RuleContent[LineIndex]) 512 513 self._FileType = FileType 514 self._State = self._Section 515 self.FileTypeList.add(FileType) 516 517 ## Parse sub-section header 518 # 519 # @param LineIndex The line index of build rule text 520 # 521 def ParseSubSectionHeader(self, LineIndex): 522 SectionType = "" 523 List = self.RuleContent[LineIndex][1:-1].split(',') 524 FamilyList = [] 525 for Section in List: 526 TokenList = Section.split('.') 527 Type = TokenList[0].strip().upper() 528 529 if SectionType == "": 530 SectionType = Type 531 elif SectionType != Type: 532 EdkLogger.error("build", FORMAT_INVALID, 533 "Two different section types are not allowed in the same sub-section", 534 File=self.RuleFile, Line=LineIndex + 1, 535 ExtraData=self.RuleContent[LineIndex]) 536 537 if len(TokenList) > 1: 538 Family = TokenList[1].strip().upper() 539 else: 540 Family = "COMMON" 541 542 if Family not in FamilyList: 543 FamilyList.append(Family) 544 545 self._FamilyList = FamilyList 546 self._TotalToolChainFamilySet.update(FamilyList) 547 self._State = SectionType.upper() 548 if 'COMMON' in FamilyList and len(FamilyList) > 1: 549 EdkLogger.error("build", FORMAT_INVALID, 550 "Specific tool chain family should not be mixed with general one", 551 File=self.RuleFile, Line=LineIndex + 1, 552 ExtraData=self.RuleContent[LineIndex]) 553 if self._State not in self._StateHandler: 554 EdkLogger.error("build", FORMAT_INVALID, File=self.RuleFile, Line=LineIndex + 1, 555 ExtraData="Unknown subsection: %s" % self.RuleContent[LineIndex]) 556 ## Parse <InputFile> sub-section 557 # 558 # @param LineIndex The line index of build rule text 559 # 560 def ParseInputFile(self, LineIndex): 561 FileList = [File.strip() for File in self.RuleContent[LineIndex].split(",")] 562 for ToolChainFamily in self._FamilyList: 563 InputFiles = self._RuleInfo[ToolChainFamily, self._State] 564 if InputFiles == None: 565 InputFiles = [] 566 self._RuleInfo[ToolChainFamily, self._State] = InputFiles 567 InputFiles.extend(FileList) 568 569 ## Parse <ExtraDependency> sub-section 570 # 571 # @param LineIndex The line index of build rule text 572 # 573 def ParseCommon(self, LineIndex): 574 for ToolChainFamily in self._FamilyList: 575 Items = self._RuleInfo[ToolChainFamily, self._State] 576 if Items == None: 577 Items = [] 578 self._RuleInfo[ToolChainFamily, self._State] = Items 579 Items.append(self.RuleContent[LineIndex]) 580 581 ## Get a build rule via [] operator 582 # 583 # @param FileExt The extension of a file 584 # @param ToolChainFamily The tool chain family name 585 # @param BuildVersion The build version number. "*" means any rule 586 # is applicalbe. 587 # 588 # @retval FileType The file type string 589 # @retval FileBuildRule The object of FileBuildRule 590 # 591 # Key = (FileExt, ModuleType, Arch, ToolChainFamily) 592 def __getitem__(self, Key): 593 if not Key: 594 return None 595 596 if Key[0] in self.Ext2FileType: 597 Type = self.Ext2FileType[Key[0]] 598 elif Key[0].upper() in self.FileTypeList: 599 Type = Key[0].upper() 600 else: 601 return None 602 603 if len(Key) > 1: 604 Key = (Type,) + Key[1:] 605 else: 606 Key = (Type,) 607 return self.RuleDatabase[Key] 608 609 _StateHandler = { 610 _SectionHeader : ParseSectionHeader, 611 _Section : ParseSection, 612 _SubSectionHeader : ParseSubSectionHeader, 613 _SubSection : ParseSubSection, 614 _InputFile : ParseInputFile, 615 _OutputFile : ParseCommon, 616 _ExtraDependency : ParseCommon, 617 _Command : ParseCommon, 618 _UnknownSection : SkipSection, 619 } 620 621# This acts like the main() function for the script, unless it is 'import'ed into another 622# script. 623if __name__ == '__main__': 624 import sys 625 EdkLogger.Initialize() 626 if len(sys.argv) > 1: 627 Br = BuildRule(sys.argv[1]) 628 print str(Br[".c", "DXE_DRIVER", "IA32", "MSFT"][1]) 629 print 630 print str(Br[".c", "DXE_DRIVER", "IA32", "INTEL"][1]) 631 print 632 print str(Br[".c", "DXE_DRIVER", "IA32", "GCC"][1]) 633 print 634 print str(Br[".ac", "ACPI_TABLE", "IA32", "MSFT"][1]) 635 print 636 print str(Br[".h", "ACPI_TABLE", "IA32", "INTEL"][1]) 637 print 638 print str(Br[".ac", "ACPI_TABLE", "IA32", "MSFT"][1]) 639 print 640 print str(Br[".s", "SEC", "IPF", "COMMON"][1]) 641 print 642 print str(Br[".s", "SEC"][1]) 643 644