1## @file 2# preprocess source file 3# 4# Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR> 5# 6# This program and the accompanying materials 7# are licensed and made available under the terms and conditions of the BSD License 8# which accompanies this distribution. The full text of the license may be found at 9# http://opensource.org/licenses/bsd-license.php 10# 11# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, 12# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. 13# 14 15## 16# Import Modules 17# 18import sys 19import Common.LongFilePathOs as os 20import re 21import CodeFragmentCollector 22import FileProfile 23from CommonDataClass import DataClass 24from Common import EdkLogger 25from EotToolError import * 26import EotGlobalData 27 28# Global Dicts 29IncludeFileListDict = {} 30IncludePathListDict = {} 31ComplexTypeDict = {} 32SUDict = {} 33 34## GetFuncDeclPattern() method 35# 36# Get the pattern of function declaration 37# 38# @return p: the pattern of function declaration 39# 40def GetFuncDeclPattern(): 41 p = re.compile(r'(EFIAPI|EFI_BOOT_SERVICE|EFI_RUNTIME_SERVICE)?\s*[_\w]+\s*\(.*\).*', re.DOTALL) 42 return p 43 44## GetArrayPattern() method 45# 46# Get the pattern of array 47# 48# @return p: the pattern of array 49# 50def GetArrayPattern(): 51 p = re.compile(r'[_\w]*\s*[\[.*\]]+') 52 return p 53 54## GetTypedefFuncPointerPattern() method 55# 56# Get the pattern of function pointer 57# 58# @return p: the pattern of function pointer 59# 60def GetTypedefFuncPointerPattern(): 61 p = re.compile('[_\w\s]*\([\w\s]*\*+\s*[_\w]+\s*\)\s*\(.*\)', re.DOTALL) 62 return p 63 64## GetDB() method 65# 66# Get global database instance 67# 68# @return EotGlobalData.gDb: the global database instance 69# 70def GetDB(): 71 return EotGlobalData.gDb 72 73## PrintErrorMsg() method 74# 75# print error message 76# 77# @param ErrorType: Type of error 78# @param Msg: Error message 79# @param TableName: table name of error found 80# @param ItemId: id of item 81# 82def PrintErrorMsg(ErrorType, Msg, TableName, ItemId): 83 Msg = Msg.replace('\n', '').replace('\r', '') 84 MsgPartList = Msg.split() 85 Msg = '' 86 for Part in MsgPartList: 87 Msg += Part 88 Msg += ' ' 89 GetDB().TblReport.Insert(ErrorType, OtherMsg = Msg, BelongsToTable = TableName, BelongsToItem = ItemId) 90 91## GetIdType() method 92# 93# Find type of input string 94# 95# @param Str: String to be parsed 96# 97# @return Type: The type of the string 98# 99def GetIdType(Str): 100 Type = DataClass.MODEL_UNKNOWN 101 Str = Str.replace('#', '# ') 102 List = Str.split() 103 if List[1] == 'include': 104 Type = DataClass.MODEL_IDENTIFIER_INCLUDE 105 elif List[1] == 'define': 106 Type = DataClass.MODEL_IDENTIFIER_MACRO_DEFINE 107 elif List[1] == 'ifdef': 108 Type = DataClass.MODEL_IDENTIFIER_MACRO_IFDEF 109 elif List[1] == 'ifndef': 110 Type = DataClass.MODEL_IDENTIFIER_MACRO_IFNDEF 111 elif List[1] == 'endif': 112 Type = DataClass.MODEL_IDENTIFIER_MACRO_ENDIF 113 elif List[1] == 'pragma': 114 Type = DataClass.MODEL_IDENTIFIER_MACRO_PROGMA 115 else: 116 Type = DataClass.MODEL_UNKNOWN 117 return Type 118 119## GetIdentifierList() method 120# 121# Get id of all files 122# 123# @return IdList: The list of all id of files 124# 125def GetIdentifierList(): 126 IdList = [] 127 128 for pp in FileProfile.PPDirectiveList: 129 Type = GetIdType(pp.Content) 130 IdPP = DataClass.IdentifierClass(-1, '', '', '', pp.Content, Type, -1, -1, pp.StartPos[0],pp.StartPos[1],pp.EndPos[0],pp.EndPos[1]) 131 IdList.append(IdPP) 132 133 for ae in FileProfile.AssignmentExpressionList: 134 IdAE = DataClass.IdentifierClass(-1, ae.Operator, '', ae.Name, ae.Value, DataClass.MODEL_IDENTIFIER_ASSIGNMENT_EXPRESSION, -1, -1, ae.StartPos[0],ae.StartPos[1],ae.EndPos[0],ae.EndPos[1]) 135 IdList.append(IdAE) 136 137 FuncDeclPattern = GetFuncDeclPattern() 138 ArrayPattern = GetArrayPattern() 139 for var in FileProfile.VariableDeclarationList: 140 DeclText = var.Declarator.strip() 141 while DeclText.startswith('*'): 142 var.Modifier += '*' 143 DeclText = DeclText.lstrip('*').strip() 144 var.Declarator = DeclText 145 if FuncDeclPattern.match(var.Declarator): 146 DeclSplitList = var.Declarator.split('(') 147 FuncName = DeclSplitList[0] 148 FuncNamePartList = FuncName.split() 149 if len(FuncNamePartList) > 1: 150 FuncName = FuncNamePartList[-1] 151 Index = 0 152 while Index < len(FuncNamePartList) - 1: 153 var.Modifier += ' ' + FuncNamePartList[Index] 154 var.Declarator = var.Declarator.lstrip().lstrip(FuncNamePartList[Index]) 155 Index += 1 156 IdVar = DataClass.IdentifierClass(-1, var.Modifier, '', var.Declarator, '', DataClass.MODEL_IDENTIFIER_FUNCTION_DECLARATION, -1, -1, var.StartPos[0],var.StartPos[1],var.EndPos[0],var.EndPos[1]) 157 IdList.append(IdVar) 158 continue 159 160 if var.Declarator.find('{') == -1: 161 for decl in var.Declarator.split(','): 162 DeclList = decl.split('=') 163 Name = DeclList[0].strip() 164 if ArrayPattern.match(Name): 165 LSBPos = var.Declarator.find('[') 166 var.Modifier += ' ' + Name[LSBPos:] 167 Name = Name[0:LSBPos] 168 169 IdVar = DataClass.IdentifierClass(-1, var.Modifier, '', Name, (len(DeclList) > 1 and [DeclList[1]]or [''])[0], DataClass.MODEL_IDENTIFIER_VARIABLE, -1, -1, var.StartPos[0],var.StartPos[1],var.EndPos[0],var.EndPos[1]) 170 IdList.append(IdVar) 171 else: 172 DeclList = var.Declarator.split('=') 173 Name = DeclList[0].strip() 174 if ArrayPattern.match(Name): 175 LSBPos = var.Declarator.find('[') 176 var.Modifier += ' ' + Name[LSBPos:] 177 Name = Name[0:LSBPos] 178 IdVar = DataClass.IdentifierClass(-1, var.Modifier, '', Name, (len(DeclList) > 1 and [DeclList[1]]or [''])[0], DataClass.MODEL_IDENTIFIER_VARIABLE, -1, -1, var.StartPos[0],var.StartPos[1],var.EndPos[0],var.EndPos[1]) 179 IdList.append(IdVar) 180 181 for enum in FileProfile.EnumerationDefinitionList: 182 LBPos = enum.Content.find('{') 183 RBPos = enum.Content.find('}') 184 Name = enum.Content[4:LBPos].strip() 185 Value = enum.Content[LBPos+1:RBPos] 186 IdEnum = DataClass.IdentifierClass(-1, '', '', Name, Value, DataClass.MODEL_IDENTIFIER_ENUMERATE, -1, -1, enum.StartPos[0],enum.StartPos[1],enum.EndPos[0],enum.EndPos[1]) 187 IdList.append(IdEnum) 188 189 for su in FileProfile.StructUnionDefinitionList: 190 Type = DataClass.MODEL_IDENTIFIER_STRUCTURE 191 SkipLen = 6 192 if su.Content.startswith('union'): 193 Type = DataClass.MODEL_IDENTIFIER_UNION 194 SkipLen = 5 195 LBPos = su.Content.find('{') 196 RBPos = su.Content.find('}') 197 if LBPos == -1 or RBPos == -1: 198 Name = su.Content[SkipLen:].strip() 199 Value = '' 200 else: 201 Name = su.Content[SkipLen:LBPos].strip() 202 Value = su.Content[LBPos+1:RBPos] 203 IdPE = DataClass.IdentifierClass(-1, '', '', Name, Value, Type, -1, -1, su.StartPos[0],su.StartPos[1],su.EndPos[0],su.EndPos[1]) 204 IdList.append(IdPE) 205 206 TdFuncPointerPattern = GetTypedefFuncPointerPattern() 207 for td in FileProfile.TypedefDefinitionList: 208 Modifier = '' 209 Name = td.ToType 210 Value = td.FromType 211 if TdFuncPointerPattern.match(td.ToType): 212 Modifier = td.FromType 213 LBPos = td.ToType.find('(') 214 TmpStr = td.ToType[LBPos+1:].strip() 215 StarPos = TmpStr.find('*') 216 if StarPos != -1: 217 Modifier += ' ' + TmpStr[0:StarPos] 218 while TmpStr[StarPos] == '*': 219 Modifier += ' ' + '*' 220 StarPos += 1 221 TmpStr = TmpStr[StarPos:].strip() 222 RBPos = TmpStr.find(')') 223 Name = TmpStr[0:RBPos] 224 Value = 'FP' + TmpStr[RBPos + 1:] 225 226 IdTd = DataClass.IdentifierClass(-1, Modifier, '', Name, Value, DataClass.MODEL_IDENTIFIER_TYPEDEF, -1, -1, td.StartPos[0],td.StartPos[1],td.EndPos[0],td.EndPos[1]) 227 IdList.append(IdTd) 228 229 for funcCall in FileProfile.FunctionCallingList: 230 IdFC = DataClass.IdentifierClass(-1, '', '', funcCall.FuncName, funcCall.ParamList, DataClass.MODEL_IDENTIFIER_FUNCTION_CALLING, -1, -1, funcCall.StartPos[0],funcCall.StartPos[1],funcCall.EndPos[0],funcCall.EndPos[1]) 231 IdList.append(IdFC) 232 return IdList 233 234## GetParamList() method 235# 236# Get a list of parameters 237# 238# @param FuncDeclarator: Function declarator 239# @param FuncNameLine: Line number of function name 240# @param FuncNameOffset: Offset of function name 241# 242# @return ParamIdList: A list of parameters 243# 244def GetParamList(FuncDeclarator, FuncNameLine = 0, FuncNameOffset = 0): 245 ParamIdList = [] 246 DeclSplitList = FuncDeclarator.split('(') 247 if len(DeclSplitList) < 2: 248 return ParamIdList 249 FuncName = DeclSplitList[0] 250 ParamStr = DeclSplitList[1].rstrip(')') 251 LineSkipped = 0 252 OffsetSkipped = 0 253 Start = 0 254 while FuncName.find('\n', Start) != -1: 255 LineSkipped += 1 256 OffsetSkipped = 0 257 Start += FuncName.find('\n', Start) 258 Start += 1 259 OffsetSkipped += len(FuncName[Start:]) 260 OffsetSkipped += 1 #skip '(' 261 ParamBeginLine = FuncNameLine + LineSkipped 262 ParamBeginOffset = OffsetSkipped 263 for p in ParamStr.split(','): 264 ListP = p.split() 265 if len(ListP) == 0: 266 continue 267 ParamName = ListP[-1] 268 DeclText = ParamName.strip() 269 RightSpacePos = p.rfind(ParamName) 270 ParamModifier = p[0:RightSpacePos] 271 if ParamName == 'OPTIONAL': 272 if ParamModifier == '': 273 ParamModifier += ' ' + 'OPTIONAL' 274 DeclText = '' 275 else: 276 ParamName = ListP[-2] 277 DeclText = ParamName.strip() 278 RightSpacePos = p.rfind(ParamName) 279 ParamModifier = p[0:RightSpacePos] 280 ParamModifier += 'OPTIONAL' 281 while DeclText.startswith('*'): 282 ParamModifier += ' ' + '*' 283 DeclText = DeclText.lstrip('*').strip() 284 ParamName = DeclText 285 286 Start = 0 287 while p.find('\n', Start) != -1: 288 LineSkipped += 1 289 OffsetSkipped = 0 290 Start += p.find('\n', Start) 291 Start += 1 292 OffsetSkipped += len(p[Start:]) 293 294 ParamEndLine = ParamBeginLine + LineSkipped 295 ParamEndOffset = OffsetSkipped 296 IdParam = DataClass.IdentifierClass(-1, ParamModifier, '', ParamName, '', DataClass.MODEL_IDENTIFIER_PARAMETER, -1, -1, ParamBeginLine, ParamBeginOffset, ParamEndLine, ParamEndOffset) 297 ParamIdList.append(IdParam) 298 ParamBeginLine = ParamEndLine 299 ParamBeginOffset = OffsetSkipped + 1 #skip ',' 300 301 return ParamIdList 302 303## GetFunctionList() 304# 305# Get a list of functions 306# 307# @return FuncObjList: A list of function objects 308# 309def GetFunctionList(): 310 FuncObjList = [] 311 for FuncDef in FileProfile.FunctionDefinitionList: 312 ParamIdList = [] 313 DeclText = FuncDef.Declarator.strip() 314 while DeclText.startswith('*'): 315 FuncDef.Modifier += '*' 316 DeclText = DeclText.lstrip('*').strip() 317 318 FuncDef.Declarator = FuncDef.Declarator.lstrip('*') 319 DeclSplitList = FuncDef.Declarator.split('(') 320 if len(DeclSplitList) < 2: 321 continue 322 323 FuncName = DeclSplitList[0] 324 FuncNamePartList = FuncName.split() 325 if len(FuncNamePartList) > 1: 326 FuncName = FuncNamePartList[-1] 327 Index = 0 328 while Index < len(FuncNamePartList) - 1: 329 FuncDef.Modifier += ' ' + FuncNamePartList[Index] 330 Index += 1 331 332 FuncObj = DataClass.FunctionClass(-1, FuncDef.Declarator, FuncDef.Modifier, FuncName.strip(), '', FuncDef.StartPos[0],FuncDef.StartPos[1],FuncDef.EndPos[0],FuncDef.EndPos[1], FuncDef.LeftBracePos[0], FuncDef.LeftBracePos[1], -1, ParamIdList, []) 333 FuncObjList.append(FuncObj) 334 335 return FuncObjList 336 337## CreateCCodeDB() method 338# 339# Create database for all c code 340# 341# @param FileNameList: A list of all c code file names 342# 343def CreateCCodeDB(FileNameList): 344 FileObjList = [] 345 ParseErrorFileList = [] 346 ParsedFiles = {} 347 for FullName in FileNameList: 348 if os.path.splitext(FullName)[1] in ('.h', '.c'): 349 if FullName.lower() in ParsedFiles: 350 continue 351 ParsedFiles[FullName.lower()] = 1 352 EdkLogger.info("Parsing " + FullName) 353 model = FullName.endswith('c') and DataClass.MODEL_FILE_C or DataClass.MODEL_FILE_H 354 collector = CodeFragmentCollector.CodeFragmentCollector(FullName) 355 try: 356 collector.ParseFile() 357 except: 358 ParseErrorFileList.append(FullName) 359 BaseName = os.path.basename(FullName) 360 DirName = os.path.dirname(FullName) 361 Ext = os.path.splitext(BaseName)[1].lstrip('.') 362 ModifiedTime = os.path.getmtime(FullName) 363 FileObj = DataClass.FileClass(-1, BaseName, Ext, DirName, FullName, model, ModifiedTime, GetFunctionList(), GetIdentifierList(), []) 364 FileObjList.append(FileObj) 365 collector.CleanFileProfileBuffer() 366 367 if len(ParseErrorFileList) > 0: 368 EdkLogger.info("Found unrecoverable error during parsing:\n\t%s\n" % "\n\t".join(ParseErrorFileList)) 369 370 Db = EotGlobalData.gDb 371 for file in FileObjList: 372 Db.InsertOneFile(file) 373 374 Db.UpdateIdentifierBelongsToFunction() 375 376## 377# 378# This acts like the main() function for the script, unless it is 'import'ed into another 379# script. 380# 381if __name__ == '__main__': 382 383 EdkLogger.Initialize() 384 EdkLogger.SetLevel(EdkLogger.QUIET) 385 CollectSourceCodeDataIntoDB(sys.argv[1]) 386 387 print 'Done!' 388