• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1## @file
2# This file is used to define comment parsing interface
3#
4# Copyright (c) 2011 - 2014, Intel Corporation. All rights reserved.<BR>
5#
6# This program and the accompanying materials are licensed and made available
7# under the terms and conditions of the BSD License which accompanies this
8# distribution. The full text of the license may be found at
9# http://opensource.org/licenses/bsd-license.php
10#
11# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
13#
14
15'''
16CommentParsing
17'''
18
19##
20# Import Modules
21#
22import re
23
24from Library.String import GetSplitValueList
25from Library.String import CleanString2
26from Library.DataType import HEADER_COMMENT_NOT_STARTED
27from Library.DataType import TAB_COMMENT_SPLIT
28from Library.DataType import HEADER_COMMENT_LICENSE
29from Library.DataType import HEADER_COMMENT_ABSTRACT
30from Library.DataType import HEADER_COMMENT_COPYRIGHT
31from Library.DataType import HEADER_COMMENT_DESCRIPTION
32from Library.DataType import TAB_SPACE_SPLIT
33from Library.DataType import TAB_COMMA_SPLIT
34from Library.DataType import SUP_MODULE_LIST
35from Library.DataType import TAB_VALUE_SPLIT
36from Library.DataType import TAB_PCD_VALIDRANGE
37from Library.DataType import TAB_PCD_VALIDLIST
38from Library.DataType import TAB_PCD_EXPRESSION
39from Library.DataType import TAB_PCD_PROMPT
40from Library.DataType import TAB_CAPHEX_START
41from Library.DataType import TAB_HEX_START
42from Library.DataType import PCD_ERR_CODE_MAX_SIZE
43from Library.ExpressionValidate import IsValidRangeExpr
44from Library.ExpressionValidate import IsValidListExpr
45from Library.ExpressionValidate import IsValidLogicalExpr
46from Object.POM.CommonObject import TextObject
47from Object.POM.CommonObject import PcdErrorObject
48import Logger.Log as Logger
49from Logger.ToolError import FORMAT_INVALID
50from Logger.ToolError import FORMAT_NOT_SUPPORTED
51from Logger import StringTable as ST
52
53## ParseHeaderCommentSection
54#
55# Parse Header comment section lines, extract Abstract, Description, Copyright
56# , License lines
57#
58# @param CommentList:   List of (Comment, LineNumber)
59# @param FileName:      FileName of the comment
60#
61def ParseHeaderCommentSection(CommentList, FileName = None, IsBinaryHeader = False):
62    Abstract = ''
63    Description = ''
64    Copyright = ''
65    License = ''
66    EndOfLine = "\n"
67    if IsBinaryHeader:
68        STR_HEADER_COMMENT_START = "@BinaryHeader"
69    else:
70        STR_HEADER_COMMENT_START = "@file"
71    HeaderCommentStage = HEADER_COMMENT_NOT_STARTED
72
73    #
74    # first find the last copyright line
75    #
76    Last = 0
77    for Index in xrange(len(CommentList)-1, 0, -1):
78        Line = CommentList[Index][0]
79        if _IsCopyrightLine(Line):
80            Last = Index
81            break
82
83    for Item in CommentList:
84        Line = Item[0]
85        LineNo = Item[1]
86
87        if not Line.startswith(TAB_COMMENT_SPLIT) and Line:
88            Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_INVALID_COMMENT_FORMAT, FileName, Item[1])
89        Comment = CleanString2(Line)[1]
90        Comment = Comment.strip()
91        #
92        # if there are blank lines between License or Description, keep them as they would be
93        # indication of different block; or in the position that Abstract should be, also keep it
94        # as it indicates that no abstract
95        #
96        if not Comment and HeaderCommentStage not in [HEADER_COMMENT_LICENSE, \
97                                                      HEADER_COMMENT_DESCRIPTION, HEADER_COMMENT_ABSTRACT]:
98            continue
99
100        if HeaderCommentStage == HEADER_COMMENT_NOT_STARTED:
101            if Comment.startswith(STR_HEADER_COMMENT_START):
102                HeaderCommentStage = HEADER_COMMENT_ABSTRACT
103            else:
104                License += Comment + EndOfLine
105        else:
106            if HeaderCommentStage == HEADER_COMMENT_ABSTRACT:
107                #
108                # in case there is no abstract and description
109                #
110                if not Comment:
111                    HeaderCommentStage = HEADER_COMMENT_DESCRIPTION
112                elif _IsCopyrightLine(Comment):
113                    Result, ErrMsg = _ValidateCopyright(Comment)
114                    ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
115                    Copyright += Comment + EndOfLine
116                    HeaderCommentStage = HEADER_COMMENT_COPYRIGHT
117                else:
118                    Abstract += Comment + EndOfLine
119                    HeaderCommentStage = HEADER_COMMENT_DESCRIPTION
120            elif HeaderCommentStage == HEADER_COMMENT_DESCRIPTION:
121                #
122                # in case there is no description
123                #
124                if _IsCopyrightLine(Comment):
125                    Result, ErrMsg = _ValidateCopyright(Comment)
126                    ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
127                    Copyright += Comment + EndOfLine
128                    HeaderCommentStage = HEADER_COMMENT_COPYRIGHT
129                else:
130                    Description += Comment + EndOfLine
131            elif HeaderCommentStage == HEADER_COMMENT_COPYRIGHT:
132                if _IsCopyrightLine(Comment):
133                    Result, ErrMsg = _ValidateCopyright(Comment)
134                    ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
135                    Copyright += Comment + EndOfLine
136                else:
137                    #
138                    # Contents after copyright line are license, those non-copyright lines in between
139                    # copyright line will be discarded
140                    #
141                    if LineNo > Last:
142                        if License:
143                            License += EndOfLine
144                        License += Comment + EndOfLine
145                        HeaderCommentStage = HEADER_COMMENT_LICENSE
146            else:
147                if not Comment and not License:
148                    continue
149                License += Comment + EndOfLine
150
151    return Abstract.strip(), Description.strip(), Copyright.strip(), License.strip()
152
153## _IsCopyrightLine
154# check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright"
155# followed by zero or more white space characters followed by a "(" character
156#
157# @param LineContent:  the line need to be checked
158# @return: True if current line is copyright line, False else
159#
160def _IsCopyrightLine (LineContent):
161    LineContent = LineContent.upper()
162    Result = False
163
164    ReIsCopyrightRe = re.compile(r"""(^|\s)COPYRIGHT *\(""", re.DOTALL)
165    if ReIsCopyrightRe.search(LineContent):
166        Result = True
167
168    return Result
169
170## ParseGenericComment
171#
172# @param GenericComment: Generic comment list, element of
173#                        (CommentLine, LineNum)
174# @param ContainerFile:  Input value for filename of Dec file
175#
176def ParseGenericComment (GenericComment, ContainerFile=None, SkipTag=None):
177    if ContainerFile:
178        pass
179    HelpTxt = None
180    HelpStr = ''
181
182    for Item in GenericComment:
183        CommentLine = Item[0]
184        Comment = CleanString2(CommentLine)[1]
185        if SkipTag is not None and Comment.startswith(SkipTag):
186            Comment = Comment.replace(SkipTag, '', 1)
187        HelpStr += Comment + '\n'
188
189    if HelpStr:
190        HelpTxt = TextObject()
191        if HelpStr.endswith('\n') and not HelpStr.endswith('\n\n') and HelpStr != '\n':
192            HelpStr = HelpStr[:-1]
193        HelpTxt.SetString(HelpStr)
194
195    return HelpTxt
196
197## ParsePcdErrorCode
198#
199# @param Value: original ErrorCode value
200# @param ContainerFile: Input value for filename of Dec file
201# @param LineNum: Line Num
202#
203def ParsePcdErrorCode (Value = None, ContainerFile = None, LineNum = None):
204    try:
205        if Value.strip().startswith((TAB_HEX_START, TAB_CAPHEX_START)):
206            Base = 16
207        else:
208            Base = 10
209        ErrorCode = long(Value, Base)
210        if ErrorCode > PCD_ERR_CODE_MAX_SIZE or ErrorCode < 0:
211            Logger.Error('Parser',
212                        FORMAT_NOT_SUPPORTED,
213                        "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value,
214                        File = ContainerFile,
215                        Line = LineNum)
216        #
217        # To delete the tailing 'L'
218        #
219        return hex(ErrorCode)[:-1]
220    except ValueError, XStr:
221        if XStr:
222            pass
223        Logger.Error('Parser',
224                    FORMAT_NOT_SUPPORTED,
225                    "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value,
226                    File = ContainerFile,
227                    Line = LineNum)
228
229## ParseDecPcdGenericComment
230#
231# @param GenericComment: Generic comment list, element of (CommentLine,
232#                         LineNum)
233# @param ContainerFile:  Input value for filename of Dec file
234#
235def ParseDecPcdGenericComment (GenericComment, ContainerFile, TokenSpaceGuidCName, CName, MacroReplaceDict):
236    HelpStr = ''
237    PromptStr = ''
238    PcdErr = None
239    PcdErrList = []
240    ValidValueNum = 0
241    ValidRangeNum = 0
242    ExpressionNum = 0
243
244    for (CommentLine, LineNum) in GenericComment:
245        Comment = CleanString2(CommentLine)[1]
246        #
247        # To replace Macro
248        #
249        MACRO_PATTERN = '[\t\s]*\$\([A-Z][_A-Z0-9]*\)'
250        MatchedStrs =  re.findall(MACRO_PATTERN, Comment)
251        for MatchedStr in MatchedStrs:
252            if MatchedStr:
253                Macro = MatchedStr.strip().lstrip('$(').rstrip(')').strip()
254                if Macro in MacroReplaceDict:
255                    Comment = Comment.replace(MatchedStr, MacroReplaceDict[Macro])
256        if Comment.startswith(TAB_PCD_VALIDRANGE):
257            if ValidValueNum > 0 or ExpressionNum > 0:
258                Logger.Error('Parser',
259                             FORMAT_NOT_SUPPORTED,
260                             ST.WRN_MULTI_PCD_RANGES,
261                             File = ContainerFile,
262                             Line = LineNum)
263            else:
264                PcdErr = PcdErrorObject()
265                PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)
266                PcdErr.SetCName(CName)
267                PcdErr.SetFileLine(Comment)
268                PcdErr.SetLineNum(LineNum)
269                ValidRangeNum += 1
270            ValidRange = Comment.replace(TAB_PCD_VALIDRANGE, "", 1).strip()
271            Valid, Cause = _CheckRangeExpression(ValidRange)
272            if Valid:
273                ValueList = ValidRange.split(TAB_VALUE_SPLIT)
274                if len(ValueList) > 1:
275                    PcdErr.SetValidValueRange((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())
276                    PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))
277                else:
278                    PcdErr.SetValidValueRange(ValidRange)
279                PcdErrList.append(PcdErr)
280            else:
281                Logger.Error("Parser",
282                         FORMAT_NOT_SUPPORTED,
283                         Cause,
284                         ContainerFile,
285                         LineNum)
286        elif Comment.startswith(TAB_PCD_VALIDLIST):
287            if ValidRangeNum > 0 or ExpressionNum > 0:
288                Logger.Error('Parser',
289                             FORMAT_NOT_SUPPORTED,
290                             ST.WRN_MULTI_PCD_RANGES,
291                             File = ContainerFile,
292                             Line = LineNum)
293            elif ValidValueNum > 0:
294                Logger.Error('Parser',
295                             FORMAT_NOT_SUPPORTED,
296                             ST.WRN_MULTI_PCD_VALIDVALUE,
297                             File = ContainerFile,
298                             Line = LineNum)
299            else:
300                PcdErr = PcdErrorObject()
301                PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)
302                PcdErr.SetCName(CName)
303                PcdErr.SetFileLine(Comment)
304                PcdErr.SetLineNum(LineNum)
305                ValidValueNum += 1
306                ValidValueExpr = Comment.replace(TAB_PCD_VALIDLIST, "", 1).strip()
307            Valid, Cause = _CheckListExpression(ValidValueExpr)
308            if Valid:
309                ValidValue = Comment.replace(TAB_PCD_VALIDLIST, "", 1).replace(TAB_COMMA_SPLIT, TAB_SPACE_SPLIT)
310                ValueList = ValidValue.split(TAB_VALUE_SPLIT)
311                if len(ValueList) > 1:
312                    PcdErr.SetValidValue((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())
313                    PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))
314                else:
315                    PcdErr.SetValidValue(ValidValue)
316                PcdErrList.append(PcdErr)
317            else:
318                Logger.Error("Parser",
319                         FORMAT_NOT_SUPPORTED,
320                         Cause,
321                         ContainerFile,
322                         LineNum)
323        elif Comment.startswith(TAB_PCD_EXPRESSION):
324            if ValidRangeNum > 0 or ValidValueNum > 0:
325                Logger.Error('Parser',
326                             FORMAT_NOT_SUPPORTED,
327                             ST.WRN_MULTI_PCD_RANGES,
328                             File = ContainerFile,
329                             Line = LineNum)
330            else:
331                PcdErr = PcdErrorObject()
332                PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)
333                PcdErr.SetCName(CName)
334                PcdErr.SetFileLine(Comment)
335                PcdErr.SetLineNum(LineNum)
336                ExpressionNum += 1
337            Expression = Comment.replace(TAB_PCD_EXPRESSION, "", 1).strip()
338            Valid, Cause = _CheckExpression(Expression)
339            if Valid:
340                ValueList = Expression.split(TAB_VALUE_SPLIT)
341                if len(ValueList) > 1:
342                    PcdErr.SetExpression((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())
343                    PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))
344                else:
345                    PcdErr.SetExpression(Expression)
346                PcdErrList.append(PcdErr)
347            else:
348                Logger.Error("Parser",
349                         FORMAT_NOT_SUPPORTED,
350                         Cause,
351                         ContainerFile,
352                         LineNum)
353        elif Comment.startswith(TAB_PCD_PROMPT):
354            if PromptStr:
355                Logger.Error('Parser',
356                             FORMAT_NOT_SUPPORTED,
357                             ST.WRN_MULTI_PCD_PROMPT,
358                             File = ContainerFile,
359                             Line = LineNum)
360            PromptStr = Comment.replace(TAB_PCD_PROMPT, "", 1).strip()
361        else:
362            if Comment:
363                HelpStr += Comment + '\n'
364
365    #
366    # remove the last EOL if the comment is of format 'FOO\n'
367    #
368    if HelpStr.endswith('\n'):
369        if HelpStr != '\n' and not HelpStr.endswith('\n\n'):
370            HelpStr = HelpStr[:-1]
371
372    return HelpStr, PcdErrList, PromptStr
373
374## ParseDecPcdTailComment
375#
376# @param TailCommentList:    Tail comment list of Pcd, item of format (Comment, LineNum)
377# @param ContainerFile:      Input value for filename of Dec file
378# @retVal SupModuleList:  The supported module type list detected
379# @retVal HelpStr:  The generic help text string detected
380#
381def ParseDecPcdTailComment (TailCommentList, ContainerFile):
382    assert(len(TailCommentList) == 1)
383    TailComment = TailCommentList[0][0]
384    LineNum = TailCommentList[0][1]
385
386    Comment = TailComment.lstrip(" #")
387
388    ReFindFirstWordRe = re.compile(r"""^([^ #]*)""", re.DOTALL)
389
390    #
391    # get first word and compare with SUP_MODULE_LIST
392    #
393    MatchObject = ReFindFirstWordRe.match(Comment)
394    if not (MatchObject and MatchObject.group(1) in SUP_MODULE_LIST):
395        return None, Comment
396
397    #
398    # parse line, it must have supported module type specified
399    #
400    if Comment.find(TAB_COMMENT_SPLIT) == -1:
401        Comment += TAB_COMMENT_SPLIT
402    SupMode, HelpStr = GetSplitValueList(Comment, TAB_COMMENT_SPLIT, 1)
403    SupModuleList = []
404    for Mod in GetSplitValueList(SupMode, TAB_SPACE_SPLIT):
405        if not Mod:
406            continue
407        elif Mod not in SUP_MODULE_LIST:
408            Logger.Error("UPT",
409                         FORMAT_INVALID,
410                         ST.WRN_INVALID_MODULE_TYPE%Mod,
411                         ContainerFile,
412                         LineNum)
413        else:
414            SupModuleList.append(Mod)
415
416    return SupModuleList, HelpStr
417
418## _CheckListExpression
419#
420# @param Expression: Pcd value list expression
421#
422def _CheckListExpression(Expression):
423    ListExpr = ''
424    if TAB_VALUE_SPLIT in Expression:
425        ListExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]
426    else:
427        ListExpr = Expression
428
429    return IsValidListExpr(ListExpr)
430
431## _CheckExpreesion
432#
433# @param Expression: Pcd value expression
434#
435def _CheckExpression(Expression):
436    Expr = ''
437    if TAB_VALUE_SPLIT in Expression:
438        Expr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]
439    else:
440        Expr = Expression
441    return IsValidLogicalExpr(Expr, True)
442
443## _CheckRangeExpression
444#
445# @param Expression:    Pcd range expression
446#
447def _CheckRangeExpression(Expression):
448    RangeExpr = ''
449    if TAB_VALUE_SPLIT in Expression:
450        RangeExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]
451    else:
452        RangeExpr = Expression
453
454    return IsValidRangeExpr(RangeExpr)
455
456## ValidateCopyright
457#
458#
459#
460def ValidateCopyright(Result, ErrType, FileName, LineNo, ErrMsg):
461    if not Result:
462        Logger.Warn("\nUPT", ErrType, FileName, LineNo, ErrMsg)
463
464## _ValidateCopyright
465#
466# @param Line:    Line that contains copyright information, # stripped
467#
468# @retval Result: True if line is conformed to Spec format, False else
469# @retval ErrMsg: the detailed error description
470#
471def _ValidateCopyright(Line):
472    if Line:
473        pass
474    Result = True
475    ErrMsg = ''
476
477    return Result, ErrMsg
478
479def GenerateTokenList (Comment):
480    #
481    # Tokenize Comment using '#' and ' ' as token seperators
482    #
483    RelplacedComment = None
484    while Comment != RelplacedComment:
485        RelplacedComment = Comment
486        Comment = Comment.replace('##', '#').replace('  ', ' ').replace(' ', '#').strip('# ')
487    return Comment.split('#')
488
489
490#
491# Comment       - Comment to parse
492# TypeTokens    - A dictionary of type token synonyms
493# RemoveTokens  - A list of tokens to remove from help text
494# ParseVariable - True for parsing [Guids].  Otherwise False
495#
496def ParseComment (Comment, UsageTokens, TypeTokens, RemoveTokens, ParseVariable):
497    #
498    # Initialize return values
499    #
500    Usage = None
501    Type = None
502    String = None
503
504    Comment = Comment[0]
505
506    NumTokens = 2
507    if ParseVariable:
508        #
509        # Remove white space around first instance of ':' from Comment if 'Variable'
510        # is in front of ':' and Variable is the 1st or 2nd token in Comment.
511        #
512        List = Comment.split(':', 1)
513        if len(List) > 1:
514            SubList = GenerateTokenList (List[0].strip())
515            if len(SubList) in [1, 2] and SubList[-1] == 'Variable':
516                if List[1].strip().find('L"') == 0:
517                    Comment = List[0].strip() + ':' + List[1].strip()
518
519        #
520        # Remove first instance of L"<VariableName> from Comment and put into String
521        # if and only if L"<VariableName>" is the 1st token, the 2nd token.  Or
522        # L"<VariableName>" is the third token immediately following 'Variable:'.
523        #
524        End = -1
525        Start = Comment.find('Variable:L"')
526        if Start >= 0:
527            String = Comment[Start + 9:]
528            End = String[2:].find('"')
529        else:
530            Start = Comment.find('L"')
531            if Start >= 0:
532                String = Comment[Start:]
533                End = String[2:].find('"')
534        if End >= 0:
535            SubList = GenerateTokenList (Comment[:Start])
536            if len(SubList) < 2:
537                Comment = Comment[:Start] + String[End + 3:]
538                String = String[:End + 3]
539                Type = 'Variable'
540                NumTokens = 1
541
542    #
543    # Initialze HelpText to Comment.
544    # Content will be remove from HelpText as matching tokens are found
545    #
546    HelpText = Comment
547
548    #
549    # Tokenize Comment using '#' and ' ' as token seperators
550    #
551    List = GenerateTokenList (Comment)
552
553    #
554    # Search first two tokens for Usage and Type and remove any matching tokens
555    # from HelpText
556    #
557    for Token in List[0:NumTokens]:
558        if Usage == None and Token in UsageTokens:
559            Usage = UsageTokens[Token]
560            HelpText = HelpText.replace(Token, '')
561    if Usage != None or not ParseVariable:
562        for Token in List[0:NumTokens]:
563            if Type == None and Token in TypeTokens:
564                Type = TypeTokens[Token]
565                HelpText = HelpText.replace(Token, '')
566            if Usage != None:
567                for Token in List[0:NumTokens]:
568                    if Token in RemoveTokens:
569                        HelpText = HelpText.replace(Token, '')
570
571    #
572    # If no Usage token is present and set Usage to UNDEFINED
573    #
574    if Usage == None:
575        Usage = 'UNDEFINED'
576
577    #
578    # If no Type token is present and set Type to UNDEFINED
579    #
580    if Type == None:
581        Type = 'UNDEFINED'
582
583    #
584    # If Type is not 'Variable:', then set String to None
585    #
586    if Type != 'Variable':
587        String = None
588
589    #
590    # Strip ' ' and '#' from the beginning of HelpText
591    # If HelpText is an empty string after all parsing is
592    # complete then set HelpText to None
593    #
594    HelpText = HelpText.lstrip('# ')
595    if HelpText == '':
596        HelpText = None
597
598    #
599    # Return parsing results
600    #
601    return Usage, Type, String, HelpText
602