• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1## @file
2# preprocess source file
3#
4#  Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR>
5#
6#  This program and the accompanying materials
7#  are licensed and made available under the terms and conditions of the BSD License
8#  which accompanies this distribution.  The full text of the license may be found at
9#  http://opensource.org/licenses/bsd-license.php
10#
11#  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12#  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
13#
14
15##
16# Import Modules
17#
18import re
19import Common.LongFilePathOs as os
20import sys
21
22import antlr3
23from CLexer import CLexer
24from CParser import CParser
25
26import FileProfile
27from CodeFragment import PP_Directive
28from ParserWarning import Warning
29
30
31##define T_CHAR_SPACE                ' '
32##define T_CHAR_NULL                 '\0'
33##define T_CHAR_CR                   '\r'
34##define T_CHAR_TAB                  '\t'
35##define T_CHAR_LF                   '\n'
36##define T_CHAR_SLASH                '/'
37##define T_CHAR_BACKSLASH            '\\'
38##define T_CHAR_DOUBLE_QUOTE         '\"'
39##define T_CHAR_SINGLE_QUOTE         '\''
40##define T_CHAR_STAR                 '*'
41##define T_CHAR_HASH                 '#'
42
43(T_CHAR_SPACE, T_CHAR_NULL, T_CHAR_CR, T_CHAR_TAB, T_CHAR_LF, T_CHAR_SLASH, \
44T_CHAR_BACKSLASH, T_CHAR_DOUBLE_QUOTE, T_CHAR_SINGLE_QUOTE, T_CHAR_STAR, T_CHAR_HASH) = \
45(' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')
46
47SEPERATOR_TUPLE = ('=', '|', ',', '{', '}')
48
49(T_COMMENT_TWO_SLASH, T_COMMENT_SLASH_STAR) = (0, 1)
50
51(T_PP_INCLUDE, T_PP_DEFINE, T_PP_OTHERS) = (0, 1, 2)
52
53## The collector for source code fragments.
54#
55# PreprocessFile method should be called prior to ParseFile
56#
57# GetNext*** procedures mean these procedures will get next token first, then make judgement.
58# Get*** procedures mean these procedures will make judgement on current token only.
59#
60class CodeFragmentCollector:
61    ## The constructor
62    #
63    #   @param  self        The object pointer
64    #   @param  FileName    The file that to be parsed
65    #
66    def __init__(self, FileName):
67        self.Profile = FileProfile.FileProfile(FileName)
68        self.Profile.FileLinesList.append(T_CHAR_LF)
69        self.FileName = FileName
70        self.CurrentLineNumber = 1
71        self.CurrentOffsetWithinLine = 0
72
73        self.__Token = ""
74        self.__SkippedChars = ""
75
76    ## __IsWhiteSpace() method
77    #
78    #   Whether char at current FileBufferPos is whitespace
79    #
80    #   @param  self        The object pointer
81    #   @param  Char        The char to test
82    #   @retval True        The char is a kind of white space
83    #   @retval False       The char is NOT a kind of white space
84    #
85    def __IsWhiteSpace(self, Char):
86        if Char in (T_CHAR_NULL, T_CHAR_CR, T_CHAR_SPACE, T_CHAR_TAB, T_CHAR_LF):
87            return True
88        else:
89            return False
90
91    ## __SkipWhiteSpace() method
92    #
93    #   Skip white spaces from current char, return number of chars skipped
94    #
95    #   @param  self        The object pointer
96    #   @retval Count       The number of chars skipped
97    #
98    def __SkipWhiteSpace(self):
99        Count = 0
100        while not self.__EndOfFile():
101            Count += 1
102            if self.__CurrentChar() in (T_CHAR_NULL, T_CHAR_CR, T_CHAR_LF, T_CHAR_SPACE, T_CHAR_TAB):
103                self.__SkippedChars += str(self.__CurrentChar())
104                self.__GetOneChar()
105
106            else:
107                Count = Count - 1
108                return Count
109
110    ## __EndOfFile() method
111    #
112    #   Judge current buffer pos is at file end
113    #
114    #   @param  self        The object pointer
115    #   @retval True        Current File buffer position is at file end
116    #   @retval False       Current File buffer position is NOT at file end
117    #
118    def __EndOfFile(self):
119        NumberOfLines = len(self.Profile.FileLinesList)
120        SizeOfLastLine = len(self.Profile.FileLinesList[-1])
121        if self.CurrentLineNumber == NumberOfLines and self.CurrentOffsetWithinLine >= SizeOfLastLine - 1:
122            return True
123        elif self.CurrentLineNumber > NumberOfLines:
124            return True
125        else:
126            return False
127
128    ## __EndOfLine() method
129    #
130    #   Judge current buffer pos is at line end
131    #
132    #   @param  self        The object pointer
133    #   @retval True        Current File buffer position is at line end
134    #   @retval False       Current File buffer position is NOT at line end
135    #
136    def __EndOfLine(self):
137        SizeOfCurrentLine = len(self.Profile.FileLinesList[self.CurrentLineNumber - 1])
138        if self.CurrentOffsetWithinLine >= SizeOfCurrentLine - 1:
139            return True
140        else:
141            return False
142
143    ## Rewind() method
144    #
145    #   Reset file data buffer to the initial state
146    #
147    #   @param  self        The object pointer
148    #
149    def Rewind(self):
150        self.CurrentLineNumber = 1
151        self.CurrentOffsetWithinLine = 0
152
153    ## __UndoOneChar() method
154    #
155    #   Go back one char in the file buffer
156    #
157    #   @param  self        The object pointer
158    #   @retval True        Successfully go back one char
159    #   @retval False       Not able to go back one char as file beginning reached
160    #
161    def __UndoOneChar(self):
162
163        if self.CurrentLineNumber == 1 and self.CurrentOffsetWithinLine == 0:
164            return False
165        elif self.CurrentOffsetWithinLine == 0:
166            self.CurrentLineNumber -= 1
167            self.CurrentOffsetWithinLine = len(self.__CurrentLine()) - 1
168        else:
169            self.CurrentOffsetWithinLine -= 1
170        return True
171
172    ## __GetOneChar() method
173    #
174    #   Move forward one char in the file buffer
175    #
176    #   @param  self        The object pointer
177    #
178    def __GetOneChar(self):
179        if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
180                self.CurrentLineNumber += 1
181                self.CurrentOffsetWithinLine = 0
182        else:
183                self.CurrentOffsetWithinLine += 1
184
185    ## __CurrentChar() method
186    #
187    #   Get the char pointed to by the file buffer pointer
188    #
189    #   @param  self        The object pointer
190    #   @retval Char        Current char
191    #
192    def __CurrentChar(self):
193        CurrentChar = self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine]
194
195        return CurrentChar
196
197    ## __NextChar() method
198    #
199    #   Get the one char pass the char pointed to by the file buffer pointer
200    #
201    #   @param  self        The object pointer
202    #   @retval Char        Next char
203    #
204    def __NextChar(self):
205        if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
206            return self.Profile.FileLinesList[self.CurrentLineNumber][0]
207        else:
208            return self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine + 1]
209
210    ## __SetCurrentCharValue() method
211    #
212    #   Modify the value of current char
213    #
214    #   @param  self        The object pointer
215    #   @param  Value       The new value of current char
216    #
217    def __SetCurrentCharValue(self, Value):
218        self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine] = Value
219
220    ## __SetCharValue() method
221    #
222    #   Modify the value of current char
223    #
224    #   @param  self        The object pointer
225    #   @param  Value       The new value of current char
226    #
227    def __SetCharValue(self, Line, Offset, Value):
228        self.Profile.FileLinesList[Line - 1][Offset] = Value
229
230    ## __CurrentLine() method
231    #
232    #   Get the list that contains current line contents
233    #
234    #   @param  self        The object pointer
235    #   @retval List        current line contents
236    #
237    def __CurrentLine(self):
238        return self.Profile.FileLinesList[self.CurrentLineNumber - 1]
239
240    ## __InsertComma() method
241    #
242    #   Insert ',' to replace PP
243    #
244    #   @param  self        The object pointer
245    #   @retval List        current line contents
246    #
247    def __InsertComma(self, Line):
248
249
250        if self.Profile.FileLinesList[Line - 1][0] != T_CHAR_HASH:
251            BeforeHashPart = str(self.Profile.FileLinesList[Line - 1]).split(T_CHAR_HASH)[0]
252            if BeforeHashPart.rstrip().endswith(T_CHAR_COMMA) or BeforeHashPart.rstrip().endswith(';'):
253                return
254
255        if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(','):
256            return
257
258        if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(';'):
259            return
260
261        if str(self.Profile.FileLinesList[Line]).lstrip().startswith(',') or str(self.Profile.FileLinesList[Line]).lstrip().startswith(';'):
262            return
263
264        self.Profile.FileLinesList[Line - 1].insert(self.CurrentOffsetWithinLine, ',')
265
266    ## PreprocessFileWithClear() method
267    #
268    # Run a preprocess for the file to clean all comments
269    #
270    # @param  self        The object pointer
271    #
272    def PreprocessFileWithClear(self):
273
274        self.Rewind()
275        InComment = False
276        DoubleSlashComment = False
277        HashComment = False
278        PPExtend = False
279        PPDirectiveObj = None
280        # HashComment in quoted string " " is ignored.
281        InString = False
282        InCharLiteral = False
283
284        self.Profile.FileLinesList = [list(s) for s in self.Profile.FileLinesListFromFile]
285        while not self.__EndOfFile():
286
287            if not InComment and self.__CurrentChar() == T_CHAR_DOUBLE_QUOTE:
288                InString = not InString
289
290            if not InComment and self.__CurrentChar() == T_CHAR_SINGLE_QUOTE:
291                InCharLiteral = not InCharLiteral
292            # meet new line, then no longer in a comment for // and '#'
293            if self.__CurrentChar() == T_CHAR_LF:
294                if HashComment and PPDirectiveObj != None:
295                    if PPDirectiveObj.Content.rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
296                        PPDirectiveObj.Content += T_CHAR_LF
297                        PPExtend = True
298                    else:
299                        PPExtend = False
300
301                EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
302
303                if InComment and DoubleSlashComment:
304                    InComment = False
305                    DoubleSlashComment = False
306
307                if InComment and HashComment and not PPExtend:
308                    InComment = False
309                    HashComment = False
310                    PPDirectiveObj.Content += T_CHAR_LF
311                    PPDirectiveObj.EndPos = EndLinePos
312                    FileProfile.PPDirectiveList.append(PPDirectiveObj)
313                    PPDirectiveObj = None
314
315                if InString or InCharLiteral:
316                    CurrentLine = "".join(self.__CurrentLine())
317                    if CurrentLine.rstrip(T_CHAR_LF).rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
318                        SlashIndex = CurrentLine.rindex(T_CHAR_BACKSLASH)
319                        self.__SetCharValue(self.CurrentLineNumber, SlashIndex, T_CHAR_SPACE)
320
321                self.CurrentLineNumber += 1
322                self.CurrentOffsetWithinLine = 0
323            # check for */ comment end
324            elif InComment and not DoubleSlashComment and not HashComment and self.__CurrentChar() == T_CHAR_STAR and self.__NextChar() == T_CHAR_SLASH:
325
326                self.__SetCurrentCharValue(T_CHAR_SPACE)
327                self.__GetOneChar()
328                self.__SetCurrentCharValue(T_CHAR_SPACE)
329                self.__GetOneChar()
330                InComment = False
331            # set comments to spaces
332            elif InComment:
333                if HashComment:
334                    # // follows hash PP directive
335                    if self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
336                        InComment = False
337                        HashComment = False
338                        PPDirectiveObj.EndPos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine - 1)
339                        FileProfile.PPDirectiveList.append(PPDirectiveObj)
340                        PPDirectiveObj = None
341                        continue
342                    else:
343                        PPDirectiveObj.Content += self.__CurrentChar()
344
345                self.__SetCurrentCharValue(T_CHAR_SPACE)
346                self.__GetOneChar()
347            # check for // comment
348            elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
349                InComment = True
350                DoubleSlashComment = True
351
352            # check for '#' comment
353            elif self.__CurrentChar() == T_CHAR_HASH and not InString and not InCharLiteral:
354                InComment = True
355                HashComment = True
356                PPDirectiveObj = PP_Directive('', (self.CurrentLineNumber, self.CurrentOffsetWithinLine), None)
357            # check for /* comment start
358            elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_STAR:
359
360                self.__SetCurrentCharValue( T_CHAR_SPACE)
361                self.__GetOneChar()
362                self.__SetCurrentCharValue( T_CHAR_SPACE)
363                self.__GetOneChar()
364                InComment = True
365            else:
366                self.__GetOneChar()
367
368        EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
369
370        if InComment and HashComment and not PPExtend:
371            PPDirectiveObj.EndPos = EndLinePos
372            FileProfile.PPDirectiveList.append(PPDirectiveObj)
373        self.Rewind()
374
375    ## ParseFile() method
376    #
377    #   Parse the file profile buffer to extract fd, fv ... information
378    #   Exception will be raised if syntax error found
379    #
380    #   @param  self        The object pointer
381    #
382    def ParseFile(self):
383        self.PreprocessFileWithClear()
384        # restore from ListOfList to ListOfString
385        self.Profile.FileLinesList = ["".join(list) for list in self.Profile.FileLinesList]
386        FileStringContents = ''
387        for fileLine in self.Profile.FileLinesList:
388            FileStringContents += fileLine
389        cStream = antlr3.StringStream(FileStringContents)
390        lexer = CLexer(cStream)
391        tStream = antlr3.CommonTokenStream(lexer)
392        parser = CParser(tStream)
393        parser.translation_unit()
394
395    ## CleanFileProfileBuffer() method
396    #
397    #   Reset all contents of the profile of a file
398    #
399    def CleanFileProfileBuffer(self):
400
401        FileProfile.PPDirectiveList = []
402        FileProfile.AssignmentExpressionList = []
403        FileProfile.FunctionDefinitionList = []
404        FileProfile.VariableDeclarationList = []
405        FileProfile.EnumerationDefinitionList = []
406        FileProfile.StructUnionDefinitionList = []
407        FileProfile.TypedefDefinitionList = []
408        FileProfile.FunctionCallingList = []
409
410    ## PrintFragments() method
411    #
412    #   Print the contents of the profile of a file
413    #
414    def PrintFragments(self):
415
416        print '################# ' + self.FileName + '#####################'
417
418        print '/****************************************/'
419        print '/*************** ASSIGNMENTS ***************/'
420        print '/****************************************/'
421        for asign in FileProfile.AssignmentExpressionList:
422            print str(asign.StartPos) + asign.Name + asign.Operator + asign.Value
423
424        print '/****************************************/'
425        print '/********* PREPROCESS DIRECTIVES ********/'
426        print '/****************************************/'
427        for pp in FileProfile.PPDirectiveList:
428            print str(pp.StartPos) + pp.Content
429
430        print '/****************************************/'
431        print '/********* VARIABLE DECLARATIONS ********/'
432        print '/****************************************/'
433        for var in FileProfile.VariableDeclarationList:
434            print str(var.StartPos) + var.Modifier + ' '+ var.Declarator
435
436        print '/****************************************/'
437        print '/********* FUNCTION DEFINITIONS *********/'
438        print '/****************************************/'
439        for func in FileProfile.FunctionDefinitionList:
440            print str(func.StartPos) + func.Modifier + ' '+ func.Declarator + ' ' + str(func.NamePos)
441
442        print '/****************************************/'
443        print '/************ ENUMERATIONS **************/'
444        print '/****************************************/'
445        for enum in FileProfile.EnumerationDefinitionList:
446            print str(enum.StartPos) + enum.Content
447
448        print '/****************************************/'
449        print '/*********** STRUCTS/UNIONS *************/'
450        print '/****************************************/'
451        for su in FileProfile.StructUnionDefinitionList:
452            print str(su.StartPos) + su.Content
453
454        print '/****************************************/'
455        print '/************** TYPEDEFS ****************/'
456        print '/****************************************/'
457        for typedef in FileProfile.TypedefDefinitionList:
458            print str(typedef.StartPos) + typedef.ToType
459
460##
461#
462# This acts like the main() function for the script, unless it is 'import'ed into another
463# script.
464#
465if __name__ == "__main__":
466
467    print "For Test."
468