• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2
3"""
4Static Analyzer qualification infrastructure.
5
6The goal is to test the analyzer against different projects, check for failures,
7compare results, and measure performance.
8
9Repository Directory will contain sources of the projects as well as the
10information on how to build them and the expected output.
11Repository Directory structure:
12   - ProjectMap file
13   - Historical Performance Data
14   - Project Dir1
15     - ReferenceOutput
16   - Project Dir2
17     - ReferenceOutput
18   ..
19
20To test the build of the analyzer one would:
21   - Copy over a copy of the Repository Directory. (TODO: Prefer to ensure that
22     the build directory does not pollute the repository to min network traffic).
23   - Build all projects, until error. Produce logs to report errors.
24   - Compare results.
25
26The files which should be kept around for failure investigations:
27   RepositoryCopy/Project DirI/ScanBuildResults
28   RepositoryCopy/Project DirI/run_static_analyzer.log
29
30Assumptions (TODO: shouldn't need to assume these.):
31   The script is being run from the Repository Directory.
32   The compiler for scan-build and scan-build are in the PATH.
33   export PATH=/Users/zaks/workspace/c2llvm/build/Release+Asserts/bin:$PATH
34
35For more logging, set the  env variables:
36   zaks:TI zaks$ export CCC_ANALYZER_LOG=1
37   zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1
38"""
39import CmpRuns
40
41import os
42import csv
43import sys
44import glob
45import shutil
46import time
47import plistlib
48from subprocess import check_call, CalledProcessError
49
50# Project map stores info about all the "registered" projects.
51ProjectMapFile = "projectMap.csv"
52
53# Names of the project specific scripts.
54# The script that needs to be executed before the build can start.
55CleanupScript = "cleanup_run_static_analyzer.sh"
56# This is a file containing commands for scan-build.
57BuildScript = "run_static_analyzer.cmd"
58
59# The log file name.
60LogFolderName = "Logs"
61BuildLogName = "run_static_analyzer.log"
62# Summary file - contains the summary of the failures. Ex: This info can be be
63# displayed when buildbot detects a build failure.
64NumOfFailuresInSummary = 10
65FailuresSummaryFileName = "failures.txt"
66# Summary of the result diffs.
67DiffsSummaryFileName = "diffs.txt"
68
69# The scan-build result directory.
70SBOutputDirName = "ScanBuildResults"
71SBOutputDirReferencePrefix = "Ref"
72
73# The list of checkers used during analyzes.
74# Currently, consists of all the non experimental checkers.
75Checkers="experimental.security.taint,core,deadcode,cplusplus,security,unix,osx,cocoa"
76
77Verbose = 1
78
79IsReferenceBuild = False
80
81# Make sure we flush the output after every print statement.
82class flushfile(object):
83    def __init__(self, f):
84        self.f = f
85    def write(self, x):
86        self.f.write(x)
87        self.f.flush()
88
89sys.stdout = flushfile(sys.stdout)
90
91def getProjectMapPath():
92    ProjectMapPath = os.path.join(os.path.abspath(os.curdir),
93                                  ProjectMapFile)
94    if not os.path.exists(ProjectMapPath):
95        print "Error: Cannot find the Project Map file " + ProjectMapPath +\
96                "\nRunning script for the wrong directory?"
97        sys.exit(-1)
98    return ProjectMapPath
99
100def getProjectDir(ID):
101    return os.path.join(os.path.abspath(os.curdir), ID)
102
103def getSBOutputDirName() :
104    if IsReferenceBuild == True :
105        return SBOutputDirReferencePrefix + SBOutputDirName
106    else :
107        return SBOutputDirName
108
109# Run pre-processing script if any.
110def runCleanupScript(Dir, PBuildLogFile):
111    ScriptPath = os.path.join(Dir, CleanupScript)
112    if os.path.exists(ScriptPath):
113        try:
114            if Verbose == 1:
115                print "  Executing: %s" % (ScriptPath,)
116            check_call("chmod +x %s" % ScriptPath, cwd = Dir,
117                                              stderr=PBuildLogFile,
118                                              stdout=PBuildLogFile,
119                                              shell=True)
120            check_call(ScriptPath, cwd = Dir, stderr=PBuildLogFile,
121                                              stdout=PBuildLogFile,
122                                              shell=True)
123        except:
124            print "Error: The pre-processing step failed. See ", \
125                  PBuildLogFile.name, " for details."
126            sys.exit(-1)
127
128# Build the project with scan-build by reading in the commands and
129# prefixing them with the scan-build options.
130def runScanBuild(Dir, SBOutputDir, PBuildLogFile):
131    BuildScriptPath = os.path.join(Dir, BuildScript)
132    if not os.path.exists(BuildScriptPath):
133        print "Error: build script is not defined: %s" % BuildScriptPath
134        sys.exit(-1)
135    SBOptions = "-plist-html -o " + SBOutputDir + " "
136    SBOptions += "-enable-checker " + Checkers + " "
137    try:
138        SBCommandFile = open(BuildScriptPath, "r")
139        SBPrefix = "scan-build " + SBOptions + " "
140        for Command in SBCommandFile:
141            SBCommand = SBPrefix + Command
142            if Verbose == 1:
143                print "  Executing: %s" % (SBCommand,)
144            check_call(SBCommand, cwd = Dir, stderr=PBuildLogFile,
145                                             stdout=PBuildLogFile,
146                                             shell=True)
147    except:
148        print "Error: scan-build failed. See ",PBuildLogFile.name,\
149              " for details."
150        raise
151
152def hasNoExtension(FileName):
153    (Root, Ext) = os.path.splitext(FileName)
154    if ((Ext == "")) :
155        return True
156    return False
157
158def isValidSingleInputFile(FileName):
159    (Root, Ext) = os.path.splitext(FileName)
160    if ((Ext == ".i") | (Ext == ".ii") |
161        (Ext == ".c") | (Ext == ".cpp") |
162        (Ext == ".m") | (Ext == "")) :
163        return True
164    return False
165
166# Run analysis on a set of preprocessed files.
167def runAnalyzePreprocessed(Dir, SBOutputDir):
168    if os.path.exists(os.path.join(Dir, BuildScript)):
169        print "Error: The preprocessed files project should not contain %s" % \
170               BuildScript
171        raise Exception()
172
173    CmdPrefix = "clang -cc1 -analyze -analyzer-output=plist -w "
174    CmdPrefix += "-analyzer-checker=" + Checkers +" -fcxx-exceptions -fblocks "
175
176    PlistPath = os.path.join(Dir, SBOutputDir, "date")
177    FailPath = os.path.join(PlistPath, "failures");
178    os.makedirs(FailPath);
179
180    for FullFileName in glob.glob(Dir + "/*"):
181        FileName = os.path.basename(FullFileName)
182        Failed = False
183
184        # Only run the analyzes on supported files.
185        if (hasNoExtension(FileName)):
186            continue
187        if (isValidSingleInputFile(FileName) == False):
188            print "Error: Invalid single input file %s." % (FullFileName,)
189            raise Exception()
190
191        # Build and call the analyzer command.
192        OutputOption = "-o " + os.path.join(PlistPath, FileName) + ".plist "
193        Command = CmdPrefix + OutputOption + os.path.join(Dir, FileName)
194        LogFile = open(os.path.join(FailPath, FileName + ".stderr.txt"), "w+b")
195        try:
196            if Verbose == 1:
197                print "  Executing: %s" % (Command,)
198            check_call(Command, cwd = Dir, stderr=LogFile,
199                                           stdout=LogFile,
200                                           shell=True)
201        except CalledProcessError, e:
202            print "Error: Analyzes of %s failed. See %s for details." \
203                  "Error code %d." % \
204                   (FullFileName, LogFile.name, e.returncode)
205            Failed = True
206        finally:
207            LogFile.close()
208
209        # If command did not fail, erase the log file.
210        if Failed == False:
211            os.remove(LogFile.name);
212
213def buildProject(Dir, SBOutputDir, IsScanBuild):
214    TBegin = time.time()
215
216    BuildLogPath = os.path.join(SBOutputDir, LogFolderName, BuildLogName)
217    print "Log file: %s" % (BuildLogPath,)
218    print "Output directory: %s" %(SBOutputDir, )
219
220    # Clean up the log file.
221    if (os.path.exists(BuildLogPath)) :
222        RmCommand = "rm " + BuildLogPath
223        if Verbose == 1:
224            print "  Executing: %s" % (RmCommand,)
225        check_call(RmCommand, shell=True)
226
227    # Clean up scan build results.
228    if (os.path.exists(SBOutputDir)) :
229        RmCommand = "rm -r " + SBOutputDir
230        if Verbose == 1:
231            print "  Executing: %s" % (RmCommand,)
232            check_call(RmCommand, shell=True)
233    assert(not os.path.exists(SBOutputDir))
234    os.makedirs(os.path.join(SBOutputDir, LogFolderName))
235
236    # Open the log file.
237    PBuildLogFile = open(BuildLogPath, "wb+")
238
239    # Build and analyze the project.
240    try:
241        runCleanupScript(Dir, PBuildLogFile)
242
243        if IsScanBuild:
244            runScanBuild(Dir, SBOutputDir, PBuildLogFile)
245        else:
246            runAnalyzePreprocessed(Dir, SBOutputDir)
247
248        if IsReferenceBuild :
249            runCleanupScript(Dir, PBuildLogFile)
250
251    finally:
252        PBuildLogFile.close()
253
254    print "Build complete (time: %.2f). See the log for more details: %s" % \
255           ((time.time()-TBegin), BuildLogPath)
256
257# A plist file is created for each call to the analyzer(each source file).
258# We are only interested on the once that have bug reports, so delete the rest.
259def CleanUpEmptyPlists(SBOutputDir):
260    for F in glob.glob(SBOutputDir + "/*/*.plist"):
261        P = os.path.join(SBOutputDir, F)
262
263        Data = plistlib.readPlist(P)
264        # Delete empty reports.
265        if not Data['files']:
266            os.remove(P)
267            continue
268
269# Given the scan-build output directory, checks if the build failed
270# (by searching for the failures directories). If there are failures, it
271# creates a summary file in the output directory.
272def checkBuild(SBOutputDir):
273    # Check if there are failures.
274    Failures = glob.glob(SBOutputDir + "/*/failures/*.stderr.txt")
275    TotalFailed = len(Failures);
276    if TotalFailed == 0:
277        CleanUpEmptyPlists(SBOutputDir)
278        Plists = glob.glob(SBOutputDir + "/*/*.plist")
279        print "Number of bug reports (non empty plist files) produced: %d" %\
280           len(Plists)
281        return;
282
283    # Create summary file to display when the build fails.
284    SummaryPath = os.path.join(SBOutputDir, LogFolderName, FailuresSummaryFileName)
285    if (Verbose > 0):
286        print "  Creating the failures summary file %s" % (SummaryPath,)
287
288    SummaryLog = open(SummaryPath, "w+")
289    try:
290        SummaryLog.write("Total of %d failures discovered.\n" % (TotalFailed,))
291        if TotalFailed > NumOfFailuresInSummary:
292            SummaryLog.write("See the first %d below.\n"
293                                                   % (NumOfFailuresInSummary,))
294        # TODO: Add a line "See the results folder for more."
295
296        FailuresCopied = NumOfFailuresInSummary
297        Idx = 0
298        for FailLogPathI in glob.glob(SBOutputDir + "/*/failures/*.stderr.txt"):
299            if Idx >= NumOfFailuresInSummary:
300                break;
301            Idx += 1
302            SummaryLog.write("\n-- Error #%d -----------\n" % (Idx,));
303            FailLogI = open(FailLogPathI, "r");
304            try:
305                shutil.copyfileobj(FailLogI, SummaryLog);
306            finally:
307                FailLogI.close()
308    finally:
309        SummaryLog.close()
310
311    print "Error: analysis failed. See ", SummaryPath
312    sys.exit(-1)
313
314# Auxiliary object to discard stdout.
315class Discarder(object):
316    def write(self, text):
317        pass # do nothing
318
319# Compare the warnings produced by scan-build.
320def runCmpResults(Dir):
321    TBegin = time.time()
322
323    RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
324    NewDir = os.path.join(Dir, SBOutputDirName)
325
326    # We have to go one level down the directory tree.
327    RefList = glob.glob(RefDir + "/*")
328    NewList = glob.glob(NewDir + "/*")
329
330    # Log folders are also located in the results dir, so ignore them.
331    RefList.remove(os.path.join(RefDir, LogFolderName))
332    NewList.remove(os.path.join(NewDir, LogFolderName))
333
334    if len(RefList) == 0 or len(NewList) == 0:
335        return False
336    assert(len(RefList) == len(NewList))
337
338    # There might be more then one folder underneath - one per each scan-build
339    # command (Ex: one for configure and one for make).
340    if (len(RefList) > 1):
341        # Assume that the corresponding folders have the same names.
342        RefList.sort()
343        NewList.sort()
344
345    # Iterate and find the differences.
346    NumDiffs = 0
347    PairList = zip(RefList, NewList)
348    for P in PairList:
349        RefDir = P[0]
350        NewDir = P[1]
351
352        assert(RefDir != NewDir)
353        if Verbose == 1:
354            print "  Comparing Results: %s %s" % (RefDir, NewDir)
355
356        DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
357        Opts = CmpRuns.CmpOptions(DiffsPath)
358        # Discard everything coming out of stdout (CmpRun produces a lot of them).
359        OLD_STDOUT = sys.stdout
360        sys.stdout = Discarder()
361        # Scan the results, delete empty plist files.
362        NumDiffs = CmpRuns.cmpScanBuildResults(RefDir, NewDir, Opts, False)
363        sys.stdout = OLD_STDOUT
364        if (NumDiffs > 0) :
365            print "Warning: %r differences in diagnostics. See %s" % \
366                  (NumDiffs, DiffsPath,)
367
368    print "Diagnostic comparison complete (time: %.2f)." % (time.time()-TBegin)
369    return (NumDiffs > 0)
370
371def updateSVN(Mode, ProjectsMap):
372    try:
373        ProjectsMap.seek(0)
374        for I in csv.reader(ProjectsMap):
375            ProjName = I[0]
376            Path = os.path.join(ProjName, getSBOutputDirName())
377
378            if Mode == "delete":
379                Command = "svn delete %s" % (Path,)
380            else:
381                Command = "svn add %s" % (Path,)
382
383            if Verbose == 1:
384                print "  Executing: %s" % (Command,)
385                check_call(Command, shell=True)
386
387        if Mode == "delete":
388            CommitCommand = "svn commit -m \"[analyzer tests] Remove " \
389                            "reference results.\""
390        else:
391            CommitCommand = "svn commit -m \"[analyzer tests] Add new " \
392                            "reference results.\""
393        if Verbose == 1:
394            print "  Executing: %s" % (CommitCommand,)
395            check_call(CommitCommand, shell=True)
396    except:
397        print "Error: SVN update failed."
398        sys.exit(-1)
399
400def testProject(ID, IsScanBuild, Dir=None):
401    print " \n\n--- Building project %s" % (ID,)
402
403    TBegin = time.time()
404
405    if Dir is None :
406        Dir = getProjectDir(ID)
407    if Verbose == 1:
408        print "  Build directory: %s." % (Dir,)
409
410    # Set the build results directory.
411    RelOutputDir = getSBOutputDirName()
412    SBOutputDir = os.path.join(Dir, RelOutputDir)
413
414    buildProject(Dir, SBOutputDir, IsScanBuild)
415
416    checkBuild(SBOutputDir)
417
418    if IsReferenceBuild == False:
419        runCmpResults(Dir)
420
421    print "Completed tests for project %s (time: %.2f)." % \
422          (ID, (time.time()-TBegin))
423
424def testAll(InIsReferenceBuild = False, UpdateSVN = False):
425    global IsReferenceBuild
426    IsReferenceBuild = InIsReferenceBuild
427
428    PMapFile = open(getProjectMapPath(), "rb")
429    try:
430        # Validate the input.
431        for I in csv.reader(PMapFile):
432            if (len(I) != 2) :
433                print "Error: Rows in the ProjectMapFile should have 3 entries."
434                raise Exception()
435            if (not ((I[1] == "1") | (I[1] == "0"))):
436                print "Error: Second entry in the ProjectMapFile should be 0 or 1."
437                raise Exception()
438
439        # When we are regenerating the reference results, we might need to
440        # update svn. Remove reference results from SVN.
441        if UpdateSVN == True:
442            assert(InIsReferenceBuild == True);
443            updateSVN("delete",  PMapFile);
444
445        # Test the projects.
446        PMapFile.seek(0)
447        for I in csv.reader(PMapFile):
448            testProject(I[0], int(I[1]))
449
450        # Add reference results to SVN.
451        if UpdateSVN == True:
452            updateSVN("add",  PMapFile);
453
454    except:
455        print "Error occurred. Premature termination."
456        raise
457    finally:
458        PMapFile.close()
459
460if __name__ == '__main__':
461    IsReference = False
462    UpdateSVN = False
463    if len(sys.argv) >= 2:
464        if sys.argv[1] == "-r":
465            IsReference = True
466        elif sys.argv[1] == "-rs":
467            IsReference = True
468            UpdateSVN = True
469        else:
470          print >> sys.stderr, 'Usage: ', sys.argv[0],\
471                             '[-r|-rs]' \
472                             'Use -r to regenerate reference output' \
473                             'Use -rs to regenerate reference output and update svn'
474
475    testAll(IsReference, UpdateSVN)
476