• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2
3from __future__ import print_function
4
5import io
6import os
7import sys
8import subprocess
9import re
10import difflib
11
12import scriptCommon
13from scriptCommon import catchPath
14
15if os.name == 'nt':
16    # Enable console colours on windows
17    os.system('')
18
19rootPath = os.path.join(catchPath, 'projects/SelfTest/Baselines')
20
21langFilenameParser = re.compile(r'(.+\.[ch]pp)')
22filelocParser = re.compile(r'''
23    .*/
24    (.+\.[ch]pp)  # filename
25    (?::|\()      # : is starting separator between filename and line number on Linux, ( on Windows
26    ([0-9]*)      # line number
27    \)?           # Windows also has an ending separator, )
28''', re.VERBOSE)
29lineNumberParser = re.compile(r' line="[0-9]*"')
30hexParser = re.compile(r'\b(0[xX][0-9a-fA-F]+)\b')
31durationsParser = re.compile(r' time="[0-9]*\.[0-9]*"')
32timestampsParser = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}Z')
33versionParser = re.compile(r'Catch v[0-9]+\.[0-9]+\.[0-9]+(-develop\.[0-9]+)?')
34nullParser = re.compile(r'\b(__null|nullptr)\b')
35exeNameParser = re.compile(r'''
36    \b
37    (CatchSelfTest|SelfTest)  # Expected executable name
38    (?:.exe)?                 # Executable name contains .exe on Windows.
39    \b
40''', re.VERBOSE)
41# This is a hack until something more reasonable is figured out
42specialCaseParser = re.compile(r'file\((\d+)\)')
43
44# errno macro expands into various names depending on platform, so we need to fix them up as well
45errnoParser = re.compile(r'''
46    \(\*__errno_location\ \(\)\)
47    |
48    \(\*__error\(\)\)
49    |
50    \(\*_errno\(\)\)
51''', re.VERBOSE)
52sinceEpochParser = re.compile(r'\d+ .+ since epoch')
53infParser = re.compile(r'''
54    \(\(float\)\(1e\+300\ \*\ 1e\+300\)\) # MSVC INFINITY macro
55    |
56    \(__builtin_inff\(\)\)                # Linux (ubuntu) INFINITY macro
57    |
58    \(__builtin_inff\ \(\)\)              # Fedora INFINITY macro
59    |
60    __builtin_huge_valf\(\)               # OSX macro
61''', re.VERBOSE)
62nanParser = re.compile(r'''
63    \(\(float\)\(\(\(float\)\(1e\+300\ \*\ 1e\+300\)\)\ \*\ 0\.0F\)\) # MSVC NAN macro
64    |
65    \(\(float\)\(INFINITY\ \*\ 0\.0F\)\) # Yet another MSVC NAN macro
66    |
67    \(__builtin_nanf\ \(""\)\)           # Linux (ubuntu) NAN macro
68    |
69    __builtin_nanf\("0x<hex\ digits>"\)  # The weird content of the brackets is there because a different parser has already ran before this one
70''', re.VERBOSE)
71
72
73if len(sys.argv) == 2:
74    cmdPath = sys.argv[1]
75else:
76    cmdPath = os.path.join(catchPath, scriptCommon.getBuildExecutable())
77
78overallResult = 0
79
80
81def diffFiles(fileA, fileB):
82    with io.open(fileA, 'r', encoding='utf-8', errors='surrogateescape') as file:
83        aLines = [line.rstrip() for line in file.readlines()]
84    with io.open(fileB, 'r', encoding='utf-8', errors='surrogateescape') as file:
85        bLines = [line.rstrip() for line in file.readlines()]
86
87    shortenedFilenameA = fileA.rsplit(os.sep, 1)[-1]
88    shortenedFilenameB = fileB.rsplit(os.sep, 1)[-1]
89
90    diff = difflib.unified_diff(aLines, bLines, fromfile=shortenedFilenameA, tofile=shortenedFilenameB, n=0)
91    return [line for line in diff if line[0] in ('+', '-')]
92
93
94def normalizeFilepath(line):
95    if catchPath in line:
96        # make paths relative to Catch root
97        line = line.replace(catchPath + os.sep, '')
98
99    m = langFilenameParser.match(line)
100    if m:
101        filepath = m.group(0)
102        # go from \ in windows paths to /
103        filepath = filepath.replace('\\', '/')
104        # remove start of relative path
105        filepath = filepath.replace('../', '')
106        line = line[:m.start()] + filepath + line[m.end():]
107
108    return line
109
110def filterLine(line, isCompact):
111    line = normalizeFilepath(line)
112
113    # strip source line numbers
114    m = filelocParser.match(line)
115    if m:
116        # note that this also strips directories, leaving only the filename
117        filename, lnum = m.groups()
118        lnum = ":<line number>" if lnum else ""
119        line = filename + lnum + line[m.end():]
120    else:
121        line = lineNumberParser.sub(" ", line)
122
123    if isCompact:
124        line = line.replace(': FAILED', ': failed')
125        line = line.replace(': PASSED', ': passed')
126
127    # strip Catch version number
128    line = versionParser.sub("<version>", line)
129
130    # replace *null* with 0
131    line = nullParser.sub("0", line)
132
133    # strip executable name
134    line = exeNameParser.sub("<exe-name>", line)
135
136    # strip hexadecimal numbers (presumably pointer values)
137    line = hexParser.sub("0x<hex digits>", line)
138
139    # strip durations and timestamps
140    line = durationsParser.sub(' time="{duration}"', line)
141    line = timestampsParser.sub('{iso8601-timestamp}', line)
142    line = specialCaseParser.sub('file:\g<1>', line)
143    line = errnoParser.sub('errno', line)
144    line = sinceEpochParser.sub('{since-epoch-report}', line)
145    line = infParser.sub('INFINITY', line)
146    line = nanParser.sub('NAN', line)
147    return line
148
149
150def approve(baseName, args):
151    global overallResult
152    args[0:0] = [cmdPath]
153    if not os.path.exists(cmdPath):
154        raise Exception("Executable doesn't exist at " + cmdPath)
155    baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
156    rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName))
157    filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
158
159    f = open(rawResultsPath, 'w')
160    subprocess.call(args, stdout=f, stderr=f)
161    f.close()
162
163    rawFile = io.open(rawResultsPath, 'r', encoding='utf-8', errors='surrogateescape')
164    filteredFile = io.open(filteredResultsPath, 'w', encoding='utf-8', errors='surrogateescape')
165    for line in rawFile:
166        filteredFile.write(filterLine(line, 'compact' in baseName).rstrip() + "\n")
167    filteredFile.close()
168    rawFile.close()
169
170    os.remove(rawResultsPath)
171    print()
172    print(baseName + ":")
173    if os.path.exists(baselinesPath):
174        diffResult = diffFiles(baselinesPath, filteredResultsPath)
175        if diffResult:
176            print('\n'.join(diffResult))
177            print("  \n****************************\n  \033[91mResults differed")
178            if len(diffResult) > overallResult:
179                overallResult = len(diffResult)
180        else:
181            os.remove(filteredResultsPath)
182            print("  \033[92mResults matched")
183        print("\033[0m")
184    else:
185        print("  first approval")
186        if overallResult == 0:
187            overallResult = 1
188
189
190print("Running approvals against executable:")
191print("  " + cmdPath)
192
193
194# ## Keep default reporters here ##
195# Standard console reporter
196approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals]", "--order", "lex", "--rng-seed", "1"])
197# console reporter, include passes, warn about No Assertions
198approve("console.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "--order", "lex", "--rng-seed", "1"])
199# console reporter, include passes, warn about No Assertions, limit failures to first 4
200approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex", "--rng-seed", "1"])
201# junit reporter, include passes, warn about No Assertions
202approve("junit.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex", "--rng-seed", "1"])
203# xml reporter, include passes, warn about No Assertions
204approve("xml.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex", "--rng-seed", "1"])
205# compact reporter, include passes, warn about No Assertions
206approve('compact.sw', ['~[!nonportable]~[!benchmark]~[approvals]', '-s', '-w', 'NoAssertions', '-r', 'compact', '--order', 'lex', "--rng-seed", "1"])
207
208if overallResult != 0:
209    print("If these differences are expected, run approve.py to approve new baselines.")
210exit(overallResult)
211