• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2
3from __future__ import print_function
4
5import io
6import os
7import sys
8import subprocess
9import re
10import difflib
11
12import scriptCommon
13from scriptCommon import catchPath
14
15if os.name == 'nt':
16    # Enable console colours on windows
17    os.system('')
18
19rootPath = os.path.join(catchPath, 'projects/SelfTest/Baselines')
20
21langFilenameParser = re.compile(r'(.+\.[ch]pp)')
22filelocParser = re.compile(r'''
23    .*/
24    (.+\.[ch]pp)  # filename
25    (?::|\()      # : is starting separator between filename and line number on Linux, ( on Windows
26    ([0-9]*)      # line number
27    \)?           # Windows also has an ending separator, )
28''', re.VERBOSE)
29lineNumberParser = re.compile(r' line="[0-9]*"')
30hexParser = re.compile(r'\b(0[xX][0-9a-fA-F]+)\b')
31durationsParser = re.compile(r' time="[0-9]*\.[0-9]*"')
32sonarqubeDurationParser = re.compile(r' duration="[0-9]+"')
33timestampsParser = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}Z')
34versionParser = re.compile(r'Catch v[0-9]+\.[0-9]+\.[0-9]+(-develop\.[0-9]+)?')
35nullParser = re.compile(r'\b(__null|nullptr)\b')
36exeNameParser = re.compile(r'''
37    \b
38    (CatchSelfTest|SelfTest)  # Expected executable name
39    (?:.exe)?                 # Executable name contains .exe on Windows.
40    \b
41''', re.VERBOSE)
42# This is a hack until something more reasonable is figured out
43specialCaseParser = re.compile(r'file\((\d+)\)')
44
45# errno macro expands into various names depending on platform, so we need to fix them up as well
46errnoParser = re.compile(r'''
47    \(\*__errno_location\ \(\)\)
48    |
49    \(\*__error\(\)\)
50    |
51    \(\*_errno\(\)\)
52''', re.VERBOSE)
53sinceEpochParser = re.compile(r'\d+ .+ since epoch')
54infParser = re.compile(r'''
55    \(\(float\)\(1e\+300\ \*\ 1e\+300\)\) # MSVC INFINITY macro
56    |
57    \(__builtin_inff\(\)\)                # Linux (ubuntu) INFINITY macro
58    |
59    \(__builtin_inff\ \(\)\)              # Fedora INFINITY macro
60    |
61    __builtin_huge_valf\(\)               # OSX macro
62''', re.VERBOSE)
63nanParser = re.compile(r'''
64    \(\(float\)\(\(\(float\)\(1e\+300\ \*\ 1e\+300\)\)\ \*\ 0\.0F\)\) # MSVC NAN macro
65    |
66    \(\(float\)\(INFINITY\ \*\ 0\.0F\)\) # Yet another MSVC NAN macro
67    |
68    \(__builtin_nanf\ \(""\)\)           # Linux (ubuntu) NAN macro
69    |
70    __builtin_nanf\("0x<hex\ digits>"\)  # The weird content of the brackets is there because a different parser has already ran before this one
71''', re.VERBOSE)
72
73
74if len(sys.argv) == 2:
75    cmdPath = sys.argv[1]
76else:
77    cmdPath = os.path.join(catchPath, scriptCommon.getBuildExecutable())
78
79overallResult = 0
80
81
82def diffFiles(fileA, fileB):
83    with io.open(fileA, 'r', encoding='utf-8', errors='surrogateescape') as file:
84        aLines = [line.rstrip() for line in file.readlines()]
85    with io.open(fileB, 'r', encoding='utf-8', errors='surrogateescape') as file:
86        bLines = [line.rstrip() for line in file.readlines()]
87
88    shortenedFilenameA = fileA.rsplit(os.sep, 1)[-1]
89    shortenedFilenameB = fileB.rsplit(os.sep, 1)[-1]
90
91    diff = difflib.unified_diff(aLines, bLines, fromfile=shortenedFilenameA, tofile=shortenedFilenameB, n=0)
92    return [line for line in diff if line[0] in ('+', '-')]
93
94
95def normalizeFilepath(line):
96    if catchPath in line:
97        # make paths relative to Catch root
98        line = line.replace(catchPath + os.sep, '')
99
100    m = langFilenameParser.match(line)
101    if m:
102        filepath = m.group(0)
103        # go from \ in windows paths to /
104        filepath = filepath.replace('\\', '/')
105        # remove start of relative path
106        filepath = filepath.replace('../', '')
107        line = line[:m.start()] + filepath + line[m.end():]
108
109    return line
110
111def filterLine(line, isCompact):
112    line = normalizeFilepath(line)
113
114    # strip source line numbers
115    m = filelocParser.match(line)
116    if m:
117        # note that this also strips directories, leaving only the filename
118        filename, lnum = m.groups()
119        lnum = ":<line number>" if lnum else ""
120        line = filename + lnum + line[m.end():]
121    else:
122        line = lineNumberParser.sub(" ", line)
123
124    if isCompact:
125        line = line.replace(': FAILED', ': failed')
126        line = line.replace(': PASSED', ': passed')
127
128    # strip Catch version number
129    line = versionParser.sub("<version>", line)
130
131    # replace *null* with 0
132    line = nullParser.sub("0", line)
133
134    # strip executable name
135    line = exeNameParser.sub("<exe-name>", line)
136
137    # strip hexadecimal numbers (presumably pointer values)
138    line = hexParser.sub("0x<hex digits>", line)
139
140    # strip durations and timestamps
141    line = durationsParser.sub(' time="{duration}"', line)
142    line = sonarqubeDurationParser.sub(' duration="{duration}"', line)
143    line = timestampsParser.sub('{iso8601-timestamp}', line)
144    line = specialCaseParser.sub('file:\g<1>', line)
145    line = errnoParser.sub('errno', line)
146    line = sinceEpochParser.sub('{since-epoch-report}', line)
147    line = infParser.sub('INFINITY', line)
148    line = nanParser.sub('NAN', line)
149    return line
150
151
152def approve(baseName, args):
153    global overallResult
154    args[0:0] = [cmdPath]
155    if not os.path.exists(cmdPath):
156        raise Exception("Executable doesn't exist at " + cmdPath)
157    baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
158    rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName))
159    filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
160
161    f = open(rawResultsPath, 'w')
162    subprocess.call(args, stdout=f, stderr=f)
163    f.close()
164
165    rawFile = io.open(rawResultsPath, 'r', encoding='utf-8', errors='surrogateescape')
166    filteredFile = io.open(filteredResultsPath, 'w', encoding='utf-8', errors='surrogateescape')
167    for line in rawFile:
168        filteredFile.write(filterLine(line, 'compact' in baseName).rstrip() + "\n")
169    filteredFile.close()
170    rawFile.close()
171
172    os.remove(rawResultsPath)
173    print()
174    print(baseName + ":")
175    if os.path.exists(baselinesPath):
176        diffResult = diffFiles(baselinesPath, filteredResultsPath)
177        if diffResult:
178            print('\n'.join(diffResult))
179            print("  \n****************************\n  \033[91mResults differed")
180            if len(diffResult) > overallResult:
181                overallResult = len(diffResult)
182        else:
183            os.remove(filteredResultsPath)
184            print("  \033[92mResults matched")
185        print("\033[0m")
186    else:
187        print("  first approval")
188        if overallResult == 0:
189            overallResult = 1
190
191
192print("Running approvals against executable:")
193print("  " + cmdPath)
194
195
196# ## Keep default reporters here ##
197# Standard console reporter
198approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals]", "--order", "lex", "--rng-seed", "1"])
199# console reporter, include passes, warn about No Assertions
200approve("console.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "--order", "lex", "--rng-seed", "1"])
201# console reporter, include passes, warn about No Assertions, limit failures to first 4
202approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex", "--rng-seed", "1"])
203# junit reporter, include passes, warn about No Assertions
204approve("junit.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex", "--rng-seed", "1"])
205# xml reporter, include passes, warn about No Assertions
206approve("xml.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex", "--rng-seed", "1"])
207# compact reporter, include passes, warn about No Assertions
208approve('compact.sw', ['~[!nonportable]~[!benchmark]~[approvals]', '-s', '-w', 'NoAssertions', '-r', 'compact', '--order', 'lex', "--rng-seed", "1"])
209# sonarqube reporter, include passes, warn about No Assertions
210approve("sonarqube.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "sonarqube", "--order", "lex", "--rng-seed", "1"])
211
212if overallResult != 0:
213    print("If these differences are expected, run approve.py to approve new baselines.")
214exit(overallResult)
215