1#!/usr/bin/env python 2 3from __future__ import print_function 4 5import io 6import os 7import sys 8import subprocess 9import re 10import difflib 11 12import scriptCommon 13from scriptCommon import catchPath 14 15if os.name == 'nt': 16 # Enable console colours on windows 17 os.system('') 18 19rootPath = os.path.join(catchPath, 'projects/SelfTest/Baselines') 20 21langFilenameParser = re.compile(r'(.+\.[ch]pp)') 22filelocParser = re.compile(r''' 23 .*/ 24 (.+\.[ch]pp) # filename 25 (?::|\() # : is starting separator between filename and line number on Linux, ( on Windows 26 ([0-9]*) # line number 27 \)? # Windows also has an ending separator, ) 28''', re.VERBOSE) 29lineNumberParser = re.compile(r' line="[0-9]*"') 30hexParser = re.compile(r'\b(0[xX][0-9a-fA-F]+)\b') 31durationsParser = re.compile(r' time="[0-9]*\.[0-9]*"') 32timestampsParser = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}Z') 33versionParser = re.compile(r'Catch v[0-9]+\.[0-9]+\.[0-9]+(-develop\.[0-9]+)?') 34nullParser = re.compile(r'\b(__null|nullptr)\b') 35exeNameParser = re.compile(r''' 36 \b 37 (CatchSelfTest|SelfTest) # Expected executable name 38 (?:.exe)? # Executable name contains .exe on Windows. 39 \b 40''', re.VERBOSE) 41# This is a hack until something more reasonable is figured out 42specialCaseParser = re.compile(r'file\((\d+)\)') 43 44# errno macro expands into various names depending on platform, so we need to fix them up as well 45errnoParser = re.compile(r''' 46 \(\*__errno_location\ \(\)\) 47 | 48 \(\*__error\(\)\) 49 | 50 \(\*_errno\(\)\) 51''', re.VERBOSE) 52sinceEpochParser = re.compile(r'\d+ .+ since epoch') 53infParser = re.compile(r''' 54 \(\(float\)\(1e\+300\ \*\ 1e\+300\)\) # MSVC INFINITY macro 55 | 56 \(__builtin_inff\(\)\) # Linux (ubuntu) INFINITY macro 57 | 58 \(__builtin_inff\ \(\)\) # Fedora INFINITY macro 59 | 60 __builtin_huge_valf\(\) # OSX macro 61''', re.VERBOSE) 62nanParser = re.compile(r''' 63 \(\(float\)\(\(\(float\)\(1e\+300\ \*\ 1e\+300\)\)\ \*\ 0\.0F\)\) # MSVC NAN macro 64 | 65 \(\(float\)\(INFINITY\ \*\ 0\.0F\)\) # Yet another MSVC NAN macro 66 | 67 \(__builtin_nanf\ \(""\)\) # Linux (ubuntu) NAN macro 68 | 69 __builtin_nanf\("0x<hex\ digits>"\) # The weird content of the brackets is there because a different parser has already ran before this one 70''', re.VERBOSE) 71 72 73if len(sys.argv) == 2: 74 cmdPath = sys.argv[1] 75else: 76 cmdPath = os.path.join(catchPath, scriptCommon.getBuildExecutable()) 77 78overallResult = 0 79 80 81def diffFiles(fileA, fileB): 82 with io.open(fileA, 'r', encoding='utf-8', errors='surrogateescape') as file: 83 aLines = [line.rstrip() for line in file.readlines()] 84 with io.open(fileB, 'r', encoding='utf-8', errors='surrogateescape') as file: 85 bLines = [line.rstrip() for line in file.readlines()] 86 87 shortenedFilenameA = fileA.rsplit(os.sep, 1)[-1] 88 shortenedFilenameB = fileB.rsplit(os.sep, 1)[-1] 89 90 diff = difflib.unified_diff(aLines, bLines, fromfile=shortenedFilenameA, tofile=shortenedFilenameB, n=0) 91 return [line for line in diff if line[0] in ('+', '-')] 92 93 94def normalizeFilepath(line): 95 if catchPath in line: 96 # make paths relative to Catch root 97 line = line.replace(catchPath + os.sep, '') 98 99 m = langFilenameParser.match(line) 100 if m: 101 filepath = m.group(0) 102 # go from \ in windows paths to / 103 filepath = filepath.replace('\\', '/') 104 # remove start of relative path 105 filepath = filepath.replace('../', '') 106 line = line[:m.start()] + filepath + line[m.end():] 107 108 return line 109 110def filterLine(line, isCompact): 111 line = normalizeFilepath(line) 112 113 # strip source line numbers 114 m = filelocParser.match(line) 115 if m: 116 # note that this also strips directories, leaving only the filename 117 filename, lnum = m.groups() 118 lnum = ":<line number>" if lnum else "" 119 line = filename + lnum + line[m.end():] 120 else: 121 line = lineNumberParser.sub(" ", line) 122 123 if isCompact: 124 line = line.replace(': FAILED', ': failed') 125 line = line.replace(': PASSED', ': passed') 126 127 # strip Catch version number 128 line = versionParser.sub("<version>", line) 129 130 # replace *null* with 0 131 line = nullParser.sub("0", line) 132 133 # strip executable name 134 line = exeNameParser.sub("<exe-name>", line) 135 136 # strip hexadecimal numbers (presumably pointer values) 137 line = hexParser.sub("0x<hex digits>", line) 138 139 # strip durations and timestamps 140 line = durationsParser.sub(' time="{duration}"', line) 141 line = timestampsParser.sub('{iso8601-timestamp}', line) 142 line = specialCaseParser.sub('file:\g<1>', line) 143 line = errnoParser.sub('errno', line) 144 line = sinceEpochParser.sub('{since-epoch-report}', line) 145 line = infParser.sub('INFINITY', line) 146 line = nanParser.sub('NAN', line) 147 return line 148 149 150def approve(baseName, args): 151 global overallResult 152 args[0:0] = [cmdPath] 153 if not os.path.exists(cmdPath): 154 raise Exception("Executable doesn't exist at " + cmdPath) 155 baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName)) 156 rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName)) 157 filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName)) 158 159 f = open(rawResultsPath, 'w') 160 subprocess.call(args, stdout=f, stderr=f) 161 f.close() 162 163 rawFile = io.open(rawResultsPath, 'r', encoding='utf-8', errors='surrogateescape') 164 filteredFile = io.open(filteredResultsPath, 'w', encoding='utf-8', errors='surrogateescape') 165 for line in rawFile: 166 filteredFile.write(filterLine(line, 'compact' in baseName).rstrip() + "\n") 167 filteredFile.close() 168 rawFile.close() 169 170 os.remove(rawResultsPath) 171 print() 172 print(baseName + ":") 173 if os.path.exists(baselinesPath): 174 diffResult = diffFiles(baselinesPath, filteredResultsPath) 175 if diffResult: 176 print('\n'.join(diffResult)) 177 print(" \n****************************\n \033[91mResults differed") 178 if len(diffResult) > overallResult: 179 overallResult = len(diffResult) 180 else: 181 os.remove(filteredResultsPath) 182 print(" \033[92mResults matched") 183 print("\033[0m") 184 else: 185 print(" first approval") 186 if overallResult == 0: 187 overallResult = 1 188 189 190print("Running approvals against executable:") 191print(" " + cmdPath) 192 193 194# ## Keep default reporters here ## 195# Standard console reporter 196approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals]", "--order", "lex", "--rng-seed", "1"]) 197# console reporter, include passes, warn about No Assertions 198approve("console.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "--order", "lex", "--rng-seed", "1"]) 199# console reporter, include passes, warn about No Assertions, limit failures to first 4 200approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex", "--rng-seed", "1"]) 201# junit reporter, include passes, warn about No Assertions 202approve("junit.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex", "--rng-seed", "1"]) 203# xml reporter, include passes, warn about No Assertions 204approve("xml.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex", "--rng-seed", "1"]) 205# compact reporter, include passes, warn about No Assertions 206approve('compact.sw', ['~[!nonportable]~[!benchmark]~[approvals]', '-s', '-w', 'NoAssertions', '-r', 'compact', '--order', 'lex', "--rng-seed", "1"]) 207 208if overallResult != 0: 209 print("If these differences are expected, run approve.py to approve new baselines.") 210exit(overallResult) 211