1# Copyright 2007 Baptiste Lepilleur and The JsonCpp Authors 2# Distributed under MIT license, or public domain if desired and 3# recognized in your jurisdiction. 4# See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE 5 6from __future__ import print_function 7from __future__ import unicode_literals 8from io import open 9from glob import glob 10import sys 11import os 12import os.path 13import optparse 14 15VALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes ' 16 17def getStatusOutput(cmd): 18 """ 19 Return int, unicode (for both Python 2 and 3). 20 Note: os.popen().close() would return None for 0. 21 """ 22 print(cmd, file=sys.stderr) 23 pipe = os.popen(cmd) 24 process_output = pipe.read() 25 try: 26 # We have been using os.popen(). When we read() the result 27 # we get 'str' (bytes) in py2, and 'str' (unicode) in py3. 28 # Ugh! There must be a better way to handle this. 29 process_output = process_output.decode('utf-8') 30 except AttributeError: 31 pass # python3 32 status = pipe.close() 33 return status, process_output 34def compareOutputs(expected, actual, message): 35 expected = expected.strip().replace('\r','').split('\n') 36 actual = actual.strip().replace('\r','').split('\n') 37 diff_line = 0 38 max_line_to_compare = min(len(expected), len(actual)) 39 for index in range(0,max_line_to_compare): 40 if expected[index].strip() != actual[index].strip(): 41 diff_line = index + 1 42 break 43 if diff_line == 0 and len(expected) != len(actual): 44 diff_line = max_line_to_compare+1 45 if diff_line == 0: 46 return None 47 def safeGetLine(lines, index): 48 index += -1 49 if index >= len(lines): 50 return '' 51 return lines[index].strip() 52 return """ Difference in %s at line %d: 53 Expected: '%s' 54 Actual: '%s' 55""" % (message, diff_line, 56 safeGetLine(expected,diff_line), 57 safeGetLine(actual,diff_line)) 58 59def safeReadFile(path): 60 try: 61 return open(path, 'rt', encoding = 'utf-8').read() 62 except IOError as e: 63 return '<File "%s" is missing: %s>' % (path,e) 64 65class FailError(Exception): 66 def __init__(self, msg): 67 super(Exception, self).__init__(msg) 68 69def runAllTests(jsontest_executable_path, input_dir = None, 70 use_valgrind=False, with_json_checker=False, 71 writerClass='StyledWriter'): 72 if not input_dir: 73 input_dir = os.path.join(os.getcwd(), 'data') 74 tests = glob(os.path.join(input_dir, '*.json')) 75 if with_json_checker: 76 all_tests = glob(os.path.join(input_dir, '../jsonchecker', '*.json')) 77 # These tests fail with strict json support, but pass with JsonCPP's 78 # extra leniency features. When adding a new exclusion to this list, 79 # remember to add the test's number and reasoning here: 80 known = ["fail{}.json".format(n) for n in [ 81 4, 9, # fail because we allow trailing commas 82 7, # fails because we allow commas after close 83 8, # fails because we allow extra close 84 10, # fails because we allow extra values after close 85 13, # fails because we allow leading zeroes in numbers 86 18, # fails because we allow deeply nested values 87 25, # fails because we allow tab characters in strings 88 27, # fails because we allow string line breaks 89 ]] 90 test_jsonchecker = [ test for test in all_tests 91 if os.path.basename(test) not in known] 92 93 else: 94 test_jsonchecker = [] 95 96 failed_tests = [] 97 valgrind_path = use_valgrind and VALGRIND_CMD or '' 98 for input_path in tests + test_jsonchecker: 99 expect_failure = os.path.basename(input_path).startswith('fail') 100 is_json_checker_test = (input_path in test_jsonchecker) or expect_failure 101 print('TESTING:', input_path, end=' ') 102 options = is_json_checker_test and '--json-checker' or '' 103 options += ' --json-writer %s'%writerClass 104 cmd = '%s%s %s "%s"' % ( valgrind_path, jsontest_executable_path, options, 105 input_path) 106 status, process_output = getStatusOutput(cmd) 107 if is_json_checker_test: 108 if expect_failure: 109 if not status: 110 print('FAILED') 111 failed_tests.append((input_path, 'Parsing should have failed:\n%s' % 112 safeReadFile(input_path))) 113 else: 114 print('OK') 115 else: 116 if status: 117 print('FAILED') 118 failed_tests.append((input_path, 'Parsing failed:\n' + process_output)) 119 else: 120 print('OK') 121 else: 122 base_path = os.path.splitext(input_path)[0] 123 actual_output = safeReadFile(base_path + '.actual') 124 actual_rewrite_output = safeReadFile(base_path + '.actual-rewrite') 125 open(base_path + '.process-output', 'wt', encoding = 'utf-8').write(process_output) 126 if status: 127 print('parsing failed') 128 failed_tests.append((input_path, 'Parsing failed:\n' + process_output)) 129 else: 130 expected_output_path = os.path.splitext(input_path)[0] + '.expected' 131 expected_output = open(expected_output_path, 'rt', encoding = 'utf-8').read() 132 detail = (compareOutputs(expected_output, actual_output, 'input') 133 or compareOutputs(expected_output, actual_rewrite_output, 'rewrite')) 134 if detail: 135 print('FAILED') 136 failed_tests.append((input_path, detail)) 137 else: 138 print('OK') 139 140 if failed_tests: 141 print() 142 print('Failure details:') 143 for failed_test in failed_tests: 144 print('* Test', failed_test[0]) 145 print(failed_test[1]) 146 print() 147 print('Test results: %d passed, %d failed.' % (len(tests)-len(failed_tests), 148 len(failed_tests))) 149 raise FailError(repr(failed_tests)) 150 else: 151 print('All %d tests passed.' % len(tests)) 152 153def main(): 154 from optparse import OptionParser 155 parser = OptionParser(usage="%prog [options] <path to jsontestrunner.exe> [test case directory]") 156 parser.add_option("--valgrind", 157 action="store_true", dest="valgrind", default=False, 158 help="run all the tests using valgrind to detect memory leaks") 159 parser.add_option("-c", "--with-json-checker", 160 action="store_true", dest="with_json_checker", default=False, 161 help="run all the tests from the official JSONChecker test suite of json.org") 162 parser.enable_interspersed_args() 163 options, args = parser.parse_args() 164 165 if len(args) < 1 or len(args) > 2: 166 parser.error('Must provides at least path to jsontestrunner executable.') 167 sys.exit(1) 168 169 jsontest_executable_path = os.path.normpath(os.path.abspath(args[0])) 170 if len(args) > 1: 171 input_path = os.path.normpath(os.path.abspath(args[1])) 172 else: 173 input_path = None 174 runAllTests(jsontest_executable_path, input_path, 175 use_valgrind=options.valgrind, 176 with_json_checker=options.with_json_checker, 177 writerClass='StyledWriter') 178 runAllTests(jsontest_executable_path, input_path, 179 use_valgrind=options.valgrind, 180 with_json_checker=options.with_json_checker, 181 writerClass='StyledStreamWriter') 182 runAllTests(jsontest_executable_path, input_path, 183 use_valgrind=options.valgrind, 184 with_json_checker=options.with_json_checker, 185 writerClass='BuiltStyledStreamWriter') 186 187if __name__ == '__main__': 188 try: 189 main() 190 except FailError: 191 sys.exit(1) 192