1#! /usr/bin/python 2 3#/****************************************************************** 4#// 5#// OpenCL Conformance Tests 6#// 7#// Copyright: (c) 2008-2009 by Apple Inc. All Rights Reserved. 8#// 9#******************************************************************/ 10 11import os, re, sys, subprocess, time, commands, tempfile, math, string 12 13DEBUG = 0 14 15log_file_name = "opencl_conformance_results_" + time.strftime("%Y-%m-%d_%H-%M", time.localtime())+ ".log" 16process_pid = 0 17 18# The amount of time between printing a "." (if no output from test) or ":" (if output) 19# to the screen while the tests are running. 20seconds_between_status_updates = 60*60*24*7 # effectively never 21 22# Help info 23def write_help_info() : 24 print("run_conformance.py test_list [CL_DEVICE_TYPE(s) to test] [partial-test-names, ...] [log=path/to/log/file/]") 25 print(" test_list - the .csv file containing the test names and commands to run the tests.") 26 print(" [partial-test-names, ...] - optional partial strings to select a subset of the tests to run.") 27 print(" [CL_DEVICE_TYPE(s) to test] - list of CL device types to test, default is CL_DEVICE_TYPE_DEFAULT.") 28 print(" [log=path/to/log/file/] - provide a path for the test log file, default is in the current directory.") 29 print(" (Note: spaces are not allowed in the log file path.") 30 31 32# Get the time formatted nicely 33def get_time() : 34 return time.strftime("%d-%b %H:%M:%S", time.localtime()) 35 36# Write text to the screen and the log file 37def write_screen_log(text) : 38 global log_file 39 print(text) 40 log_file.write(text+"\n") 41 42# Load the tests from a csv formated file of the form name,command 43def get_tests(filename, devices_to_test): 44 tests = [] 45 if (os.path.exists(filename) == False): 46 print("FAILED: test_list \"" + filename + "\" does not exist.") 47 print("") 48 write_help_info() 49 sys.exit(-1) 50 file = open(filename, 'r') 51 for line in file.readlines(): 52 comment = re.search("^#.*", line) 53 if (comment): 54 continue 55 device_specific_match = re.search("^\s*(.+?)\s*,\s*(.+?)\s*,\s*(.+?)\s*$", line) 56 if (device_specific_match): 57 if (device_specific_match.group(1) in devices_to_test): 58 test_path = string.replace(device_specific_match.group(3), '/', os.sep) 59 test_name = string.replace(device_specific_match.group(2), '/', os.sep) 60 tests.append((test_name, test_path)) 61 else: 62 print("Skipping " + device_specific_match.group(2) + " because " + device_specific_match.group(1) + " is not in the list of devices to test.") 63 continue 64 match = re.search("^\s*(.+?)\s*,\s*(.+?)\s*$", line) 65 if (match): 66 test_path = string.replace(match.group(2), '/', os.sep) 67 test_name = string.replace(match.group(1), '/', os.sep) 68 tests.append((test_name, test_path)) 69 return tests 70 71 72def run_test_checking_output(current_directory, test_dir, log_file): 73 global process_pid, seconds_between_status_updates 74 failures_this_run = 0 75 start_time = time.time() 76 # Create a temporary file for capturing the output from the test 77 (output_fd, output_name) = tempfile.mkstemp() 78 if ( not os.path.exists(output_name)) : 79 write_screen_log("\n ==> ERROR: could not create temporary file %s ." % output_name) 80 os.close(output_fd) 81 return -1 82 # Execute the test 83 program_to_run = test_dir_without_args = test_dir.split(None, 1)[0] 84 if ( os.sep == '\\' ) : program_to_run += ".exe" 85 if (os.path.exists(current_directory + os.sep + program_to_run)) : 86 os.chdir(os.path.dirname(current_directory+os.sep+test_dir_without_args) ) 87 try: 88 if (DEBUG): p = subprocess.Popen("", stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True) 89 else : p = subprocess.Popen(current_directory + os.sep + test_dir, stderr=output_fd, stdout=output_fd, shell=True) 90 except OSError: 91 write_screen_log("\n ==> ERROR: failed to execute test. Failing test. : " + str(OSError)) 92 os.close(output_fd) 93 return -1 94 else: 95 write_screen_log("\n ==> ERROR: test file (" + current_directory + os.sep + program_to_run +") does not exist. Failing test.") 96 os.close(output_fd) 97 return -1 98 # Set the global pid so we can kill it if this is aborted 99 process_pid = p.pid 100 # Read one character at a time from the temporary output file while the process is running. 101 # When we get an end-of-line, look for errors and write the results to the log file. 102 # This allows us to process the file as it is being produced. 103 # Keep track of the state for reading 104 # Whether we are done, if we have more to read, and where in the file we last read 105 done = False 106 more_to_read = True 107 pointer = 0 108 pointer_at_last_user_update = 0 109 output_this_run = False 110 try: 111 read_output = open(output_name, 'r') 112 except IOError: 113 write_screen_log("\n ==> ERROR: could not open output file from test.") 114 os.close(output_fd) 115 return -1 116 line = "" 117 while (not done or more_to_read): 118 os.fsync(output_fd) 119 # Determine if we should display some output 120 elapsed_time = (time.time() - start_time) 121 if (elapsed_time > seconds_between_status_updates): 122 start_time = time.time() 123 # If we've received output from the test since the last update, display a # 124 if (pointer != pointer_at_last_user_update): 125 sys.stdout.write(":") 126 else: 127 sys.stdout.write(".") 128 pointer_at_last_user_update = pointer 129 sys.stdout.flush() 130 # Check if we're done 131 p.poll() 132 if (not done and p.returncode != None): 133 if (p.returncode < 0): 134 if (not output_this_run): 135 print "" 136 output_this_run = True 137 write_screen_log(" ==> ERROR: test killed/crashed: " + str(p.returncode)+ ".") 138 done = True 139 # Try reading 140 try: 141 read_output.seek(pointer) 142 char_read = read_output.read(1) 143 except IOError: 144 time.sleep(1) 145 continue 146 # If we got a full line then process it 147 if (char_read == "\n"): 148 # Look for failures and report them as such 149 match = re.search(".*(FAILED|ERROR).*", line) 150 if (match): 151 if (not output_this_run): 152 print "" 153 output_this_run = True 154 print(" ==> " + line.replace('\n','')) 155 match = re.search(".*FAILED.*", line) 156 if (match): 157 failures_this_run = failures_this_run + 1 158 match = re.search(".*(PASSED).*", line) 159 if (match): 160 if (not output_this_run): 161 print "" 162 output_this_run = True 163 print(" " + line.replace('\n','')) 164 # Write it to the log 165 log_file.write(" " + line +"\n") 166 log_file.flush() 167 line = "" 168 pointer = pointer + 1 169 # If we are at the end of the file, then re-open it to get new data 170 elif (char_read == ""): 171 more_to_read = False 172 read_output.close() 173 time.sleep(1) 174 try: 175 os.fsync(output_fd) 176 read_output = open(output_name, 'r') 177 # See if there is more to read. This happens if the process ends and we have data left. 178 read_output.seek(pointer) 179 if (read_output.read(1) != ""): 180 more_to_read = True 181 except IOError: 182 write_screen_log("\n ==> ERROR: could not reopen output file from test.") 183 return -1 184 done = True 185 else: 186 line = line + char_read 187 pointer = pointer + 1 188 # Now we are done, so write out any remaining data in the file: 189 # This should only happen if the process exited with an error. 190 os.fsync(output_fd) 191 while (read_output.read(1) != ""): 192 log_file.write(read_output.read(1)) 193 # Return the total number of failures 194 if (p.returncode == 0 and failures_this_run > 0): 195 write_screen_log("\n ==> ERROR: Test returned 0, but number of FAILED lines reported is " + str(failures_this_run) +".") 196 return failures_this_run 197 return p.returncode 198 199 200def run_tests(tests) : 201 global curent_directory 202 global process_pid 203 # Run the tests 204 failures = 0 205 previous_test = None 206 test_number = 1 207 for test in tests: 208 # Print the name of the test we're running and the time 209 (test_name, test_dir) = test 210 if (test_dir != previous_test): 211 print("========== " + test_dir) 212 log_file.write("========================================================================================\n") 213 log_file.write("========================================================================================\n") 214 log_file.write("(" + get_time() + ") Running Tests: " + test_dir +"\n") 215 log_file.write("========================================================================================\n") 216 log_file.write("========================================================================================\n") 217 previous_test = test_dir 218 print("("+get_time()+") BEGIN " + test_name.ljust(40) +": "), 219 log_file.write(" ----------------------------------------------------------------------------------------\n") 220 log_file.write(" (" + get_time() + ") Running Sub Test: " + test_name + "\n") 221 log_file.write(" ----------------------------------------------------------------------------------------\n") 222 log_file.flush() 223 sys.stdout.flush() 224 225 # Run the test 226 result = 0 227 start_time = time.time() 228 try: 229 process_pid = 0 230 result = run_test_checking_output(current_directory, test_dir, log_file) 231 except KeyboardInterrupt: 232 # Catch an interrupt from the user 233 write_screen_log("\nFAILED: Execution interrupted. Killing test process, but not aborting full test run.") 234 os.kill(process_pid, 9) 235 answer = raw_input("Abort all tests? (y/n)") 236 if (answer.find("y") != -1): 237 write_screen_log("\nUser chose to abort all tests.") 238 log_file.close() 239 sys.exit(-1) 240 else: 241 write_screen_log("\nUser chose to continue with other tests. Reporting this test as failed.") 242 result = 1 243 run_time = (time.time() - start_time) 244 245 # Move print the finish status 246 if (result == 0): 247 print("("+get_time()+") PASSED " + test_name.ljust(40) +": (" + str(int(run_time)).rjust(3) + "s, test " + str(test_number).rjust(3) + os.sep + str(len(tests)) +")"), 248 else: 249 print("("+get_time()+") FAILED " + test_name.ljust(40) +": (" + str(int(run_time)).rjust(3) + "s, test " + str(test_number).rjust(3) + os.sep + str(len(tests)) +")"), 250 251 test_number = test_number + 1 252 log_file.write(" ----------------------------------------------------------------------------------------\n") 253 log_file.flush() 254 255 print("") 256 if (result != 0): 257 log_file.write(" *******************************************************************************************\n") 258 log_file.write(" * ("+get_time()+") Test " + test_name + " ==> FAILED: " + str(result)+"\n") 259 log_file.write(" *******************************************************************************************\n") 260 failures = failures + 1 261 else: 262 log_file.write(" ("+get_time()+") Test " + test_name +" passed in " + str(run_time) + "s\n") 263 264 log_file.write(" ----------------------------------------------------------------------------------------\n") 265 log_file.write("\n") 266 return failures 267 268 269 270 271 272# ######################## 273# Begin OpenCL conformance run script 274# ######################## 275 276if (len(sys.argv) < 2): 277 write_help_info() 278 sys.exit(-1) 279 280 281current_directory = os.getcwd() 282# Open the log file 283for arg in sys.argv: 284 match = re.search("log=(\S+)", arg) 285 if (match): 286 log_file_name = match.group(1).rstrip('/') + os.sep + log_file_name 287try: 288 log_file = open(log_file_name, "w") 289except IOError: 290 print "Could not open log file " + log_file_name 291 292# Determine which devices to test 293device_types = ["CL_DEVICE_TYPE_DEFAULT", "CL_DEVICE_TYPE_CPU", "CL_DEVICE_TYPE_GPU", "CL_DEVICE_TYPE_ACCELERATOR", "CL_DEVICE_TYPE_ALL"] 294devices_to_test = [] 295for device in device_types: 296 if device in sys.argv[2:]: 297 devices_to_test.append(device) 298if (len(devices_to_test) == 0): 299 devices_to_test = ["CL_DEVICE_TYPE_DEFAULT"] 300write_screen_log("Testing on: " + str(devices_to_test)) 301 302# Get the tests 303tests = get_tests(sys.argv[1], devices_to_test) 304 305# If tests are specified on the command line then run just those ones 306tests_to_use = [] 307num_of_patterns_to_match = 0 308for arg in sys.argv[2:]: 309 if arg in device_types: 310 continue 311 if re.search("log=(\S+)", arg): 312 continue 313 num_of_patterns_to_match = num_of_patterns_to_match + 1 314 found_it = False 315 for test in tests: 316 (test_name, test_dir) = test 317 if (test_name.find(arg) != -1 or test_dir.find(arg) != -1): 318 found_it = True 319 if (test not in tests_to_use): 320 tests_to_use.append(test) 321 if (found_it == False): 322 print("Failed to find a test matching " + arg) 323if (len(tests_to_use) == 0): 324 if (num_of_patterns_to_match > 0): 325 print("FAILED: Failed to find any tests matching the given command-line options.") 326 print("") 327 write_help_info() 328 sys.exit(-1) 329else: 330 tests = tests_to_use[:] 331 332write_screen_log("Test execution arguments: " + str(sys.argv)) 333write_screen_log("Logging to file " + log_file_name +".") 334write_screen_log("Loaded tests from " + sys.argv[1] + ", total of " + str(len(tests)) + " tests selected to run:") 335for (test_name, test_command) in tests: 336 write_screen_log(test_name.ljust(50) + " (" + test_command +")") 337 338# Run the tests 339total_failures = 0 340for device_to_test in devices_to_test: 341 os.environ['CL_DEVICE_TYPE'] = device_to_test 342 write_screen_log("========================================================================================") 343 write_screen_log("========================================================================================") 344 write_screen_log(("Setting CL_DEVICE_TYPE to " + device_to_test).center(90)) 345 write_screen_log("========================================================================================") 346 write_screen_log("========================================================================================") 347 failures = run_tests(tests) 348 write_screen_log("========================================================================================") 349 if (failures == 0): 350 write_screen_log(">> TEST on " + device_to_test + " PASSED") 351 else: 352 write_screen_log(">> TEST on " + device_to_test + " FAILED (" + str(failures) + " FAILURES)") 353 write_screen_log("========================================================================================") 354 total_failures = total_failures + failures 355 356write_screen_log("("+get_time()+") Testing complete. " + str(total_failures) + " failures for " + str(len(tests)) + " tests.") 357log_file.close() 358