1""" 2TestCommon.py: a testing framework for commands and scripts 3 with commonly useful error handling 4 5The TestCommon module provides a simple, high-level interface for writing 6tests of executable commands and scripts, especially commands and scripts 7that interact with the file system. All methods throw exceptions and 8exit on failure, with useful error messages. This makes a number of 9explicit checks unnecessary, making the test scripts themselves simpler 10to write and easier to read. 11 12The TestCommon class is a subclass of the TestCmd class. In essence, 13TestCommon is a wrapper that handles common TestCmd error conditions in 14useful ways. You can use TestCommon directly, or subclass it for your 15program and add additional (or override) methods to tailor it to your 16program's specific needs. Alternatively, the TestCommon class serves 17as a useful example of how to define your own TestCmd subclass. 18 19As a subclass of TestCmd, TestCommon provides access to all of the 20variables and methods from the TestCmd module. Consequently, you can 21use any variable or method documented in the TestCmd module without 22having to explicitly import TestCmd. 23 24A TestCommon environment object is created via the usual invocation: 25 26 import TestCommon 27 test = TestCommon.TestCommon() 28 29You can use all of the TestCmd keyword arguments when instantiating a 30TestCommon object; see the TestCmd documentation for details. 31 32Here is an overview of the methods and keyword arguments that are 33provided by the TestCommon class: 34 35 test.must_be_writable('file1', ['file2', ...]) 36 37 test.must_contain('file', 'required text\n') 38 39 test.must_contain_all_lines(output, lines, ['title', find]) 40 41 test.must_contain_any_line(output, lines, ['title', find]) 42 43 test.must_exist('file1', ['file2', ...]) 44 45 test.must_match('file', "expected contents\n") 46 47 test.must_not_be_writable('file1', ['file2', ...]) 48 49 test.must_not_contain('file', 'banned text\n') 50 51 test.must_not_contain_any_line(output, lines, ['title', find]) 52 53 test.must_not_exist('file1', ['file2', ...]) 54 55 test.run(options = "options to be prepended to arguments", 56 stdout = "expected standard output from the program", 57 stderr = "expected error output from the program", 58 status = expected_status, 59 match = match_function) 60 61The TestCommon module also provides the following variables 62 63 TestCommon.python_executable 64 TestCommon.exe_suffix 65 TestCommon.obj_suffix 66 TestCommon.shobj_prefix 67 TestCommon.shobj_suffix 68 TestCommon.lib_prefix 69 TestCommon.lib_suffix 70 TestCommon.dll_prefix 71 TestCommon.dll_suffix 72 73""" 74 75# Copyright 2000-2010 Steven Knight 76# This module is free software, and you may redistribute it and/or modify 77# it under the same terms as Python itself, so long as this copyright message 78# and disclaimer are retained in their original form. 79# 80# IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, 81# SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF 82# THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH 83# DAMAGE. 84# 85# THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT 86# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 87# PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, 88# AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, 89# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. 90 91__author__ = "Steven Knight <knight at baldmt dot com>" 92__revision__ = "TestCommon.py 0.37.D001 2010/01/11 16:55:50 knight" 93__version__ = "0.37" 94 95import copy 96import os 97import os.path 98import stat 99import string 100import sys 101import types 102import UserList 103 104from TestCmd import * 105from TestCmd import __all__ 106 107__all__.extend([ 'TestCommon', 108 'exe_suffix', 109 'obj_suffix', 110 'shobj_prefix', 111 'shobj_suffix', 112 'lib_prefix', 113 'lib_suffix', 114 'dll_prefix', 115 'dll_suffix', 116 ]) 117 118# Variables that describe the prefixes and suffixes on this system. 119if sys.platform == 'win32': 120 exe_suffix = '.exe' 121 obj_suffix = '.obj' 122 shobj_suffix = '.obj' 123 shobj_prefix = '' 124 lib_prefix = '' 125 lib_suffix = '.lib' 126 dll_prefix = '' 127 dll_suffix = '.dll' 128 module_prefix = '' 129 module_suffix = '.dll' 130elif sys.platform == 'cygwin': 131 exe_suffix = '.exe' 132 obj_suffix = '.o' 133 shobj_suffix = '.os' 134 shobj_prefix = '' 135 lib_prefix = 'lib' 136 lib_suffix = '.a' 137 dll_prefix = '' 138 dll_suffix = '.dll' 139 module_prefix = '' 140 module_suffix = '.dll' 141elif string.find(sys.platform, 'irix') != -1: 142 exe_suffix = '' 143 obj_suffix = '.o' 144 shobj_suffix = '.o' 145 shobj_prefix = '' 146 lib_prefix = 'lib' 147 lib_suffix = '.a' 148 dll_prefix = 'lib' 149 dll_suffix = '.so' 150 module_prefix = 'lib' 151 module_prefix = '.so' 152elif string.find(sys.platform, 'darwin') != -1: 153 exe_suffix = '' 154 obj_suffix = '.o' 155 shobj_suffix = '.os' 156 shobj_prefix = '' 157 lib_prefix = 'lib' 158 lib_suffix = '.a' 159 dll_prefix = 'lib' 160 dll_suffix = '.dylib' 161 module_prefix = '' 162 module_suffix = '.so' 163elif string.find(sys.platform, 'sunos') != -1: 164 exe_suffix = '' 165 obj_suffix = '.o' 166 shobj_suffix = '.os' 167 shobj_prefix = 'so_' 168 lib_prefix = 'lib' 169 lib_suffix = '.a' 170 dll_prefix = 'lib' 171 dll_suffix = '.dylib' 172 module_prefix = '' 173 module_suffix = '.so' 174else: 175 exe_suffix = '' 176 obj_suffix = '.o' 177 shobj_suffix = '.os' 178 shobj_prefix = '' 179 lib_prefix = 'lib' 180 lib_suffix = '.a' 181 dll_prefix = 'lib' 182 dll_suffix = '.so' 183 module_prefix = 'lib' 184 module_suffix = '.so' 185 186def is_List(e): 187 return type(e) is types.ListType \ 188 or isinstance(e, UserList.UserList) 189 190def is_writable(f): 191 mode = os.stat(f)[stat.ST_MODE] 192 return mode & stat.S_IWUSR 193 194def separate_files(flist): 195 existing = [] 196 missing = [] 197 for f in flist: 198 if os.path.exists(f): 199 existing.append(f) 200 else: 201 missing.append(f) 202 return existing, missing 203 204def _failed(self, status = 0): 205 if self.status is None or status is None: 206 return None 207 try: 208 return _status(self) not in status 209 except TypeError: 210 # status wasn't an iterable 211 return _status(self) != status 212 213def _status(self): 214 return self.status 215 216class TestCommon(TestCmd): 217 218 # Additional methods from the Perl Test::Cmd::Common module 219 # that we may wish to add in the future: 220 # 221 # $test->subdir('subdir', ...); 222 # 223 # $test->copy('src_file', 'dst_file'); 224 225 def __init__(self, **kw): 226 """Initialize a new TestCommon instance. This involves just 227 calling the base class initialization, and then changing directory 228 to the workdir. 229 """ 230 apply(TestCmd.__init__, [self], kw) 231 os.chdir(self.workdir) 232 233 def must_be_writable(self, *files): 234 """Ensures that the specified file(s) exist and are writable. 235 An individual file can be specified as a list of directory names, 236 in which case the pathname will be constructed by concatenating 237 them. Exits FAILED if any of the files does not exist or is 238 not writable. 239 """ 240 files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files) 241 existing, missing = separate_files(files) 242 unwritable = filter(lambda x, iw=is_writable: not iw(x), existing) 243 if missing: 244 print "Missing files: `%s'" % string.join(missing, "', `") 245 if unwritable: 246 print "Unwritable files: `%s'" % string.join(unwritable, "', `") 247 self.fail_test(missing + unwritable) 248 249 def must_contain(self, file, required, mode = 'rb'): 250 """Ensures that the specified file contains the required text. 251 """ 252 file_contents = self.read(file, mode) 253 contains = (string.find(file_contents, required) != -1) 254 if not contains: 255 print "File `%s' does not contain required string." % file 256 print self.banner('Required string ') 257 print required 258 print self.banner('%s contents ' % file) 259 print file_contents 260 self.fail_test(not contains) 261 262 def must_contain_all_lines(self, output, lines, title=None, find=None): 263 """Ensures that the specified output string (first argument) 264 contains all of the specified lines (second argument). 265 266 An optional third argument can be used to describe the type 267 of output being searched, and only shows up in failure output. 268 269 An optional fourth argument can be used to supply a different 270 function, of the form "find(line, output), to use when searching 271 for lines in the output. 272 """ 273 if find is None: 274 find = lambda o, l: string.find(o, l) != -1 275 missing = [] 276 for line in lines: 277 if not find(output, line): 278 missing.append(line) 279 280 if missing: 281 if title is None: 282 title = 'output' 283 sys.stdout.write("Missing expected lines from %s:\n" % title) 284 for line in missing: 285 sys.stdout.write(' ' + repr(line) + '\n') 286 sys.stdout.write(self.banner(title + ' ')) 287 sys.stdout.write(output) 288 self.fail_test() 289 290 def must_contain_any_line(self, output, lines, title=None, find=None): 291 """Ensures that the specified output string (first argument) 292 contains at least one of the specified lines (second argument). 293 294 An optional third argument can be used to describe the type 295 of output being searched, and only shows up in failure output. 296 297 An optional fourth argument can be used to supply a different 298 function, of the form "find(line, output), to use when searching 299 for lines in the output. 300 """ 301 if find is None: 302 find = lambda o, l: string.find(o, l) != -1 303 for line in lines: 304 if find(output, line): 305 return 306 307 if title is None: 308 title = 'output' 309 sys.stdout.write("Missing any expected line from %s:\n" % title) 310 for line in lines: 311 sys.stdout.write(' ' + repr(line) + '\n') 312 sys.stdout.write(self.banner(title + ' ')) 313 sys.stdout.write(output) 314 self.fail_test() 315 316 def must_contain_lines(self, lines, output, title=None): 317 # Deprecated; retain for backwards compatibility. 318 return self.must_contain_all_lines(output, lines, title) 319 320 def must_exist(self, *files): 321 """Ensures that the specified file(s) must exist. An individual 322 file be specified as a list of directory names, in which case the 323 pathname will be constructed by concatenating them. Exits FAILED 324 if any of the files does not exist. 325 """ 326 files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files) 327 missing = filter(lambda x: not os.path.exists(x), files) 328 if missing: 329 print "Missing files: `%s'" % string.join(missing, "', `") 330 self.fail_test(missing) 331 332 def must_match(self, file, expect, mode = 'rb'): 333 """Matches the contents of the specified file (first argument) 334 against the expected contents (second argument). The expected 335 contents are a list of lines or a string which will be split 336 on newlines. 337 """ 338 file_contents = self.read(file, mode) 339 try: 340 self.fail_test(not self.match(file_contents, expect)) 341 except KeyboardInterrupt: 342 raise 343 except: 344 print "Unexpected contents of `%s'" % file 345 self.diff(expect, file_contents, 'contents ') 346 raise 347 348 def must_not_contain(self, file, banned, mode = 'rb'): 349 """Ensures that the specified file doesn't contain the banned text. 350 """ 351 file_contents = self.read(file, mode) 352 contains = (string.find(file_contents, banned) != -1) 353 if contains: 354 print "File `%s' contains banned string." % file 355 print self.banner('Banned string ') 356 print banned 357 print self.banner('%s contents ' % file) 358 print file_contents 359 self.fail_test(contains) 360 361 def must_not_contain_any_line(self, output, lines, title=None, find=None): 362 """Ensures that the specified output string (first argument) 363 does not contain any of the specified lines (second argument). 364 365 An optional third argument can be used to describe the type 366 of output being searched, and only shows up in failure output. 367 368 An optional fourth argument can be used to supply a different 369 function, of the form "find(line, output), to use when searching 370 for lines in the output. 371 """ 372 if find is None: 373 find = lambda o, l: string.find(o, l) != -1 374 unexpected = [] 375 for line in lines: 376 if find(output, line): 377 unexpected.append(line) 378 379 if unexpected: 380 if title is None: 381 title = 'output' 382 sys.stdout.write("Unexpected lines in %s:\n" % title) 383 for line in unexpected: 384 sys.stdout.write(' ' + repr(line) + '\n') 385 sys.stdout.write(self.banner(title + ' ')) 386 sys.stdout.write(output) 387 self.fail_test() 388 389 def must_not_contain_lines(self, lines, output, title=None): 390 return self.must_not_contain_any_line(output, lines, title) 391 392 def must_not_exist(self, *files): 393 """Ensures that the specified file(s) must not exist. 394 An individual file be specified as a list of directory names, in 395 which case the pathname will be constructed by concatenating them. 396 Exits FAILED if any of the files exists. 397 """ 398 files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files) 399 existing = filter(os.path.exists, files) 400 if existing: 401 print "Unexpected files exist: `%s'" % string.join(existing, "', `") 402 self.fail_test(existing) 403 404 def must_not_be_writable(self, *files): 405 """Ensures that the specified file(s) exist and are not writable. 406 An individual file can be specified as a list of directory names, 407 in which case the pathname will be constructed by concatenating 408 them. Exits FAILED if any of the files does not exist or is 409 writable. 410 """ 411 files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files) 412 existing, missing = separate_files(files) 413 writable = filter(is_writable, existing) 414 if missing: 415 print "Missing files: `%s'" % string.join(missing, "', `") 416 if writable: 417 print "Writable files: `%s'" % string.join(writable, "', `") 418 self.fail_test(missing + writable) 419 420 def _complete(self, actual_stdout, expected_stdout, 421 actual_stderr, expected_stderr, status, match): 422 """ 423 Post-processes running a subcommand, checking for failure 424 status and displaying output appropriately. 425 """ 426 if _failed(self, status): 427 expect = '' 428 if status != 0: 429 expect = " (expected %s)" % str(status) 430 print "%s returned %s%s" % (self.program, str(_status(self)), expect) 431 print self.banner('STDOUT ') 432 print actual_stdout 433 print self.banner('STDERR ') 434 print actual_stderr 435 self.fail_test() 436 if not expected_stdout is None and not match(actual_stdout, expected_stdout): 437 self.diff(expected_stdout, actual_stdout, 'STDOUT ') 438 if actual_stderr: 439 print self.banner('STDERR ') 440 print actual_stderr 441 self.fail_test() 442 if not expected_stderr is None and not match(actual_stderr, expected_stderr): 443 print self.banner('STDOUT ') 444 print actual_stdout 445 self.diff(expected_stderr, actual_stderr, 'STDERR ') 446 self.fail_test() 447 448 def start(self, program = None, 449 interpreter = None, 450 arguments = None, 451 universal_newlines = None, 452 **kw): 453 """ 454 Starts a program or script for the test environment. 455 456 This handles the "options" keyword argument and exceptions. 457 """ 458 options = kw.pop('options', None) 459 if options: 460 if arguments is None: 461 arguments = options 462 else: 463 arguments = options + " " + arguments 464 465 try: 466 return apply(TestCmd.start, 467 (self, program, interpreter, arguments, universal_newlines), 468 kw) 469 except KeyboardInterrupt: 470 raise 471 except Exception, e: 472 print self.banner('STDOUT ') 473 try: 474 print self.stdout() 475 except IndexError: 476 pass 477 print self.banner('STDERR ') 478 try: 479 print self.stderr() 480 except IndexError: 481 pass 482 cmd_args = self.command_args(program, interpreter, arguments) 483 sys.stderr.write('Exception trying to execute: %s\n' % cmd_args) 484 raise e 485 486 def finish(self, popen, stdout = None, stderr = '', status = 0, **kw): 487 """ 488 Finishes and waits for the process being run under control of 489 the specified popen argument. Additional arguments are similar 490 to those of the run() method: 491 492 stdout The expected standard output from 493 the command. A value of None means 494 don't test standard output. 495 496 stderr The expected error output from 497 the command. A value of None means 498 don't test error output. 499 500 status The expected exit status from the 501 command. A value of None means don't 502 test exit status. 503 """ 504 apply(TestCmd.finish, (self, popen,), kw) 505 match = kw.get('match', self.match) 506 self._complete(self.stdout(), stdout, 507 self.stderr(), stderr, status, match) 508 509 def run(self, options = None, arguments = None, 510 stdout = None, stderr = '', status = 0, **kw): 511 """Runs the program under test, checking that the test succeeded. 512 513 The arguments are the same as the base TestCmd.run() method, 514 with the addition of: 515 516 options Extra options that get appended to the beginning 517 of the arguments. 518 519 stdout The expected standard output from 520 the command. A value of None means 521 don't test standard output. 522 523 stderr The expected error output from 524 the command. A value of None means 525 don't test error output. 526 527 status The expected exit status from the 528 command. A value of None means don't 529 test exit status. 530 531 By default, this expects a successful exit (status = 0), does 532 not test standard output (stdout = None), and expects that error 533 output is empty (stderr = ""). 534 """ 535 if options: 536 if arguments is None: 537 arguments = options 538 else: 539 arguments = options + " " + arguments 540 kw['arguments'] = arguments 541 match = kw.pop('match', self.match) 542 apply(TestCmd.run, [self], kw) 543 self._complete(self.stdout(), stdout, 544 self.stderr(), stderr, status, match) 545 546 def skip_test(self, message="Skipping test.\n"): 547 """Skips a test. 548 549 Proper test-skipping behavior is dependent on the external 550 TESTCOMMON_PASS_SKIPS environment variable. If set, we treat 551 the skip as a PASS (exit 0), and otherwise treat it as NO RESULT. 552 In either case, we print the specified message as an indication 553 that the substance of the test was skipped. 554 555 (This was originally added to support development under Aegis. 556 Technically, skipping a test is a NO RESULT, but Aegis would 557 treat that as a test failure and prevent the change from going to 558 the next step. Since we ddn't want to force anyone using Aegis 559 to have to install absolutely every tool used by the tests, we 560 would actually report to Aegis that a skipped test has PASSED 561 so that the workflow isn't held up.) 562 """ 563 if message: 564 sys.stdout.write(message) 565 sys.stdout.flush() 566 pass_skips = os.environ.get('TESTCOMMON_PASS_SKIPS') 567 if pass_skips in [None, 0, '0']: 568 # skip=1 means skip this function when showing where this 569 # result came from. They only care about the line where the 570 # script called test.skip_test(), not the line number where 571 # we call test.no_result(). 572 self.no_result(skip=1) 573 else: 574 # We're under the development directory for this change, 575 # so this is an Aegis invocation; pass the test (exit 0). 576 self.pass_test() 577 578# Local Variables: 579# tab-width:4 580# indent-tabs-mode:nil 581# End: 582# vim: set expandtab tabstop=4 shiftwidth=4: 583