1import os, shutil, logging, re 2from autotest_lib.client.bin import test, utils 3from autotest_lib.client.common_lib import error 4 5class npb(test.test): 6 """ 7 This module runs the NAS Parallel Benchmarks on the client machine 8 9 @note: Since we use gfortran to complie these benchmarks, this test might 10 not be able to run on older Operating Systems. 11 @see: http://www.nas.nasa.gov/Resources/Software/npb.html 12 """ 13 version = 1 14 def initialize(self, tests=''): 15 # Initialize failure counter 16 self.n_fail = 0 17 # Get the parameters for run_once() 18 self.tests = tests 19 # Ratio is the reason between 1 and the number of CPUs of the system. 20 self.ratio = 1.0 / utils.count_cpus() 21 logging.debug('Ratio (1/n_cpus) found for this system: %s' % self.ratio) 22 23 24 def setup(self, tarball='NPB3.3.tar.gz'): 25 tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir) 26 utils.extract_tarball_to_dir(tarball, self.srcdir) 27 os.chdir(self.srcdir) 28 # Prepare the makefile and benchmarks to generate. 29 utils.system('patch -p1 < ../enable-all-tests.patch') 30 utils.system('cd NPB3.3-OMP && make suite') 31 32 33 def run_once(self): 34 """ 35 Run each benchmark twice, with different number of threads. 36 37 A sanity check is made on each benchmark executed: 38 The ratio between the times 39 time_ratio = time_one_thrd / time_full_thrds 40 41 Has to be contained inside an envelope: 42 upper_bound = full_thrds * (1 + (1/n_cpus)) 43 lower_bound = full_thrds * (1 - (1/n_cpus)) 44 45 Otherwise, we throw an exception (this test might be running under a 46 virtual machine and sanity check failure might mean bugs on smp 47 implementation). 48 """ 49 os.chdir(self.srcdir) 50 51 # get the tests to run 52 test_list = self.tests.split() 53 54 if len(test_list) == 0: 55 raise error.TestError('No tests (benchmarks) provided. Exit.') 56 57 for itest in test_list: 58 itest_cmd = os.path.join('NPB3.3-OMP/bin/', itest) 59 try: 60 itest = utils.run(itest_cmd) 61 except: 62 logging.error('NPB benchmark %s has failed. Output: %s', 63 itest_cmd, itest.stdout) 64 self.n_fail += 1 65 logging.debug(itest.stdout) 66 67 # Get the number of threads that the test ran 68 # (which is supposed to be equal to the number of system cores) 69 m = re.search('Total threads\s*=\s*(.*)\n', itest.stdout) 70 71 # Gather benchmark results 72 ts = re.search('Time in seconds\s*=\s*(.*)\n', itest.stdout) 73 mt = re.search('Mop/s total\s*=\s*(.*)\n', itest.stdout) 74 mp = re.search('Mop/s/thread\s*=\s*(.*)\n', itest.stdout) 75 76 time_seconds = float(ts.groups()[0]) 77 mops_total = float(mt.groups()[0]) 78 mops_per_thread = float(mp.groups()[0]) 79 80 logging.info('Test: %s', itest_cmd) 81 logging.info('Time (s): %s', time_seconds) 82 logging.info('Total operations executed (mops/s): %s', mops_total) 83 logging.info('Total operations per thread (mops/s/thread): %s', 84 mops_per_thread) 85 86 self.write_test_keyval({'test': itest_cmd}) 87 self.write_test_keyval({'time_seconds': time_seconds}) 88 self.write_test_keyval({'mops_total': mops_total}) 89 self.write_test_keyval({'mops_per_thread': mops_per_thread}) 90 91 # A little extra sanity check comes handy 92 if int(m.groups()[0]) != utils.count_cpus(): 93 raise error.TestError("NPB test suite evaluated the number " 94 "of threads incorrectly: System appears " 95 "to have %s cores, but %s threads were " 96 "executed.") 97 98 # We will use this integer with float point vars later. 99 full_thrds = float(m.groups()[0]) 100 101 # get duration for full_threads running. 102 m = re.search('Time in seconds\s*=\s*(.*)\n', itest.stdout) 103 time_full_thrds = float(m.groups()[0]) 104 105 # repeat the execution with single thread. 106 itest_single_cmd = ''.join(['OMP_NUM_THREADS=1 ', itest_cmd]) 107 try: 108 itest_single = utils.run(itest_single_cmd) 109 except: 110 logging.error('NPB benchmark single thread %s has failed. ' 111 'Output: %s', 112 itest_single_cmd, 113 itest_single.stdout) 114 self.n_fail += 1 115 116 m = re.search('Time in seconds\s*=\s*(.*)\n', itest_single.stdout) 117 time_one_thrd = float(m.groups()[0]) 118 119 # check durations 120 ratio = self.ratio 121 time_ratio = float(time_one_thrd / time_full_thrds) 122 upper_bound = full_thrds * (1 + ratio) 123 lower_bound = full_thrds * (1 - ratio) 124 logging.debug('Time ratio for %s: %s', itest_cmd, time_ratio) 125 logging.debug('Upper bound: %s', upper_bound) 126 logging.debug('Lower bound: %s', lower_bound) 127 128 violates_upper_bound = time_ratio > upper_bound 129 violates_lower_bound = time_ratio < lower_bound 130 if violates_upper_bound or violates_lower_bound: 131 logging.error('NPB benchmark %s failed sanity check ' 132 '- time ratio outside bounds' % itest_cmd) 133 self.n_fail += 1 134 else: 135 logging.debug('NPB benchmark %s sanity check PASS' % itest_cmd) 136 137 138 def cleanup(self): 139 """ 140 Raise TestError if failures were detected during test execution. 141 """ 142 if self.n_fail != 0: 143 raise error.TestError('NPB test failed.') 144 else: 145 logging.info('NPB test passed.') 146