1""" 2DO NOT import this file directly - import client/bin/utils.py, 3which will mix this in 4 5Convenience functions for use by tests or whomever. 6 7Note that this file is mixed in by utils.py - note very carefully the 8precedence order defined there 9""" 10import os, shutil, commands, pickle, glob 11import math, re, fnmatch, logging, multiprocessing 12from autotest_lib.client.common_lib import error, utils, magic 13 14 15def grep(pattern, file): 16 """ 17 This is mainly to fix the return code inversion from grep 18 Also handles compressed files. 19 20 returns 1 if the pattern is present in the file, 0 if not. 21 """ 22 command = 'grep "%s" > /dev/null' % pattern 23 ret = cat_file_to_cmd(file, command, ignore_status=True) 24 return not ret 25 26 27def difflist(list1, list2): 28 """returns items in list2 that are not in list1""" 29 diff = []; 30 for x in list2: 31 if x not in list1: 32 diff.append(x) 33 return diff 34 35 36def cat_file_to_cmd(file, command, ignore_status=0, return_output=False): 37 """ 38 equivalent to 'cat file | command' but knows to use 39 zcat or bzcat if appropriate 40 """ 41 if not os.path.isfile(file): 42 raise NameError('invalid file %s to cat to command %s' 43 % (file, command)) 44 45 if return_output: 46 run_cmd = utils.system_output 47 else: 48 run_cmd = utils.system 49 50 if magic.guess_type(file) == 'application/x-bzip2': 51 cat = 'bzcat' 52 elif magic.guess_type(file) == 'application/x-gzip': 53 cat = 'zcat' 54 else: 55 cat = 'cat' 56 return run_cmd('%s %s | %s' % (cat, file, command), 57 ignore_status=ignore_status) 58 59 60def extract_tarball_to_dir(tarball, dir): 61 """ 62 Extract a tarball to a specified directory name instead of whatever 63 the top level of a tarball is - useful for versioned directory names, etc 64 """ 65 if os.path.exists(dir): 66 if os.path.isdir(dir): 67 shutil.rmtree(dir) 68 else: 69 os.remove(dir) 70 pwd = os.getcwd() 71 os.chdir(os.path.dirname(os.path.abspath(dir))) 72 newdir = extract_tarball(tarball) 73 os.rename(newdir, dir) 74 os.chdir(pwd) 75 76 77def extract_tarball(tarball): 78 """Returns the directory extracted by the tarball.""" 79 extracted = cat_file_to_cmd(tarball, 'tar xvf - 2>/dev/null', 80 return_output=True).splitlines() 81 82 dir = None 83 84 for line in extracted: 85 if line.startswith('./'): 86 line = line[2:] 87 if not line or line == '.': 88 continue 89 topdir = line.split('/')[0] 90 if os.path.isdir(topdir): 91 if dir: 92 assert(dir == topdir) 93 else: 94 dir = topdir 95 if dir: 96 return dir 97 else: 98 raise NameError('extracting tarball produced no dir') 99 100 101def hash_file(filename, size=None, method="md5"): 102 """ 103 Calculate the hash of filename. 104 If size is not None, limit to first size bytes. 105 Throw exception if something is wrong with filename. 106 Can be also implemented with bash one-liner (assuming size%1024==0): 107 dd if=filename bs=1024 count=size/1024 | sha1sum - 108 109 @param filename: Path of the file that will have its hash calculated. 110 @param method: Method used to calculate the hash. Supported methods: 111 * md5 112 * sha1 113 @returns: Hash of the file, if something goes wrong, return None. 114 """ 115 chunksize = 4096 116 fsize = os.path.getsize(filename) 117 118 if not size or size > fsize: 119 size = fsize 120 f = open(filename, 'rb') 121 122 try: 123 hash = utils.hash(method) 124 except ValueError: 125 logging.error("Unknown hash type %s, returning None", method) 126 127 while size > 0: 128 if chunksize > size: 129 chunksize = size 130 data = f.read(chunksize) 131 if len(data) == 0: 132 logging.debug("Nothing left to read but size=%d", size) 133 break 134 hash.update(data) 135 size -= len(data) 136 f.close() 137 return hash.hexdigest() 138 139 140def unmap_url_cache(cachedir, url, expected_hash, method="md5"): 141 """ 142 Downloads a file from a URL to a cache directory. If the file is already 143 at the expected position and has the expected hash, let's not download it 144 again. 145 146 @param cachedir: Directory that might hold a copy of the file we want to 147 download. 148 @param url: URL for the file we want to download. 149 @param expected_hash: Hash string that we expect the file downloaded to 150 have. 151 @param method: Method used to calculate the hash string (md5, sha1). 152 """ 153 # Let's convert cachedir to a canonical path, if it's not already 154 cachedir = os.path.realpath(cachedir) 155 if not os.path.isdir(cachedir): 156 try: 157 os.makedirs(cachedir) 158 except: 159 raise ValueError('Could not create cache directory %s' % cachedir) 160 file_from_url = os.path.basename(url) 161 file_local_path = os.path.join(cachedir, file_from_url) 162 163 file_hash = None 164 failure_counter = 0 165 while not file_hash == expected_hash: 166 if os.path.isfile(file_local_path): 167 file_hash = hash_file(file_local_path, method) 168 if file_hash == expected_hash: 169 # File is already at the expected position and ready to go 170 src = file_from_url 171 else: 172 # Let's download the package again, it's corrupted... 173 logging.error("Seems that file %s is corrupted, trying to " 174 "download it again", file_from_url) 175 src = url 176 failure_counter += 1 177 else: 178 # File is not there, let's download it 179 src = url 180 if failure_counter > 1: 181 raise EnvironmentError("Consistently failed to download the " 182 "package %s. Aborting further download " 183 "attempts. This might mean either the " 184 "network connection has problems or the " 185 "expected hash string that was determined " 186 "for this file is wrong", file_from_url) 187 file_path = utils.unmap_url(cachedir, src, cachedir) 188 189 return file_path 190 191 192def force_copy(src, dest): 193 """Replace dest with a new copy of src, even if it exists""" 194 if os.path.isfile(dest): 195 os.remove(dest) 196 if os.path.isdir(dest): 197 dest = os.path.join(dest, os.path.basename(src)) 198 shutil.copyfile(src, dest) 199 return dest 200 201 202def force_link(src, dest): 203 """Link src to dest, overwriting it if it exists""" 204 return utils.system("ln -sf %s %s" % (src, dest)) 205 206 207def file_contains_pattern(file, pattern): 208 """Return true if file contains the specified egrep pattern""" 209 if not os.path.isfile(file): 210 raise NameError('file %s does not exist' % file) 211 return not utils.system('egrep -q "' + pattern + '" ' + file, ignore_status=True) 212 213 214def list_grep(list, pattern): 215 """True if any item in list matches the specified pattern.""" 216 compiled = re.compile(pattern) 217 for line in list: 218 match = compiled.search(line) 219 if (match): 220 return 1 221 return 0 222 223 224def get_os_vendor(): 225 """Try to guess what's the os vendor 226 """ 227 if os.path.isfile('/etc/SuSE-release'): 228 return 'SUSE' 229 230 issue = '/etc/issue' 231 232 if not os.path.isfile(issue): 233 return 'Unknown' 234 235 if file_contains_pattern(issue, 'Red Hat'): 236 return 'Red Hat' 237 elif file_contains_pattern(issue, 'Fedora'): 238 return 'Fedora Core' 239 elif file_contains_pattern(issue, 'SUSE'): 240 return 'SUSE' 241 elif file_contains_pattern(issue, 'Ubuntu'): 242 return 'Ubuntu' 243 elif file_contains_pattern(issue, 'Debian'): 244 return 'Debian' 245 else: 246 return 'Unknown' 247 248 249def get_cc(): 250 try: 251 return os.environ['CC'] 252 except KeyError: 253 return 'gcc' 254 255 256def get_vmlinux(): 257 """Return the full path to vmlinux 258 259 Ahem. This is crap. Pray harder. Bad Martin. 260 """ 261 vmlinux = '/boot/vmlinux-%s' % utils.system_output('uname -r') 262 if os.path.isfile(vmlinux): 263 return vmlinux 264 vmlinux = '/lib/modules/%s/build/vmlinux' % utils.system_output('uname -r') 265 if os.path.isfile(vmlinux): 266 return vmlinux 267 return None 268 269 270def get_systemmap(): 271 """Return the full path to System.map 272 273 Ahem. This is crap. Pray harder. Bad Martin. 274 """ 275 map = '/boot/System.map-%s' % utils.system_output('uname -r') 276 if os.path.isfile(map): 277 return map 278 map = '/lib/modules/%s/build/System.map' % utils.system_output('uname -r') 279 if os.path.isfile(map): 280 return map 281 return None 282 283 284def get_modules_dir(): 285 """Return the modules dir for the running kernel version""" 286 kernel_version = utils.system_output('uname -r') 287 return '/lib/modules/%s/kernel' % kernel_version 288 289 290_CPUINFO_RE = re.compile(r'^(?P<key>[^\t]*)\t*: ?(?P<value>.*)$') 291 292 293def get_cpuinfo(): 294 """Read /proc/cpuinfo and convert to a list of dicts.""" 295 cpuinfo = [] 296 with open('/proc/cpuinfo', 'r') as f: 297 cpu = {} 298 for line in f: 299 line = line.strip() 300 if not line: 301 cpuinfo.append(cpu) 302 cpu = {} 303 continue 304 match = _CPUINFO_RE.match(line) 305 cpu[match.group('key')] = match.group('value') 306 if cpu: 307 # cpuinfo usually ends in a blank line, so this shouldn't happen. 308 cpuinfo.append(cpu) 309 return cpuinfo 310 311 312def get_cpu_arch(): 313 """Work out which CPU architecture we're running on""" 314 f = open('/proc/cpuinfo', 'r') 315 cpuinfo = f.readlines() 316 f.close() 317 if list_grep(cpuinfo, '^cpu.*(RS64|POWER3|Broadband Engine)'): 318 return 'power' 319 elif list_grep(cpuinfo, '^cpu.*POWER4'): 320 return 'power4' 321 elif list_grep(cpuinfo, '^cpu.*POWER5'): 322 return 'power5' 323 elif list_grep(cpuinfo, '^cpu.*POWER6'): 324 return 'power6' 325 elif list_grep(cpuinfo, '^cpu.*POWER7'): 326 return 'power7' 327 elif list_grep(cpuinfo, '^cpu.*PPC970'): 328 return 'power970' 329 elif list_grep(cpuinfo, 'ARM'): 330 return 'arm' 331 elif list_grep(cpuinfo, '^flags.*:.* lm .*'): 332 return 'x86_64' 333 elif list_grep(cpuinfo, 'CPU.*implementer.*0x41'): 334 return 'arm' 335 else: 336 return 'i386' 337 338 339def get_arm_soc_family(): 340 """Work out which ARM SoC we're running on""" 341 f = open('/proc/cpuinfo', 'r') 342 cpuinfo = f.readlines() 343 f.close() 344 if list_grep(cpuinfo, 'EXYNOS5'): 345 return 'exynos5' 346 elif list_grep(cpuinfo, 'Tegra'): 347 return 'tegra' 348 elif list_grep(cpuinfo, 'Rockchip'): 349 return 'rockchip' 350 return 'arm' 351 352 353def get_cpu_soc_family(): 354 """Like get_cpu_arch, but for ARM, returns the SoC family name""" 355 family = get_cpu_arch() 356 if family == 'arm': 357 family = get_arm_soc_family() 358 return family 359 360 361INTEL_UARCH_TABLE = { 362 '06_1C': 'Atom', 363 '06_26': 'Atom', 364 '06_36': 'Atom', 365 '06_4C': 'Braswell', 366 '06_3D': 'Broadwell', 367 '06_0D': 'Dothan', 368 '06_3A': 'IvyBridge', 369 '06_3E': 'IvyBridge', 370 '06_3C': 'Haswell', 371 '06_3F': 'Haswell', 372 '06_45': 'Haswell', 373 '06_46': 'Haswell', 374 '06_0F': 'Merom', 375 '06_16': 'Merom', 376 '06_17': 'Nehalem', 377 '06_1A': 'Nehalem', 378 '06_1D': 'Nehalem', 379 '06_1E': 'Nehalem', 380 '06_1F': 'Nehalem', 381 '06_2E': 'Nehalem', 382 '06_2A': 'SandyBridge', 383 '06_2D': 'SandyBridge', 384 '06_4E': 'Skylake', 385 '0F_03': 'Prescott', 386 '0F_04': 'Prescott', 387 '0F_06': 'Presler', 388 '06_25': 'Westmere', 389 '06_2C': 'Westmere', 390 '06_2F': 'Westmere', 391} 392 393 394def get_intel_cpu_uarch(numeric=False): 395 """Return the Intel microarchitecture we're running on, or None. 396 397 Returns None if this is not an Intel CPU. Returns the family and model as 398 underscore-separated hex (per Intel manual convention) if the uarch is not 399 known, or if numeric is True. 400 """ 401 if not get_current_kernel_arch().startswith('x86'): 402 return None 403 cpuinfo = get_cpuinfo()[0] 404 if cpuinfo['vendor_id'] != 'GenuineIntel': 405 return None 406 family_model = '%02X_%02X' % (int(cpuinfo['cpu family']), 407 int(cpuinfo['model'])) 408 if numeric: 409 return family_model 410 return INTEL_UARCH_TABLE.get(family_model, family_model) 411 412 413def get_current_kernel_arch(): 414 """Get the machine architecture, now just a wrap of 'uname -m'.""" 415 return os.popen('uname -m').read().rstrip() 416 417 418def get_file_arch(filename): 419 # -L means follow symlinks 420 file_data = utils.system_output('file -L ' + filename) 421 if file_data.count('80386'): 422 return 'i386' 423 return None 424 425 426def count_cpus(): 427 """number of CPUs in the local machine according to /proc/cpuinfo""" 428 try: 429 return multiprocessing.cpu_count() 430 except Exception as e: 431 logging.exception('can not get cpu count from' 432 ' multiprocessing.cpu_count()') 433 cpuinfo = get_cpuinfo() 434 # Returns at least one cpu. Check comment #1 in crosbug.com/p/9582. 435 return len(cpuinfo) or 1 436 437 438def cpu_online_map(): 439 """ 440 Check out the available cpu online map 441 """ 442 cpuinfo = get_cpuinfo() 443 cpus = [] 444 for cpu in cpuinfo: 445 cpus.append(cpu['processor']) # grab cpu number 446 return cpus 447 448 449def get_cpu_family(): 450 cpuinfo = get_cpuinfo()[0] 451 return int(cpuinfo['cpu_family']) 452 453 454def get_cpu_vendor(): 455 cpuinfo = get_cpuinfo() 456 vendors = [cpu['vendor_id'] for cpu in cpuinfo] 457 for v in vendors[1:]: 458 if v != vendors[0]: 459 raise error.TestError('multiple cpu vendors found: ' + str(vendors)) 460 return vendors[0] 461 462 463def probe_cpus(): 464 """ 465 This routine returns a list of cpu devices found under 466 /sys/devices/system/cpu. 467 """ 468 cmd = 'find /sys/devices/system/cpu/ -maxdepth 1 -type d -name cpu*' 469 return utils.system_output(cmd).splitlines() 470 471 472# Returns total memory in kb 473def read_from_meminfo(key): 474 meminfo = utils.system_output('grep %s /proc/meminfo' % key) 475 return int(re.search(r'\d+', meminfo).group(0)) 476 477 478def memtotal(): 479 return read_from_meminfo('MemTotal') 480 481 482def freememtotal(): 483 return read_from_meminfo('MemFree') 484 485def usable_memtotal(): 486 # Reserved 5% for OS use 487 return int(read_from_meminfo('MemFree') * 0.95) 488 489 490def rounded_memtotal(): 491 # Get total of all physical mem, in kbytes 492 usable_kbytes = memtotal() 493 # usable_kbytes is system's usable DRAM in kbytes, 494 # as reported by memtotal() from device /proc/meminfo memtotal 495 # after Linux deducts 1.5% to 5.1% for system table overhead 496 # Undo the unknown actual deduction by rounding up 497 # to next small multiple of a big power-of-two 498 # eg 12GB - 5.1% gets rounded back up to 12GB 499 mindeduct = 0.015 # 1.5 percent 500 maxdeduct = 0.055 # 5.5 percent 501 # deduction range 1.5% .. 5.5% supports physical mem sizes 502 # 6GB .. 12GB in steps of .5GB 503 # 12GB .. 24GB in steps of 1 GB 504 # 24GB .. 48GB in steps of 2 GB ... 505 # Finer granularity in physical mem sizes would require 506 # tighter spread between min and max possible deductions 507 508 # increase mem size by at least min deduction, without rounding 509 min_kbytes = int(usable_kbytes / (1.0 - mindeduct)) 510 # increase mem size further by 2**n rounding, by 0..roundKb or more 511 round_kbytes = int(usable_kbytes / (1.0 - maxdeduct)) - min_kbytes 512 # find least binary roundup 2**n that covers worst-cast roundKb 513 mod2n = 1 << int(math.ceil(math.log(round_kbytes, 2))) 514 # have round_kbytes <= mod2n < round_kbytes*2 515 # round min_kbytes up to next multiple of mod2n 516 phys_kbytes = min_kbytes + mod2n - 1 517 phys_kbytes = phys_kbytes - (phys_kbytes % mod2n) # clear low bits 518 return phys_kbytes 519 520 521def sysctl(key, value=None): 522 """Generic implementation of sysctl, to read and write. 523 524 @param key: A location under /proc/sys 525 @param value: If not None, a value to write into the sysctl. 526 527 @return The single-line sysctl value as a string. 528 """ 529 path = '/proc/sys/%s' % key 530 if value is not None: 531 utils.write_one_line(path, str(value)) 532 return utils.read_one_line(path) 533 534 535def sysctl_kernel(key, value=None): 536 """(Very) partial implementation of sysctl, for kernel params""" 537 if value is not None: 538 # write 539 utils.write_one_line('/proc/sys/kernel/%s' % key, str(value)) 540 else: 541 # read 542 out = utils.read_one_line('/proc/sys/kernel/%s' % key) 543 return int(re.search(r'\d+', out).group(0)) 544 545 546def _convert_exit_status(sts): 547 if os.WIFSIGNALED(sts): 548 return -os.WTERMSIG(sts) 549 elif os.WIFEXITED(sts): 550 return os.WEXITSTATUS(sts) 551 else: 552 # impossible? 553 raise RuntimeError("Unknown exit status %d!" % sts) 554 555 556def where_art_thy_filehandles(): 557 """Dump the current list of filehandles""" 558 os.system("ls -l /proc/%d/fd >> /dev/tty" % os.getpid()) 559 560 561def print_to_tty(string): 562 """Output string straight to the tty""" 563 open('/dev/tty', 'w').write(string + '\n') 564 565 566def dump_object(object): 567 """Dump an object's attributes and methods 568 569 kind of like dir() 570 """ 571 for item in object.__dict__.iteritems(): 572 print item 573 try: 574 (key, value) = item 575 dump_object(value) 576 except: 577 continue 578 579 580def environ(env_key): 581 """return the requested environment variable, or '' if unset""" 582 if (os.environ.has_key(env_key)): 583 return os.environ[env_key] 584 else: 585 return '' 586 587 588def prepend_path(newpath, oldpath): 589 """prepend newpath to oldpath""" 590 if (oldpath): 591 return newpath + ':' + oldpath 592 else: 593 return newpath 594 595 596def append_path(oldpath, newpath): 597 """append newpath to oldpath""" 598 if (oldpath): 599 return oldpath + ':' + newpath 600 else: 601 return newpath 602 603 604_TIME_OUTPUT_RE = re.compile( 605 r'([\d\.]*)user ([\d\.]*)system ' 606 r'(\d*):([\d\.]*)elapsed (\d*)%CPU') 607 608 609def avgtime_print(dir): 610 """ Calculate some benchmarking statistics. 611 Input is a directory containing a file called 'time'. 612 File contains one-per-line results of /usr/bin/time. 613 Output is average Elapsed, User, and System time in seconds, 614 and average CPU percentage. 615 """ 616 user = system = elapsed = cpu = count = 0 617 with open(dir + "/time") as f: 618 for line in f: 619 try: 620 m = _TIME_OUTPUT_RE.match(line); 621 user += float(m.group(1)) 622 system += float(m.group(2)) 623 elapsed += (float(m.group(3)) * 60) + float(m.group(4)) 624 cpu += float(m.group(5)) 625 count += 1 626 except: 627 raise ValueError("badly formatted times") 628 629 return "Elapsed: %0.2fs User: %0.2fs System: %0.2fs CPU: %0.0f%%" % \ 630 (elapsed / count, user / count, system / count, cpu / count) 631 632 633def to_seconds(time_string): 634 """Converts a string in M+:SS.SS format to S+.SS""" 635 elts = time_string.split(':') 636 if len(elts) == 1: 637 return time_string 638 return str(int(elts[0]) * 60 + float(elts[1])) 639 640 641_TIME_OUTPUT_RE_2 = re.compile(r'(.*?)user (.*?)system (.*?)elapsed') 642 643 644def extract_all_time_results(results_string): 645 """Extract user, system, and elapsed times into a list of tuples""" 646 results = [] 647 for result in _TIME_OUTPUT_RE_2.findall(results_string): 648 results.append(tuple([to_seconds(elt) for elt in result])) 649 return results 650 651 652def running_config(): 653 """ 654 Return path of config file of the currently running kernel 655 """ 656 version = utils.system_output('uname -r') 657 for config in ('/proc/config.gz', \ 658 '/boot/config-%s' % version, 659 '/lib/modules/%s/build/.config' % version): 660 if os.path.isfile(config): 661 return config 662 return None 663 664 665def check_for_kernel_feature(feature): 666 config = running_config() 667 668 if not config: 669 raise TypeError("Can't find kernel config file") 670 671 if magic.guess_type(config) == 'application/x-gzip': 672 grep = 'zgrep' 673 else: 674 grep = 'grep' 675 grep += ' ^CONFIG_%s= %s' % (feature, config) 676 677 if not utils.system_output(grep, ignore_status=True): 678 raise ValueError("Kernel doesn't have a %s feature" % (feature)) 679 680 681def check_glibc_ver(ver): 682 glibc_ver = commands.getoutput('ldd --version').splitlines()[0] 683 glibc_ver = re.search(r'(\d+\.\d+(\.\d+)?)', glibc_ver).group() 684 if utils.compare_versions(glibc_ver, ver) == -1: 685 raise error.TestError("Glibc too old (%s). Glibc >= %s is needed." % 686 (glibc_ver, ver)) 687 688def check_kernel_ver(ver): 689 kernel_ver = utils.system_output('uname -r') 690 kv_tmp = re.split(r'[-]', kernel_ver)[0:3] 691 # In compare_versions, if v1 < v2, return value == -1 692 if utils.compare_versions(kv_tmp[0], ver) == -1: 693 raise error.TestError("Kernel too old (%s). Kernel > %s is needed." % 694 (kernel_ver, ver)) 695 696 697def human_format(number): 698 # Convert number to kilo / mega / giga format. 699 if number < 1024: 700 return "%d" % number 701 kilo = float(number) / 1024.0 702 if kilo < 1024: 703 return "%.2fk" % kilo 704 meg = kilo / 1024.0 705 if meg < 1024: 706 return "%.2fM" % meg 707 gig = meg / 1024.0 708 return "%.2fG" % gig 709 710 711def numa_nodes(): 712 node_paths = glob.glob('/sys/devices/system/node/node*') 713 nodes = [int(re.sub(r'.*node(\d+)', r'\1', x)) for x in node_paths] 714 return (sorted(nodes)) 715 716 717def node_size(): 718 nodes = max(len(numa_nodes()), 1) 719 return ((memtotal() * 1024) / nodes) 720 721 722def pickle_load(filename): 723 return pickle.load(open(filename, 'r')) 724 725 726# Return the kernel version and build timestamp. 727def running_os_release(): 728 return os.uname()[2:4] 729 730 731def running_os_ident(): 732 (version, timestamp) = running_os_release() 733 return version + '::' + timestamp 734 735 736def running_os_full_version(): 737 (version, timestamp) = running_os_release() 738 return version 739 740 741# much like find . -name 'pattern' 742def locate(pattern, root=os.getcwd()): 743 for path, dirs, files in os.walk(root): 744 for f in files: 745 if fnmatch.fnmatch(f, pattern): 746 yield os.path.abspath(os.path.join(path, f)) 747 748 749def freespace(path): 750 """Return the disk free space, in bytes""" 751 s = os.statvfs(path) 752 return s.f_bavail * s.f_bsize 753 754 755def disk_block_size(path): 756 """Return the disk block size, in bytes""" 757 return os.statvfs(path).f_bsize 758 759 760_DISK_PARTITION_3_RE = re.compile(r'^(/dev/hd[a-z]+)3', re.M) 761 762def get_disks(): 763 df_output = utils.system_output('df') 764 return _DISK_PARTITION_3_RE.findall(df_output) 765 766 767def get_disk_size(disk_name): 768 """ 769 Return size of disk in byte. Return 0 in Error Case 770 771 @param disk_name: disk name to find size 772 """ 773 device = os.path.basename(disk_name) 774 for line in file('/proc/partitions'): 775 try: 776 _, _, blocks, name = re.split(r' +', line.strip()) 777 except ValueError: 778 continue 779 if name == device: 780 return 1024 * int(blocks) 781 return 0 782 783 784def get_disk_size_gb(disk_name): 785 """ 786 Return size of disk in GB (10^9). Return 0 in Error Case 787 788 @param disk_name: disk name to find size 789 """ 790 return int(get_disk_size(disk_name) / (10.0 ** 9) + 0.5) 791 792 793def get_disk_model(disk_name): 794 """ 795 Return model name for internal storage device 796 797 @param disk_name: disk name to find model 798 """ 799 cmd1 = 'udevadm info --query=property --name=%s' % disk_name 800 cmd2 = 'grep -E "ID_(NAME|MODEL)="' 801 cmd3 = 'cut -f 2 -d"="' 802 cmd = ' | '.join([cmd1, cmd2, cmd3]) 803 return utils.system_output(cmd) 804 805 806_DISK_DEV_RE = re.compile(r'/dev/sd[a-z]|/dev/mmcblk[0-9]*') 807 808 809def get_disk_from_filename(filename): 810 """ 811 Return the disk device the filename is on. 812 If the file is on tmpfs or other special file systems, 813 return None. 814 815 @param filename: name of file, full path. 816 """ 817 818 if not os.path.exists(filename): 819 raise error.TestError('file %s missing' % filename) 820 821 if filename[0] != '/': 822 raise error.TestError('This code works only with full path') 823 824 m = _DISK_DEV_RE.match(filename) 825 while not m: 826 if filename[0] != '/': 827 return None 828 if filename == '/dev/root': 829 cmd = 'rootdev -d -s' 830 elif filename.startswith('/dev/mapper'): 831 cmd = 'dmsetup table "%s"' % os.path.basename(filename) 832 dmsetup_output = utils.system_output(cmd).split(' ') 833 if dmsetup_output[2] == 'verity': 834 maj_min = dmsetup_output[4] 835 elif dmsetup_output[2] == 'crypt': 836 maj_min = dmsetup_output[6] 837 cmd = 'realpath "/dev/block/%s"' % maj_min 838 elif filename.startswith('/dev/loop'): 839 cmd = 'losetup -O BACK-FILE "%s" | tail -1' % filename 840 else: 841 cmd = 'df "%s" | tail -1 | cut -f 1 -d" "' % filename 842 filename = utils.system_output(cmd) 843 m = _DISK_DEV_RE.match(filename) 844 return m.group(0) 845 846 847def get_disk_firmware_version(disk_name): 848 """ 849 Return firmware version for internal storage device. (empty string for eMMC) 850 851 @param disk_name: disk name to find model 852 """ 853 cmd1 = 'udevadm info --query=property --name=%s' % disk_name 854 cmd2 = 'grep -E "ID_REVISION="' 855 cmd3 = 'cut -f 2 -d"="' 856 cmd = ' | '.join([cmd1, cmd2, cmd3]) 857 return utils.system_output(cmd) 858 859 860def is_disk_scsi(disk_name): 861 """ 862 Return true if disk is a scsi device, return false otherwise 863 864 @param disk_name: disk name check 865 """ 866 return re.match('/dev/sd[a-z]+', disk_name) 867 868 869def is_disk_harddisk(disk_name): 870 """ 871 Return true if disk is a harddisk, return false otherwise 872 873 @param disk_name: disk name check 874 """ 875 cmd1 = 'udevadm info --query=property --name=%s' % disk_name 876 cmd2 = 'grep -E "ID_ATA_ROTATION_RATE_RPM="' 877 cmd3 = 'cut -f 2 -d"="' 878 cmd = ' | '.join([cmd1, cmd2, cmd3]) 879 880 rtt = utils.system_output(cmd) 881 882 # eMMC will not have this field; rtt == '' 883 # SSD will have zero rotation rate; rtt == '0' 884 # For harddisk rtt > 0 885 return rtt and int(rtt) > 0 886 887 888def verify_hdparm_feature(disk_name, feature): 889 """ 890 Check for feature support for SCSI disk using hdparm 891 892 @param disk_name: target disk 893 @param feature: hdparm output string of the feature 894 """ 895 cmd = 'hdparm -I %s | grep -q "%s"' % (disk_name, feature) 896 ret = utils.system(cmd, ignore_status=True) 897 if ret == 0: 898 return True 899 elif ret == 1: 900 return False 901 else: 902 raise error.TestFail('Error running command %s' % cmd) 903 904 905def get_storage_error_msg(disk_name, reason): 906 """ 907 Get Error message for storage test which include disk model. 908 and also include the firmware version for the SCSI disk 909 910 @param disk_name: target disk 911 @param reason: Reason of the error. 912 """ 913 914 msg = reason 915 916 model = get_disk_model(disk_name) 917 msg += ' Disk model: %s' % model 918 919 if is_disk_scsi(disk_name): 920 fw = get_disk_firmware_version(disk_name) 921 msg += ' firmware: %s' % fw 922 923 return msg 924 925 926def load_module(module_name, params=None): 927 # Checks if a module has already been loaded 928 if module_is_loaded(module_name): 929 return False 930 931 cmd = '/sbin/modprobe ' + module_name 932 if params: 933 cmd += ' ' + params 934 utils.system(cmd) 935 return True 936 937 938def unload_module(module_name): 939 """ 940 Removes a module. Handles dependencies. If even then it's not possible 941 to remove one of the modules, it will trhow an error.CmdError exception. 942 943 @param module_name: Name of the module we want to remove. 944 """ 945 l_raw = utils.system_output("/bin/lsmod").splitlines() 946 lsmod = [x for x in l_raw if x.split()[0] == module_name] 947 if len(lsmod) > 0: 948 line_parts = lsmod[0].split() 949 if len(line_parts) == 4: 950 submodules = line_parts[3].split(",") 951 for submodule in submodules: 952 unload_module(submodule) 953 utils.system("/sbin/modprobe -r %s" % module_name) 954 logging.info("Module %s unloaded", module_name) 955 else: 956 logging.info("Module %s is already unloaded", module_name) 957 958 959def module_is_loaded(module_name): 960 module_name = module_name.replace('-', '_') 961 modules = utils.system_output('/bin/lsmod').splitlines() 962 for module in modules: 963 if module.startswith(module_name) and module[len(module_name)] == ' ': 964 return True 965 return False 966 967 968def get_loaded_modules(): 969 lsmod_output = utils.system_output('/bin/lsmod').splitlines()[1:] 970 return [line.split(None, 1)[0] for line in lsmod_output] 971 972 973def get_huge_page_size(): 974 output = utils.system_output('grep Hugepagesize /proc/meminfo') 975 return int(output.split()[1]) # Assumes units always in kB. :( 976 977 978def get_num_huge_pages(): 979 raw_hugepages = utils.system_output('/sbin/sysctl vm.nr_hugepages') 980 return int(raw_hugepages.split()[2]) 981 982 983def set_num_huge_pages(num): 984 utils.system('/sbin/sysctl vm.nr_hugepages=%d' % num) 985 986 987def ping_default_gateway(): 988 """Ping the default gateway.""" 989 990 network = open('/etc/sysconfig/network') 991 m = re.search('GATEWAY=(\S+)', network.read()) 992 993 if m: 994 gw = m.group(1) 995 cmd = 'ping %s -c 5 > /dev/null' % gw 996 return utils.system(cmd, ignore_status=True) 997 998 raise error.TestError('Unable to find default gateway') 999 1000 1001def drop_caches(): 1002 """Writes back all dirty pages to disk and clears all the caches.""" 1003 utils.system("sync") 1004 # We ignore failures here as this will fail on 2.6.11 kernels. 1005 utils.system("echo 3 > /proc/sys/vm/drop_caches", ignore_status=True) 1006 1007 1008def process_is_alive(name_pattern): 1009 """ 1010 'pgrep name' misses all python processes and also long process names. 1011 'pgrep -f name' gets all shell commands with name in args. 1012 So look only for command whose initial pathname ends with name. 1013 Name itself is an egrep pattern, so it can use | etc for variations. 1014 """ 1015 return utils.system("pgrep -f '^([^ /]*/)*(%s)([ ]|$)'" % name_pattern, 1016 ignore_status=True) == 0 1017 1018 1019def get_hwclock_seconds(utc=True): 1020 """ 1021 Return the hardware clock in seconds as a floating point value. 1022 Use Coordinated Universal Time if utc is True, local time otherwise. 1023 Raise a ValueError if unable to read the hardware clock. 1024 """ 1025 cmd = '/sbin/hwclock --debug' 1026 if utc: 1027 cmd += ' --utc' 1028 hwclock_output = utils.system_output(cmd, ignore_status=True) 1029 match = re.search(r'= ([0-9]+) seconds since .+ (-?[0-9.]+) seconds$', 1030 hwclock_output, re.DOTALL) 1031 if match: 1032 seconds = int(match.group(1)) + float(match.group(2)) 1033 logging.debug('hwclock seconds = %f', seconds) 1034 return seconds 1035 1036 raise ValueError('Unable to read the hardware clock -- ' + 1037 hwclock_output) 1038 1039 1040def set_wake_alarm(alarm_time): 1041 """ 1042 Set the hardware RTC-based wake alarm to 'alarm_time'. 1043 """ 1044 utils.write_one_line('/sys/class/rtc/rtc0/wakealarm', str(alarm_time)) 1045 1046 1047def set_power_state(state): 1048 """ 1049 Set the system power state to 'state'. 1050 """ 1051 utils.write_one_line('/sys/power/state', state) 1052 1053 1054def standby(): 1055 """ 1056 Power-on suspend (S1) 1057 """ 1058 set_power_state('standby') 1059 1060 1061def suspend_to_ram(): 1062 """ 1063 Suspend the system to RAM (S3) 1064 """ 1065 set_power_state('mem') 1066 1067 1068def suspend_to_disk(): 1069 """ 1070 Suspend the system to disk (S4) 1071 """ 1072 set_power_state('disk') 1073