1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4"""Module to deal with result cache.""" 5 6from __future__ import print_function 7 8import glob 9import hashlib 10import os 11import pickle 12import re 13import tempfile 14import json 15 16from cros_utils import command_executer 17from cros_utils import misc 18 19from image_checksummer import ImageChecksummer 20 21import results_report 22import test_flag 23 24SCRATCH_DIR = os.path.expanduser('~/cros_scratch') 25RESULTS_FILE = 'results.txt' 26MACHINE_FILE = 'machine.txt' 27AUTOTEST_TARBALL = 'autotest.tbz2' 28PERF_RESULTS_FILE = 'perf-results.txt' 29CACHE_KEYS_FILE = 'cache_keys.txt' 30 31 32class Result(object): 33 """Class for holding the results of a single test run. 34 35 This class manages what exactly is stored inside the cache without knowing 36 what the key of the cache is. For runs with perf, it stores perf.data, 37 perf.report, etc. The key generation is handled by the ResultsCache class. 38 """ 39 40 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 41 self.chromeos_root = label.chromeos_root 42 self._logger = logger 43 self.ce = cmd_exec or command_executer.GetCommandExecuter( 44 self._logger, log_level=log_level) 45 self.temp_dir = None 46 self.label = label 47 self.results_dir = None 48 self.log_level = log_level 49 self.machine = machine 50 self.perf_data_files = [] 51 self.perf_report_files = [] 52 self.results_file = [] 53 self.chrome_version = '' 54 self.err = None 55 self.chroot_results_dir = '' 56 self.test_name = '' 57 self.keyvals = None 58 self.board = None 59 self.suite = None 60 self.retval = None 61 self.out = None 62 63 def CopyFilesTo(self, dest_dir, files_to_copy): 64 file_index = 0 65 for file_to_copy in files_to_copy: 66 if not os.path.isdir(dest_dir): 67 command = 'mkdir -p %s' % dest_dir 68 self.ce.RunCommand(command) 69 dest_file = os.path.join( 70 dest_dir, ('%s.%s' % (os.path.basename(file_to_copy), file_index))) 71 ret = self.ce.CopyFiles(file_to_copy, dest_file, recursive=False) 72 if ret: 73 raise IOError('Could not copy results file: %s' % file_to_copy) 74 75 def CopyResultsTo(self, dest_dir): 76 self.CopyFilesTo(dest_dir, self.perf_data_files) 77 self.CopyFilesTo(dest_dir, self.perf_report_files) 78 if len(self.perf_data_files) or len(self.perf_report_files): 79 self._logger.LogOutput('Perf results files stored in %s.' % dest_dir) 80 81 def GetNewKeyvals(self, keyvals_dict): 82 # Initialize 'units' dictionary. 83 units_dict = {} 84 for k in keyvals_dict: 85 units_dict[k] = '' 86 results_files = self.GetDataMeasurementsFiles() 87 for f in results_files: 88 # Make sure we can find the results file 89 if os.path.exists(f): 90 data_filename = f 91 else: 92 # Otherwise get the base filename and create the correct 93 # path for it. 94 _, f_base = misc.GetRoot(f) 95 data_filename = os.path.join(self.chromeos_root, 'chroot/tmp', 96 self.temp_dir, f_base) 97 if data_filename.find('.json') > 0: 98 raw_dict = dict() 99 if os.path.exists(data_filename): 100 with open(data_filename, 'r') as data_file: 101 raw_dict = json.load(data_file) 102 103 if 'charts' in raw_dict: 104 raw_dict = raw_dict['charts'] 105 for k1 in raw_dict: 106 field_dict = raw_dict[k1] 107 for k2 in field_dict: 108 result_dict = field_dict[k2] 109 key = k1 + '__' + k2 110 if 'value' in result_dict: 111 keyvals_dict[key] = result_dict['value'] 112 elif 'values' in result_dict: 113 values = result_dict['values'] 114 if ('type' in result_dict and 115 result_dict['type'] == 'list_of_scalar_values' and values and 116 values != 'null'): 117 keyvals_dict[key] = sum(values) / float(len(values)) 118 else: 119 keyvals_dict[key] = values 120 units_dict[key] = result_dict['units'] 121 else: 122 if os.path.exists(data_filename): 123 with open(data_filename, 'r') as data_file: 124 lines = data_file.readlines() 125 for line in lines: 126 tmp_dict = json.loads(line) 127 graph_name = tmp_dict['graph'] 128 graph_str = (graph_name + '__') if graph_name else '' 129 key = graph_str + tmp_dict['description'] 130 keyvals_dict[key] = tmp_dict['value'] 131 units_dict[key] = tmp_dict['units'] 132 133 return keyvals_dict, units_dict 134 135 def AppendTelemetryUnits(self, keyvals_dict, units_dict): 136 """keyvals_dict is the dict of key-value used to generate Crosperf reports. 137 138 units_dict is a dictionary of the units for the return values in 139 keyvals_dict. We need to associate the units with the return values, 140 for Telemetry tests, so that we can include the units in the reports. 141 This function takes each value in keyvals_dict, finds the corresponding 142 unit in the units_dict, and replaces the old value with a list of the 143 old value and the units. This later gets properly parsed in the 144 ResultOrganizer class, for generating the reports. 145 """ 146 147 results_dict = {} 148 for k in keyvals_dict: 149 # We don't want these lines in our reports; they add no useful data. 150 if k == '' or k == 'telemetry_Crosperf': 151 continue 152 val = keyvals_dict[k] 153 units = units_dict[k] 154 new_val = [val, units] 155 results_dict[k] = new_val 156 return results_dict 157 158 def GetKeyvals(self): 159 results_in_chroot = os.path.join(self.chromeos_root, 'chroot', 'tmp') 160 if not self.temp_dir: 161 self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot) 162 command = 'cp -r {0}/* {1}'.format(self.results_dir, self.temp_dir) 163 self.ce.RunCommand(command, print_to_console=False) 164 165 command = ('python generate_test_report --no-color --csv %s' % 166 (os.path.join('/tmp', os.path.basename(self.temp_dir)))) 167 _, out, _ = self.ce.ChrootRunCommandWOutput( 168 self.chromeos_root, command, print_to_console=False) 169 keyvals_dict = {} 170 tmp_dir_in_chroot = misc.GetInsideChrootPath(self.chromeos_root, 171 self.temp_dir) 172 for line in out.splitlines(): 173 tokens = re.split('=|,', line) 174 key = tokens[-2] 175 if key.startswith(tmp_dir_in_chroot): 176 key = key[len(tmp_dir_in_chroot) + 1:] 177 value = tokens[-1] 178 keyvals_dict[key] = value 179 180 # Check to see if there is a perf_measurements file and get the 181 # data from it if so. 182 keyvals_dict, units_dict = self.GetNewKeyvals(keyvals_dict) 183 if self.suite == 'telemetry_Crosperf': 184 # For telemtry_Crosperf results, append the units to the return 185 # results, for use in generating the reports. 186 keyvals_dict = self.AppendTelemetryUnits(keyvals_dict, units_dict) 187 return keyvals_dict 188 189 def GetResultsDir(self): 190 mo = re.search(r'Results placed in (\S+)', self.out) 191 if mo: 192 result = mo.group(1) 193 return result 194 raise RuntimeError('Could not find results directory.') 195 196 def FindFilesInResultsDir(self, find_args): 197 if not self.results_dir: 198 return None 199 200 command = 'find %s %s' % (self.results_dir, find_args) 201 ret, out, _ = self.ce.RunCommandWOutput(command, print_to_console=False) 202 if ret: 203 raise RuntimeError('Could not run find command!') 204 return out 205 206 def GetResultsFile(self): 207 return self.FindFilesInResultsDir('-name results-chart.json').splitlines() 208 209 def GetPerfDataFiles(self): 210 return self.FindFilesInResultsDir('-name perf.data').splitlines() 211 212 def GetPerfReportFiles(self): 213 return self.FindFilesInResultsDir('-name perf.data.report').splitlines() 214 215 def GetDataMeasurementsFiles(self): 216 result = self.FindFilesInResultsDir('-name perf_measurements').splitlines() 217 if not result: 218 result = \ 219 self.FindFilesInResultsDir('-name results-chart.json').splitlines() 220 return result 221 222 def GeneratePerfReportFiles(self): 223 perf_report_files = [] 224 for perf_data_file in self.perf_data_files: 225 # Generate a perf.report and store it side-by-side with the perf.data 226 # file. 227 chroot_perf_data_file = misc.GetInsideChrootPath(self.chromeos_root, 228 perf_data_file) 229 perf_report_file = '%s.report' % perf_data_file 230 if os.path.exists(perf_report_file): 231 raise RuntimeError( 232 'Perf report file already exists: %s' % perf_report_file) 233 chroot_perf_report_file = misc.GetInsideChrootPath( 234 self.chromeos_root, perf_report_file) 235 perf_path = os.path.join(self.chromeos_root, 'chroot', 'usr/bin/perf') 236 237 perf_file = '/usr/sbin/perf' 238 if os.path.exists(perf_path): 239 perf_file = '/usr/bin/perf' 240 241 command = ('%s report ' 242 '-n ' 243 '--symfs /build/%s ' 244 '--vmlinux /build/%s/usr/lib/debug/boot/vmlinux ' 245 '--kallsyms /build/%s/boot/System.map-* ' 246 '-i %s --stdio ' 247 '> %s' % (perf_file, self.board, self.board, self.board, 248 chroot_perf_data_file, chroot_perf_report_file)) 249 self.ce.ChrootRunCommand(self.chromeos_root, command) 250 251 # Add a keyval to the dictionary for the events captured. 252 perf_report_files.append( 253 misc.GetOutsideChrootPath(self.chromeos_root, 254 chroot_perf_report_file)) 255 return perf_report_files 256 257 def GatherPerfResults(self): 258 report_id = 0 259 for perf_report_file in self.perf_report_files: 260 with open(perf_report_file, 'r') as f: 261 report_contents = f.read() 262 for group in re.findall(r'Events: (\S+) (\S+)', report_contents): 263 num_events = group[0] 264 event_name = group[1] 265 key = 'perf_%s_%s' % (report_id, event_name) 266 value = str(misc.UnitToNumber(num_events)) 267 self.keyvals[key] = value 268 269 def PopulateFromRun(self, out, err, retval, test, suite): 270 self.board = self.label.board 271 self.out = out 272 self.err = err 273 self.retval = retval 274 self.test_name = test 275 self.suite = suite 276 self.chroot_results_dir = self.GetResultsDir() 277 self.results_dir = misc.GetOutsideChrootPath(self.chromeos_root, 278 self.chroot_results_dir) 279 self.results_file = self.GetResultsFile() 280 self.perf_data_files = self.GetPerfDataFiles() 281 # Include all perf.report data in table. 282 self.perf_report_files = self.GeneratePerfReportFiles() 283 # TODO(asharif): Do something similar with perf stat. 284 285 # Grab keyvals from the directory. 286 self.ProcessResults() 287 288 def ProcessJsonResults(self): 289 # Open and parse the json results file generated by telemetry/test_that. 290 if not self.results_file: 291 raise IOError('No results file found.') 292 filename = self.results_file[0] 293 if not filename.endswith('.json'): 294 raise IOError('Attempt to call json on non-json file: %s' % filename) 295 296 if not os.path.exists(filename): 297 return {} 298 299 keyvals = {} 300 with open(filename, 'r') as f: 301 raw_dict = json.load(f) 302 if 'charts' in raw_dict: 303 raw_dict = raw_dict['charts'] 304 for k, field_dict in raw_dict.iteritems(): 305 for item in field_dict: 306 keyname = k + '__' + item 307 value_dict = field_dict[item] 308 if 'value' in value_dict: 309 result = value_dict['value'] 310 elif 'values' in value_dict: 311 values = value_dict['values'] 312 if not values: 313 continue 314 if ('type' in value_dict and 315 value_dict['type'] == 'list_of_scalar_values' and 316 values != 'null'): 317 result = sum(values) / float(len(values)) 318 else: 319 result = values 320 units = value_dict['units'] 321 new_value = [result, units] 322 keyvals[keyname] = new_value 323 return keyvals 324 325 def ProcessResults(self, use_cache=False): 326 # Note that this function doesn't know anything about whether there is a 327 # cache hit or miss. It should process results agnostic of the cache hit 328 # state. 329 if self.results_file and self.results_file[0].find( 330 'results-chart.json') != -1: 331 self.keyvals = self.ProcessJsonResults() 332 else: 333 if not use_cache: 334 print('\n ** WARNING **: Had to use deprecated output-method to ' 335 'collect results.\n') 336 self.keyvals = self.GetKeyvals() 337 self.keyvals['retval'] = self.retval 338 # Generate report from all perf.data files. 339 # Now parse all perf report files and include them in keyvals. 340 self.GatherPerfResults() 341 342 def GetChromeVersionFromCache(self, cache_dir): 343 # Read chrome_version from keys file, if present. 344 chrome_version = '' 345 keys_file = os.path.join(cache_dir, CACHE_KEYS_FILE) 346 if os.path.exists(keys_file): 347 with open(keys_file, 'r') as f: 348 lines = f.readlines() 349 for l in lines: 350 if l.startswith('Google Chrome '): 351 chrome_version = l 352 if chrome_version.endswith('\n'): 353 chrome_version = chrome_version[:-1] 354 break 355 return chrome_version 356 357 def PopulateFromCacheDir(self, cache_dir, test, suite): 358 self.test_name = test 359 self.suite = suite 360 # Read in everything from the cache directory. 361 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 362 self.out = pickle.load(f) 363 self.err = pickle.load(f) 364 self.retval = pickle.load(f) 365 366 # Untar the tarball to a temporary directory 367 self.temp_dir = tempfile.mkdtemp(dir=os.path.join(self.chromeos_root, 368 'chroot', 'tmp')) 369 370 command = ('cd %s && tar xf %s' % 371 (self.temp_dir, os.path.join(cache_dir, AUTOTEST_TARBALL))) 372 ret = self.ce.RunCommand(command, print_to_console=False) 373 if ret: 374 raise RuntimeError('Could not untar cached tarball') 375 self.results_dir = self.temp_dir 376 self.results_file = self.GetDataMeasurementsFiles() 377 self.perf_data_files = self.GetPerfDataFiles() 378 self.perf_report_files = self.GetPerfReportFiles() 379 self.chrome_version = self.GetChromeVersionFromCache(cache_dir) 380 self.ProcessResults(use_cache=True) 381 382 def CleanUp(self, rm_chroot_tmp): 383 if rm_chroot_tmp and self.results_dir: 384 dirname, basename = misc.GetRoot(self.results_dir) 385 if basename.find('test_that_results_') != -1: 386 command = 'rm -rf %s' % self.results_dir 387 else: 388 command = 'rm -rf %s' % dirname 389 self.ce.RunCommand(command) 390 if self.temp_dir: 391 command = 'rm -rf %s' % self.temp_dir 392 self.ce.RunCommand(command) 393 394 def StoreToCacheDir(self, cache_dir, machine_manager, key_list): 395 # Create the dir if it doesn't exist. 396 temp_dir = tempfile.mkdtemp() 397 398 # Store to the temp directory. 399 with open(os.path.join(temp_dir, RESULTS_FILE), 'w') as f: 400 pickle.dump(self.out, f) 401 pickle.dump(self.err, f) 402 pickle.dump(self.retval, f) 403 404 if not test_flag.GetTestMode(): 405 with open(os.path.join(temp_dir, CACHE_KEYS_FILE), 'w') as f: 406 f.write('%s\n' % self.label.name) 407 f.write('%s\n' % self.label.chrome_version) 408 f.write('%s\n' % self.machine.checksum_string) 409 for k in key_list: 410 f.write(k) 411 f.write('\n') 412 413 if self.results_dir: 414 tarball = os.path.join(temp_dir, AUTOTEST_TARBALL) 415 command = ('cd %s && ' 416 'tar ' 417 '--exclude=var/spool ' 418 '--exclude=var/log ' 419 '-cjf %s .' % (self.results_dir, tarball)) 420 ret = self.ce.RunCommand(command) 421 if ret: 422 raise RuntimeError("Couldn't store autotest output directory.") 423 # Store machine info. 424 # TODO(asharif): Make machine_manager a singleton, and don't pass it into 425 # this function. 426 with open(os.path.join(temp_dir, MACHINE_FILE), 'w') as f: 427 f.write(machine_manager.machine_checksum_string[self.label.name]) 428 429 if os.path.exists(cache_dir): 430 command = 'rm -rf {0}'.format(cache_dir) 431 self.ce.RunCommand(command) 432 433 command = 'mkdir -p {0} && '.format(os.path.dirname(cache_dir)) 434 command += 'chmod g+x {0} && '.format(temp_dir) 435 command += 'mv {0} {1}'.format(temp_dir, cache_dir) 436 ret = self.ce.RunCommand(command) 437 if ret: 438 command = 'rm -rf {0}'.format(temp_dir) 439 self.ce.RunCommand(command) 440 raise RuntimeError('Could not move dir %s to dir %s' % (temp_dir, 441 cache_dir)) 442 443 @classmethod 444 def CreateFromRun(cls, 445 logger, 446 log_level, 447 label, 448 machine, 449 out, 450 err, 451 retval, 452 test, 453 suite='telemetry_Crosperf'): 454 if suite == 'telemetry': 455 result = TelemetryResult(logger, label, log_level, machine) 456 else: 457 result = cls(logger, label, log_level, machine) 458 result.PopulateFromRun(out, err, retval, test, suite) 459 return result 460 461 @classmethod 462 def CreateFromCacheHit(cls, 463 logger, 464 log_level, 465 label, 466 machine, 467 cache_dir, 468 test, 469 suite='telemetry_Crosperf'): 470 if suite == 'telemetry': 471 result = TelemetryResult(logger, label, log_level, machine) 472 else: 473 result = cls(logger, label, log_level, machine) 474 try: 475 result.PopulateFromCacheDir(cache_dir, test, suite) 476 477 except RuntimeError as e: 478 logger.LogError('Exception while using cache: %s' % e) 479 return None 480 return result 481 482 483class TelemetryResult(Result): 484 """Class to hold the results of a single Telemetry run.""" 485 486 def __init__(self, logger, label, log_level, machine, cmd_exec=None): 487 super(TelemetryResult, self).__init__(logger, label, log_level, machine, 488 cmd_exec) 489 490 def PopulateFromRun(self, out, err, retval, test, suite): 491 self.out = out 492 self.err = err 493 self.retval = retval 494 495 self.ProcessResults() 496 497 # pylint: disable=arguments-differ 498 def ProcessResults(self): 499 # The output is: 500 # url,average_commit_time (ms),... 501 # www.google.com,33.4,21.2,... 502 # We need to convert to this format: 503 # {"www.google.com:average_commit_time (ms)": "33.4", 504 # "www.google.com:...": "21.2"} 505 # Added note: Occasionally the output comes back 506 # with "JSON.stringify(window.automation.GetResults())" on 507 # the first line, and then the rest of the output as 508 # described above. 509 510 lines = self.out.splitlines() 511 self.keyvals = {} 512 513 if lines: 514 if lines[0].startswith('JSON.stringify'): 515 lines = lines[1:] 516 517 if not lines: 518 return 519 labels = lines[0].split(',') 520 for line in lines[1:]: 521 fields = line.split(',') 522 if len(fields) != len(labels): 523 continue 524 for i in xrange(1, len(labels)): 525 key = '%s %s' % (fields[0], labels[i]) 526 value = fields[i] 527 self.keyvals[key] = value 528 self.keyvals['retval'] = self.retval 529 530 def PopulateFromCacheDir(self, cache_dir, test, suite): 531 self.test_name = test 532 self.suite = suite 533 with open(os.path.join(cache_dir, RESULTS_FILE), 'r') as f: 534 self.out = pickle.load(f) 535 self.err = pickle.load(f) 536 self.retval = pickle.load(f) 537 538 self.chrome_version = \ 539 super(TelemetryResult, self).GetChromeVersionFromCache(cache_dir) 540 self.ProcessResults() 541 542 543class CacheConditions(object): 544 """Various Cache condition values, for export.""" 545 546 # Cache hit only if the result file exists. 547 CACHE_FILE_EXISTS = 0 548 549 # Cache hit if the checksum of cpuinfo and totalmem of 550 # the cached result and the new run match. 551 MACHINES_MATCH = 1 552 553 # Cache hit if the image checksum of the cached result and the new run match. 554 CHECKSUMS_MATCH = 2 555 556 # Cache hit only if the cached result was successful 557 RUN_SUCCEEDED = 3 558 559 # Never a cache hit. 560 FALSE = 4 561 562 # Cache hit if the image path matches the cached image path. 563 IMAGE_PATH_MATCH = 5 564 565 # Cache hit if the uuid of hard disk mataches the cached one 566 567 SAME_MACHINE_MATCH = 6 568 569 570class ResultsCache(object): 571 """Class to handle the cache for storing/retrieving test run results. 572 573 This class manages the key of the cached runs without worrying about what 574 is exactly stored (value). The value generation is handled by the Results 575 class. 576 """ 577 CACHE_VERSION = 6 578 579 def __init__(self): 580 # Proper initialization happens in the Init function below. 581 self.chromeos_image = None 582 self.chromeos_root = None 583 self.test_name = None 584 self.iteration = None 585 self.test_args = None 586 self.profiler_args = None 587 self.board = None 588 self.cache_conditions = None 589 self.machine_manager = None 590 self.machine = None 591 self._logger = None 592 self.ce = None 593 self.label = None 594 self.share_cache = None 595 self.suite = None 596 self.log_level = None 597 self.show_all = None 598 self.run_local = None 599 600 def Init(self, chromeos_image, chromeos_root, test_name, iteration, test_args, 601 profiler_args, machine_manager, machine, board, cache_conditions, 602 logger_to_use, log_level, label, share_cache, suite, 603 show_all_results, run_local): 604 self.chromeos_image = chromeos_image 605 self.chromeos_root = chromeos_root 606 self.test_name = test_name 607 self.iteration = iteration 608 self.test_args = test_args 609 self.profiler_args = profiler_args 610 self.board = board 611 self.cache_conditions = cache_conditions 612 self.machine_manager = machine_manager 613 self.machine = machine 614 self._logger = logger_to_use 615 self.ce = command_executer.GetCommandExecuter( 616 self._logger, log_level=log_level) 617 self.label = label 618 self.share_cache = share_cache 619 self.suite = suite 620 self.log_level = log_level 621 self.show_all = show_all_results 622 self.run_local = run_local 623 624 def GetCacheDirForRead(self): 625 matching_dirs = [] 626 for glob_path in self.FormCacheDir(self.GetCacheKeyList(True)): 627 matching_dirs += glob.glob(glob_path) 628 629 if matching_dirs: 630 # Cache file found. 631 return matching_dirs[0] 632 return None 633 634 def GetCacheDirForWrite(self, get_keylist=False): 635 cache_path = self.FormCacheDir(self.GetCacheKeyList(False))[0] 636 if get_keylist: 637 args_str = '%s_%s_%s' % (self.test_args, self.profiler_args, 638 self.run_local) 639 version, image = results_report.ParseChromeosImage( 640 self.label.chromeos_image) 641 keylist = [ 642 version, image, self.label.board, self.machine.name, self.test_name, 643 str(self.iteration), args_str 644 ] 645 return cache_path, keylist 646 return cache_path 647 648 def FormCacheDir(self, list_of_strings): 649 cache_key = ' '.join(list_of_strings) 650 cache_dir = misc.GetFilenameFromString(cache_key) 651 if self.label.cache_dir: 652 cache_home = os.path.abspath(os.path.expanduser(self.label.cache_dir)) 653 cache_path = [os.path.join(cache_home, cache_dir)] 654 else: 655 cache_path = [os.path.join(SCRATCH_DIR, cache_dir)] 656 657 if len(self.share_cache): 658 for path in [x.strip() for x in self.share_cache.split(',')]: 659 if os.path.exists(path): 660 cache_path.append(os.path.join(path, cache_dir)) 661 else: 662 self._logger.LogFatal('Unable to find shared cache: %s' % path) 663 664 return cache_path 665 666 def GetCacheKeyList(self, read): 667 if read and CacheConditions.MACHINES_MATCH not in self.cache_conditions: 668 machine_checksum = '*' 669 else: 670 machine_checksum = self.machine_manager.machine_checksum[self.label.name] 671 if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions: 672 checksum = '*' 673 elif self.label.image_type == 'trybot': 674 checksum = hashlib.md5(self.label.chromeos_image).hexdigest() 675 elif self.label.image_type == 'official': 676 checksum = '*' 677 else: 678 checksum = ImageChecksummer().Checksum(self.label, self.log_level) 679 680 if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions: 681 image_path_checksum = '*' 682 else: 683 image_path_checksum = hashlib.md5(self.chromeos_image).hexdigest() 684 685 machine_id_checksum = '' 686 if read and CacheConditions.SAME_MACHINE_MATCH not in self.cache_conditions: 687 machine_id_checksum = '*' 688 else: 689 if self.machine and self.machine.name in self.label.remote: 690 machine_id_checksum = self.machine.machine_id_checksum 691 else: 692 for machine in self.machine_manager.GetMachines(self.label): 693 if machine.name == self.label.remote[0]: 694 machine_id_checksum = machine.machine_id_checksum 695 break 696 697 temp_test_args = '%s %s %s' % (self.test_args, self.profiler_args, 698 self.run_local) 699 test_args_checksum = hashlib.md5(temp_test_args).hexdigest() 700 return (image_path_checksum, self.test_name, str(self.iteration), 701 test_args_checksum, checksum, machine_checksum, machine_id_checksum, 702 str(self.CACHE_VERSION)) 703 704 def ReadResult(self): 705 if CacheConditions.FALSE in self.cache_conditions: 706 cache_dir = self.GetCacheDirForWrite() 707 command = 'rm -rf %s' % (cache_dir,) 708 self.ce.RunCommand(command) 709 return None 710 cache_dir = self.GetCacheDirForRead() 711 712 if not cache_dir: 713 return None 714 715 if not os.path.isdir(cache_dir): 716 return None 717 718 if self.log_level == 'verbose': 719 self._logger.LogOutput('Trying to read from cache dir: %s' % cache_dir) 720 result = Result.CreateFromCacheHit(self._logger, self.log_level, self.label, 721 self.machine, cache_dir, self.test_name, 722 self.suite) 723 if not result: 724 return None 725 726 if (result.retval == 0 or 727 CacheConditions.RUN_SUCCEEDED not in self.cache_conditions): 728 return result 729 730 return None 731 732 def StoreResult(self, result): 733 cache_dir, keylist = self.GetCacheDirForWrite(get_keylist=True) 734 result.StoreToCacheDir(cache_dir, self.machine_manager, keylist) 735 736 737class MockResultsCache(ResultsCache): 738 """Class for mock testing, corresponding to ResultsCache class.""" 739 740 def Init(self, *args): 741 pass 742 743 def ReadResult(self): 744 return None 745 746 def StoreResult(self, result): 747 pass 748 749 750class MockResult(Result): 751 """Class for mock testing, corresponding to Result class.""" 752 753 def PopulateFromRun(self, out, err, retval, test, suite): 754 self.out = out 755 self.err = err 756 self.retval = retval 757