1import json 2import os 3 4from autotest_lib.server.hosts import file_store 5from autotest_lib.client.common_lib import utils 6from autotest_lib.tko import tast 7from autotest_lib.tko import utils as tko_utils 8 9 10class HostKeyvalError(Exception): 11 """Raised when the host keyval cannot be read.""" 12 13 14class job(object): 15 """Represents a job.""" 16 17 def __init__(self, dir, user, label, machine, queued_time, started_time, 18 finished_time, machine_owner, machine_group, aborted_by, 19 aborted_on, keyval_dict): 20 self.dir = dir 21 self.tests = [] 22 self.user = user 23 self.label = label 24 self.machine = machine 25 self.queued_time = queued_time 26 self.started_time = started_time 27 self.finished_time = finished_time 28 self.machine_owner = machine_owner 29 self.machine_group = machine_group 30 self.aborted_by = aborted_by 31 self.aborted_on = aborted_on 32 self.keyval_dict = keyval_dict 33 self.afe_parent_job_id = None 34 self.build_version = None 35 self.suite = None 36 self.board = None 37 self.job_idx = None 38 # id of the corresponding tko_task_references entry. 39 # This table is used to refer to skylab task / afe job corresponding to 40 # this tko_job. 41 self.task_reference_id = None 42 43 @staticmethod 44 def read_keyval(dir): 45 """ 46 Read job keyval files. 47 48 @param dir: String name of directory containing job keyval files. 49 50 @return A dictionary containing job keyvals. 51 52 """ 53 dir = os.path.normpath(dir) 54 top_dir = tko_utils.find_toplevel_job_dir(dir) 55 if not top_dir: 56 top_dir = dir 57 assert(dir.startswith(top_dir)) 58 59 # Pull in and merge all the keyval files, with higher-level 60 # overriding values in the lower-level ones. 61 keyval = {} 62 while True: 63 try: 64 upper_keyval = utils.read_keyval(dir) 65 # HACK: exclude hostname from the override - this is a special 66 # case where we want lower to override higher. 67 if 'hostname' in upper_keyval and 'hostname' in keyval: 68 del upper_keyval['hostname'] 69 keyval.update(upper_keyval) 70 except IOError: 71 pass # If the keyval can't be read just move on to the next. 72 if dir == top_dir: 73 break 74 else: 75 assert(dir != '/') 76 dir = os.path.dirname(dir) 77 return keyval 78 79 80class kernel(object): 81 """Represents a kernel.""" 82 83 def __init__(self, base, patches, kernel_hash): 84 self.base = base 85 self.patches = patches 86 self.kernel_hash = kernel_hash 87 88 89 @staticmethod 90 def compute_hash(base, hashes): 91 """Compute a hash given the base string and hashes for each patch. 92 93 @param base: A string representing the kernel base. 94 @param hashes: A list of hashes, where each hash is associated with a 95 patch of this kernel. 96 97 @return A string representing the computed hash. 98 99 """ 100 key_string = ','.join([base] + hashes) 101 return utils.hash('md5', key_string).hexdigest() 102 103 104class test(object): 105 """Represents a test.""" 106 107 def __init__(self, subdir, testname, status, reason, test_kernel, 108 machine, started_time, finished_time, iterations, 109 attributes, perf_values, labels): 110 self.subdir = subdir 111 self.testname = testname 112 self.status = status 113 self.reason = reason 114 self.kernel = test_kernel 115 self.machine = machine 116 self.started_time = started_time 117 self.finished_time = finished_time 118 self.iterations = iterations 119 self.attributes = attributes 120 self.perf_values = perf_values 121 self.labels = labels 122 123 124 @staticmethod 125 def load_iterations(keyval_path): 126 """Abstract method to load a list of iterations from a keyval file. 127 128 @param keyval_path: String path to a keyval file. 129 130 @return A list of iteration objects. 131 132 """ 133 raise NotImplementedError 134 135 136 @staticmethod 137 def load_perf_values(perf_values_file): 138 """Loads perf values from a perf measurements file. 139 140 @param perf_values_file: The string path to a perf measurements file. 141 142 @return A list of perf_value_iteration objects. 143 144 """ 145 raise NotImplementedError 146 147 148 @classmethod 149 def parse_test(cls, job, subdir, testname, status, reason, test_kernel, 150 started_time, finished_time, existing_instance=None): 151 """ 152 Parse test result files to construct a complete test instance. 153 154 Given a job and the basic metadata about the test that can be 155 extracted from the status logs, parse the test result files (keyval 156 files and perf measurement files) and use them to construct a complete 157 test instance. 158 159 @param job: A job object. 160 @param subdir: The string subdirectory name for the given test. 161 @param testname: The name of the test. 162 @param status: The status of the test. 163 @param reason: The reason string for the test. 164 @param test_kernel: The kernel of the test. 165 @param started_time: The start time of the test. 166 @param finished_time: The finish time of the test. 167 @param existing_instance: An existing test instance. 168 169 @return A test instance that has the complete information. 170 171 """ 172 tko_utils.dprint("parsing test %s %s" % (subdir, testname)) 173 174 if tast.is_tast_test(testname): 175 attributes, perf_values = tast.load_tast_test_aux_results(job, 176 testname) 177 iterations = [] 178 elif subdir: 179 # Grab iterations from the results keyval. 180 iteration_keyval = os.path.join(job.dir, subdir, 181 'results', 'keyval') 182 iterations = cls.load_iterations(iteration_keyval) 183 184 # Grab perf values from the perf measurements file. 185 perf_values_file = os.path.join(job.dir, subdir, 186 'results', 'results-chart.json') 187 perf_values = {} 188 if os.path.exists(perf_values_file): 189 with open(perf_values_file, 'r') as fp: 190 contents = fp.read() 191 if contents: 192 perf_values = json.loads(contents) 193 194 # Grab test attributes from the subdir keyval. 195 test_keyval = os.path.join(job.dir, subdir, 'keyval') 196 attributes = test.load_attributes(test_keyval) 197 else: 198 iterations = [] 199 perf_values = {} 200 attributes = {} 201 202 # Grab test+host attributes from the host keyval. 203 host_keyval = cls.parse_host_keyval(job.dir, job.machine) 204 attributes.update(dict(('host-%s' % k, v) 205 for k, v in host_keyval.iteritems())) 206 207 if existing_instance: 208 def constructor(*args, **dargs): 209 """Initializes an existing test instance.""" 210 existing_instance.__init__(*args, **dargs) 211 return existing_instance 212 else: 213 constructor = cls 214 215 return constructor(subdir, testname, status, reason, test_kernel, 216 job.machine, started_time, finished_time, 217 iterations, attributes, perf_values, []) 218 219 220 @classmethod 221 def parse_partial_test(cls, job, subdir, testname, reason, test_kernel, 222 started_time): 223 """ 224 Create a test instance representing a partial test result. 225 226 Given a job and the basic metadata available when a test is 227 started, create a test instance representing the partial result. 228 Assume that since the test is not complete there are no results files 229 actually available for parsing. 230 231 @param job: A job object. 232 @param subdir: The string subdirectory name for the given test. 233 @param testname: The name of the test. 234 @param reason: The reason string for the test. 235 @param test_kernel: The kernel of the test. 236 @param started_time: The start time of the test. 237 238 @return A test instance that has partial test information. 239 240 """ 241 tko_utils.dprint('parsing partial test %s %s' % (subdir, testname)) 242 243 return cls(subdir, testname, 'RUNNING', reason, test_kernel, 244 job.machine, started_time, None, [], {}, [], []) 245 246 247 @staticmethod 248 def load_attributes(keyval_path): 249 """ 250 Load test attributes from a test keyval path. 251 252 Load the test attributes into a dictionary from a test 253 keyval path. Does not assume that the path actually exists. 254 255 @param keyval_path: The string path to a keyval file. 256 257 @return A dictionary representing the test keyvals. 258 259 """ 260 if not os.path.exists(keyval_path): 261 return {} 262 return utils.read_keyval(keyval_path) 263 264 265 @staticmethod 266 def _parse_keyval(job_dir, sub_keyval_path): 267 """ 268 Parse a file of keyvals. 269 270 @param job_dir: The string directory name of the associated job. 271 @param sub_keyval_path: Path to a keyval file relative to job_dir. 272 273 @return A dictionary representing the keyvals. 274 275 """ 276 # The "real" job dir may be higher up in the directory tree. 277 job_dir = tko_utils.find_toplevel_job_dir(job_dir) 278 if not job_dir: 279 return {} # We can't find a top-level job dir with job keyvals. 280 281 # The keyval is <job_dir>/`sub_keyval_path` if it exists. 282 keyval_path = os.path.join(job_dir, sub_keyval_path) 283 if os.path.isfile(keyval_path): 284 return utils.read_keyval(keyval_path) 285 else: 286 return {} 287 288 289 @staticmethod 290 def _is_multimachine(job_dir): 291 """ 292 Determine whether the job is a multi-machine job. 293 294 @param job_dir: The string directory name of the associated job. 295 296 @return True, if the job is a multi-machine job, or False if not. 297 298 """ 299 machines_path = os.path.join(job_dir, '.machines') 300 if os.path.exists(machines_path): 301 with open(machines_path, 'r') as fp: 302 line_count = len(fp.read().splitlines()) 303 if line_count > 1: 304 return True 305 return False 306 307 308 @staticmethod 309 def parse_host_keyval(job_dir, hostname): 310 """ 311 Parse host keyvals. 312 313 @param job_dir: The string directory name of the associated job. 314 @param hostname: The string hostname. 315 316 @return A dictionary representing the host keyvals. 317 318 @raises HostKeyvalError if the host keyval is not found. 319 320 """ 321 keyval_path = os.path.join('host_keyvals', hostname) 322 hostinfo_path = os.path.join(job_dir, 'host_info_store', 323 hostname + '.store') 324 # Skylab uses hostinfo. If this is not present, try falling back to the 325 # host keyval file (moblab), or an empty host keyval for multi-machine 326 # tests (jetstream). 327 if os.path.exists(hostinfo_path): 328 tko_utils.dprint('Reading keyvals from hostinfo.') 329 return _parse_hostinfo_keyval(hostinfo_path) 330 elif os.path.exists(os.path.join(job_dir, keyval_path)): 331 tko_utils.dprint('Reading keyvals from %s.' % keyval_path) 332 return test._parse_keyval(job_dir, keyval_path) 333 elif test._is_multimachine(job_dir): 334 tko_utils.dprint('Multimachine job, no keyvals.') 335 return {} 336 raise HostKeyvalError('Host keyval not found') 337 338 339 @staticmethod 340 def parse_job_keyval(job_dir): 341 """ 342 Parse job keyvals. 343 344 @param job_dir: The string directory name of the associated job. 345 346 @return A dictionary representing the job keyvals. 347 348 """ 349 # The job keyval is <job_dir>/keyval if it exists. 350 return test._parse_keyval(job_dir, 'keyval') 351 352 353def _parse_hostinfo_keyval(hostinfo_path): 354 """ 355 Parse host keyvals from hostinfo. 356 357 @param hostinfo_path: The string path to the host info store file. 358 359 @return A dictionary representing the host keyvals. 360 361 """ 362 store = file_store.FileStore(hostinfo_path) 363 hostinfo = store.get() 364 # TODO(ayatane): Investigate if urllib.quote is better. 365 label_string = ','.join(label.replace(':', '%3A') 366 for label in hostinfo.labels) 367 return {'labels': label_string, 'platform': hostinfo.model} 368 369 370class patch(object): 371 """Represents a patch.""" 372 373 def __init__(self, spec, reference, hash): 374 self.spec = spec 375 self.reference = reference 376 self.hash = hash 377 378 379class iteration(object): 380 """Represents an iteration.""" 381 382 def __init__(self, index, attr_keyval, perf_keyval): 383 self.index = index 384 self.attr_keyval = attr_keyval 385 self.perf_keyval = perf_keyval 386 387 388 @staticmethod 389 def parse_line_into_dicts(line, attr_dict, perf_dict): 390 """ 391 Abstract method to parse a keyval line and insert it into a dictionary. 392 393 @param line: The string line to parse. 394 @param attr_dict: Dictionary of generic iteration attributes. 395 @param perf_dict: Dictionary of iteration performance results. 396 397 """ 398 raise NotImplementedError 399 400 401 @classmethod 402 def load_from_keyval(cls, keyval_path): 403 """ 404 Load a list of iterations from an iteration keyval file. 405 406 Keyval data from separate iterations is separated by blank 407 lines. Makes use of the parse_line_into_dicts method to 408 actually parse the individual lines. 409 410 @param keyval_path: The string path to a keyval file. 411 412 @return A list of iteration objects. 413 414 """ 415 if not os.path.exists(keyval_path): 416 return [] 417 418 iterations = [] 419 index = 1 420 attr, perf = {}, {} 421 for line in file(keyval_path): 422 line = line.strip() 423 if line: 424 cls.parse_line_into_dicts(line, attr, perf) 425 else: 426 iterations.append(cls(index, attr, perf)) 427 index += 1 428 attr, perf = {}, {} 429 if attr or perf: 430 iterations.append(cls(index, attr, perf)) 431 return iterations 432 433 434class perf_value_iteration(object): 435 """Represents a perf value iteration.""" 436 437 def __init__(self, index, perf_measurements): 438 """ 439 Initializes the perf values for a particular test iteration. 440 441 @param index: The integer iteration number. 442 @param perf_measurements: A list of dictionaries, where each dictionary 443 contains the information for a measured perf metric from the 444 current iteration. 445 446 """ 447 self.index = index 448 self.perf_measurements = perf_measurements 449 450 451 def add_measurement(self, measurement): 452 """ 453 Appends to the list of perf measurements for this iteration. 454 455 @param measurement: A dictionary containing information for a measured 456 perf metric. 457 458 """ 459 self.perf_measurements.append(measurement) 460 461 462 @staticmethod 463 def parse_line_into_dict(line): 464 """ 465 Abstract method to parse an individual perf measurement line. 466 467 @param line: A string line from the perf measurement output file. 468 469 @return A dicionary representing the information for a measured perf 470 metric from one line of the perf measurement output file, or an 471 empty dictionary if the line cannot be parsed successfully. 472 473 """ 474 raise NotImplementedError 475