1import json 2import os 3 4from autotest_lib.server.hosts import file_store 5from autotest_lib.client.common_lib import utils 6from autotest_lib.tko import tast 7from autotest_lib.tko import utils as tko_utils 8 9 10class HostKeyvalError(Exception): 11 """Raised when the host keyval cannot be read.""" 12 13 14class job(object): 15 """Represents a job.""" 16 17 def __init__(self, dir, user, label, machine, queued_time, started_time, 18 finished_time, machine_owner, machine_group, aborted_by, 19 aborted_on, keyval_dict): 20 self.dir = dir 21 self.tests = [] 22 self.user = user 23 self.label = label 24 self.machine = machine 25 self.queued_time = queued_time 26 self.started_time = started_time 27 self.finished_time = finished_time 28 self.machine_owner = machine_owner 29 self.machine_group = machine_group 30 self.aborted_by = aborted_by 31 self.aborted_on = aborted_on 32 self.keyval_dict = keyval_dict 33 self.afe_parent_job_id = None 34 self.build_version = None 35 self.suite = None 36 self.board = None 37 self.job_idx = None 38 # id of the corresponding tko_task_references entry. 39 # This table is used to refer to skylab task / afe job corresponding to 40 # this tko_job. 41 self.task_reference_id = None 42 43 @staticmethod 44 def read_keyval(dir): 45 """ 46 Read job keyval files. 47 48 @param dir: String name of directory containing job keyval files. 49 50 @return A dictionary containing job keyvals. 51 52 """ 53 dir = os.path.normpath(dir) 54 top_dir = tko_utils.find_toplevel_job_dir(dir) 55 if not top_dir: 56 top_dir = dir 57 assert(dir.startswith(top_dir)) 58 59 # Pull in and merge all the keyval files, with higher-level 60 # overriding values in the lower-level ones. 61 keyval = {} 62 while True: 63 try: 64 upper_keyval = utils.read_keyval(dir) 65 # HACK: exclude hostname from the override - this is a special 66 # case where we want lower to override higher. 67 if 'hostname' in upper_keyval and 'hostname' in keyval: 68 del upper_keyval['hostname'] 69 keyval.update(upper_keyval) 70 except IOError: 71 pass # If the keyval can't be read just move on to the next. 72 if dir == top_dir: 73 break 74 else: 75 assert(dir != '/') 76 dir = os.path.dirname(dir) 77 return keyval 78 79 80class kernel(object): 81 """Represents a kernel.""" 82 83 def __init__(self, base, patches, kernel_hash): 84 self.base = base 85 self.patches = patches 86 self.kernel_hash = kernel_hash 87 88 89 @staticmethod 90 def compute_hash(base, hashes): 91 """Compute a hash given the base string and hashes for each patch. 92 93 @param base: A string representing the kernel base. 94 @param hashes: A list of hashes, where each hash is associated with a 95 patch of this kernel. 96 97 @return A string representing the computed hash. 98 99 """ 100 key_string = ','.join([base] + hashes) 101 return utils.hash('md5', key_string).hexdigest() 102 103 104class test(object): 105 """Represents a test.""" 106 107 def __init__(self, subdir, testname, status, reason, test_kernel, 108 machine, started_time, finished_time, iterations, 109 attributes, perf_values, labels): 110 self.subdir = subdir 111 self.testname = testname 112 self.status = status 113 self.reason = reason 114 self.kernel = test_kernel 115 self.machine = machine 116 self.started_time = started_time 117 self.finished_time = finished_time 118 self.iterations = iterations 119 self.attributes = attributes 120 self.perf_values = perf_values 121 self.labels = labels 122 123 124 @staticmethod 125 def load_iterations(keyval_path): 126 """Abstract method to load a list of iterations from a keyval file. 127 128 @param keyval_path: String path to a keyval file. 129 130 @return A list of iteration objects. 131 132 """ 133 raise NotImplementedError 134 135 136 @classmethod 137 def parse_test(cls, job, subdir, testname, status, reason, test_kernel, 138 started_time, finished_time, existing_instance=None): 139 """ 140 Parse test result files to construct a complete test instance. 141 142 Given a job and the basic metadata about the test that can be 143 extracted from the status logs, parse the test result files (keyval 144 files and perf measurement files) and use them to construct a complete 145 test instance. 146 147 @param job: A job object. 148 @param subdir: The string subdirectory name for the given test. 149 @param testname: The name of the test. 150 @param status: The status of the test. 151 @param reason: The reason string for the test. 152 @param test_kernel: The kernel of the test. 153 @param started_time: The start time of the test. 154 @param finished_time: The finish time of the test. 155 @param existing_instance: An existing test instance. 156 157 @return A test instance that has the complete information. 158 159 """ 160 tko_utils.dprint("parsing test %s %s" % (subdir, testname)) 161 162 if tast.is_tast_test(testname): 163 attributes, perf_values = tast.load_tast_test_aux_results(job, 164 testname) 165 iterations = [] 166 elif subdir: 167 # Grab iterations from the results keyval. 168 iteration_keyval = os.path.join(job.dir, subdir, 169 'results', 'keyval') 170 iterations = cls.load_iterations(iteration_keyval) 171 172 # Grab perf values from the perf measurements file. 173 perf_values_file = os.path.join(job.dir, subdir, 174 'results', 'results-chart.json') 175 perf_values = {} 176 if os.path.exists(perf_values_file): 177 with open(perf_values_file, 'r') as fp: 178 contents = fp.read() 179 if contents: 180 perf_values = json.loads(contents) 181 182 # Grab test attributes from the subdir keyval. 183 test_keyval = os.path.join(job.dir, subdir, 'keyval') 184 attributes = test.load_attributes(test_keyval) 185 else: 186 iterations = [] 187 perf_values = {} 188 attributes = {} 189 190 # Grab test+host attributes from the host keyval. 191 host_keyval = cls.parse_host_keyval(job.dir, job.machine) 192 attributes.update(dict(('host-%s' % k, v) 193 for k, v in host_keyval.iteritems())) 194 195 if existing_instance: 196 def constructor(*args, **dargs): 197 """Initializes an existing test instance.""" 198 existing_instance.__init__(*args, **dargs) 199 return existing_instance 200 else: 201 constructor = cls 202 203 return constructor(subdir, testname, status, reason, test_kernel, 204 job.machine, started_time, finished_time, 205 iterations, attributes, perf_values, []) 206 207 208 @classmethod 209 def parse_partial_test(cls, job, subdir, testname, reason, test_kernel, 210 started_time): 211 """ 212 Create a test instance representing a partial test result. 213 214 Given a job and the basic metadata available when a test is 215 started, create a test instance representing the partial result. 216 Assume that since the test is not complete there are no results files 217 actually available for parsing. 218 219 @param job: A job object. 220 @param subdir: The string subdirectory name for the given test. 221 @param testname: The name of the test. 222 @param reason: The reason string for the test. 223 @param test_kernel: The kernel of the test. 224 @param started_time: The start time of the test. 225 226 @return A test instance that has partial test information. 227 228 """ 229 tko_utils.dprint('parsing partial test %s %s' % (subdir, testname)) 230 231 return cls(subdir, testname, 'RUNNING', reason, test_kernel, 232 job.machine, started_time, None, [], {}, [], []) 233 234 235 @staticmethod 236 def load_attributes(keyval_path): 237 """ 238 Load test attributes from a test keyval path. 239 240 Load the test attributes into a dictionary from a test 241 keyval path. Does not assume that the path actually exists. 242 243 @param keyval_path: The string path to a keyval file. 244 245 @return A dictionary representing the test keyvals. 246 247 """ 248 if not os.path.exists(keyval_path): 249 return {} 250 return utils.read_keyval(keyval_path) 251 252 253 @staticmethod 254 def _parse_keyval(job_dir, sub_keyval_path): 255 """ 256 Parse a file of keyvals. 257 258 @param job_dir: The string directory name of the associated job. 259 @param sub_keyval_path: Path to a keyval file relative to job_dir. 260 261 @return A dictionary representing the keyvals. 262 263 """ 264 # The "real" job dir may be higher up in the directory tree. 265 job_dir = tko_utils.find_toplevel_job_dir(job_dir) 266 if not job_dir: 267 return {} # We can't find a top-level job dir with job keyvals. 268 269 # The keyval is <job_dir>/`sub_keyval_path` if it exists. 270 keyval_path = os.path.join(job_dir, sub_keyval_path) 271 if os.path.isfile(keyval_path): 272 return utils.read_keyval(keyval_path) 273 else: 274 return {} 275 276 277 @staticmethod 278 def _is_multimachine(job_dir): 279 """ 280 Determine whether the job is a multi-machine job. 281 282 @param job_dir: The string directory name of the associated job. 283 284 @return True, if the job is a multi-machine job, or False if not. 285 286 """ 287 machines_path = os.path.join(job_dir, '.machines') 288 if os.path.exists(machines_path): 289 with open(machines_path, 'r') as fp: 290 line_count = len(fp.read().splitlines()) 291 if line_count > 1: 292 return True 293 return False 294 295 296 @staticmethod 297 def parse_host_keyval(job_dir, hostname): 298 """ 299 Parse host keyvals. 300 301 @param job_dir: The string directory name of the associated job. 302 @param hostname: The string hostname. 303 304 @return A dictionary representing the host keyvals. 305 306 @raises HostKeyvalError if the host keyval is not found. 307 308 """ 309 keyval_path = os.path.join('host_keyvals', hostname) 310 hostinfo_path = os.path.join(job_dir, 'host_info_store', 311 hostname + '.store') 312 # Skylab uses hostinfo. If this is not present, try falling back to the 313 # host keyval file (moblab), or an empty host keyval for multi-machine 314 # tests (jetstream). 315 if os.path.exists(hostinfo_path): 316 tko_utils.dprint('Reading keyvals from hostinfo.') 317 return _parse_hostinfo_keyval(hostinfo_path) 318 elif os.path.exists(os.path.join(job_dir, keyval_path)): 319 tko_utils.dprint('Reading keyvals from %s.' % keyval_path) 320 return test._parse_keyval(job_dir, keyval_path) 321 elif test._is_multimachine(job_dir): 322 tko_utils.dprint('Multimachine job, no keyvals.') 323 return {} 324 raise HostKeyvalError('Host keyval not found') 325 326 327 @staticmethod 328 def parse_job_keyval(job_dir): 329 """ 330 Parse job keyvals. 331 332 @param job_dir: The string directory name of the associated job. 333 334 @return A dictionary representing the job keyvals. 335 336 """ 337 # The job keyval is <job_dir>/keyval if it exists. 338 return test._parse_keyval(job_dir, 'keyval') 339 340 341def _parse_hostinfo_keyval(hostinfo_path): 342 """ 343 Parse host keyvals from hostinfo. 344 345 @param hostinfo_path: The string path to the host info store file. 346 347 @return A dictionary representing the host keyvals. 348 349 """ 350 store = file_store.FileStore(hostinfo_path) 351 hostinfo = store.get() 352 # TODO(ayatane): Investigate if urllib.quote is better. 353 label_string = ','.join(label.replace(':', '%3A') 354 for label in hostinfo.labels) 355 return {'labels': label_string, 'platform': hostinfo.model} 356 357 358class patch(object): 359 """Represents a patch.""" 360 361 def __init__(self, spec, reference, hash): 362 self.spec = spec 363 self.reference = reference 364 self.hash = hash 365 366 367class iteration(object): 368 """Represents an iteration.""" 369 370 def __init__(self, index, attr_keyval, perf_keyval): 371 self.index = index 372 self.attr_keyval = attr_keyval 373 self.perf_keyval = perf_keyval 374 375 376 @staticmethod 377 def parse_line_into_dicts(line, attr_dict, perf_dict): 378 """ 379 Abstract method to parse a keyval line and insert it into a dictionary. 380 381 @param line: The string line to parse. 382 @param attr_dict: Dictionary of generic iteration attributes. 383 @param perf_dict: Dictionary of iteration performance results. 384 385 """ 386 raise NotImplementedError 387 388 389 @classmethod 390 def load_from_keyval(cls, keyval_path): 391 """ 392 Load a list of iterations from an iteration keyval file. 393 394 Keyval data from separate iterations is separated by blank 395 lines. Makes use of the parse_line_into_dicts method to 396 actually parse the individual lines. 397 398 @param keyval_path: The string path to a keyval file. 399 400 @return A list of iteration objects. 401 402 """ 403 if not os.path.exists(keyval_path): 404 return [] 405 406 iterations = [] 407 index = 1 408 attr, perf = {}, {} 409 for line in file(keyval_path): 410 line = line.strip() 411 if line: 412 cls.parse_line_into_dicts(line, attr, perf) 413 else: 414 iterations.append(cls(index, attr, perf)) 415 index += 1 416 attr, perf = {}, {} 417 if attr or perf: 418 iterations.append(cls(index, attr, perf)) 419 return iterations 420 421 422class perf_value_iteration(object): 423 """Represents a perf value iteration.""" 424 425 def __init__(self, index, perf_measurements): 426 """ 427 Initializes the perf values for a particular test iteration. 428 429 @param index: The integer iteration number. 430 @param perf_measurements: A list of dictionaries, where each dictionary 431 contains the information for a measured perf metric from the 432 current iteration. 433 434 """ 435 self.index = index 436 self.perf_measurements = perf_measurements 437 438 439 @staticmethod 440 def parse_line_into_dict(line): 441 """ 442 Abstract method to parse an individual perf measurement line. 443 444 @param line: A string line from the perf measurement output file. 445 446 @return A dicionary representing the information for a measured perf 447 metric from one line of the perf measurement output file, or an 448 empty dictionary if the line cannot be parsed successfully. 449 450 """ 451 raise NotImplementedError 452