1import json 2import os 3 4from autotest_lib.server.hosts import file_store 5from autotest_lib.client.common_lib import utils 6from autotest_lib.tko import tast 7from autotest_lib.tko import utils as tko_utils 8 9 10class job(object): 11 """Represents a job.""" 12 13 def __init__(self, dir, user, label, machine, queued_time, started_time, 14 finished_time, machine_owner, machine_group, aborted_by, 15 aborted_on, keyval_dict): 16 self.dir = dir 17 self.tests = [] 18 self.user = user 19 self.label = label 20 self.machine = machine 21 self.queued_time = queued_time 22 self.started_time = started_time 23 self.finished_time = finished_time 24 self.machine_owner = machine_owner 25 self.machine_group = machine_group 26 self.aborted_by = aborted_by 27 self.aborted_on = aborted_on 28 self.keyval_dict = keyval_dict 29 self.afe_parent_job_id = None 30 self.build_version = None 31 self.suite = None 32 self.board = None 33 self.job_idx = None 34 # id of the corresponding tko_task_references entry. 35 # This table is used to refer to skylab task / afe job corresponding to 36 # this tko_job. 37 self.task_reference_id = None 38 39 @staticmethod 40 def read_keyval(dir): 41 """ 42 Read job keyval files. 43 44 @param dir: String name of directory containing job keyval files. 45 46 @return A dictionary containing job keyvals. 47 48 """ 49 dir = os.path.normpath(dir) 50 top_dir = tko_utils.find_toplevel_job_dir(dir) 51 if not top_dir: 52 top_dir = dir 53 assert(dir.startswith(top_dir)) 54 55 # Pull in and merge all the keyval files, with higher-level 56 # overriding values in the lower-level ones. 57 keyval = {} 58 while True: 59 try: 60 upper_keyval = utils.read_keyval(dir) 61 # HACK: exclude hostname from the override - this is a special 62 # case where we want lower to override higher. 63 if 'hostname' in upper_keyval and 'hostname' in keyval: 64 del upper_keyval['hostname'] 65 keyval.update(upper_keyval) 66 except IOError: 67 pass # If the keyval can't be read just move on to the next. 68 if dir == top_dir: 69 break 70 else: 71 assert(dir != '/') 72 dir = os.path.dirname(dir) 73 return keyval 74 75 76class kernel(object): 77 """Represents a kernel.""" 78 79 def __init__(self, base, patches, kernel_hash): 80 self.base = base 81 self.patches = patches 82 self.kernel_hash = kernel_hash 83 84 85 @staticmethod 86 def compute_hash(base, hashes): 87 """Compute a hash given the base string and hashes for each patch. 88 89 @param base: A string representing the kernel base. 90 @param hashes: A list of hashes, where each hash is associated with a 91 patch of this kernel. 92 93 @return A string representing the computed hash. 94 95 """ 96 key_string = ','.join([base] + hashes) 97 return utils.hash('md5', key_string).hexdigest() 98 99 100class test(object): 101 """Represents a test.""" 102 103 def __init__(self, subdir, testname, status, reason, test_kernel, 104 machine, started_time, finished_time, iterations, 105 attributes, perf_values, labels): 106 self.subdir = subdir 107 self.testname = testname 108 self.status = status 109 self.reason = reason 110 self.kernel = test_kernel 111 self.machine = machine 112 self.started_time = started_time 113 self.finished_time = finished_time 114 self.iterations = iterations 115 self.attributes = attributes 116 self.perf_values = perf_values 117 self.labels = labels 118 119 120 @staticmethod 121 def load_iterations(keyval_path): 122 """Abstract method to load a list of iterations from a keyval file. 123 124 @param keyval_path: String path to a keyval file. 125 126 @return A list of iteration objects. 127 128 """ 129 raise NotImplementedError 130 131 132 @staticmethod 133 def load_perf_values(perf_values_file): 134 """Loads perf values from a perf measurements file. 135 136 @param perf_values_file: The string path to a perf measurements file. 137 138 @return A list of perf_value_iteration objects. 139 140 """ 141 raise NotImplementedError 142 143 144 @classmethod 145 def parse_test(cls, job, subdir, testname, status, reason, test_kernel, 146 started_time, finished_time, existing_instance=None): 147 """ 148 Parse test result files to construct a complete test instance. 149 150 Given a job and the basic metadata about the test that can be 151 extracted from the status logs, parse the test result files (keyval 152 files and perf measurement files) and use them to construct a complete 153 test instance. 154 155 @param job: A job object. 156 @param subdir: The string subdirectory name for the given test. 157 @param testname: The name of the test. 158 @param status: The status of the test. 159 @param reason: The reason string for the test. 160 @param test_kernel: The kernel of the test. 161 @param started_time: The start time of the test. 162 @param finished_time: The finish time of the test. 163 @param existing_instance: An existing test instance. 164 165 @return A test instance that has the complete information. 166 167 """ 168 tko_utils.dprint("parsing test %s %s" % (subdir, testname)) 169 170 if tast.is_tast_test(testname): 171 attributes, perf_values = tast.load_tast_test_aux_results(job, 172 testname) 173 iterations = [] 174 elif subdir: 175 # Grab iterations from the results keyval. 176 iteration_keyval = os.path.join(job.dir, subdir, 177 'results', 'keyval') 178 iterations = cls.load_iterations(iteration_keyval) 179 180 # Grab perf values from the perf measurements file. 181 perf_values_file = os.path.join(job.dir, subdir, 182 'results', 'results-chart.json') 183 perf_values = {} 184 if os.path.exists(perf_values_file): 185 with open(perf_values_file, 'r') as fp: 186 contents = fp.read() 187 if contents: 188 perf_values = json.loads(contents) 189 190 # Grab test attributes from the subdir keyval. 191 test_keyval = os.path.join(job.dir, subdir, 'keyval') 192 attributes = test.load_attributes(test_keyval) 193 else: 194 iterations = [] 195 perf_values = {} 196 attributes = {} 197 198 # Grab test+host attributes from the host keyval. 199 host_keyval = cls.parse_host_keyval(job.dir, job.machine) 200 attributes.update(dict(('host-%s' % k, v) 201 for k, v in host_keyval.iteritems())) 202 203 if existing_instance: 204 def constructor(*args, **dargs): 205 """Initializes an existing test instance.""" 206 existing_instance.__init__(*args, **dargs) 207 return existing_instance 208 else: 209 constructor = cls 210 211 return constructor(subdir, testname, status, reason, test_kernel, 212 job.machine, started_time, finished_time, 213 iterations, attributes, perf_values, []) 214 215 216 @classmethod 217 def parse_partial_test(cls, job, subdir, testname, reason, test_kernel, 218 started_time): 219 """ 220 Create a test instance representing a partial test result. 221 222 Given a job and the basic metadata available when a test is 223 started, create a test instance representing the partial result. 224 Assume that since the test is not complete there are no results files 225 actually available for parsing. 226 227 @param job: A job object. 228 @param subdir: The string subdirectory name for the given test. 229 @param testname: The name of the test. 230 @param reason: The reason string for the test. 231 @param test_kernel: The kernel of the test. 232 @param started_time: The start time of the test. 233 234 @return A test instance that has partial test information. 235 236 """ 237 tko_utils.dprint('parsing partial test %s %s' % (subdir, testname)) 238 239 return cls(subdir, testname, 'RUNNING', reason, test_kernel, 240 job.machine, started_time, None, [], {}, [], []) 241 242 243 @staticmethod 244 def load_attributes(keyval_path): 245 """ 246 Load test attributes from a test keyval path. 247 248 Load the test attributes into a dictionary from a test 249 keyval path. Does not assume that the path actually exists. 250 251 @param keyval_path: The string path to a keyval file. 252 253 @return A dictionary representing the test keyvals. 254 255 """ 256 if not os.path.exists(keyval_path): 257 return {} 258 return utils.read_keyval(keyval_path) 259 260 261 @staticmethod 262 def _parse_keyval(job_dir, sub_keyval_path): 263 """ 264 Parse a file of keyvals. 265 266 @param job_dir: The string directory name of the associated job. 267 @param sub_keyval_path: Path to a keyval file relative to job_dir. 268 269 @return A dictionary representing the keyvals. 270 271 """ 272 # The "real" job dir may be higher up in the directory tree. 273 job_dir = tko_utils.find_toplevel_job_dir(job_dir) 274 if not job_dir: 275 return {} # We can't find a top-level job dir with job keyvals. 276 277 # The keyval is <job_dir>/`sub_keyval_path` if it exists. 278 keyval_path = os.path.join(job_dir, sub_keyval_path) 279 if os.path.isfile(keyval_path): 280 return utils.read_keyval(keyval_path) 281 else: 282 return {} 283 284 285 @staticmethod 286 def parse_host_keyval(job_dir, hostname): 287 """ 288 Parse host keyvals. 289 290 @param job_dir: The string directory name of the associated job. 291 @param hostname: The string hostname. 292 293 @return A dictionary representing the host keyvals. 294 295 """ 296 keyval_path = os.path.join('host_keyvals', hostname) 297 # The host keyval is <job_dir>/host_keyvals/<hostname> if it exists. 298 # Otherwise we're running on Skylab which uses hostinfo. 299 if not os.path.exists(os.path.join(job_dir, keyval_path)): 300 tko_utils.dprint("trying to use hostinfo") 301 try: 302 return _parse_hostinfo_keyval(job_dir, hostname) 303 except Exception as e: 304 # If anything goes wrong, log it and just use the old flow. 305 tko_utils.dprint("tried using hostinfo: %s" % e) 306 return test._parse_keyval(job_dir, keyval_path) 307 308 309 @staticmethod 310 def parse_job_keyval(job_dir): 311 """ 312 Parse job keyvals. 313 314 @param job_dir: The string directory name of the associated job. 315 316 @return A dictionary representing the job keyvals. 317 318 """ 319 # The job keyval is <job_dir>/keyval if it exists. 320 return test._parse_keyval(job_dir, 'keyval') 321 322 323def _parse_hostinfo_keyval(job_dir, hostname): 324 """ 325 Parse host keyvals from hostinfo. 326 327 @param job_dir: The string directory name of the associated job. 328 @param hostname: The string hostname. 329 330 @return A dictionary representing the host keyvals. 331 332 """ 333 # The hostinfo path looks like: 334 # host_info_store/chromeos6-row4-rack11-host6.store 335 # 336 # TODO(ayatane): We should pass hostinfo path explicitly. 337 subdir = 'host_info_store' 338 hostinfo_path = os.path.join(job_dir, subdir, hostname + '.store') 339 store = file_store.FileStore(hostinfo_path) 340 hostinfo = store.get() 341 # TODO(ayatane): Investigate if urllib.quote is better. 342 label_string = ','.join(label.replace(':', '%3A') 343 for label in hostinfo.labels) 344 return {'labels': label_string, 'platform': hostinfo.model} 345 346 347class patch(object): 348 """Represents a patch.""" 349 350 def __init__(self, spec, reference, hash): 351 self.spec = spec 352 self.reference = reference 353 self.hash = hash 354 355 356class iteration(object): 357 """Represents an iteration.""" 358 359 def __init__(self, index, attr_keyval, perf_keyval): 360 self.index = index 361 self.attr_keyval = attr_keyval 362 self.perf_keyval = perf_keyval 363 364 365 @staticmethod 366 def parse_line_into_dicts(line, attr_dict, perf_dict): 367 """ 368 Abstract method to parse a keyval line and insert it into a dictionary. 369 370 @param line: The string line to parse. 371 @param attr_dict: Dictionary of generic iteration attributes. 372 @param perf_dict: Dictionary of iteration performance results. 373 374 """ 375 raise NotImplementedError 376 377 378 @classmethod 379 def load_from_keyval(cls, keyval_path): 380 """ 381 Load a list of iterations from an iteration keyval file. 382 383 Keyval data from separate iterations is separated by blank 384 lines. Makes use of the parse_line_into_dicts method to 385 actually parse the individual lines. 386 387 @param keyval_path: The string path to a keyval file. 388 389 @return A list of iteration objects. 390 391 """ 392 if not os.path.exists(keyval_path): 393 return [] 394 395 iterations = [] 396 index = 1 397 attr, perf = {}, {} 398 for line in file(keyval_path): 399 line = line.strip() 400 if line: 401 cls.parse_line_into_dicts(line, attr, perf) 402 else: 403 iterations.append(cls(index, attr, perf)) 404 index += 1 405 attr, perf = {}, {} 406 if attr or perf: 407 iterations.append(cls(index, attr, perf)) 408 return iterations 409 410 411class perf_value_iteration(object): 412 """Represents a perf value iteration.""" 413 414 def __init__(self, index, perf_measurements): 415 """ 416 Initializes the perf values for a particular test iteration. 417 418 @param index: The integer iteration number. 419 @param perf_measurements: A list of dictionaries, where each dictionary 420 contains the information for a measured perf metric from the 421 current iteration. 422 423 """ 424 self.index = index 425 self.perf_measurements = perf_measurements 426 427 428 def add_measurement(self, measurement): 429 """ 430 Appends to the list of perf measurements for this iteration. 431 432 @param measurement: A dictionary containing information for a measured 433 perf metric. 434 435 """ 436 self.perf_measurements.append(measurement) 437 438 439 @staticmethod 440 def parse_line_into_dict(line): 441 """ 442 Abstract method to parse an individual perf measurement line. 443 444 @param line: A string line from the perf measurement output file. 445 446 @return A dicionary representing the information for a measured perf 447 metric from one line of the perf measurement output file, or an 448 empty dictionary if the line cannot be parsed successfully. 449 450 """ 451 raise NotImplementedError 452