1#!/usr/bin/python3 2 3"""A script that provides convertion between models.job and a protocol 4buffer object. 5 6This script contains only one class that takes an job instance and 7convert it into a protocol buffer object. The class will also be 8responsible for serializing the job instance via protocol buffers. 9 10""" 11 12# import python libraries 13from __future__ import division 14from __future__ import print_function 15 16import datetime 17import time 18import logging 19 20# import autotest libraries 21from autotest_lib.tko import models 22from autotest_lib.tko import tko_pb2 23from autotest_lib.tko import utils 24import six 25 26__author__ = 'darrenkuo@google.com (Darren Kuo)' 27 28mktime = time.mktime 29datetime = datetime.datetime 30 31class JobSerializer(object): 32 """A class that takes a job object of the tko module and package 33 it with a protocol buffer. 34 35 This class will take a model.job object as input and create a 36 protocol buffer to include all the content of the job object. This 37 protocol buffer object will be serialized into a binary file. 38 """ 39 40 def __init__(self): 41 42 self.job_type_dict = {'dir':str, 'tests':list, 'user':str, 43 'label':str, 'machine':str, 44 'queued_time':datetime, 45 'started_time':datetime, 46 'finished_time':datetime, 47 'machine_owner':str, 48 'machine_group':str, 'aborted_by':str, 49 'aborted_on':datetime, 50 'keyval_dict':dict, 51 'afe_parent_job_id':str, 52 'build_version':str, 53 'suite':str, 54 'board':str} 55 56 self.test_type_dict = {'subdir':str, 'testname':str, 57 'status':str, 'reason':str, 58 'kernel':models.kernel, 'machine':str, 59 'started_time':datetime, 60 'finished_time':datetime, 61 'iterations':list, 'attributes':dict, 62 'labels':list} 63 64 self.kernel_type_dict = {'base':str, 'kernel_hash':str} 65 66 self.iteration_type_dict = {'index':int, 'attr_keyval':dict, 67 'perf_keyval':dict} 68 69 70 def deserialize_from_binary(self, infile): 71 """Takes in a binary file name and returns a tko job object. 72 73 The method first deserialize the binary into a protocol buffer 74 job object and then converts the job object into a tko job 75 object. 76 77 78 @param infile: the name of the binary file that will be deserialized. 79 80 @return: a tko job that is represented by the binary file will 81 be returned. 82 """ 83 84 job_pb = tko_pb2.Job() 85 86 binary = open(infile, 'r') 87 try: 88 job_pb.ParseFromString(binary.read()) 89 finally: 90 binary.close() 91 92 return self.get_tko_job(job_pb) 93 94 95 def serialize_to_binary(self, the_job, tag, binaryfilename): 96 """Serializes the tko job object into a binary by using a 97 protocol buffer. 98 99 The method takes a tko job object and constructs a protocol 100 buffer job object. Then invokes the built-in serializing 101 function on the object to get a binary string. The string is 102 then written to outfile. 103 104 Precondition: Assumes that all the information about the job 105 is already in the job object. Any fields that is None will be 106 provided a default value. 107 108 @param the_job: the tko job object that will be serialized. 109 tag: contains the job name and the afe_job_id 110 binaryfilename: the name of the file that will be written to 111 @param tag: The job tag string. 112 @param binaryfilename: The output filename. 113 114 @return: the filename of the file that contains the 115 binary of the serialized object. 116 """ 117 118 pb_job = tko_pb2.Job() 119 self.set_pb_job(the_job, pb_job, tag) 120 121 out = open(binaryfilename, 'wb') 122 try: 123 out.write(pb_job.SerializeToString()) 124 finally: 125 out.close() 126 127 128 def set_afe_job_id_and_tag(self, pb_job, tag): 129 """Sets the pb job's afe_job_id and tag field. 130 131 @param 132 pb_job: the pb job that will have it's fields set. 133 tag: used to set pb_job.tag and pb_job.afe_job_id. 134 """ 135 pb_job.tag = tag 136 pb_job.afe_job_id = utils.get_afe_job_id(tag) 137 138 139 # getter setter methods 140 def get_tko_job(self, job): 141 """Creates a a new tko job object from the pb job object. 142 143 Uses getter methods on the pb objects to extract all the 144 attributes and finally constructs a tko job object using the 145 models.job constructor. 146 147 @param 148 job: a pb job where data is being extracted from. 149 150 @return a tko job object. 151 """ 152 153 fields_dict = self.get_trivial_attr(job, self.job_type_dict) 154 155 fields_dict['tests'] = [self.get_tko_test(test) for test in job.tests] 156 157 fields_dict['keyval_dict'] = dict((keyval.name, keyval.value) 158 for keyval in job.keyval_dict) 159 160 newjob = models.job(fields_dict['dir'], fields_dict['user'], 161 fields_dict['label'], 162 fields_dict['machine'], 163 fields_dict['queued_time'], 164 fields_dict['started_time'], 165 fields_dict['finished_time'], 166 fields_dict['machine_owner'], 167 fields_dict['machine_group'], 168 fields_dict['aborted_by'], 169 fields_dict['aborted_on'], 170 fields_dict['keyval_dict']) 171 172 newjob.tests.extend(fields_dict['tests']) 173 174 return newjob 175 176 177 def set_pb_job(self, tko_job, pb_job, tag): 178 """Set the fields for the new job object. 179 180 Method takes in a tko job and an empty protocol buffer job 181 object. Then safely sets all the appropriate field by first 182 testing if the value in the original object is None. 183 184 @param 185 tko_job: a tko job instance that will have it's values 186 transfered to the new job 187 pb_job: a new instance of the job class provided in the 188 protocol buffer. 189 tag: used to set pb_job.tag and pb_job.afe_job_id. 190 """ 191 192 self.set_trivial_attr(tko_job, pb_job, self.job_type_dict) 193 self.set_afe_job_id_and_tag(pb_job, tag) 194 if hasattr(tko_job, 'index'): 195 pb_job.job_idx = tko_job.index 196 197 for test in tko_job.tests: 198 newtest = pb_job.tests.add() 199 self.set_pb_test(test, newtest) 200 201 for key, val in six.iteritems(tko_job.keyval_dict): 202 newkeyval = pb_job.keyval_dict.add() 203 newkeyval.name = key 204 newkeyval.value = str(val) 205 206 207 def get_tko_test(self, test): 208 """Creates a tko test from pb_test. 209 210 Extracts data from pb_test by calling helper methods and 211 creates a tko test using the models.test constructor. 212 213 @param: 214 test: a pb_test where fields will be extracted from. 215 216 @return a new instance of models.test 217 """ 218 fields_dict = self.get_trivial_attr(test, self.test_type_dict) 219 220 fields_dict['kernel'] = self.get_tko_kernel(test.kernel) 221 222 fields_dict['iterations'] = [self.get_tko_iteration(iteration) 223 for iteration in test.iterations] 224 225 fields_dict['attributes'] = dict((keyval.name, keyval.value) 226 for keyval in test.attributes) 227 228 fields_dict['labels'] = list(test.labels) 229 230 # The constructor for models.test accepts a "perf_values" list that 231 # represents performance values of the test. The empty list argument 232 # in the constructor call below represents this value and makes this 233 # code adhere properly to the models.test constructor argument list. 234 # However, the effect of the empty list is that perf values are 235 # ignored in the job_serializer module. This is ok for now because 236 # autotest does not use the current module. If job_serializer is used 237 # in the future, we need to modify the "tko.proto" protobuf file to 238 # understand the notion of perf_values, then modify this file 239 # accordingly to use it. 240 return models.test(fields_dict['subdir'], 241 fields_dict['testname'], 242 fields_dict['status'], 243 fields_dict['reason'], 244 fields_dict['kernel'], 245 fields_dict['machine'], 246 fields_dict['started_time'], 247 fields_dict['finished_time'], 248 fields_dict['iterations'], 249 fields_dict['attributes'], 250 [], 251 fields_dict['labels']) 252 253 254 def set_pb_test(self, tko_test, pb_test): 255 """Sets the various fields of test object of the tko protocol. 256 257 Method takes a tko test and a new test of the protocol buffer and 258 transfers the values in the tko test to the new test. 259 260 @param 261 tko_test: a tko test instance. 262 pb_test: an empty protocol buffer test instance. 263 264 """ 265 266 self.set_trivial_attr(tko_test, pb_test, self.test_type_dict) 267 268 self.set_pb_kernel(tko_test.kernel, pb_test.kernel) 269 if hasattr(tko_test, 'test_idx'): 270 pb_test.test_idx = tko_test.test_idx 271 272 for current_iteration in tko_test.iterations: 273 pb_iteration = pb_test.iterations.add() 274 self.set_pb_iteration(current_iteration, pb_iteration) 275 276 for key, val in six.iteritems(tko_test.attributes): 277 newkeyval = pb_test.attributes.add() 278 newkeyval.name = key 279 newkeyval.value = str(val) 280 281 for current_label in tko_test.labels: 282 pb_test.labels.append(current_label) 283 284 285 def get_tko_kernel(self, kernel): 286 """Constructs a new tko kernel object from a pb kernel object. 287 288 Uses all the getter methods on the pb kernel object to extract 289 the attributes and constructs a new tko kernel object using 290 the model.kernel constructor. 291 292 @param 293 kernel: a pb kernel object where data will be extracted. 294 295 @return a new tko kernel object. 296 """ 297 298 fields_dict = self.get_trivial_attr(kernel, self.kernel_type_dict) 299 300 return models.kernel(fields_dict['base'], [], fields_dict['kernel_hash']) 301 302 303 def set_pb_kernel(self, tko_kernel, pb_kernel): 304 """Set a specific kernel of a test. 305 306 Takes the same form of all the other setting methods. It 307 seperates the string variables from the int variables and set 308 them safely. 309 310 @param 311 tko_kernel: a tko kernel. 312 pb_kernel: an empty protocol buffer kernel. 313 314 """ 315 316 self.set_trivial_attr(tko_kernel, pb_kernel, self.kernel_type_dict) 317 318 319 def get_tko_iteration(self, iteration): 320 """Creates a new tko iteration with the data in the provided 321 pb iteration. 322 323 Uses the data in the pb iteration and the models.iteration 324 constructor to create a new tko iterations 325 326 @param 327 iteration: a pb iteration instance 328 329 @return a tko iteration instance with the same data. 330 """ 331 332 fields_dict = self.get_trivial_attr(iteration, 333 self.iteration_type_dict) 334 335 fields_dict['attr_keyval'] = dict((keyval.name, keyval.value) 336 for keyval in iteration.attr_keyval) 337 338 fields_dict['perf_keyval'] = dict((keyval.name, keyval.value) 339 for keyval in iteration.perf_keyval) 340 341 return models.iteration(fields_dict['index'], 342 fields_dict['attr_keyval'], 343 fields_dict['perf_keyval']) 344 345 346 def set_pb_iteration(self, tko_iteration, pb_iteration): 347 """Sets all fields for a particular iteration. 348 349 Takes same form as all the other setting methods. Sets int, 350 str and datetime variables safely. 351 352 @param 353 tko_iteration: a tko test iteration. 354 pb_iteration: an empty pb test iteration. 355 356 """ 357 358 self.set_trivial_attr(tko_iteration, pb_iteration, 359 self.iteration_type_dict) 360 361 for key, val in six.iteritems(tko_iteration.attr_keyval): 362 newkeyval = pb_iteration.attr_keyval.add() 363 newkeyval.name = key 364 newkeyval.value = str(val) 365 366 for key, val in six.iteritems(tko_iteration.perf_keyval): 367 newkeyval = pb_iteration.perf_keyval.add() 368 newkeyval.name = key 369 newkeyval.value = str(val) 370 371 372 def get_trivial_attr(self, obj, objdict): 373 """Get all trivial attributes from the object. 374 375 This function is used to extract attributes from a pb job. The 376 dictionary specifies the types of each attribute in each tko 377 class. 378 379 @param 380 obj: the pb object that is being extracted. 381 objdict: the dict that specifies the type. 382 383 @return a dict of each attr name and it's corresponding value. 384 """ 385 386 resultdict = {} 387 for field, field_type in objdict.items(): 388 value = getattr(obj, field) 389 # six.integer_types is a tuple, so we can't check 390 # "if field_type in (str, six.integer_types)" 391 if field_type == str or field_type in six.integer_types: 392 resultdict[field] = field_type(value) 393 elif field_type == datetime: 394 resultdict[field] = ( 395 datetime.fromtimestamp(value/1000.0)) 396 397 return resultdict 398 399 400 def set_trivial_attr(self, tko_obj, pb_obj, objdict): 401 """Sets all the easy attributes appropriately according to the 402 type. 403 404 This function is used to set all the trivial attributes 405 provided by objdict, the dictionary that specifies the types 406 of each attribute in each tko class. 407 408 @param 409 tko_obj: the original object that has the data being copied. 410 pb_obj: the new pb object that is being copied into. 411 objdict: specifies the type of each attribute in the class we 412 are working with. 413 414 """ 415 for attr, attr_type in six.iteritems(objdict): 416 if attr_type == datetime: 417 t = getattr(tko_obj, attr) 418 if not t: 419 self.set_attr_safely(pb_obj, attr, t, int) 420 else: 421 t = mktime(t.timetuple()) + 1e-6 * t.microsecond 422 if six.PY2: 423 setattr(pb_obj, attr, long(t*1000)) 424 else: 425 setattr(pb_obj, attr, int(t*1000)) 426 else: 427 value = getattr(tko_obj, attr) 428 self.set_attr_safely(pb_obj, attr, value, attr_type) 429 430 431 def set_attr_safely(self, var, attr, value, vartype): 432 """Sets a particular attribute of var if the provided value is 433 not None. 434 435 Checks if value is None. If not, set the attribute of the var 436 to be the default value. This is necessary for the special 437 required fields of the protocol buffer. 438 439 @param 440 var: the variable of which one of the attribute is being set. 441 attr: the attribute that is being set. 442 value: the value that is being checked 443 vartype: the expected type of the attr 444 445 """ 446 # In py2, there is int and long, in py3 its only int. 447 supported_types = six.integer_types + (str,) 448 if vartype in supported_types: 449 if value is None: 450 value = vartype() 451 elif not isinstance(value, vartype): 452 logging.warning('Unexpected type %s for attr %s, should be %s', 453 type(value), attr, vartype) 454 455 setattr(var, attr, vartype(value)) 456