1#!/usr/bin/python 2 3"""A script that provides convertion between models.job and a protocol 4buffer object. 5 6This script contains only one class that takes an job instance and 7convert it into a protocol buffer object. The class will also be 8responsible for serializing the job instance via protocol buffers. 9 10""" 11 12# import python libraries 13import datetime 14import time 15 16# import autotest libraries 17from autotest_lib.tko import models 18from autotest_lib.tko import tko_pb2 19from autotest_lib.tko import utils 20 21__author__ = 'darrenkuo@google.com (Darren Kuo)' 22 23mktime = time.mktime 24datetime = datetime.datetime 25 26class JobSerializer(object): 27 """A class that takes a job object of the tko module and package 28 it with a protocol buffer. 29 30 This class will take a model.job object as input and create a 31 protocol buffer to include all the content of the job object. This 32 protocol buffer object will be serialized into a binary file. 33 """ 34 35 def __init__(self): 36 37 self.job_type_dict = {'dir':str, 'tests':list, 'user':str, 38 'label':str, 'machine':str, 39 'queued_time':datetime, 40 'started_time':datetime, 41 'finished_time':datetime, 42 'machine_owner':str, 43 'machine_group':str, 'aborted_by':str, 44 'aborted_on':datetime, 45 'keyval_dict':dict, 46 'afe_parent_job_id':str, 47 'build_version':str, 48 'suite':str, 49 'board':str} 50 51 self.test_type_dict = {'subdir':str, 'testname':str, 52 'status':str, 'reason':str, 53 'kernel':models.kernel, 'machine':str, 54 'started_time':datetime, 55 'finished_time':datetime, 56 'iterations':list, 'attributes':dict, 57 'labels':list} 58 59 self.kernel_type_dict = {'base':str, 'kernel_hash':str} 60 61 self.iteration_type_dict = {'index':int, 'attr_keyval':dict, 62 'perf_keyval':dict} 63 64 65 def deserialize_from_binary(self, infile): 66 """Takes in a binary file name and returns a tko job object. 67 68 The method first deserialize the binary into a protocol buffer 69 job object and then converts the job object into a tko job 70 object. 71 72 73 @param infile: the name of the binary file that will be deserialized. 74 75 @return: a tko job that is represented by the binary file will 76 be returned. 77 """ 78 79 job_pb = tko_pb2.Job() 80 81 binary = open(infile, 'r') 82 try: 83 job_pb.ParseFromString(binary.read()) 84 finally: 85 binary.close() 86 87 return self.get_tko_job(job_pb) 88 89 90 def serialize_to_binary(self, the_job, tag, binaryfilename): 91 """Serializes the tko job object into a binary by using a 92 protocol buffer. 93 94 The method takes a tko job object and constructs a protocol 95 buffer job object. Then invokes the native serializing 96 function on the object to get a binary string. The string is 97 then written to outfile. 98 99 Precondition: Assumes that all the information about the job 100 is already in the job object. Any fields that is None will be 101 provided a default value. 102 103 @param the_job: the tko job object that will be serialized. 104 tag: contains the job name and the afe_job_id 105 binaryfilename: the name of the file that will be written to 106 @param tag: The job tag string. 107 @param binaryfilename: The output filename. 108 109 @return: the filename of the file that contains the 110 binary of the serialized object. 111 """ 112 113 pb_job = tko_pb2.Job() 114 self.set_pb_job(the_job, pb_job, tag) 115 116 out = open(binaryfilename, 'wb') 117 try: 118 out.write(pb_job.SerializeToString()) 119 finally: 120 out.close() 121 122 123 def set_afe_job_id_and_tag(self, pb_job, tag): 124 """Sets the pb job's afe_job_id and tag field. 125 126 @param 127 pb_job: the pb job that will have it's fields set. 128 tag: used to set pb_job.tag and pb_job.afe_job_id. 129 """ 130 pb_job.tag = tag 131 pb_job.afe_job_id = utils.get_afe_job_id(tag) 132 133 134 # getter setter methods 135 def get_tko_job(self, job): 136 """Creates a a new tko job object from the pb job object. 137 138 Uses getter methods on the pb objects to extract all the 139 attributes and finally constructs a tko job object using the 140 models.job constructor. 141 142 @param 143 job: a pb job where data is being extracted from. 144 145 @return a tko job object. 146 """ 147 148 fields_dict = self.get_trivial_attr(job, self.job_type_dict) 149 150 fields_dict['tests'] = [self.get_tko_test(test) for test in job.tests] 151 152 fields_dict['keyval_dict'] = dict((keyval.name, keyval.value) 153 for keyval in job.keyval_dict) 154 155 newjob = models.job(fields_dict['dir'], fields_dict['user'], 156 fields_dict['label'], 157 fields_dict['machine'], 158 fields_dict['queued_time'], 159 fields_dict['started_time'], 160 fields_dict['finished_time'], 161 fields_dict['machine_owner'], 162 fields_dict['machine_group'], 163 fields_dict['aborted_by'], 164 fields_dict['aborted_on'], 165 fields_dict['keyval_dict']) 166 167 newjob.tests.extend(fields_dict['tests']) 168 169 return newjob 170 171 172 def set_pb_job(self, tko_job, pb_job, tag): 173 """Set the fields for the new job object. 174 175 Method takes in a tko job and an empty protocol buffer job 176 object. Then safely sets all the appropriate field by first 177 testing if the value in the original object is None. 178 179 @param 180 tko_job: a tko job instance that will have it's values 181 transfered to the new job 182 pb_job: a new instance of the job class provided in the 183 protocol buffer. 184 tag: used to set pb_job.tag and pb_job.afe_job_id. 185 """ 186 187 self.set_trivial_attr(tko_job, pb_job, self.job_type_dict) 188 self.set_afe_job_id_and_tag(pb_job, tag) 189 if hasattr(tko_job, 'index'): 190 pb_job.job_idx = tko_job.index 191 192 for test in tko_job.tests: 193 newtest = pb_job.tests.add() 194 self.set_pb_test(test, newtest) 195 196 for key, val in tko_job.keyval_dict.iteritems(): 197 newkeyval = pb_job.keyval_dict.add() 198 newkeyval.name = key 199 newkeyval.value = str(val) 200 201 202 def get_tko_test(self, test): 203 """Creates a tko test from pb_test. 204 205 Extracts data from pb_test by calling helper methods and 206 creates a tko test using the models.test constructor. 207 208 @param: 209 test: a pb_test where fields will be extracted from. 210 211 @return a new instance of models.test 212 """ 213 fields_dict = self.get_trivial_attr(test, self.test_type_dict) 214 215 fields_dict['kernel'] = self.get_tko_kernel(test.kernel) 216 217 fields_dict['iterations'] = [self.get_tko_iteration(iteration) 218 for iteration in test.iterations] 219 220 fields_dict['attributes'] = dict((keyval.name, keyval.value) 221 for keyval in test.attributes) 222 223 fields_dict['labels'] = list(test.labels) 224 225 # The constructor for models.test accepts a "perf_values" list that 226 # represents performance values of the test. The empty list argument 227 # in the constructor call below represents this value and makes this 228 # code adhere properly to the models.test constructor argument list. 229 # However, the effect of the empty list is that perf values are 230 # ignored in the job_serializer module. This is ok for now because 231 # autotest does not use the current module. If job_serializer is used 232 # in the future, we need to modify the "tko.proto" protobuf file to 233 # understand the notion of perf_values, then modify this file 234 # accordingly to use it. 235 return models.test(fields_dict['subdir'], 236 fields_dict['testname'], 237 fields_dict['status'], 238 fields_dict['reason'], 239 fields_dict['kernel'], 240 fields_dict['machine'], 241 fields_dict['started_time'], 242 fields_dict['finished_time'], 243 fields_dict['iterations'], 244 fields_dict['attributes'], 245 [], 246 fields_dict['labels']) 247 248 249 def set_pb_test(self, tko_test, pb_test): 250 """Sets the various fields of test object of the tko protocol. 251 252 Method takes a tko test and a new test of the protocol buffer and 253 transfers the values in the tko test to the new test. 254 255 @param 256 tko_test: a tko test instance. 257 pb_test: an empty protocol buffer test instance. 258 259 """ 260 261 self.set_trivial_attr(tko_test, pb_test, self.test_type_dict) 262 263 self.set_pb_kernel(tko_test.kernel, pb_test.kernel) 264 if hasattr(tko_test, 'test_idx'): 265 pb_test.test_idx = tko_test.test_idx 266 267 for current_iteration in tko_test.iterations: 268 pb_iteration = pb_test.iterations.add() 269 self.set_pb_iteration(current_iteration, pb_iteration) 270 271 for key, val in tko_test.attributes.iteritems(): 272 newkeyval = pb_test.attributes.add() 273 newkeyval.name = key 274 newkeyval.value = str(val) 275 276 for current_label in tko_test.labels: 277 pb_test.labels.append(current_label) 278 279 280 def get_tko_kernel(self, kernel): 281 """Constructs a new tko kernel object from a pb kernel object. 282 283 Uses all the getter methods on the pb kernel object to extract 284 the attributes and constructs a new tko kernel object using 285 the model.kernel constructor. 286 287 @param 288 kernel: a pb kernel object where data will be extracted. 289 290 @return a new tko kernel object. 291 """ 292 293 fields_dict = self.get_trivial_attr(kernel, self.kernel_type_dict) 294 295 return models.kernel(fields_dict['base'], [], fields_dict['kernel_hash']) 296 297 298 def set_pb_kernel(self, tko_kernel, pb_kernel): 299 """Set a specific kernel of a test. 300 301 Takes the same form of all the other setting methods. It 302 seperates the string variables from the int variables and set 303 them safely. 304 305 @param 306 tko_kernel: a tko kernel. 307 pb_kernel: an empty protocol buffer kernel. 308 309 """ 310 311 self.set_trivial_attr(tko_kernel, pb_kernel, self.kernel_type_dict) 312 313 314 def get_tko_iteration(self, iteration): 315 """Creates a new tko iteration with the data in the provided 316 pb iteration. 317 318 Uses the data in the pb iteration and the models.iteration 319 constructor to create a new tko iterations 320 321 @param 322 iteration: a pb iteration instance 323 324 @return a tko iteration instance with the same data. 325 """ 326 327 fields_dict = self.get_trivial_attr(iteration, 328 self.iteration_type_dict) 329 330 fields_dict['attr_keyval'] = dict((keyval.name, keyval.value) 331 for keyval in iteration.attr_keyval) 332 333 fields_dict['perf_keyval'] = dict((keyval.name, keyval.value) 334 for keyval in iteration.perf_keyval) 335 336 return models.iteration(fields_dict['index'], 337 fields_dict['attr_keyval'], 338 fields_dict['perf_keyval']) 339 340 341 def set_pb_iteration(self, tko_iteration, pb_iteration): 342 """Sets all fields for a particular iteration. 343 344 Takes same form as all the other setting methods. Sets int, 345 str and datetime variables safely. 346 347 @param 348 tko_iteration: a tko test iteration. 349 pb_iteration: an empty pb test iteration. 350 351 """ 352 353 self.set_trivial_attr(tko_iteration, pb_iteration, 354 self.iteration_type_dict) 355 356 for key, val in tko_iteration.attr_keyval.iteritems(): 357 newkeyval = pb_iteration.attr_keyval.add() 358 newkeyval.name = key 359 newkeyval.value = str(val) 360 361 for key, val in tko_iteration.perf_keyval.iteritems(): 362 newkeyval = pb_iteration.perf_keyval.add() 363 newkeyval.name = key 364 newkeyval.value = str(val) 365 366 367 def get_trivial_attr(self, obj, objdict): 368 """Get all trivial attributes from the object. 369 370 This function is used to extract attributes from a pb job. The 371 dictionary specifies the types of each attribute in each tko 372 class. 373 374 @param 375 obj: the pb object that is being extracted. 376 objdict: the dict that specifies the type. 377 378 @return a dict of each attr name and it's corresponding value. 379 """ 380 381 resultdict = {} 382 for field, field_type in objdict.items(): 383 value = getattr(obj, field) 384 if field_type in (str, int, long): 385 resultdict[field] = field_type(value) 386 elif field_type == datetime: 387 resultdict[field] = ( 388 datetime.fromtimestamp(value/1000.0)) 389 390 return resultdict 391 392 393 def set_trivial_attr(self, tko_obj, pb_obj, objdict): 394 """Sets all the easy attributes appropriately according to the 395 type. 396 397 This function is used to set all the trivial attributes 398 provided by objdict, the dictionary that specifies the types 399 of each attribute in each tko class. 400 401 @param 402 tko_obj: the original object that has the data being copied. 403 pb_obj: the new pb object that is being copied into. 404 objdict: specifies the type of each attribute in the class we 405 are working with. 406 407 """ 408 for attr, attr_type in objdict.iteritems(): 409 if attr_type == datetime: 410 t = getattr(tko_obj, attr) 411 if not t: 412 self.set_attr_safely(pb_obj, attr, t, int) 413 else: 414 t = mktime(t.timetuple()) + 1e-6 * t.microsecond 415 setattr(pb_obj, attr, long(t*1000)) 416 else: 417 value = getattr(tko_obj, attr) 418 self.set_attr_safely(pb_obj, attr, value, attr_type) 419 420 421 def set_attr_safely(self, var, attr, value, vartype): 422 """Sets a particular attribute of var if the provided value is 423 not None. 424 425 Checks if value is None. If not, set the attribute of the var 426 to be the default value. This is necessary for the special 427 required fields of the protocol buffer. 428 429 @param 430 var: the variable of which one of the attribute is being set. 431 attr: the attribute that is being set. 432 value: the value that is being checked 433 vartype: the expected type of the attr 434 435 """ 436 437 supported_types = [int, long, str] 438 if vartype in supported_types: 439 if value is None: 440 value = vartype() 441 else: 442 assert isinstance(value, vartype), ( 443 'Unexpected type %s for attr %s, should be %s' % 444 (type(value), attr, vartype)) 445 446 setattr(var, attr, value) 447