• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import re, os
2
3from autotest_lib.client.common_lib import utils as common_utils
4from autotest_lib.tko import utils as tko_utils, models, status_lib
5from autotest_lib.tko.parsers import base
6
7
8class NoHostnameError(Exception):
9    pass
10
11
12class BoardLabelError(Exception):
13    pass
14
15
16class job(models.job):
17    def __init__(self, dir):
18        job_dict = job.load_from_dir(dir)
19        super(job, self).__init__(dir, **job_dict)
20
21
22    @classmethod
23    def load_from_dir(cls, dir):
24        keyval = cls.read_keyval(dir)
25        tko_utils.dprint(str(keyval))
26
27        user = keyval.get("user", None)
28        label = keyval.get("label", None)
29        queued_time = tko_utils.get_timestamp(keyval, "job_queued")
30        started_time = tko_utils.get_timestamp(keyval, "job_started")
31        finished_time = tko_utils.get_timestamp(keyval, "job_finished")
32        machine = cls.determine_hostname(keyval, dir)
33        machine_group = cls.determine_machine_group(machine, dir)
34        machine_owner = keyval.get("owner", None)
35
36        aborted_by = keyval.get("aborted_by", None)
37        aborted_at = tko_utils.get_timestamp(keyval, "aborted_on")
38
39        return {"user": user, "label": label, "machine": machine,
40                "queued_time": queued_time, "started_time": started_time,
41                "finished_time": finished_time, "machine_owner": machine_owner,
42                "machine_group": machine_group, "aborted_by": aborted_by,
43                "aborted_on": aborted_at, "keyval_dict": keyval}
44
45
46    @classmethod
47    def determine_hostname(cls, keyval, job_dir):
48        host_group_name = keyval.get("host_group_name", None)
49        machine = keyval.get("hostname", "")
50        is_multimachine = "," in machine
51
52        # determine what hostname to use
53        if host_group_name:
54            if is_multimachine or not machine:
55                tko_utils.dprint("Using host_group_name %r instead of "
56                                 "machine name." % host_group_name)
57                machine = host_group_name
58        elif is_multimachine:
59            try:
60                machine = job.find_hostname(job_dir) # find a unique hostname
61            except NoHostnameError:
62                pass  # just use the comma-separated name
63
64        tko_utils.dprint("MACHINE NAME: %s" % machine)
65        return machine
66
67
68    @classmethod
69    def determine_machine_group(cls, hostname, job_dir):
70        machine_groups = set()
71        for individual_hostname in hostname.split(","):
72            host_keyval = models.test.parse_host_keyval(job_dir,
73                                                        individual_hostname)
74            if not host_keyval:
75                tko_utils.dprint('Unable to parse host keyval for %s'
76                                 % individual_hostname)
77            elif 'labels' in host_keyval:
78                # Use board label as machine group. This is to avoid the
79                # confusion of multiple boards mapping to the same platform in
80                # wmatrix. With this change, wmatrix will group tests with the
81                # same board, rather than the same platform.
82                labels = host_keyval['labels'].split(',')
83                board_labels = [l[8:] for l in labels
84                               if l.startswith('board%3A')]
85                if board_labels and len(board_labels) == 1:
86                    machine_groups.add(board_labels[0])
87                else:
88                    error = ('Failed to retrieve board label from host labels: '
89                             '%s' % host_keyval['labels'])
90                    tko_utils.dprint(error)
91                    raise BoardLabelError(error)
92            elif "platform" in host_keyval:
93                machine_groups.add(host_keyval["platform"])
94        machine_group = ",".join(sorted(machine_groups))
95        tko_utils.dprint("MACHINE GROUP: %s" % machine_group)
96        return machine_group
97
98
99    @staticmethod
100    def find_hostname(path):
101        hostname = os.path.join(path, "sysinfo", "hostname")
102        try:
103            machine = open(hostname).readline().rstrip()
104            return machine
105        except Exception:
106            tko_utils.dprint("Could not read a hostname from "
107                             "sysinfo/hostname")
108
109        uname = os.path.join(path, "sysinfo", "uname_-a")
110        try:
111            machine = open(uname).readline().split()[1]
112            return machine
113        except Exception:
114            tko_utils.dprint("Could not read a hostname from "
115                             "sysinfo/uname_-a")
116
117        raise NoHostnameError("Unable to find a machine name")
118
119
120class kernel(models.kernel):
121    def __init__(self, job, verify_ident=None):
122        kernel_dict = kernel.load_from_dir(job.dir, verify_ident)
123        super(kernel, self).__init__(**kernel_dict)
124
125
126    @staticmethod
127    def load_from_dir(dir, verify_ident=None):
128        # try and load the booted kernel version
129        attributes = False
130        i = 1
131        build_dir = os.path.join(dir, "build")
132        while True:
133            if not os.path.exists(build_dir):
134                break
135            build_log = os.path.join(build_dir, "debug", "build_log")
136            attributes = kernel.load_from_build_log(build_log)
137            if attributes:
138                break
139            i += 1
140            build_dir = os.path.join(dir, "build.%d" % (i))
141
142        if not attributes:
143            if verify_ident:
144                base = verify_ident
145            else:
146                base = kernel.load_from_sysinfo(dir)
147            patches = []
148            hashes = []
149        else:
150            base, patches, hashes = attributes
151        tko_utils.dprint("kernel.__init__() found kernel version %s"
152                         % base)
153
154        # compute the kernel hash
155        if base == "UNKNOWN":
156            kernel_hash = "UNKNOWN"
157        else:
158            kernel_hash = kernel.compute_hash(base, hashes)
159
160        return {"base": base, "patches": patches,
161                "kernel_hash": kernel_hash}
162
163
164    @staticmethod
165    def load_from_sysinfo(path):
166        for subdir in ("reboot1", ""):
167            uname_path = os.path.join(path, "sysinfo", subdir,
168                                      "uname_-a")
169            if not os.path.exists(uname_path):
170                continue
171            uname = open(uname_path).readline().split()
172            return re.sub("-autotest$", "", uname[2])
173        return "UNKNOWN"
174
175
176    @staticmethod
177    def load_from_build_log(path):
178        if not os.path.exists(path):
179            return None
180
181        base, patches, hashes = "UNKNOWN", [], []
182        for line in file(path):
183            head, rest = line.split(": ", 1)
184            rest = rest.split()
185            if head == "BASE":
186                base = rest[0]
187            elif head == "PATCH":
188                patches.append(patch(*rest))
189                hashes.append(rest[2])
190        return base, patches, hashes
191
192
193class test(models.test):
194    def __init__(self, subdir, testname, status, reason, test_kernel,
195                 machine, started_time, finished_time, iterations,
196                 attributes, labels):
197        # for backwards compatibility with the original parser
198        # implementation, if there is no test version we need a NULL
199        # value to be used; also, if there is a version it should
200        # be terminated by a newline
201        if "version" in attributes:
202            attributes["version"] = str(attributes["version"])
203        else:
204            attributes["version"] = None
205
206        super(test, self).__init__(subdir, testname, status, reason,
207                                   test_kernel, machine, started_time,
208                                   finished_time, iterations,
209                                   attributes, labels)
210
211
212    @staticmethod
213    def load_iterations(keyval_path):
214        return iteration.load_from_keyval(keyval_path)
215
216
217class patch(models.patch):
218    def __init__(self, spec, reference, hash):
219        tko_utils.dprint("PATCH::%s %s %s" % (spec, reference, hash))
220        super(patch, self).__init__(spec, reference, hash)
221        self.spec = spec
222        self.reference = reference
223        self.hash = hash
224
225
226class iteration(models.iteration):
227    @staticmethod
228    def parse_line_into_dicts(line, attr_dict, perf_dict):
229        key, value = line.split("=", 1)
230        perf_dict[key] = value
231
232
233class status_line(object):
234    def __init__(self, indent, status, subdir, testname, reason,
235                 optional_fields):
236        # pull out the type & status of the line
237        if status == "START":
238            self.type = "START"
239            self.status = None
240        elif status.startswith("END "):
241            self.type = "END"
242            self.status = status[4:]
243        else:
244            self.type = "STATUS"
245            self.status = status
246        assert (self.status is None or
247                self.status in status_lib.statuses)
248
249        # save all the other parameters
250        self.indent = indent
251        self.subdir = self.parse_name(subdir)
252        self.testname = self.parse_name(testname)
253        self.reason = reason
254        self.optional_fields = optional_fields
255
256
257    @staticmethod
258    def parse_name(name):
259        if name == "----":
260            return None
261        return name
262
263
264    @staticmethod
265    def is_status_line(line):
266        return re.search(r"^\t*(\S[^\t]*\t){3}", line) is not None
267
268
269    @classmethod
270    def parse_line(cls, line):
271        if not status_line.is_status_line(line):
272            return None
273        match = re.search(r"^(\t*)(.*)$", line, flags=re.DOTALL)
274        if not match:
275            # A more useful error message than:
276            #  AttributeError: 'NoneType' object has no attribute 'groups'
277            # to help us debug WTF happens on occasion here.
278            raise RuntimeError("line %r could not be parsed." % line)
279        indent, line = match.groups()
280        indent = len(indent)
281
282        # split the line into the fixed and optional fields
283        parts = line.rstrip("\n").split("\t")
284
285        part_index = 3
286        status, subdir, testname = parts[0:part_index]
287
288        # all optional parts should be of the form "key=value". once we've found
289        # a non-matching part, treat it and the rest of the parts as the reason.
290        optional_fields = {}
291        while part_index < len(parts):
292            kv = re.search(r"^(\w+)=(.+)", parts[part_index])
293            if not kv:
294                break
295
296            optional_fields[kv.group(1)] = kv.group(2)
297            part_index += 1
298
299        reason = "\t".join(parts[part_index:])
300
301        # build up a new status_line and return it
302        return cls(indent, status, subdir, testname, reason,
303                   optional_fields)
304
305
306class parser(base.parser):
307    @staticmethod
308    def make_job(dir):
309        return job(dir)
310
311
312    def state_iterator(self, buffer):
313        new_tests = []
314        boot_count = 0
315        group_subdir = None
316        sought_level = 0
317        stack = status_lib.status_stack()
318        current_kernel = kernel(self.job)
319        boot_in_progress = False
320        alert_pending = None
321        started_time = None
322
323        while not self.finished or buffer.size():
324            # stop processing once the buffer is empty
325            if buffer.size() == 0:
326                yield new_tests
327                new_tests = []
328                continue
329
330            # parse the next line
331            line = buffer.get()
332            tko_utils.dprint('\nSTATUS: ' + line.strip())
333            line = status_line.parse_line(line)
334            if line is None:
335                tko_utils.dprint('non-status line, ignoring')
336                continue # ignore non-status lines
337
338            # have we hit the job start line?
339            if (line.type == "START" and not line.subdir and
340                not line.testname):
341                sought_level = 1
342                tko_utils.dprint("found job level start "
343                                 "marker, looking for level "
344                                 "1 groups now")
345                continue
346
347            # have we hit the job end line?
348            if (line.type == "END" and not line.subdir and
349                not line.testname):
350                tko_utils.dprint("found job level end "
351                                 "marker, looking for level "
352                                 "0 lines now")
353                sought_level = 0
354
355            # START line, just push another layer on to the stack
356            # and grab the start time if this is at the job level
357            # we're currently seeking
358            if line.type == "START":
359                group_subdir = None
360                stack.start()
361                if line.indent == sought_level:
362                    started_time = \
363                                 tko_utils.get_timestamp(
364                        line.optional_fields, "timestamp")
365                tko_utils.dprint("start line, ignoring")
366                continue
367            # otherwise, update the status on the stack
368            else:
369                tko_utils.dprint("GROPE_STATUS: %s" %
370                                 [stack.current_status(),
371                                  line.status, line.subdir,
372                                  line.testname, line.reason])
373                stack.update(line.status)
374
375            if line.status == "ALERT":
376                tko_utils.dprint("job level alert, recording")
377                alert_pending = line.reason
378                continue
379
380            # ignore Autotest.install => GOOD lines
381            if (line.testname == "Autotest.install" and
382                line.status == "GOOD"):
383                tko_utils.dprint("Successful Autotest "
384                                 "install, ignoring")
385                continue
386
387            # ignore END lines for a reboot group
388            if (line.testname == "reboot" and line.type == "END"):
389                tko_utils.dprint("reboot group, ignoring")
390                continue
391
392            # convert job-level ABORTs into a 'CLIENT_JOB' test, and
393            # ignore other job-level events
394            if line.testname is None:
395                if (line.status == "ABORT" and
396                    line.type != "END"):
397                    line.testname = "CLIENT_JOB"
398                else:
399                    tko_utils.dprint("job level event, "
400                                    "ignoring")
401                    continue
402
403            # use the group subdir for END lines
404            if line.type == "END":
405                line.subdir = group_subdir
406
407            # are we inside a block group?
408            if (line.indent != sought_level and
409                line.status != "ABORT" and
410                not line.testname.startswith('reboot.')):
411                if line.subdir:
412                    tko_utils.dprint("set group_subdir: "
413                                     + line.subdir)
414                    group_subdir = line.subdir
415                tko_utils.dprint("ignoring incorrect indent "
416                                 "level %d != %d," %
417                                 (line.indent, sought_level))
418                continue
419
420            # use the subdir as the testname, except for
421            # boot.* and kernel.* tests
422            if (line.testname is None or
423                not re.search(r"^(boot(\.\d+)?$|kernel\.)",
424                              line.testname)):
425                if line.subdir and '.' in line.subdir:
426                    line.testname = line.subdir
427
428            # has a reboot started?
429            if line.testname == "reboot.start":
430                started_time = tko_utils.get_timestamp(
431                    line.optional_fields, "timestamp")
432                tko_utils.dprint("reboot start event, "
433                                 "ignoring")
434                boot_in_progress = True
435                continue
436
437            # has a reboot finished?
438            if line.testname == "reboot.verify":
439                line.testname = "boot.%d" % boot_count
440                tko_utils.dprint("reboot verified")
441                boot_in_progress = False
442                verify_ident = line.reason.strip()
443                current_kernel = kernel(self.job, verify_ident)
444                boot_count += 1
445
446            if alert_pending:
447                line.status = "ALERT"
448                line.reason = alert_pending
449                alert_pending = None
450
451            # create the actual test object
452            finished_time = tko_utils.get_timestamp(
453                line.optional_fields, "timestamp")
454            final_status = stack.end()
455            tko_utils.dprint("Adding: "
456                             "%s\nSubdir:%s\nTestname:%s\n%s" %
457                             (final_status, line.subdir,
458                              line.testname, line.reason))
459            new_test = test.parse_test(self.job, line.subdir,
460                                       line.testname,
461                                       final_status, line.reason,
462                                       current_kernel,
463                                       started_time,
464                                       finished_time)
465            started_time = None
466            new_tests.append(new_test)
467
468        # the job is finished, but we never came back from reboot
469        if boot_in_progress:
470            testname = "boot.%d" % boot_count
471            reason = "machine did not return from reboot"
472            tko_utils.dprint(("Adding: ABORT\nSubdir:----\n"
473                              "Testname:%s\n%s")
474                             % (testname, reason))
475            new_test = test.parse_test(self.job, None, testname,
476                                       "ABORT", reason,
477                                       current_kernel, None, None)
478            new_tests.append(new_test)
479        yield new_tests
480