report/dejagnu/summary.py

# Copyright 2011 Google Inc. All Rights Reserved.
# Author: kbaclawski@google.com (Krystian Baclawski)
#

from collections import defaultdict
from collections import namedtuple
from datetime import datetime
from fnmatch import fnmatch
from itertools import groupby
import logging
import os.path
import re


class DejaGnuTestResult(namedtuple('Result', 'name variant result flaky')):
  """Stores the result of a single test case."""

  # avoid adding __dict__ to the class
  __slots__ = ()

  LINE_RE = re.compile(r'([A-Z]+):\s+([\w/+.-]+)(.*)')

  @classmethod
  def FromLine(cls, line):
    """Alternate constructor which takes a string and parses it."""
    try:
      attrs, line = line.split('|', 1)

      if attrs.strip() != 'flaky':
        return None

      line = line.strip()
      flaky = True
    except ValueError:
      flaky = False

    fields = cls.LINE_RE.match(line.strip())

    if fields:
      result, path, variant = fields.groups()

      # some of the tests are generated in build dir and are issued from there,
      # because every test run is performed in randomly named tmp directory we
      # need to remove random part
      try:
        # assume that 2nd field is a test path
        path_parts = path.split('/')

        index = path_parts.index('testsuite')
        path = '/'.join(path_parts[index + 1:])
      except ValueError:
        path = '/'.join(path_parts)

      # Remove junk from test description.
      variant = variant.strip(', ')

      substitutions = [
          # remove include paths - they contain name of tmp directory
          ('-I\S+', ''),
          # compress white spaces
          ('\s+', ' ')
      ]

      for pattern, replacement in substitutions:
        variant = re.sub(pattern, replacement, variant)

      # Some tests separate last component of path by space, so actual filename
      # ends up in description instead of path part. Correct that.
      try:
        first, rest = variant.split(' ', 1)
      except ValueError:
        pass
      else:
        if first.endswith('.o'):
          path = os.path.join(path, first)
          variant = rest

      # DejaGNU framework errors don't contain path part at all, so description
      # part has to be reconstructed.
      if not any(os.path.basename(path).endswith('.%s' % suffix)
                 for suffix in ['h', 'c', 'C', 'S', 'H', 'cc', 'i', 'o']):
        variant = '%s %s' % (path, variant)
        path = ''

      # Some tests are picked up from current directory (presumably DejaGNU
      # generates some test files). Remove the prefix for these files.
      if path.startswith('./'):
        path = path[2:]

      return cls(path, variant or '', result, flaky=flaky)

  def __str__(self):
    """Returns string representation of a test result."""
    if self.flaky:
      fmt = 'flaky | '
    else:
      fmt = ''
    fmt += '{2}: {0}'
    if self.variant:
      fmt += ' {1}'
    return fmt.format(*self)


class DejaGnuTestRun(object):
  """Container for test results that were a part of single test run.

  The class stores also metadata related to the test run.

  Attributes:
    board: Name of DejaGNU board, which was used to run the tests.
    date: The date when the test run was started.
    target: Target triple.
    host: Host triple.
    tool: The tool that was tested (e.g. gcc, binutils, g++, etc.)
    results: a list of DejaGnuTestResult objects.
  """

  __slots__ = ('board', 'date', 'target', 'host', 'tool', 'results')

  def __init__(self, **kwargs):
    assert all(name in self.__slots__ for name in kwargs)

    self.results = set()
    self.date = kwargs.get('date', datetime.now())

    for name in ('board', 'target', 'tool', 'host'):
      setattr(self, name, kwargs.get(name, 'unknown'))

  @classmethod
  def FromFile(cls, filename):
    """Alternate constructor - reads a DejaGNU output file."""
    test_run = cls()
    test_run.FromDejaGnuOutput(filename)
    test_run.CleanUpTestResults()
    return test_run

  @property
  def summary(self):
    """Returns a summary as {ResultType -> Count} dictionary."""
    summary = defaultdict(int)

    for r in self.results:
      summary[r.result] += 1

    return summary

  def _ParseBoard(self, fields):
    self.board = fields.group(1).strip()

  def _ParseDate(self, fields):
    self.date = datetime.strptime(fields.group(2).strip(), '%a %b %d %X %Y')

  def _ParseTarget(self, fields):
    self.target = fields.group(2).strip()

  def _ParseHost(self, fields):
    self.host = fields.group(2).strip()

  def _ParseTool(self, fields):
    self.tool = fields.group(1).strip()

  def FromDejaGnuOutput(self, filename):
    """Read in and parse DejaGNU output file."""

    logging.info('Reading "%s" DejaGNU output file.', filename)

    with open(filename, 'r') as report:
      lines = [line.strip() for line in report.readlines() if line.strip()]

    parsers = ((re.compile(r'Running target (.*)'), self._ParseBoard),
               (re.compile(r'Test Run By (.*) on (.*)'), self._ParseDate),
               (re.compile(r'=== (.*) tests ==='), self._ParseTool),
               (re.compile(r'Target(\s+)is (.*)'), self._ParseTarget),
               (re.compile(r'Host(\s+)is (.*)'), self._ParseHost))

    for line in lines:
      result = DejaGnuTestResult.FromLine(line)

      if result:
        self.results.add(result)
      else:
        for regexp, parser in parsers:
          fields = regexp.match(line)
          if fields:
            parser(fields)
            break

    logging.debug('DejaGNU output file parsed successfully.')
    logging.debug(self)

  def CleanUpTestResults(self):
    """Remove certain test results considered to be spurious.

    1) Large number of test reported as UNSUPPORTED are also marked as
       UNRESOLVED. If that's the case remove latter result.
    2) If a test is performed on compiler output and for some reason compiler
       fails, we don't want to report all failures that depend on the former.
    """
    name_key = lambda v: v.name
    results_by_name = sorted(self.results, key=name_key)

    for name, res_iter in groupby(results_by_name, key=name_key):
      results = set(res_iter)

      # If DejaGnu was unable to compile a test it will create following result:
      failed = DejaGnuTestResult(name, '(test for excess errors)', 'FAIL',
                                 False)

      # If a test compilation failed, remove all results that are dependent.
      if failed in results:
        dependants = set(filter(lambda r: r.result != 'FAIL', results))

        self.results -= dependants

        for res in dependants:
          logging.info('Removed {%s} dependance.', res)

      # Remove all UNRESOLVED results that were also marked as UNSUPPORTED.
      unresolved = [res._replace(result='UNRESOLVED')
                    for res in results if res.result == 'UNSUPPORTED']

      for res in unresolved:
        if res in self.results:
          self.results.remove(res)
          logging.info('Removed {%s} duplicate.', res)

  def _IsApplicable(self, manifest):
    """Checks if test results need to be reconsidered based on the manifest."""
    check_list = [(self.tool, manifest.tool), (self.board, manifest.board)]

    return all(fnmatch(text, pattern) for text, pattern in check_list)

  def SuppressTestResults(self, manifests):
    """Suppresses all test results listed in manifests."""

    # Get a set of tests results that are going to be suppressed if they fail.
    manifest_results = set()

    for manifest in filter(self._IsApplicable, manifests):
      manifest_results |= set(manifest.results)

    suppressed_results = self.results & manifest_results

    for result in sorted(suppressed_results):
      logging.debug('Result suppressed for {%s}.', result)

      new_result = '!' + result.result

      # Mark result suppression as applied.
      manifest_results.remove(result)

      # Rewrite test result.
      self.results.remove(result)
      self.results.add(result._replace(result=new_result))

    for result in sorted(manifest_results):
      logging.warning('Result {%s} listed in manifest but not suppressed.',
                      result)

  def __str__(self):
    return '{0}, {1} @{2} on {3}'.format(self.target, self.tool, self.board,
                                         self.date)