• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Methods related to outputting script results in a human-readable format.
5
6Also probably a good example of how to *not* write HTML.
7"""
8
9from __future__ import print_function
10
11import collections
12import logging
13import sys
14import tempfile
15from typing import Any, Dict, IO, List, Optional, OrderedDict, Set, Union
16
17import six
18
19from unexpected_passes_common import data_types
20
21FULL_PASS = 'Fully passed in the following'
22PARTIAL_PASS = 'Partially passed in the following'
23NEVER_PASS = 'Never passed in the following'
24
25HTML_HEADER = """\
26<!DOCTYPE html>
27<html>
28<head>
29<meta content="width=device-width">
30<style>
31.collapsible_group {
32  background-color: #757575;
33  border: none;
34  color: white;
35  font-size:20px;
36  outline: none;
37  text-align: left;
38  width: 100%;
39}
40.active_collapsible_group, .collapsible_group:hover {
41  background-color: #474747;
42}
43.highlighted_collapsible_group {
44  background-color: #008000;
45  border: none;
46  color: white;
47  font-size:20px;
48  outline: none;
49  text-align: left;
50  width: 100%;
51}
52.active_highlighted_collapsible_group, .highlighted_collapsible_group:hover {
53  background-color: #004d00;
54}
55.content {
56  background-color: #e1e4e8;
57  display: none;
58  padding: 0 25px;
59}
60button {
61  user-select: text;
62}
63h1 {
64  background-color: black;
65  color: white;
66}
67</style>
68</head>
69<body>
70"""
71
72HTML_FOOTER = """\
73<script>
74function OnClickImpl(element) {
75  let sibling = element.nextElementSibling;
76  if (sibling.style.display === "block") {
77    sibling.style.display = "none";
78  } else {
79    sibling.style.display = "block";
80  }
81}
82
83function OnClick() {
84  this.classList.toggle("active_collapsible_group");
85  OnClickImpl(this);
86}
87
88function OnClickHighlighted() {
89  this.classList.toggle("active_highlighted_collapsible_group");
90  OnClickImpl(this);
91}
92
93// Repeatedly bubble up the highlighted_collapsible_group class as long as all
94// siblings are highlighted.
95let found_element_to_convert = false;
96do {
97  found_element_to_convert = false;
98  // Get an initial list of all highlighted_collapsible_groups.
99  let highlighted_collapsible_groups = document.getElementsByClassName(
100      "highlighted_collapsible_group");
101  let highlighted_list = [];
102  for (elem of highlighted_collapsible_groups) {
103    highlighted_list.push(elem);
104  }
105
106  // Bubble up the highlighted_collapsible_group class.
107  while (highlighted_list.length) {
108    elem = highlighted_list.shift();
109    if (elem.tagName == 'BODY') {
110      continue;
111    }
112    if (elem.classList.contains("content")) {
113      highlighted_list.push(elem.previousElementSibling);
114      continue;
115    }
116    if (elem.classList.contains("collapsible_group")) {
117      found_element_to_convert = true;
118      elem.classList.add("highlighted_collapsible_group");
119      elem.classList.remove("collapsible_group");
120    }
121
122    sibling_elements = elem.parentElement.children;
123    let found_non_highlighted_group = false;
124    for (e of sibling_elements) {
125      if (e.classList.contains("collapsible_group")) {
126        found_non_highlighted_group = true;
127        break
128      }
129    }
130    if (!found_non_highlighted_group) {
131      highlighted_list.push(elem.parentElement);
132    }
133  }
134} while (found_element_to_convert);
135
136// Apply OnClick listeners so [highlighted_]collapsible_groups properly
137// shrink/expand.
138let collapsible_groups = document.getElementsByClassName("collapsible_group");
139for (element of collapsible_groups) {
140  element.addEventListener("click", OnClick);
141}
142
143highlighted_collapsible_groups = document.getElementsByClassName(
144    "highlighted_collapsible_group");
145for (element of highlighted_collapsible_groups) {
146  element.addEventListener("click", OnClickHighlighted);
147}
148</script>
149</body>
150</html>
151"""
152
153SECTION_STALE = 'Stale Expectations (Passed 100% Everywhere, Can Remove)'
154SECTION_SEMI_STALE = ('Semi Stale Expectations (Passed 100% In Some Places, '
155                      'But Not Everywhere - Can Likely Be Modified But Not '
156                      'Necessarily Removed)')
157SECTION_ACTIVE = ('Active Expectations (Failed At Least Once Everywhere, '
158                  'Likely Should Be Left Alone)')
159SECTION_UNMATCHED = ('Unmatched Results (An Expectation Existed When The Test '
160                     'Ran, But No Matching One Currently Exists OR The '
161                     'Expectation Is Too New)')
162SECTION_UNUSED = ('Unused Expectations (Indicative Of The Configuration No '
163                  'Longer Being Tested Or Tags Changing)')
164
165MAX_BUGS_PER_LINE = 5
166MAX_CHARACTERS_PER_CL_LINE = 72
167
168ElementType = Union[Dict[str, Any], List[str], str]
169# Sample:
170# {
171#   expectation_file: {
172#     test_name: {
173#       expectation_summary: {
174#         builder_name: {
175#           'Fully passed in the following': [
176#             step1,
177#           ],
178#           'Partially passed in the following': {
179#             step2: [
180#               failure_link,
181#             ],
182#           },
183#           'Never passed in the following': [
184#             step3,
185#           ],
186#         }
187#       }
188#     }
189#   }
190# }
191FullOrNeverPassValue = List[str]
192PartialPassValue = Dict[str, List[str]]
193PassValue = Union[FullOrNeverPassValue, PartialPassValue]
194BuilderToPassMap = Dict[str, Dict[str, PassValue]]
195ExpectationToBuilderMap = Dict[str, BuilderToPassMap]
196TestToExpectationMap = Dict[str, ExpectationToBuilderMap]
197ExpectationFileStringDict = Dict[str, TestToExpectationMap]
198# Sample:
199# {
200#   test_name: {
201#     builder_name: {
202#       step_name: [
203#         individual_result_string_1,
204#         individual_result_string_2,
205#         ...
206#       ],
207#       ...
208#     },
209#     ...
210#   },
211#   ...
212# }
213StepToResultsMap = Dict[str, List[str]]
214BuilderToStepMap = Dict[str, StepToResultsMap]
215TestToBuilderStringDict = Dict[str, BuilderToStepMap]
216# Sample:
217# {
218#   result_output.FULL_PASS: {
219#     builder_name: [
220#       step_name (total passes / total builds)
221#     ],
222#   },
223#   result_output.NEVER_PASS: {
224#     builder_name: [
225#       step_name (total passes / total builds)
226#     ],
227#   },
228#   result_output.PARTIAL_PASS: {
229#     builder_name: {
230#       step_name (total passes / total builds): [
231#         failure links,
232#       ],
233#     },
234#   },
235# }
236FullOrNeverPassStepValue = List[str]
237PartialPassStepValue = Dict[str, List[str]]
238PassStepValue = Union[FullOrNeverPassStepValue, PartialPassStepValue]
239
240UnmatchedResultsType = Dict[str, data_types.ResultListType]
241UnusedExpectation = Dict[str, List[data_types.Expectation]]
242
243RemovedUrlsType = Union[List[str], Set[str]]
244
245
246def OutputResults(stale_dict: data_types.TestExpectationMap,
247                  semi_stale_dict: data_types.TestExpectationMap,
248                  active_dict: data_types.TestExpectationMap,
249                  unmatched_results: UnmatchedResultsType,
250                  unused_expectations: UnusedExpectation,
251                  output_format: str,
252                  file_handle: Optional[IO] = None) -> None:
253  """Outputs script results to |file_handle|.
254
255  Args:
256    stale_dict: A data_types.TestExpectationMap containing all the stale
257        expectations.
258    semi_stale_dict: A data_types.TestExpectationMap containing all the
259        semi-stale expectations.
260    active_dict: A data_types.TestExpectationmap containing all the active
261        expectations.
262    ummatched_results: Any unmatched results found while filling
263        |test_expectation_map|, as returned by
264        queries.FillExpectationMapFor[Ci|Try]Builders().
265    unused_expectations: A dict from expectation file (str) to list of
266        unmatched Expectations that were pulled out of |test_expectation_map|
267    output_format: A string denoting the format to output to. Valid values are
268        "print" and "html".
269    file_handle: An optional open file-like object to output to. If not
270        specified, a suitable default will be used.
271  """
272  assert isinstance(stale_dict, data_types.TestExpectationMap)
273  assert isinstance(semi_stale_dict, data_types.TestExpectationMap)
274  assert isinstance(active_dict, data_types.TestExpectationMap)
275  logging.info('Outputting results in format %s', output_format)
276  stale_str_dict = _ConvertTestExpectationMapToStringDict(stale_dict)
277  semi_stale_str_dict = _ConvertTestExpectationMapToStringDict(semi_stale_dict)
278  active_str_dict = _ConvertTestExpectationMapToStringDict(active_dict)
279  unmatched_results_str_dict = _ConvertUnmatchedResultsToStringDict(
280      unmatched_results)
281  unused_expectations_str_list = _ConvertUnusedExpectationsToStringDict(
282      unused_expectations)
283
284  if output_format == 'print':
285    file_handle = file_handle or sys.stdout
286    if stale_dict:
287      file_handle.write(SECTION_STALE + '\n')
288      RecursivePrintToFile(stale_str_dict, 0, file_handle)
289    if semi_stale_dict:
290      file_handle.write(SECTION_SEMI_STALE + '\n')
291      RecursivePrintToFile(semi_stale_str_dict, 0, file_handle)
292    if active_dict:
293      file_handle.write(SECTION_ACTIVE + '\n')
294      RecursivePrintToFile(active_str_dict, 0, file_handle)
295
296    if unused_expectations_str_list:
297      file_handle.write('\n' + SECTION_UNUSED + '\n')
298      RecursivePrintToFile(unused_expectations_str_list, 0, file_handle)
299    if unmatched_results_str_dict:
300      file_handle.write('\n' + SECTION_UNMATCHED + '\n')
301      RecursivePrintToFile(unmatched_results_str_dict, 0, file_handle)
302
303  elif output_format == 'html':
304    should_close_file = False
305    if not file_handle:
306      should_close_file = True
307      file_handle = tempfile.NamedTemporaryFile(delete=False,
308                                                suffix='.html',
309                                                mode='w')
310
311    file_handle.write(HTML_HEADER)
312    if stale_dict:
313      file_handle.write('<h1>' + SECTION_STALE + '</h1>\n')
314      _RecursiveHtmlToFile(stale_str_dict, file_handle)
315    if semi_stale_dict:
316      file_handle.write('<h1>' + SECTION_SEMI_STALE + '</h1>\n')
317      _RecursiveHtmlToFile(semi_stale_str_dict, file_handle)
318    if active_dict:
319      file_handle.write('<h1>' + SECTION_ACTIVE + '</h1>\n')
320      _RecursiveHtmlToFile(active_str_dict, file_handle)
321
322    if unused_expectations_str_list:
323      file_handle.write('\n<h1>' + SECTION_UNUSED + "</h1>\n")
324      _RecursiveHtmlToFile(unused_expectations_str_list, file_handle)
325    if unmatched_results_str_dict:
326      file_handle.write('\n<h1>' + SECTION_UNMATCHED + '</h1>\n')
327      _RecursiveHtmlToFile(unmatched_results_str_dict, file_handle)
328
329    file_handle.write(HTML_FOOTER)
330    if should_close_file:
331      file_handle.close()
332    print('Results available at file://%s' % file_handle.name)
333  else:
334    raise RuntimeError('Unsupported output format %s' % output_format)
335
336
337def RecursivePrintToFile(element: ElementType, depth: int,
338                         file_handle: IO) -> None:
339  """Recursively prints |element| as text to |file_handle|.
340
341  Args:
342    element: A dict, list, or str/unicode to output.
343    depth: The current depth of the recursion as an int.
344    file_handle: An open file-like object to output to.
345  """
346  if element is None:
347    element = str(element)
348  if isinstance(element, six.string_types):
349    file_handle.write(('  ' * depth) + element + '\n')
350  elif isinstance(element, dict):
351    for k, v in element.items():
352      RecursivePrintToFile(k, depth, file_handle)
353      RecursivePrintToFile(v, depth + 1, file_handle)
354  elif isinstance(element, list):
355    for i in element:
356      RecursivePrintToFile(i, depth, file_handle)
357  else:
358    raise RuntimeError('Given unhandled type %s' % type(element))
359
360
361def _RecursiveHtmlToFile(element: ElementType, file_handle: IO) -> None:
362  """Recursively outputs |element| as HTMl to |file_handle|.
363
364  Iterables will be output as a collapsible section containing any of the
365  iterable's contents.
366
367  Any link-like text will be turned into anchor tags.
368
369  Args:
370    element: A dict, list, or str/unicode to output.
371    file_handle: An open file-like object to output to.
372  """
373  if isinstance(element, six.string_types):
374    file_handle.write('<p>%s</p>\n' % _LinkifyString(element))
375  elif isinstance(element, dict):
376    for k, v in element.items():
377      html_class = 'collapsible_group'
378      # This allows us to later (in JavaScript) recursively highlight sections
379      # that are likely of interest to the user, i.e. whose expectations can be
380      # modified.
381      if k and FULL_PASS in k:
382        html_class = 'highlighted_collapsible_group'
383      file_handle.write('<button type="button" class="%s">%s</button>\n' %
384                        (html_class, k))
385      file_handle.write('<div class="content">\n')
386      _RecursiveHtmlToFile(v, file_handle)
387      file_handle.write('</div>\n')
388  elif isinstance(element, list):
389    for i in element:
390      _RecursiveHtmlToFile(i, file_handle)
391  else:
392    raise RuntimeError('Given unhandled type %s' % type(element))
393
394
395def _LinkifyString(s: str) -> str:
396  """Turns instances of links into anchor tags.
397
398  Args:
399    s: The string to linkify.
400
401  Returns:
402    A copy of |s| with instances of links turned into anchor tags pointing to
403    the link.
404  """
405  for component in s.split():
406    if component.startswith('http'):
407      component = component.strip(',.!')
408      s = s.replace(component, '<a href="%s">%s</a>' % (component, component))
409  return s
410
411
412def _ConvertTestExpectationMapToStringDict(
413    test_expectation_map: data_types.TestExpectationMap
414) -> ExpectationFileStringDict:
415  """Converts |test_expectation_map| to a dict of strings for reporting.
416
417  Args:
418    test_expectation_map: A data_types.TestExpectationMap.
419
420  Returns:
421    A string dictionary representation of |test_expectation_map| in the
422    following format:
423    {
424      expectation_file: {
425        test_name: {
426          expectation_summary: {
427            builder_name: {
428              'Fully passed in the following': [
429                step1,
430              ],
431              'Partially passed in the following': {
432                step2: [
433                  failure_link,
434                ],
435              },
436              'Never passed in the following': [
437                step3,
438              ],
439            }
440          }
441        }
442      }
443    }
444  """
445  assert isinstance(test_expectation_map, data_types.TestExpectationMap)
446  output_dict = {}
447  # This initially looks like a good target for using
448  # data_types.TestExpectationMap's iterators since there are many nested loops.
449  # However, we need to reset state in different loops, and the alternative of
450  # keeping all the state outside the loop and resetting under certain
451  # conditions ends up being less readable than just using nested loops.
452  for expectation_file, expectation_map in test_expectation_map.items():
453    output_dict[expectation_file] = {}
454
455    for expectation, builder_map in expectation_map.items():
456      test_name = expectation.test
457      expectation_str = _FormatExpectation(expectation)
458      output_dict[expectation_file].setdefault(test_name, {})
459      output_dict[expectation_file][test_name][expectation_str] = {}
460
461      for builder_name, step_map in builder_map.items():
462        output_dict[expectation_file][test_name][expectation_str][
463            builder_name] = {}
464        fully_passed = []
465        partially_passed = {}
466        never_passed = []
467
468        for step_name, stats in step_map.items():
469          if stats.NeverNeededExpectation(expectation):
470            fully_passed.append(AddStatsToStr(step_name, stats))
471          elif stats.AlwaysNeededExpectation(expectation):
472            never_passed.append(AddStatsToStr(step_name, stats))
473          else:
474            assert step_name not in partially_passed
475            partially_passed[step_name] = stats
476
477        output_builder_map = output_dict[expectation_file][test_name][
478            expectation_str][builder_name]
479        if fully_passed:
480          output_builder_map[FULL_PASS] = fully_passed
481        if partially_passed:
482          output_builder_map[PARTIAL_PASS] = {}
483          for step_name, stats in partially_passed.items():
484            s = AddStatsToStr(step_name, stats)
485            output_builder_map[PARTIAL_PASS][s] = list(stats.failure_links)
486        if never_passed:
487          output_builder_map[NEVER_PASS] = never_passed
488  return output_dict
489
490
491def _ConvertUnmatchedResultsToStringDict(unmatched_results: UnmatchedResultsType
492                                         ) -> TestToBuilderStringDict:
493  """Converts |unmatched_results| to a dict of strings for reporting.
494
495  Args:
496    unmatched_results: A dict mapping builder names (string) to lists of
497        data_types.Result who did not have a matching expectation.
498
499  Returns:
500    A string dictionary representation of |unmatched_results| in the following
501    format:
502    {
503      test_name: {
504        builder_name: {
505          step_name: [
506            individual_result_string_1,
507            individual_result_string_2,
508            ...
509          ],
510          ...
511        },
512        ...
513      },
514      ...
515    }
516  """
517  output_dict = {}
518  for builder, results in unmatched_results.items():
519    for r in results:
520      builder_map = output_dict.setdefault(r.test, {})
521      step_map = builder_map.setdefault(builder, {})
522      result_str = 'Got "%s" on %s with tags [%s]' % (
523          r.actual_result, data_types.BuildLinkFromBuildId(
524              r.build_id), ' '.join(r.tags))
525      step_map.setdefault(r.step, []).append(result_str)
526  return output_dict
527
528
529def _ConvertUnusedExpectationsToStringDict(
530    unused_expectations: UnusedExpectation) -> Dict[str, List[str]]:
531  """Converts |unused_expectations| to a dict of strings for reporting.
532
533  Args:
534    unused_expectations: A dict mapping expectation file (str) to lists of
535        data_types.Expectation who did not have any matching results.
536
537  Returns:
538    A string dictionary representation of |unused_expectations| in the following
539    format:
540    {
541      expectation_file: [
542        expectation1,
543        expectation2,
544      ],
545    }
546    The expectations are in a format similar to what would be present as a line
547    in an expectation file.
548  """
549  output_dict = {}
550  for expectation_file, expectations in unused_expectations.items():
551    expectation_str_list = []
552    for e in expectations:
553      expectation_str_list.append(e.AsExpectationFileString())
554    output_dict[expectation_file] = expectation_str_list
555  return output_dict
556
557
558def _FormatExpectation(expectation: data_types.Expectation) -> str:
559  return '"%s" expectation on "%s"' % (' '.join(
560      expectation.expected_results), ' '.join(expectation.tags))
561
562
563def AddStatsToStr(s: str, stats: data_types.BuildStats) -> str:
564  return '%s %s' % (s, stats.GetStatsAsString())
565
566
567def OutputAffectedUrls(removed_urls: RemovedUrlsType,
568                       orphaned_urls: Optional[RemovedUrlsType] = None,
569                       bug_file_handle: Optional[IO] = None) -> None:
570  """Outputs URLs of affected expectations for easier consumption by the user.
571
572  Outputs the following:
573
574  1. A string suitable for passing to Chrome via the command line to
575     open all bugs in the browser.
576  2. A string suitable for copying into the CL description to associate the CL
577     with all the affected bugs.
578  3. A string containing any bugs that should be closable since there are no
579     longer any associated expectations.
580
581  Args:
582    removed_urls: A set or list of strings containing bug URLs.
583    orphaned_urls: A subset of |removed_urls| whose bugs no longer have any
584        corresponding expectations.
585    bug_file_handle: An optional open file-like object to write CL description
586        bug information to. If not specified, will print to the terminal.
587  """
588  removed_urls = list(removed_urls)
589  removed_urls.sort()
590  orphaned_urls = orphaned_urls or []
591  orphaned_urls = list(orphaned_urls)
592  orphaned_urls.sort()
593  _OutputAffectedUrls(removed_urls, orphaned_urls)
594  _OutputUrlsForClDescription(removed_urls,
595                              orphaned_urls,
596                              file_handle=bug_file_handle)
597
598
599def _OutputAffectedUrls(affected_urls: List[str],
600                        orphaned_urls: List[str],
601                        file_handle: Optional[IO] = None) -> None:
602  """Outputs |urls| for opening in a browser as affected bugs.
603
604  Args:
605    affected_urls: A list of strings containing URLs to output.
606    orphaned_urls: A list of strings containing URLs to output as closable.
607    file_handle: A file handle to write the string to. Defaults to stdout.
608  """
609  _OutputUrlsForCommandLine(affected_urls, "Affected bugs", file_handle)
610  if orphaned_urls:
611    _OutputUrlsForCommandLine(orphaned_urls, "Closable bugs", file_handle)
612
613
614def _OutputUrlsForCommandLine(urls: List[str],
615                              description: str,
616                              file_handle: Optional[IO] = None) -> None:
617  """Outputs |urls| for opening in a browser.
618
619  The output string is meant to be passed to a browser via the command line in
620  order to open all URLs in that browser, e.g.
621
622  `google-chrome https://crbug.com/1234 https://crbug.com/2345`
623
624  Args:
625    urls: A list of strings containing URLs to output.
626    description: A description of the URLs to be output.
627    file_handle: A file handle to write the string to. Defaults to stdout.
628  """
629  file_handle = file_handle or sys.stdout
630
631  def _StartsWithHttp(url: str) -> bool:
632    return url.startswith('https://') or url.startswith('http://')
633
634  urls = [u if _StartsWithHttp(u) else 'https://%s' % u for u in urls]
635  file_handle.write('%s: %s\n' % (description, ' '.join(urls)))
636
637
638def _OutputUrlsForClDescription(affected_urls: List[str],
639                                orphaned_urls: List[str],
640                                file_handle: Optional[IO] = None) -> None:
641  """Outputs |urls| for use in a CL description.
642
643  Output adheres to the line length recommendation and max number of bugs per
644  line supported in Gerrit.
645
646  Args:
647    affected_urls: A list of strings containing URLs to output.
648    orphaned_urls: A list of strings containing URLs to output as closable.
649    file_handle: A file handle to write the string to. Defaults to stdout.
650  """
651
652  def AddBugTypeToOutputString(urls, prefix):
653    output_str = ''
654    current_line = ''
655    bugs_on_line = 0
656
657    urls = collections.deque(urls)
658
659    while len(urls):
660      current_bug = urls.popleft()
661      current_bug = current_bug.split('crbug.com/', 1)[1]
662      # Handles cases like crbug.com/angleproject/1234.
663      current_bug = current_bug.replace('/', ':')
664
665      # First bug on the line.
666      if not current_line:
667        current_line = '%s %s' % (prefix, current_bug)
668      # Bug or length limit hit for line.
669      elif (
670          len(current_line) + len(current_bug) + 2 > MAX_CHARACTERS_PER_CL_LINE
671          or bugs_on_line >= MAX_BUGS_PER_LINE):
672        output_str += current_line + '\n'
673        bugs_on_line = 0
674        current_line = '%s %s' % (prefix, current_bug)
675      # Can add to current line.
676      else:
677        current_line += ', %s' % current_bug
678
679      bugs_on_line += 1
680
681    output_str += current_line + '\n'
682    return output_str
683
684  file_handle = file_handle or sys.stdout
685  affected_but_not_closable = set(affected_urls) - set(orphaned_urls)
686  affected_but_not_closable = list(affected_but_not_closable)
687  affected_but_not_closable.sort()
688
689  output_str = ''
690  if affected_but_not_closable:
691    output_str += AddBugTypeToOutputString(affected_but_not_closable, 'Bug:')
692  if orphaned_urls:
693    output_str += AddBugTypeToOutputString(orphaned_urls, 'Fixed:')
694
695  file_handle.write('Affected bugs for CL description:\n%s' % output_str)
696