• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2023 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from __future__ import annotations
6
7import json
8import logging
9from typing import TYPE_CHECKING, Iterable, Optional
10
11from crossbench.probes import probe
12from crossbench.probes.json import JsonResultProbe, JsonResultProbeContext
13from crossbench.probes.metric import MetricsMerger
14from crossbench.probes.results import (EmptyProbeResult, ProbeResult,
15                                       ProbeResultDict)
16
17if TYPE_CHECKING:
18  from crossbench.runner.actions import Actions
19  from crossbench.runner.groups.browsers import BrowsersRunGroup
20  from crossbench.runner.groups.repetitions import RepetitionsRunGroup
21  from crossbench.runner.groups.stories import StoriesRunGroup
22  from crossbench.runner.run import Run
23  from crossbench.types import Json, JsonDict, JsonList
24
25
26class InternalProbe(probe.Probe):
27  IS_GENERAL_PURPOSE = False
28
29  @property
30  def is_internal(self) -> bool:
31    return True
32
33
34class InternalJsonResultProbe(JsonResultProbe, InternalProbe):
35  IS_GENERAL_PURPOSE = False
36  FLATTEN = False
37
38  def get_context(self, run: Run) -> InternalJsonResultProbeContext:
39    return InternalJsonResultProbeContext(self, run)
40
41
42class InternalJsonResultProbeContext(
43    JsonResultProbeContext[InternalJsonResultProbe]):
44
45  def stop(self) -> None:
46    # Only extract data in the late teardown phase.
47    pass
48
49  def teardown(self) -> ProbeResult:
50    self._json_data = self.extract_json(self.run)  # pylint: disable=no-member
51    return super().teardown()
52
53
54class LogProbe(InternalProbe):
55  """
56  Runner-internal meta-probe: Collects the python logging data from the runner
57  itself.
58  """
59  NAME = "cb.log"
60
61  def get_context(self, run: Run) -> LogProbeContext:
62    return LogProbeContext(self, run)
63
64
65class LogProbeContext(probe.ProbeContext[LogProbe]):
66
67  def __init__(self, probe_instance: LogProbe, run: Run) -> None:
68    super().__init__(probe_instance, run)
69    self._log_handler: Optional[logging.Handler] = None
70
71  def setup(self) -> None:
72    log_formatter = logging.Formatter(
73        "%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] "
74        "[%(name)s]  %(message)s")
75    self._log_handler = logging.FileHandler(self.result_path)
76    self._log_handler.setFormatter(log_formatter)
77    self._log_handler.setLevel(logging.DEBUG)
78    logging.getLogger().addHandler(self._log_handler)
79
80  def start(self) -> None:
81    pass
82
83  def stop(self) -> None:
84    pass
85
86  def teardown(self) -> ProbeResult:
87    assert self._log_handler
88    logging.getLogger().removeHandler(self._log_handler)
89    self._log_handler = None
90    return ProbeResult(file=(self.local_result_path,))
91
92
93class SystemDetailsProbe(InternalJsonResultProbe):
94  """
95  Runner-internal meta-probe: Collects the browser's system/platform details.
96  """
97  NAME = "cb.system.details"
98
99  def to_json(self, actions: Actions) -> Json:
100    return actions.run.browser_platform.system_details()
101
102  def merge_repetitions(self, group: RepetitionsRunGroup) -> ProbeResult:
103    return EmptyProbeResult()
104
105
106class ErrorsProbe(InternalJsonResultProbe):
107  """
108  Runner-internal meta-probe: Collects all errors from running stories and/or
109  from merging probe data.
110  """
111  NAME = "cb.errors"
112
113  def to_json(self, actions: Actions) -> Json:
114    return actions.run.exceptions.to_json()
115
116  def merge_repetitions(self, group: RepetitionsRunGroup) -> ProbeResult:
117    return self._merge_group(group, (run.results for run in group.runs))
118
119  def merge_stories(self, group: StoriesRunGroup) -> ProbeResult:
120    return self._merge_group(
121        group, (rep_group.results for rep_group in group.repetitions_groups))
122
123  def merge_browsers(self, group: BrowsersRunGroup) -> ProbeResult:
124    return self._merge_group(
125        group, (story_group.results for story_group in group.story_groups))
126
127  def _merge_group(self, group,
128                   results_iter: Iterable[ProbeResultDict]) -> ProbeResult:
129    merged_errors = []
130
131    for results in results_iter:
132      result = results[self]
133      if not result:
134        continue
135      source_file = result.json
136      assert source_file.is_file()
137      with source_file.open(encoding="utf-8") as f:
138        repetition_errors = json.load(f)
139        assert isinstance(repetition_errors, list)
140        merged_errors.extend(repetition_errors)
141
142    group_errors = group.exceptions.to_json()
143    assert isinstance(group_errors, list)
144    merged_errors.extend(group_errors)
145
146    if not merged_errors:
147      return EmptyProbeResult()
148
149    return self.write_group_result(group, merged_errors)
150
151
152class DurationsProbe(InternalJsonResultProbe):
153  """
154  Runner-internal meta-probe: Collects timing information for various components
155  of the runner (and the times spent in individual stories as well).
156  """
157  NAME = "cb.durations"
158
159  def to_json(self, actions: Actions) -> Json:
160    return actions.run.durations.to_json()
161
162  def merge_stories(self, group: StoriesRunGroup) -> ProbeResult:
163    merged = MetricsMerger.merge_json_list(
164        (repetitions_group.results[self].json
165         for repetitions_group in group.repetitions_groups),
166        merge_duplicate_paths=True)
167    return self.write_group_result(group, merged, csv_formatter=None)
168
169  def merge_browsers(self, group: BrowsersRunGroup) -> ProbeResult:
170    merged = MetricsMerger.merge_json_list(
171        (story_group.results[self].json for story_group in group.story_groups),
172        merge_duplicate_paths=True)
173    return self.write_group_result(group, merged, csv_formatter=None)
174
175
176class ResultsSummaryProbe(InternalJsonResultProbe):
177  """
178  Runner-internal meta-probe: Collects a summary results.json with all the Run
179  information, including all paths to the results of all attached Probes.
180  """
181  NAME = "cb.results"
182  # Given that this is  a meta-Probe that summarizes the data from other
183  # probes we exclude it from the default results lists.
184  PRODUCES_DATA = False
185
186  @property
187  def is_attached(self) -> bool:
188    return True
189
190  def to_json(self, actions: Actions) -> JsonDict:
191    return actions.run.details_json()
192
193  def merge_repetitions(self, group: RepetitionsRunGroup) -> ProbeResult:
194    repetitions: JsonList = []
195    browser: Optional[JsonDict] = None
196
197    for run in group.runs:
198      source_file = run.results[self].json
199      assert source_file.is_file()
200      with source_file.open(encoding="utf-8") as f:
201        repetition_data = json.load(f)
202      if browser is None:
203        browser = repetition_data["browser"]
204        del browser["log"]
205      repetitions.append({
206          "cwd": repetition_data["cwd"],
207          "probes": repetition_data["probes"],
208          "success": repetition_data["success"],
209          "errors": repetition_data["errors"],
210      })
211
212    merged_data: JsonDict = {
213        "cwd": str(group.path),
214        "story": group.story.details_json(),
215        "browser": browser,
216        "group": group.info,
217        "repetitions": repetitions,
218        "probes": group.results.to_json(),
219        "success": group.is_success,
220        "errors": group.exceptions.error_messages(),
221    }
222    return self.write_group_result(group, merged_data, csv_formatter=None)
223
224  def merge_stories(self, group: StoriesRunGroup) -> ProbeResult:
225    stories: JsonDict = {}
226    browser = None
227
228    for repetitions_group in group.repetitions_groups:
229      source_file = repetitions_group.results[self].json
230      assert source_file.is_file()
231      with source_file.open(encoding="utf-8") as f:
232        merged_story_data = json.load(f)
233      if browser is None:
234        browser = merged_story_data["browser"]
235      story_info = merged_story_data["story"]
236      stories[story_info["name"]] = {
237          "cwd": merged_story_data["cwd"],
238          "duration": story_info["duration"],
239          "probes": merged_story_data["probes"],
240          "errors": merged_story_data["errors"],
241      }
242
243    merged_data: JsonDict = {
244        "cwd": str(group.path),
245        "browser": browser,
246        "stories": stories,
247        "group": group.info,
248        "probes": group.results.to_json(),
249        "success": group.is_success,
250        "errors": group.exceptions.error_messages(),
251    }
252    return self.write_group_result(group, merged_data, csv_formatter=None)
253
254  def merge_browsers(self, group: BrowsersRunGroup) -> ProbeResult:
255    browsers: JsonDict = {}
256    for story_group in group.story_groups:
257      source_file = story_group.results[self].json
258      assert source_file.is_file()
259      with source_file.open(encoding="utf-8") as f:
260        merged_browser_data = json.load(f)
261      browser_info = merged_browser_data["browser"]
262      browsers[browser_info["unique_name"]] = {
263          "cwd": merged_browser_data["cwd"],
264          "probes": merged_browser_data["probes"],
265          "errors": merged_browser_data["errors"],
266      }
267
268    merged_data: JsonDict = {
269        "cwd": str(group.path),
270        "browsers": browsers,
271        "probes": group.results.to_json(),
272        "success": group.is_success,
273        "errors": group.exceptions.error_messages(),
274    }
275    return self.write_group_result(group, merged_data, csv_formatter=None)
276