• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2024 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from __future__ import annotations
6
7import abc
8import argparse
9import dataclasses
10import functools
11import logging
12import re
13from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence
14
15from crossbench.browsers.attributes import BrowserAttributes
16from crossbench.browsers.browser import Browser
17from crossbench.env import HostEnvironment
18from crossbench.parse import ObjectParser
19from crossbench.probes.json import JsonResultProbe, JsonResultProbeContext
20from crossbench.probes.probe import ProbeConfigParser
21from crossbench.probes.result_location import ResultLocation
22
23if TYPE_CHECKING:
24  from crossbench.runner.actions import Actions
25  from crossbench.runner.run import Run
26  from crossbench.types import Json
27
28
29class ChromeHistogramMetric(abc.ABC):
30  """
31  Stores enough information to log a single metric from a diff between two UMA
32  histograms.
33  """
34
35  def __init__(self, name: str, histogram_name: str) -> None:
36    super().__init__()
37    self._name = name
38    self._histogram_name = histogram_name
39
40  @property
41  def name(self) -> str:
42    return self._name
43
44  @property
45  def histogram_name(self) -> str:
46    return self._histogram_name
47
48  @abc.abstractmethod
49  def compute(self, delta: ChromeHistogramSample,
50              baseline: ChromeHistogramSample) -> float:
51    pass
52
53
54class ChromeHistogramCountMetric(ChromeHistogramMetric):
55
56  def __init__(self, histogram_name: str):
57    super().__init__(f"{histogram_name}_count", histogram_name)
58
59  def compute(self, delta: ChromeHistogramSample,
60              baseline: ChromeHistogramSample) -> float:
61    return delta.diff_count(baseline)
62
63
64class ChromeHistogramMeanMetric(ChromeHistogramMetric):
65
66  def __init__(self, histogram_name: str):
67    super().__init__(f"{histogram_name}_mean", histogram_name)
68
69  def compute(self, delta: ChromeHistogramSample,
70              baseline: ChromeHistogramSample) -> float:
71    return delta.diff_mean(baseline)
72
73
74class ChromeHistogramPercentileMetric(ChromeHistogramMetric):
75
76  def __init__(self, histogram_name: str, percentile: int):
77    super().__init__(f"{histogram_name}_p{percentile}", histogram_name)
78    self._percentile = percentile
79
80  def compute(self, delta: ChromeHistogramSample,
81              baseline: ChromeHistogramSample) -> float:
82    return delta.diff_percentile(baseline, self._percentile)
83
84
85PERCENTILE_METRIC_RE = re.compile(r"^p(\d+)$")
86
87
88def parse_histogram_metrics(value: Any,
89                            name: str = "value"
90                           ) -> Sequence[ChromeHistogramMetric]:
91  result: List[ChromeHistogramMetric] = []
92  d = ObjectParser.dict(value, name)
93  for k, v in d.items():
94    histogram_name = ObjectParser.any_str(k, f"{name} name")
95    metrics = ObjectParser.non_empty_sequence(
96        v, f"{name} {histogram_name} metrics")
97    for x in metrics:
98      metric = ObjectParser.any_str(x)
99      if metric == "count":
100        result.append(ChromeHistogramCountMetric(histogram_name))
101      elif metric == "mean":
102        result.append(ChromeHistogramMeanMetric(histogram_name))
103      else:
104        m = re.match(PERCENTILE_METRIC_RE, metric)
105        if not m:
106          raise argparse.ArgumentTypeError(
107              f"{name} {histogram_name} {metric} is not a valid metric")
108        percentile = int(m[1])
109        if percentile < 0 or percentile > 100:
110          raise argparse.ArgumentTypeError(
111              f"{name} {histogram_name} {metric} is not a valid percentile")
112        result.append(
113            ChromeHistogramPercentileMetric(histogram_name, percentile))
114  return result
115
116
117class ChromeHistogramsProbe(JsonResultProbe):
118  """
119  Probe that collects UMA histogram metrics from Chrome.
120  """
121  NAME = "chrome_histograms"
122  RESULT_LOCATION = ResultLocation.LOCAL
123
124  @classmethod
125  def config_parser(cls) -> ProbeConfigParser:
126    parser = super().config_parser()
127    parser.add_argument(
128        "metrics",
129        required=True,
130        type=parse_histogram_metrics,
131        help=("Required dictionary of Chrome UMA histogram metric names. "
132              "Histograms are recorded before and after a test and any "
133              "differences logged."))
134    return parser
135
136  def __init__(self, metrics: Sequence[ChromeHistogramMetric]) -> None:
137    super().__init__()
138    self._metrics = metrics
139
140  @property
141  def metrics(self) -> Sequence[ChromeHistogramMetric]:
142    return self._metrics
143
144  def validate_browser(self, env: HostEnvironment, browser: Browser) -> None:
145    super().validate_browser(env, browser)
146    self.expect_browser(browser, BrowserAttributes.CHROMIUM_BASED)
147
148  def to_json(self, actions: Actions) -> Json:
149    raise NotImplementedError("should not be called, data comes from context")
150
151  def get_context(self, run: Run) -> ChromeHistogramsProbeContext:
152    return ChromeHistogramsProbeContext(self, run)
153
154
155@dataclasses.dataclass
156class ChromeHistogramBucket:
157  min: int
158  max: int
159  count: int
160
161
162ChromeHistogramBuckets = List[ChromeHistogramBucket]
163
164
165class ChromeHistogramSample:
166  """
167  Stores the contents of one UMA histogram and provides helpers to generate
168  metrics based on the difference between two samples.
169  """
170
171  # Generated by https://source.chromium.org/chromium/chromium/src/+/main:base/metrics/sample_vector.cc;l=520;drc=de573334f8fa97f9a7c99577611302736d2490b6
172  # Example histogram body lines (with whitespace shortened):
173  # "1326111536  -------------------O                              (19 = 63.3%)"
174  # "114   ---O                                              (3 = 3.1%) {92.7%}"
175  # "12  ... "
176  # "1000..."
177  _BUCKET_RE = re.compile(
178      r"^(-?\d+) *(?:(?:-*O "  # Bucket min and ASCII bar
179      r"+\((\d+) = \d+\.\d%\)(?: \{\d+\.\d%\}"  # Count and optional sum %
180      r")?)|(?:\.\.\. ))$"  # Or a "..." line
181  )
182
183  # Generated by https://source.chromium.org/chromium/chromium/src/+/main:base/metrics/sample_vector.cc;l=538;drc=de573334f8fa97f9a7c99577611302736d2490b6
184  # Example histogram header lines:
185  # "Histogram: UKM.InitSequence recorded 1 samples, mean = 1.0 (flags = 0x41)"
186  # "Histogram: WebUI.CreatedForUrl recorded 30 samples (flags = 0x41)"
187  _HEADER_RE = re.compile(r"^Histogram: +.* recorded (\d+) samples"
188                          r"(?:, mean = (-?\d+\.\d+))?"
189                          r" \(flags = (0x[0-9A-Fa-f]+)\)$")
190
191  @classmethod
192  def from_json(cls, histogram_dict: Dict) -> ChromeHistogramSample:
193    name = ObjectParser.any_str(histogram_dict["name"], "histogram name")
194    header = ObjectParser.any_str(histogram_dict["header"], "histogram header")
195    body = ObjectParser.any_str(histogram_dict["body"], "histogram body")
196
197    m = re.match(cls._HEADER_RE, header)
198    if not m:
199      raise argparse.ArgumentTypeError(
200          f"{name} histogram header has invalid data: {header}")
201    count = int(m.group(1))
202    mean = float(m.group(2)) if m.group(2) is not None else None
203    flags = int(m.group(3), 16)
204
205    bucket_counts: Dict[int, int] = {}
206    bucket_maxes: Dict[int, int] = {}
207    prev_min: Optional[int] = None
208    for i, line in enumerate(body.splitlines(), start=1):
209      m = re.match(cls._BUCKET_RE, line)
210      if not m:
211        raise argparse.ArgumentTypeError(
212            f"{name} histogram body line {i} has invalid data: {line}")
213
214      bucket_min = int(m.group(1))
215
216      # Previous bucket's max is this bucket's min.
217      if prev_min is not None:
218        bucket_maxes[prev_min] = bucket_min
219      prev_min = bucket_min
220
221      if bucket_count_str := m.group(2):
222        bucket_count = int(bucket_count_str)
223        if bucket_count > 0:
224          bucket_counts[bucket_min] = bucket_count
225    return ChromeHistogramSample(name, count, mean, flags, bucket_counts,
226                                 bucket_maxes)
227
228  def __init__(self,
229               name: str,
230               count: int = 0,
231               mean: Optional[float] = 0,
232               flags: int = 0,
233               bucket_counts: Optional[Dict[int, int]] = None,
234               bucket_maxes: Optional[Dict[int, int]] = None):
235    self._name = name
236    self._count = count
237    self._mean = mean
238    self._flags = flags
239    self._bucket_counts = bucket_counts or {}
240    self._bucket_maxes = bucket_maxes or {}
241    bucket_sum = sum(self._bucket_counts.values())
242    if count != bucket_sum:
243      raise ValueError(f"Histogram {name} has {count} total samples, "
244                       f"but buckets add to {bucket_sum}")
245
246  @property
247  def mean(self) -> Optional[float]:
248    return self._mean
249
250  @property
251  def count(self) -> int:
252    return self._count
253
254  def bucket_max(self, bucket_min: int) -> Optional[int]:
255    return self._bucket_maxes.get(bucket_min)
256
257  def bucket_count(self, bucket_min: int) -> int:
258    return self._bucket_counts.get(bucket_min, 0)
259
260  def diff_buckets(self,
261                   baseline: ChromeHistogramSample) -> ChromeHistogramBuckets:
262    buckets: ChromeHistogramBuckets = []
263    for bucket_min, bucket_count in self._bucket_counts.items():
264      bucket_count = bucket_count - baseline.bucket_count(bucket_min)
265      bucket_max: Optional[int] = self._bucket_maxes.get(bucket_min)
266      buckets.append(
267          ChromeHistogramBucket(bucket_min, bucket_max, bucket_count))
268    return buckets
269
270  def diff_percentile(self, baseline: ChromeHistogramSample,
271                      percentile: int) -> float:
272    if percentile < 0 or percentile > 100:
273      raise ValueError(f"{percentile} is not a valid percentile")
274    buckets = self.diff_buckets(baseline)
275    count = functools.reduce(lambda s, b: b.count + s, buckets, 0)
276    if count == 0:
277      raise ValueError(
278          f"{self._name} can not compute percentile without any samples")
279    target = count * percentile / 100
280    for bucket in buckets:
281      if target <= bucket.count:
282        if bucket.max is None:
283          return bucket.min
284        # Assume all samples are evenly distributed within the bucket.
285        # NB: 0 <= target <= bucket_count
286        t = target / (bucket.count + 1)
287        return bucket.min * (1 - t) + bucket.max * t
288      target -= bucket.count
289    raise ValueError("overflowed histogram buckets looking for percentile")
290
291  def diff_mean(self, baseline: ChromeHistogramSample) -> float:
292    count = self._count - baseline.count
293    if count <= 0:
294      raise ValueError(f"{self._name} can not compute mean without any samples")
295    if self._mean is None or baseline.mean is None:
296      raise ValueError(
297          f"{self._name} has no mean reported, is it an enum histogram?")
298
299    return (self._mean * self._count - baseline.mean * baseline.count) / count
300
301  def diff_count(self, baseline: ChromeHistogramSample) -> int:
302    return self._count - baseline.count
303
304  @property
305  def name(self) -> str:
306    return self._name
307
308
309class ChromeHistogramsProbeContext(JsonResultProbeContext[ChromeHistogramsProbe]
310                                  ):
311
312  # JS code that overrides the chrome.send response handler and requests
313  # histograms.
314  HISTOGRAM_SEND = """
315function webUIResponse(id, isSuccess, response) {
316  if (id === "crossbench_histograms_1") {
317    window.crossbench_histograms = response;
318  }
319}
320window.cr.webUIResponse = webUIResponse;
321chrome.send("requestHistograms", ["crossbench_histograms_1", "", true]);
322"""
323
324  # JS code that checks if there is a histogram response.
325  HISTOGRAM_WAIT = "return !!window.crossbench_histograms"
326
327  # JS code that returns the histograms response.
328  HISTOGRAM_DATA = "return window.crossbench_histograms"
329
330  def __init__(self, probe: ChromeHistogramsProbe, run: Run) -> None:
331    super().__init__(probe, run)
332    self._baseline: Optional[Dict[str, ChromeHistogramSample]] = None
333    self._delta: Optional[Dict[str, ChromeHistogramSample]] = None
334
335  def dump_histograms(self, name: str) -> Dict[str, ChromeHistogramSample]:
336    with self.run.actions(
337        f"Probe({self.probe.name}) dump histograms {name}") as actions:
338      actions.show_url("chrome://histograms")
339      actions.js(self.HISTOGRAM_SEND)
340      actions.wait_js_condition(self.HISTOGRAM_WAIT, 0.1, 10.0)
341      data = actions.js(self.HISTOGRAM_DATA)
342      histogram_list = ObjectParser.sequence(data)
343      histograms: Dict[str, ChromeHistogramSample] = {}
344      for histogram_dict in histogram_list:
345        histogram = ChromeHistogramSample.from_json(
346            ObjectParser.dict(histogram_dict))
347        histograms[histogram.name] = histogram
348      return histograms
349
350  def start(self) -> None:
351    self._baseline = self.dump_histograms("start")
352    super().start()
353
354  def stop(self) -> None:
355    self._delta = self.dump_histograms("stop")
356    super().stop()
357
358  def to_json(self, actions: Actions) -> Json:
359    del actions
360    assert self._baseline, "Did not extract start histograms"
361    assert self._delta, "Did not extract end histograms"
362    json = {}
363    for metric in self.probe.metrics:
364      baseline = self._baseline.get(
365          metric.histogram_name, ChromeHistogramSample(metric.histogram_name))
366      delta = self._delta.get(metric.histogram_name,
367                              ChromeHistogramSample(metric.histogram_name))
368      try:
369        json[metric.name] = metric.compute(delta, baseline)
370      except Exception as e:  # pylint: disable=broad-exception-caught
371        logging.warning("Failed to log metric %s: %s", metric.name, e)
372    return json
373