• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2022 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from __future__ import annotations
6
7import dataclasses
8import datetime as dt
9import enum
10import logging
11import os
12import urllib.request
13from typing import (TYPE_CHECKING, Any, Callable, Dict, Iterable, List,
14                    Optional, Tuple, Union)
15from urllib.parse import urlparse
16
17import colorama
18
19from crossbench import compat, helper, plt
20
21if TYPE_CHECKING:
22  from crossbench.browsers.browser import Browser
23  from crossbench.path import LocalPath
24  from crossbench.plt.base import CmdArg, Platform
25  from crossbench.probes.probe import Probe
26
27
28def merge_bool(name: str, left: Optional[bool],
29               right: Optional[bool]) -> Optional[bool]:
30  if left is None:
31    return right
32  if right is None:
33    return left
34  if left != right:
35    raise ValueError(f"Conflicting merge values for {name}: "
36                     f"{left} vs. {right}")
37  return left
38
39
40Number = Union[float, int]
41
42
43def merge_number_max(name: str, left: Optional[Number],
44                     right: Optional[Number]) -> Optional[Number]:
45  del name
46  if left is None:
47    return right
48  if right is None:
49    return left
50  return max(left, right)
51
52
53def merge_number_min(name: str, left: Optional[Number],
54                     right: Optional[Number]) -> Optional[Number]:
55  del name
56  if left is None:
57    return right
58  if right is None:
59    return left
60  return min(left, right)
61
62
63def merge_str_list(name: str, left: Optional[List[str]],
64                   right: Optional[List[str]]) -> Optional[List[str]]:
65  del name
66  if left is None:
67    return right
68  if right is None:
69    return left
70  return left + right
71
72
73@dataclasses.dataclass(frozen=True)
74class HostEnvironmentConfig:
75  IGNORE = None
76
77  disk_min_free_space_gib: Optional[float] = IGNORE
78  power_use_battery: Optional[bool] = IGNORE
79  screen_brightness_percent: Optional[int] = IGNORE
80  cpu_max_usage_percent: Optional[float] = IGNORE
81  cpu_min_relative_speed: Optional[float] = IGNORE
82  system_allow_monitoring: Optional[bool] = IGNORE
83  browser_allow_existing_process: Optional[bool] = IGNORE
84  browser_allow_background: Optional[bool] = IGNORE
85  browser_is_headless: Optional[bool] = IGNORE
86  require_probes: Optional[bool] = IGNORE
87  system_forbidden_process_names: Optional[List[str]] = IGNORE
88  screen_allow_autobrightness: Optional[bool] = IGNORE
89
90  def merge(self, other: HostEnvironmentConfig) -> HostEnvironmentConfig:
91    mergers: Dict[str, Callable[[str, Any, Any], Any]] = {
92        "disk_min_free_space_gib": merge_number_max,
93        "power_use_battery": merge_bool,
94        "screen_brightness_percent": merge_number_max,
95        "cpu_max_usage_percent": merge_number_min,
96        "cpu_min_relative_speed": merge_number_max,
97        "system_allow_monitoring": merge_bool,
98        "browser_allow_existing_process": merge_bool,
99        "browser_allow_background": merge_bool,
100        "browser_is_headless": merge_bool,
101        "require_probes": merge_bool,
102        "system_forbidden_process_names": merge_str_list,
103        "screen_allow_autobrightness": merge_bool,
104    }
105    kwargs = {}
106    for name, merger in mergers.items():
107      self_value = getattr(self, name)
108      other_value = getattr(other, name)
109      kwargs[name] = merger(name, self_value, other_value)
110    return HostEnvironmentConfig(**kwargs)
111
112
113@enum.unique
114class ValidationMode(compat.StrEnumWithHelp):
115  THROW = ("throw", "Strict mode, throw and abort on env issues")
116  PROMPT = ("prompt", "Prompt to accept potential env issues")
117  WARN = ("warn", "Only display a warning for env issue")
118  SKIP = ("skip", "Don't perform any env validation")
119
120
121class ValidationError(Exception):
122  pass
123
124
125_config_default = HostEnvironmentConfig()
126_config_strict = HostEnvironmentConfig(
127    cpu_max_usage_percent=98,
128    cpu_min_relative_speed=1,
129    system_allow_monitoring=False,
130    browser_allow_existing_process=False,
131    require_probes=True,
132)
133_config_battery = _config_strict.merge(
134    HostEnvironmentConfig(power_use_battery=True))
135_config_power = _config_strict.merge(
136    HostEnvironmentConfig(power_use_battery=False))
137_config_catan = _config_strict.merge(
138    HostEnvironmentConfig(
139        screen_brightness_percent=65,
140        system_forbidden_process_names=["terminal", "iterm2"],
141        screen_allow_autobrightness=False))
142
143STALE_RESULT_ICONS = {
144    75: "��",
145    100: "��",
146    125: "��",
147    150: "��",
148    200: "��",
149    250: "��",
150    500: "��",
151    1000: "����‍♂️",
152}
153
154
155class HostEnvironment:
156  """
157  HostEnvironment can check and enforce certain settings on a host
158  where we run benchmarks.
159
160  Modes:
161    skip:     Do not perform any checks
162    warn:     Only warn about mismatching host conditions
163    enforce:  Tries to auto-enforce conditions and warns about others.
164    prompt:   Interactive mode to skip over certain conditions
165    fail:     Fast-fail on mismatch
166  """
167
168  CONFIGS = {
169      "default": _config_default,
170      "strict": _config_strict,
171      "battery": _config_battery,
172      "power": _config_power,
173      "catan": _config_catan,
174  }
175
176  def __init__(self,
177               platform: Platform,
178               out_dir: LocalPath,
179               browsers: Iterable[Browser],
180               probes: Iterable[Probe],
181               repetitions: int,
182               config: Optional[HostEnvironmentConfig] = None,
183               validation_mode: ValidationMode = ValidationMode.THROW):
184    self._wait_until = dt.datetime.now()
185    self._config = config or HostEnvironmentConfig()
186    self._out_dir = out_dir
187    self._browsers = tuple(browsers)
188    self._probes = tuple(probes)
189    self._repetitions = repetitions
190    self._platform = platform
191    self._validation_mode = validation_mode
192
193  @property
194  def platform(self) -> Platform:
195    return self._platform
196
197  @property
198  def repetitions(self) -> int:
199    return self._repetitions
200
201  @property
202  def browsers(self) -> Tuple[Browser, ...]:
203    return self._browsers
204
205  @property
206  def config(self) -> HostEnvironmentConfig:
207    return self._config
208
209  @property
210  def validation_mode(self) -> ValidationMode:
211    return self._validation_mode
212
213  def _add_min_delay(self, seconds: float) -> None:
214    end_time = dt.datetime.now() + dt.timedelta(seconds=seconds)
215    self._wait_until = max(self._wait_until, end_time)
216
217  def _wait_min_time(self) -> None:
218    delta = self._wait_until - dt.datetime.now()
219    if delta > dt.timedelta(0):
220      self._platform.sleep(delta)
221
222  def handle_validation_warning(self, message: str) -> None:
223    message = f"Runner/Host environment requests cannot be fulfilled: {message}"
224    self.handle_warning(message)
225
226  def handle_warning(self,
227                     message: str,
228                     allow_interactive: bool = True) -> None:
229    """Process a warning, depending on the requested mode, this will
230    - throw an error,
231    - log a warning,
232    - prompts for continue [Yn], or
233    - skips (and just debug logs) a warning.
234    If returned True (in the prompt mode) the env validation may continue.
235    """
236    if self._validation_mode == ValidationMode.SKIP:
237      logging.debug("Ignoring %s", message)
238      return
239    if self._validation_mode == ValidationMode.WARN:
240      logging.warning(message)
241      return
242    if self._validation_mode == ValidationMode.PROMPT:
243      if allow_interactive:
244        result = input(f"{colorama.Fore.RED}{message} Continue?"
245                       f"{colorama.Fore.RESET} [Yn]")
246        # Accept <enter> as default input to continue.
247        if result.lower() != "n":
248          return
249    elif self._validation_mode != ValidationMode.THROW:
250      raise ValueError(
251          f"Unknown environment validation mode={self._validation_mode}")
252    raise ValidationError(message)
253
254  def validate_url(self,
255                   url: str,
256                   platform: plt.Platform = plt.PLATFORM) -> bool:
257    if self._validation_mode == ValidationMode.SKIP:
258      return True
259    result = urlparse(url)
260    if result.scheme == "file":
261      return platform.exists(result.path)
262    if platform.is_remote and result.hostname in ("localhost", "127.0.0.1"):
263      # TODO: support remote URL verification, for now we just assume that
264      # checking a live site is ok.
265      return True
266    try:
267      if not all([result.scheme in ["http", "https"], result.netloc]):
268        return False
269      if self._validation_mode != ValidationMode.PROMPT:
270        return True
271      with urllib.request.urlopen(url, timeout=5) as request:
272        if request.getcode() == 200:
273          return True
274        logging.debug("Could not load URL '%s', got %s", url, request)
275    except urllib.error.URLError as e:
276      logging.debug("Could not parse URL '%s' got error: %s", url, e)
277    return False
278
279  def _check_system_monitoring(self) -> None:
280    # TODO(cbruni): refactor to use list_... and disable_system_monitoring api
281    if self._platform.is_macos:
282      self._check_crowdstrike()
283
284  def _check_crowdstrike(self) -> None:
285    """Crowdstrike security monitoring (for googlers go/crowdstrike-falcon) can
286    have quite terrible overhead for each file-access. Disable it to reduce
287    flakiness. """
288    is_disabled = False
289    force_disable = self._config.system_allow_monitoring is False
290    try:
291      # TODO(cbruni): refactor to use list_... and disable_system_monitoring api
292      is_disabled = self._platform.check_system_monitoring(force_disable)
293      if force_disable:
294        # Add cool-down period, crowdstrike caused CPU usage spikes
295        self._add_min_delay(5)
296    except plt.SubprocessError as e:
297      self.handle_validation_warning(
298          "Could not disable go/crowdstrike-falcon monitor which can cause"
299          f" high background CPU usage: {e}")
300      return
301    if not is_disabled:
302      self.handle_validation_warning(
303          "Crowdstrike monitoring is running, "
304          "which can impact startup performance drastically.\n"
305          "Use the following command to disable it manually:\n"
306          "sudo /Applications/Falcon.app/Contents/Resources/falconctl unload\n")
307
308  def _check_disk_space(self) -> None:
309    limit = self._config.disk_min_free_space_gib
310    if limit is HostEnvironmentConfig.IGNORE:
311      return
312    # Check the remaining disk space on the FS where we write the results.
313    usage = self._platform.disk_usage(self._out_dir)
314    free_gib = round(usage.free / 1024 / 1024 / 1024, 2)
315    if free_gib < limit:
316      self.handle_validation_warning(
317          f"Only {free_gib}GiB disk space left, expected at least {limit}GiB.")
318
319  def _check_power(self) -> None:
320    use_battery = self._config.power_use_battery
321    if use_battery is HostEnvironmentConfig.IGNORE:
322      return
323    battery_probes = []
324    # Certain probes may require battery power:
325    for probe in self._probes:
326      if probe.BATTERY_ONLY:
327        battery_probes.append(probe)
328    if not use_battery and battery_probes:
329      probes_str = ",".join(probe.name for probe in battery_probes)
330      self.handle_validation_warning(
331          "Requested battery_power=False, "
332          f"but probes={probes_str} require battery power.")
333    sys_use_battery = self._platform.is_battery_powered
334    if sys_use_battery != use_battery:
335      self.handle_validation_warning(
336          f"Expected battery_power={use_battery}, "
337          f"but the system reported battery_power={sys_use_battery}")
338
339  def _check_cpu_usage(self) -> None:
340    max_cpu_usage = self._config.cpu_max_usage_percent
341    if max_cpu_usage is HostEnvironmentConfig.IGNORE:
342      return
343    cpu_usage_percent = round(100 * self._platform.cpu_usage(), 1)
344    if cpu_usage_percent > max_cpu_usage:
345      self.handle_validation_warning(
346          f"CPU usage={cpu_usage_percent}% is higher than "
347          f"requested max={max_cpu_usage}%.")
348
349  def _check_cpu_temperature(self) -> None:
350    min_relative_speed = self._config.cpu_min_relative_speed
351    if min_relative_speed is HostEnvironmentConfig.IGNORE:
352      return
353    cpu_speed = self._platform.get_relative_cpu_speed()
354    if cpu_speed < min_relative_speed:
355      self.handle_validation_warning(
356          "CPU thermal throttling is active. "
357          f"Relative speed is {cpu_speed}, "
358          f"but expected at least {min_relative_speed}.")
359
360  def _check_forbidden_system_process(self) -> None:
361    # Verify that no terminals are running.
362    # They introduce too much overhead. (As measured with powermetrics)
363    system_forbidden_process_names = self._config.system_forbidden_process_names
364    if system_forbidden_process_names is HostEnvironmentConfig.IGNORE:
365      return
366    process_found = self._platform.process_running(
367        system_forbidden_process_names)
368    if process_found:
369      self.handle_validation_warning(
370          f"Process:{process_found} found."
371          "Make sure not to have a terminal opened. Use SSH.")
372
373  def _check_screen_autobrightness(self) -> None:
374    auto_brightness = self._config.screen_allow_autobrightness
375    if auto_brightness is not False:
376      return
377    if self._platform.check_autobrightness():
378      self.handle_validation_warning(
379          "Auto-brightness was found to be ON. "
380          "Deactivate it in 'System Preferences/Displays'")
381
382  def _check_cpu_power_mode(self) -> bool:
383    # TODO Implement checks for performance mode
384    return True
385
386  def _check_running_binaries(self) -> None:
387    if self._config.browser_allow_existing_process:
388      return
389    grouped_browsers: Dict[plt.Platform, List[Browser]] = helper.group_by(
390        self.browsers, key=lambda browser: browser.platform)
391    for platform, browsers in grouped_browsers.items():
392      self._check_running_binaries_on_platform(platform, browsers)
393
394  def _check_running_binaries_on_platform(
395      self, platform: plt.Platform, platform_browsers: List[Browser]) -> None:
396    browser_binaries: Dict[str, List[Browser]] = helper.group_by(
397        platform_browsers, key=lambda browser: os.fspath(browser.path))
398    own_pid = os.getpid()
399    for proc_info in platform.processes(["cmdline", "exe", "pid", "name"]):
400      if not browser_binaries:
401        return
402      # Skip over this python script which might have the binary path as
403      # part of the command line invocation.
404      if proc_info["pid"] == own_pid:
405        continue
406      cmdline = " ".join(proc_info.get("cmdline") or "")
407      exe = proc_info.get("exe") or proc_info.get("name")
408      if not exe:
409        continue
410      # Windows uses some intermediate processes that contains the binary name
411      # on the command line.
412      if (platform.is_win and
413          proc_info.get("name") in ("cmd.exe", "vpython3.exe")):
414        continue
415      for binary, browsers in list(browser_binaries.items()):
416        # Add a white-space to get less false-positives
417        if f"{binary} " not in cmdline and binary != exe:
418          continue
419        # Use the first in the group
420        browser: Browser = browsers[0]
421        logging.debug("Binary=%s Platform=%s", binary, platform)
422        logging.debug("PS status output:")
423        logging.debug("proc(pid=%s, name=%s, cmd=%s)", proc_info["pid"],
424                      proc_info["name"], cmdline)
425        self.handle_validation_warning(
426            f"{browser.app_name} {browser.version} "
427            f"seems to be already running on {platform}.")
428        # Avoid re-checking the same binary once we've allowed it to be running.
429        del browser_binaries[binary]
430
431  def _check_screen_brightness(self) -> None:
432    brightness = self._config.screen_brightness_percent
433    if brightness is HostEnvironmentConfig.IGNORE:
434      return
435    assert 0 <= brightness <= 100, f"Invalid brightness={brightness}"
436    self._platform.set_main_display_brightness(brightness)
437    current = self._platform.get_main_display_brightness()
438    if current != brightness:
439      self.handle_validation_warning(
440          f"Requested main display brightness={brightness}%, "
441          "but got {brightness}%")
442
443  def _check_headless(self) -> None:
444    # TODO: migrate to full viewport support
445    requested_headless = self._config.browser_is_headless
446    if requested_headless is HostEnvironmentConfig.IGNORE:
447      return
448    if self._platform.is_linux and not requested_headless:
449      # Check that the system can run browsers with a UI.
450      if not self._platform.has_display:
451        self.handle_validation_warning(
452            "Requested browser_is_headless=False, "
453            "but no DISPLAY is available to run with a UI.")
454    # Check that browsers are running in the requested display mode:
455    for browser in self.browsers:
456      if browser.viewport.is_headless != requested_headless:
457        self.handle_validation_warning(
458            f"Requested browser_is_headless={requested_headless},"
459            f"but browser {browser.unique_name} has conflicting "
460            f"headless={browser.viewport.is_headless}.")
461
462  def _check_probes(self) -> None:
463    for probe in self._probes:
464      try:
465        probe.validate_env(self)
466      except Exception as e:
467        raise ValidationError(
468            f"Probe='{probe.NAME}' validation failed: {e}") from e
469    require_probes = self._config.require_probes
470    if require_probes is HostEnvironmentConfig.IGNORE:
471      return
472    if self._config.require_probes and not self._probes:
473      self.handle_validation_warning("No probes specified.")
474
475  def _check_results_dir(self) -> None:
476    results_dir = self._out_dir.parent
477    if not results_dir.exists():
478      return
479    results = [path for path in results_dir.iterdir() if path.is_dir()]
480    num_results = len(results)
481    if num_results < 20:
482      return
483    message = (f"Found {num_results} existing crossbench results. "
484               f"Consider cleaning stale results in '{results_dir}'")
485    for count, icon in reversed(STALE_RESULT_ICONS.items()):
486      if num_results > count:
487        message = f"{icon} {message}"
488        break
489    if num_results > 50:
490      logging.error(message)
491    else:
492      logging.warning(message)
493
494  def _check_macos_terminal(self) -> None:
495    if not self._platform.is_macos or (
496        self._platform.environ.get("TERM_PROGRAM") != "Apple_Terminal"):
497      return
498    any_not_headless = any(
499        not browser.viewport.is_headless for browser in self.browsers)
500    if any_not_headless:
501      self.handle_validation_warning(
502          "Terminal.app does not launch apps in the foreground.\n"
503          "Please use iTerm.app for a better experience.")
504
505  def check_browser_focused(self, browser: Browser) -> None:
506    if (self._config.browser_allow_background or not browser.pid or
507        browser.viewport.is_headless):
508      return
509    info = browser.platform.foreground_process()
510    if not info:
511      return
512    if info["pid"] != browser.pid:
513      self.handle_warning(
514          f"Browser(name={browser.unique_name} pid={browser.pid})) "
515          "was not in the foreground at the end of the benchmark. "
516          "Background apps and tabs can be heavily throttled.",
517          allow_interactive=False)
518
519  def setup(self) -> None:
520    self.validate()
521
522  def validate(self) -> None:
523    logging.info("-" * 80)
524    if self._validation_mode == ValidationMode.SKIP:
525      logging.info("VALIDATE ENVIRONMENT: SKIP")
526      return
527    message = "VALIDATE ENVIRONMENT"
528    if self._validation_mode != ValidationMode.WARN:
529      message += " (--env-validation=warn for soft warnings)"
530    message += ": %s"
531    logging.info(message, self._validation_mode.name)
532    self._check_system_monitoring()
533    self._check_power()
534    self._check_disk_space()
535    self._check_cpu_usage()
536    self._check_cpu_temperature()
537    self._check_cpu_power_mode()
538    self._check_running_binaries()
539    self._check_screen_brightness()
540    self._check_headless()
541    self._check_results_dir()
542    self._check_probes()
543    self._wait_min_time()
544    self._check_forbidden_system_process()
545    self._check_screen_autobrightness()
546    self._check_macos_terminal()
547
548  def check_installed(self,
549                      binaries: Iterable[str],
550                      message: str = "Missing binaries: {}") -> None:
551    assert not isinstance(binaries, str), "Expected iterable of strings."
552    missing_binaries = list(
553        binary for binary in binaries if not self._platform.which(binary))
554    if missing_binaries:
555      self.handle_validation_warning(message.format(missing_binaries))
556
557  def check_sh_success(self,
558                       *args: CmdArg,
559                       message: str = "Could not execute: {}") -> None:
560    assert args, "Missing sh arguments"
561    try:
562      assert self._platform.sh_stdout(*args, quiet=True)
563    except plt.SubprocessError as e:
564      self.handle_validation_warning(message.format(e))
565