• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2022 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5from __future__ import annotations
6
7import atexit
8import logging
9import os
10import signal
11import subprocess
12import tempfile
13from typing import TYPE_CHECKING, Dict, List, Optional, TextIO, Tuple, Union
14
15from crossbench import helper
16from crossbench.probes.probe import (Probe, ProbeConfigParser, ProbeContext,
17                                     ProbeMissingDataError)
18from crossbench.probes.result_location import ResultLocation
19from crossbench.probes.results import (EmptyProbeResult, LocalProbeResult,
20                                       ProbeResult)
21
22if TYPE_CHECKING:
23  from crossbench.browsers.browser import Viewport
24  from crossbench.env import HostEnvironment
25  from crossbench.path import LocalPath
26  from crossbench.runner.groups.browsers import BrowsersRunGroup
27  from crossbench.runner.groups.repetitions import RepetitionsRunGroup
28  from crossbench.runner.run import Run
29  from crossbench.stories.story import Story
30
31
32class VideoProbe(Probe):
33  """
34  General-purpose Probe that collects screen-recordings.
35
36  It can also produce a timestrip png and creates merged versions of these files
37  for visually comparing various browsers / variants / cb.stories
38  """
39  NAME = "video"
40  RESULT_LOCATION = ResultLocation.BROWSER
41  VIDEO_QUALITY = ["-vcodec", "libx264", "-crf", "20"]
42  IMAGE_FORMAT = "png"
43  TIMESTRIP_FILE_SUFFIX = f".timestrip.{IMAGE_FORMAT}"
44  FRAMERATE = 60
45
46  @classmethod
47  def config_parser(cls) -> ProbeConfigParser:
48    parser = super().config_parser()
49    parser.add_argument(
50        "generate_timestrip",
51        aliases=("timestrip",),
52        type=bool,
53        default=True,
54        help="Produce a timestrip png")
55    parser.add_argument(
56        "merge_runs",
57        type=bool,
58        default=True,
59        help="Merge videos from multiple runs")
60    return parser
61
62  def __init__(self,
63               generate_timestrip: bool = True,
64               merge_runs: bool = True) -> None:
65    super().__init__()
66    self._duration = None
67    self._generate_timestrip = generate_timestrip
68    self._merge_runs = merge_runs
69
70  @property
71  def result_path_name(self) -> str:
72    return f"{self.name}.mp4"
73
74  @property
75  def generate_timestrip(self) -> bool:
76    return self._generate_timestrip
77
78  @property
79  def merge_runs(self) -> bool:
80    return self._merge_runs
81
82  def validate_env(self, env: HostEnvironment) -> None:
83    super().validate_env(env)
84    if env.repetitions > 10:
85      env.handle_warning(
86          f"Probe={self.NAME} might not be able to merge so many "
87          f"repetitions={env.repetitions}.")
88    env.check_installed(
89        binaries=("ffmpeg",), message="Missing binaries for video probe: {}")
90    # Check that ffmpeg can be executed
91    env.check_sh_success("ffmpeg", "-version")
92    env.check_installed(
93        binaries=("montage",),
94        message="Missing 'montage' binary, please install imagemagick.")
95    # Check that montage can be executed
96    env.check_sh_success("montage", "--version")
97    self._pre_check_viewport_size(env)
98
99  def _pre_check_viewport_size(self, env: HostEnvironment) -> None:
100    first_viewport: Viewport = env.browsers[0].viewport
101    for browser in env.browsers:
102      viewport: Viewport = browser.viewport
103      if viewport.is_headless:
104        env.handle_warning(
105            f"Cannot record video for headless browser: {browser}")
106      # TODO: support fullscreen / maximised
107      if not viewport.has_size:
108        env.handle_warning(
109            "Can only record video for browsers with explicit viewport sizes, "
110            f"but got {viewport} for {browser}.")
111      if viewport.x < 10 or viewport.y < 50:
112        env.handle_warning(
113            f"Viewport for '{browser}' might include toolbar: {viewport}")
114      if viewport != first_viewport:
115        env.handle_warning(
116            "Video recording requires same viewport size for all browsers.\n"
117            f"Viewport size for {browser} is {viewport}, "
118            f"which differs from first viewport {first_viewport}. ")
119
120  def get_context(self, run: Run) -> VideoProbeContext:
121    return VideoProbeContext(self, run)
122
123  def merge_repetitions(self, group: RepetitionsRunGroup) -> ProbeResult:
124    if not self.merge_runs:
125      return LocalProbeResult()
126    runs = tuple(group.runs)
127    if len(runs) == 1:
128      # In the simple case just copy the files
129      run_files = runs[0].results[self].file_list
130      group_files = [group.path / f.name for f in run_files]
131      for src, dest in zip(run_files, group_files):
132        self.host_platform.copy(src, dest)
133      return LocalProbeResult(file=group_files)
134
135    video_file = group.get_local_probe_result_path(self)
136    group_files = [video_file]
137    logging.info("VIDEO merge page repetitions")
138    browser = group.browser
139    video_file_inputs: List[Union[str, LocalPath]] = []
140    for run in runs:
141      video_file_inputs += ["-i", run.results[self].file_list[0]]
142    draw_text = ("fontfile='/Library/Fonts/Arial.ttf':"
143                 f"text='{browser.app_name} {browser.label}':"
144                 "fontsize=h/15:"
145                 "y=h-line_h-10:x=10:"
146                 "box=1:boxborderw=20:boxcolor=white")
147    self.host_platform.sh(
148        "ffmpeg", "-hide_banner", \
149        *video_file_inputs, \
150        "-filter_complex",
151        f"hstack=inputs={len(runs)},"
152        f"drawtext={draw_text},"
153        "scale=3000:-2", *self.VIDEO_QUALITY, video_file)
154
155    if self._generate_timestrip:
156      timeline_strip_file = video_file.with_suffix(self.TIMESTRIP_FILE_SUFFIX)
157      logging.info("TIMESTRIP merge page repetitions")
158      timeline_strips = (run.results[self].file_list[1] for run in runs)
159      self.host_platform.sh("montage", *timeline_strips, "-tile", "1x",
160                            "-gravity", "NorthWest", "-geometry", "x100",
161                            timeline_strip_file)
162      group_files.append(timeline_strip_file)
163
164    return LocalProbeResult(file=group_files)
165
166  def merge_browsers(self, group: BrowsersRunGroup) -> ProbeResult:
167    """Merge story videos from multiple browser/configurations"""
168    if not self.merge_runs:
169      return LocalProbeResult()
170    groups = list(group.repetitions_groups)
171    if len(groups) <= 1:
172      return EmptyProbeResult()
173    grouped: Dict[Story, List[RepetitionsRunGroup]] = helper.group_by(
174        groups, key=lambda repetitions_group: repetitions_group.story)
175
176    result_dir = group.get_local_probe_result_path(self)
177    result_dir = result_dir / result_dir.stem
178    result_dir.mkdir(parents=True)
179    return LocalProbeResult(
180        file=(self._merge_stories_for_browser(result_dir, story,
181                                              repetitions_groups)
182              for story, repetitions_groups in grouped.items()))
183
184  def _merge_stories_for_browser(
185      self, result_dir: LocalPath, story: Story,
186      repetitions_groups: List[RepetitionsRunGroup]) -> LocalPath:
187    story = repetitions_groups[0].story
188    result_path = result_dir / f"{story.name}_combined.mp4"
189
190    if len(repetitions_groups) == 1:
191      # In the simple case just copy files
192      input_file = repetitions_groups[0].results[self].file_list[0]
193      self.host_platform.copy(input_file, result_path)
194      return result_path
195
196    input_files: List[str] = []
197    for repetitions_group in repetitions_groups:
198      result_files = repetitions_group.results[self].file_list
199      input_files += ["-i", os.fspath(result_files[0])]
200    try:
201      self.host_platform.sh("ffmpeg", "-hide_banner", *input_files,
202                            "-filter_complex",
203                            f"vstack=inputs={len(repetitions_groups)}",
204                            *self.VIDEO_QUALITY, result_path)
205    except Exception as e:
206      logging.error("Merging multiple browser video failed. "
207                    "Different screen orientations are not supported yet.")
208      logging.debug("Browser video merging failed: %e", e)
209      raise e
210    return result_path
211
212
213class VideoProbeContext(ProbeContext[VideoProbe]):
214  IMAGE_FORMAT = "png"
215  FFMPEG_TIMELINE_TEXT = (
216      "drawtext="
217      "fontfile=/Library/Fonts/Arial.ttf:"
218      "text='%{eif\\:t\\:d}.%{eif\\:t*100-floor(t)*100\\:d}s':"
219      "fontsize=h/16:"
220      "y=h-line_h-5:x=5:"
221      "box=1:boxborderw=15:boxcolor=white")
222
223  def __init__(self, probe: VideoProbe, run: Run) -> None:
224    super().__init__(probe, run)
225    self._record_process: Optional[subprocess.Popen] = None
226    self._recorder_log_file: Optional[TextIO] = None
227
228  def start(self) -> None:
229    browser = self.run.browser
230    cmd = self._record_cmd(browser.viewport)
231    logging.debug("Screen recorder cmd: %s", cmd)
232    if self.browser_platform.is_remote:
233      self._recorder_log_file = None
234    else:
235      self._recorder_log_file = self.local_result_path.with_suffix(
236          ".recorder.log").open(
237              "w", encoding="utf-8")
238    self._record_process = self.browser_platform.popen(
239        *cmd,
240        stdin=subprocess.PIPE,
241        stderr=subprocess.STDOUT,
242        stdout=self._recorder_log_file)
243    if self._record_process.poll():
244      raise ValueError("Could not start screen recorder")
245    atexit.register(self.stop_process)
246    # TODO: Add common start-story-delay on runner for these cases.
247    self.host_platform.sleep(1)
248
249  def _record_cmd(self, viewport: Viewport) -> Tuple[str, ...]:
250    if self.browser_platform.is_linux:
251      env_display = os.environ.get("DISPLAY", ":0.0")
252      return ("ffmpeg", "-hide_banner", "-video_size",
253              f"{viewport.width}x{viewport.height}", "-f", "x11grab",
254              "-framerate", str(self.probe.FRAMERATE), "-i",
255              f"{env_display}+{viewport.x},{viewport.y}", str(self.result_path))
256    if self.browser_platform.is_macos:
257      return ("/usr/sbin/screencapture", "-v",
258              f"-R{viewport.x},{viewport.y},{viewport.width},{viewport.height}",
259              str(self.result_path))
260    if self.browser_platform.is_android:
261      return ("screenrecord", str(self.result_path))
262    raise ValueError("Invalid platform")
263
264  def stop(self) -> None:
265    assert self._record_process, "screencapture stopped early."
266    if self.browser_platform.is_macos:
267      assert not self._record_process.poll(), (
268          "screencapture stopped early. "
269          "Please ensure that the parent application has screen recording "
270          "permissions")
271      # The mac screencapture stops on the first (arbitrary) input.
272      self._record_process.communicate(input=b"stop")
273    elif self.browser_platform.is_android:
274      self._record_process.send_signal(signal.SIGINT)
275    else:
276      self._record_process.terminate()
277
278  def teardown(self) -> ProbeResult:
279    assert self._record_process, "Screen recorder stopped early."
280    if self._recorder_log_file:
281      self._recorder_log_file.close()
282    self.stop_process()
283    if not self.browser_platform.is_file(self.result_path):
284      raise ProbeMissingDataError(
285          f"No screen recording video found at: {self.result_path}")
286    # Copy files
287    browser_result = self.browser_result(file=(self.result_path,))
288    self._default_result_path = browser_result.file
289    assert self.host_platform.exists(self.result_path)
290
291    if not self.probe.generate_timestrip:
292      return LocalProbeResult(file=(self.local_result_path,))
293
294    with tempfile.TemporaryDirectory() as tmp_dir:
295      self._convert_to_constant_framerate()
296      timestrip_file = self._create_time_strip(
297          self.host_platform.local_path(tmp_dir))
298    return LocalProbeResult(file=(self.local_result_path, timestrip_file))
299
300  def stop_process(self) -> None:
301    if self._record_process:
302      helper.wait_and_kill(self._record_process, timeout=5)
303      self._record_process = None
304
305  def _convert_to_constant_framerate(self):
306    # On some platforms (android for certain) we get VFR videos which confuse
307    # the next video extraction / conversion steps.
308    vrf_video_result = (
309        self.local_result_path.parent / f"vfr_{self.result_path.name}")
310    self.local_result_path.rename(vrf_video_result)
311    self.host_platform.sh(
312        "ffmpeg", "-hide_banner", \
313        "-fflags", "+igndts", \
314        "-i", vrf_video_result, \
315        "-filter:v", "fps=60", \
316        "-fps_mode:v", "cfr",
317        # Use the decoder timebase.
318        "-copytb", "0", \
319        *self.probe.VIDEO_QUALITY,
320        self.result_path
321    )
322    if not self.local_result_path.exists() or self.local_result_path.stat(
323    ).st_size == 0:
324      vrf_video_result.rename(self.result_path)
325      logging.error("Could not generate constant FPS video: %s",
326                    self.result_path)
327    else:
328      vrf_video_result.unlink()
329
330  def _create_time_strip(self, tmpdir: LocalPath) -> LocalPath:
331    logging.info("TIMESTRIP")
332    progress_dir = tmpdir / "progress"
333    progress_dir.mkdir(parents=True, exist_ok=True)
334    timeline_dir = tmpdir / "timeline"
335    timeline_dir.mkdir(exist_ok=True)
336    # Try detect scene changes / steps
337    self.host_platform.sh(
338        "ffmpeg", "-hide_banner", "-i", self.result_path, \
339        "-filter_complex", "scale=3000:-2,"
340        "select='gt(scene\\,0.011)'," + self.FFMPEG_TIMELINE_TEXT, \
341        "-fps_mode", "cfr", \
342        "-framerate", str(self.probe.FRAMERATE), \
343          f"{progress_dir}/%02d.{self.IMAGE_FORMAT}")
344    # Extract at regular intervals of 100ms, assuming 60fps input
345    every_nth_frame = self.probe.FRAMERATE / 20
346    safe_duration = 10
347    safe_duration = 2
348    self.host_platform.sh(
349        "ffmpeg", "-hide_banner", \
350        "-i", self.result_path, \
351        "-filter_complex",
352        f"trim=duration={safe_duration},"
353        "scale=3000:-2,"
354        f"select=not(mod(n\\,{every_nth_frame}))," + self.FFMPEG_TIMELINE_TEXT,
355        f"{timeline_dir}/%02d.{self.IMAGE_FORMAT}")
356
357    timeline_strip_file = self.local_result_path.with_suffix(
358        self.probe.TIMESTRIP_FILE_SUFFIX)
359    self.runner.platform.sh("montage", f"{timeline_dir}/*.{self.IMAGE_FORMAT}",
360                            "-tile", "x1", "-gravity", "NorthWest", "-geometry",
361                            "x100", timeline_strip_file)
362    return timeline_strip_file
363