1# Copyright 2024 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5from __future__ import annotations 6 7import dataclasses 8import datetime as dt 9import shlex 10import subprocess 11from typing import TYPE_CHECKING 12 13import crossbench.path as pth 14from crossbench.action_runner.action import all as i_action 15from crossbench.action_runner.basic_action_runner import BasicActionRunner 16from crossbench.action_runner.display_rectangle import DisplayRectangle 17from crossbench.action_runner.element_not_found_error import \ 18 ElementNotFoundError 19from crossbench.benchmarks.loading.point import Point 20from crossbench.parse import NumberParser 21 22if TYPE_CHECKING: 23 from typing import Optional, Tuple, Type 24 25 from crossbench.runner.actions import Actions 26 from crossbench.runner.run import Run 27 28SCRIPTS_DIR = pth.LocalPath(__file__).parent / "chromeos_scripts" 29 30 31class ChromeOSViewportInfo: 32 33 def __init__(self, device_pixel_ratio, window_outer_width, window_inner_width, 34 window_inner_height, screen_width, screen_height, 35 screen_avail_width, screen_avail_height, window_offset_x, 36 window_offset_y, 37 element_rect: Optional[DisplayRectangle]) -> None: 38 39 # The actual screen width and height in pixels. 40 # Corrects for any zoom/scaling factors. 41 # 80 is a common factor of most display pixel widths, so use it as a common 42 # factor to ensure integer division. 43 screen_width_pixels = round( 44 screen_width * device_pixel_ratio / 45 (window_outer_width / window_inner_width) / 80) * 80 46 47 # 60 is a common factor of most display pixel heights, so use it as a common 48 # factor to ensure integer division. 49 screen_height_pixels = round( 50 screen_height * device_pixel_ratio / 51 (window_outer_width / window_inner_width) / 60) * 60 52 53 self._actual_pixel_ratio = screen_width_pixels / screen_avail_width 54 55 screen_avail_width = round(self.css_to_native_distance(screen_avail_width)) 56 screen_avail_height = round( 57 self.css_to_native_distance(screen_avail_height)) 58 59 window_inner_width = round(self.css_to_native_distance(window_inner_width)) 60 window_inner_height = round( 61 self.css_to_native_distance(window_inner_height)) 62 63 window_offset_x = round(self.css_to_native_distance(window_offset_x)) 64 65 window_offset_y = round(self.css_to_native_distance(window_offset_y)) 66 window_offset_y += (screen_avail_height - window_inner_height) 67 68 visible_width = min(window_inner_width, 69 screen_avail_width - window_offset_x) 70 visible_height = min(window_inner_height, 71 round(screen_avail_height - window_offset_y)) 72 73 self._native_screen = DisplayRectangle( 74 Point(0, 0), screen_width_pixels, screen_height_pixels) 75 76 self._browser_viewable = DisplayRectangle( 77 Point(window_offset_x, window_offset_y), visible_width, visible_height) 78 79 self._element_rect: Optional[DisplayRectangle] = None 80 if element_rect: 81 self._element_rect = self._dom_rect_to_native_rect(element_rect) 82 83 @property 84 def browser_viewable(self) -> DisplayRectangle: 85 return self._browser_viewable 86 87 @property 88 def native_screen(self) -> DisplayRectangle: 89 return self._native_screen 90 91 @property 92 def element_rect(self) -> Optional[DisplayRectangle]: 93 return self._element_rect 94 95 def _dom_rect_to_native_rect(self, 96 dom_rect: DisplayRectangle) -> DisplayRectangle: 97 browser_viewable = self.browser_viewable 98 correct_ratio_rect = dom_rect * self._actual_pixel_ratio 99 100 adjusted_left = correct_ratio_rect.left + browser_viewable.left 101 adjusted_top = correct_ratio_rect.top + browser_viewable.top 102 adjusted_width = min(correct_ratio_rect.width, 103 self._native_screen.width - correct_ratio_rect.left) 104 adjusted_height = min(correct_ratio_rect.height, 105 self._native_screen.height - correct_ratio_rect.top) 106 107 return DisplayRectangle( 108 Point(adjusted_left, adjusted_top), adjusted_width, adjusted_height) 109 110 def css_to_native_distance(self, distance: float) -> float: 111 return distance * self._actual_pixel_ratio 112 113 114@dataclasses.dataclass(frozen=True) 115# Stores the configuration of the touchscreen device for the Chromebook. 116class TouchDevice: 117 # The path of the device. 118 device_path: str 119 # The maximum X value for a touch input. 120 x_max: int 121 # The maximum Y value for a touch input. 122 y_max: int 123 124 @classmethod 125 def parse_str(cls: Type[TouchDevice], config: str) -> TouchDevice: 126 # The first line of output is always 'Performing autotest_lib import' 127 # Followed by the output we care about. 128 touch_device_values = config.splitlines()[1].split(" ") 129 130 return TouchDevice(touch_device_values[0], 131 NumberParser.positive_zero_int(touch_device_values[1]), 132 NumberParser.positive_zero_int(touch_device_values[2])) 133 134 def __str__(self) -> str: 135 return f"{self.device_path} {self.x_max} {self.y_max}" 136 137 def is_valid_tap_position(self, position: Point) -> bool: 138 return (0 <= position.x and position.x <= self.x_max and 0 <= position.y and 139 position.y <= self.y_max) 140 141 142@dataclasses.dataclass(frozen=True) 143class ChromeOSTouchEvent: 144 touch_device: TouchDevice 145 146 # The viewport in which the start and end positions lie. 147 viewport: DisplayRectangle 148 # The start position in terms of the device's screen resolution 149 start_position: Point 150 # The end position in terms of the device's screen resolution 151 end_position: Optional[Point] = None 152 153 duration: dt.timedelta = dt.timedelta() 154 155 # Touch event data recorded with evemu-record on a dedede. 156 # This has been tested to work on dedede, brya, and volteer. 157 # Some devices, however, may use a different x-y orientation 158 # (such as kukui in landscape mode) and are not currently supported. 159 _TAP_DOWN = """E: <time> 0003 0039 0 160E: <time> 0003 0035 <x> 161E: <time> 0003 0036 <y> 162E: <time> 0001 014a 1 163E: <time> 0003 0000 <x> 164E: <time> 0003 0001 <y> 165E: <time> 0000 0000 0 166""" 167 168 _TAP_POSITION = """E: <time> 0003 0035 <x> 169E: <time> 0003 0036 <y> 170E: <time> 0003 0000 <x> 171E: <time> 0003 0001 <y> 172E: <time> 0000 0000 0 173""" 174 175 _TAP_UP = """E: <time> 0003 0039 -1 176E: <time> 0001 014a 0 177E: <time> 0000 0000 0 178""" 179 180 # For swipes, simulate the touch panel updating the position 60 times a 181 # second. 182 # This was chosen arbitrarily, but should balance a realistic swipe action 183 # with the size of the playback file that needs to be pushed to the device. 184 _TOUCH_UPDATE_HERTZ = 60 185 186 def __str__(self) -> str: 187 # Not sure why, but evemu-playback does not like it when the event time 188 # starts at 0.X 189 current_event_time_seconds: float = 1.0 190 playback_script: str = "" 191 192 start_position: Point = self._rereference_to_touch_coordinates( 193 self.viewport, self.start_position) 194 195 playback_script += self._format_script_block(self._TAP_DOWN, 196 current_event_time_seconds, 197 start_position) 198 199 # Shortcut for long taps 200 if not self.end_position: 201 current_event_time_seconds += self.duration.total_seconds() 202 playback_script += self._format_script_block(self._TAP_UP, 203 current_event_time_seconds, 204 start_position) 205 return playback_script 206 207 end_position: Point = self._rereference_to_touch_coordinates( 208 self.viewport, self.end_position) 209 210 num_position_updates: int = round(self.duration.total_seconds() * 211 self._TOUCH_UPDATE_HERTZ) 212 assert num_position_updates > 0, "Choose a longer scroll duration." 213 214 increment_distance_x = (end_position.x - 215 start_position.x) / num_position_updates 216 increment_distance_y = (end_position.y - 217 start_position.y) / num_position_updates 218 219 current_position_x = start_position.x 220 current_position_y = start_position.y 221 222 for _ in range(num_position_updates): 223 current_event_time_seconds += 1.0 / self._TOUCH_UPDATE_HERTZ 224 current_position_x += increment_distance_x 225 current_position_y += increment_distance_y 226 playback_script += self._format_script_block( 227 self._TAP_POSITION, current_event_time_seconds, 228 Point(round(current_position_x), round(current_position_y))) 229 230 playback_script += self._format_script_block(self._TAP_UP, 231 current_event_time_seconds, 232 end_position) 233 return playback_script 234 235 def _rereference(self, original: int, original_max: int, new_max: int) -> int: 236 return round(float(original / original_max) * new_max) 237 238 def _rereference_to_touch_coordinates(self, 239 original_viewport: DisplayRectangle, 240 point: Point) -> Point: 241 x = self._rereference(point.x, original_viewport.width, 242 self.touch_device.x_max) 243 y = self._rereference(point.y, original_viewport.height, 244 self.touch_device.y_max) 245 246 return Point(x, y) 247 248 def _format_script_block(self, script_block: str, time: float, 249 position: Point) -> str: 250 if not self.touch_device.is_valid_tap_position(position): 251 raise ValueError(f"Cannot tap on out of bounds position: {position}") 252 253 return script_block.replace("<x>", str(round(position.x))).replace( 254 "<y>", str(round(position.y))).replace("<time>", f"{time:.6f}") 255 256 257class ChromeOSInputActionRunner(BasicActionRunner): 258 259 def __init__(self): 260 super().__init__() 261 self._touch_device: Optional[TouchDevice] = None 262 self._remote_tmp_file = "" 263 264 def click_touch(self, run: Run, action: i_action.ClickAction) -> None: 265 if self._touch_device is None: 266 self._touch_device = self._setup_touch_device(run) 267 268 with run.actions("ClickAction", measure=False) as actions: 269 270 click_location, viewport = self._get_click_location(actions, action) 271 272 if not click_location: 273 return 274 275 self._execute_touch_playback( 276 run, 277 ChromeOSTouchEvent( 278 self._touch_device, 279 viewport.native_screen, 280 click_location, 281 end_position=None, 282 duration=action.duration)) 283 284 def click_mouse(self, run: Run, action: i_action.ClickAction) -> None: 285 with run.actions("ClickAction", measure=False) as actions: 286 287 click_location, viewport = self._get_click_location(actions, action) 288 289 if not click_location: 290 return 291 292 browser_platform = run.browser_platform 293 self._remote_tmp_file = browser_platform.mktemp() 294 script = (SCRIPTS_DIR / "mouse.py").read_text() 295 browser_platform.set_file_contents(self._remote_tmp_file, script) 296 297 run.browser_platform.sh("python3", self._remote_tmp_file, 298 str(viewport.native_screen.width), 299 str(viewport.native_screen.height), 300 str(action.duration.total_seconds()), 301 str(click_location.x), str(click_location.y)) 302 303 def scroll_touch(self, run: Run, action: i_action.ScrollAction) -> None: 304 if self._touch_device is None: 305 self._touch_device = self._setup_touch_device(run) 306 307 with run.actions("ScrollAction", measure=False) as actions: 308 309 viewport_info: ChromeOSViewportInfo = self._get_viewport_info( 310 actions, action.selector, False) 311 312 scroll_area: DisplayRectangle = viewport_info.browser_viewable 313 314 total_scroll_distance = viewport_info.css_to_native_distance( 315 action.distance) 316 317 if action.selector: 318 if not viewport_info.element_rect: 319 if action.required: 320 raise ElementNotFoundError(action.selector) 321 return 322 scroll_area = viewport_info.element_rect 323 324 max_swipe_distance = scroll_area.bottom - scroll_area.top 325 326 remaining_distance = abs(total_scroll_distance) 327 328 while remaining_distance > 0: 329 330 current_distance = min(max_swipe_distance, remaining_distance) 331 332 # The duration for this swipe should be only a fraction of the total 333 # duration since the entire distance may not be covered in one swipe. 334 current_duration = (current_distance / 335 abs(total_scroll_distance)) * action.duration 336 337 if total_scroll_distance > 0: 338 # If scrolling down, the swipe should start at the bottom and end 339 # above. 340 y_start = scroll_area.bottom 341 y_end = scroll_area.bottom - current_distance 342 343 else: 344 # If scrolling up, the swipe should start at the top and end below. 345 y_start = scroll_area.top 346 y_end = scroll_area.top + current_distance 347 348 self._execute_touch_playback( 349 run, 350 ChromeOSTouchEvent( 351 self._touch_device, 352 viewport_info.native_screen, 353 Point(scroll_area.middle.x, y_start), 354 end_position=Point(scroll_area.middle.x, y_end), 355 duration=current_duration)) 356 357 remaining_distance -= current_distance 358 359 def text_input_keyboard(self, run: Run, 360 action: i_action.TextInputAction) -> None: 361 browser_platform = run.browser_platform 362 self._remote_tmp_file = browser_platform.mktemp() 363 script = (SCRIPTS_DIR / "text_input.py").read_text() 364 browser_platform.set_file_contents(self._remote_tmp_file, script) 365 366 try: 367 typing_process = browser_platform.popen( 368 "python3", self._remote_tmp_file, bufsize=0, stdin=subprocess.PIPE) 369 370 self._rate_limit_keystrokes( 371 run, action, lambda run, actions, text: typing_process.stdin.write( 372 text.encode("utf-8"))) 373 finally: 374 typing_process.stdin.close() 375 typing_process.wait(timeout=action.timeout.total_seconds()) 376 377 def _get_click_location( 378 self, actions: Actions, action: i_action.ClickAction 379 ) -> Tuple[Optional[Point], ChromeOSViewportInfo]: 380 viewport_info: ChromeOSViewportInfo = self._get_viewport_info( 381 actions, action.selector, action.scroll_into_view) 382 383 if action.selector: 384 element_rect = viewport_info.element_rect 385 if not element_rect: 386 if action.required: 387 raise ElementNotFoundError(action.selector) 388 return (None, viewport_info) 389 click_location: Point = element_rect.middle 390 else: 391 click_location = action.coordinates 392 393 assert click_location, "Invalid click location click action." 394 395 return (click_location, viewport_info) 396 397 def _get_viewport_info(self, 398 actions: Actions, 399 selector: Optional[str], 400 scroll_into_view=False) -> ChromeOSViewportInfo: 401 402 script = "" 403 if selector: 404 selector, script = self.get_selector_script(selector) 405 406 script += (SCRIPTS_DIR / "get_window_positions.js").read_text() 407 408 (found_element, pixel_ratio, outer_width, inner_width, inner_height, 409 screen_width, screen_height, avail_width, avail_height, screen_x, screen_y, 410 element_left, element_top, element_width, element_height) = actions.js( 411 script, arguments=[selector, scroll_into_view]) 412 413 element_rect: Optional[DisplayRectangle] = None 414 415 if found_element: 416 element_rect = DisplayRectangle( 417 Point(element_left, element_top), element_width, element_height) 418 419 viewport_info: ChromeOSViewportInfo = ChromeOSViewportInfo( 420 device_pixel_ratio=pixel_ratio, 421 window_outer_width=outer_width, 422 window_inner_width=inner_width, 423 window_inner_height=inner_height, 424 screen_width=screen_width, 425 screen_height=screen_height, 426 screen_avail_width=avail_width, 427 screen_avail_height=avail_height, 428 window_offset_x=screen_x, 429 window_offset_y=screen_y, 430 element_rect=element_rect) 431 432 return viewport_info 433 434 def _query_touch_device(self, run: Run) -> str: 435 try: 436 with (SCRIPTS_DIR / "query_touch_device.py").open() as file: 437 return run.browser_platform.sh_stdout("python3", "-", stdin=file) 438 except Exception as e: 439 raise RuntimeError( 440 "Failed to query touchscreen information from device.") from e 441 442 def _setup_touch_device(self, run: Run) -> TouchDevice: 443 self._remote_tmp_file = run.browser_platform.mktemp() 444 445 touch_device_output = self._query_touch_device(run) 446 447 return TouchDevice.parse_str(touch_device_output) 448 449 def _execute_touch_playback(self, run: Run, 450 touch_event: ChromeOSTouchEvent) -> None: 451 # Ideally the touch event data could just be sent to |input| of evemu-play, 452 # but after a lot of testing, evemu-play *only* behaves when input is 453 # redirected from a file such as with: 454 # 'evemu-play touch-device < input-file.txt' 455 # Using a pipe to redirect the input *does not work*: 456 # 'cat input-file.txt | evemu-play touch-device' 457 458 # Because of this weird behavior, create a temp file on the device first 459 # that contains the touch events. 460 461 touch_event_cmds = str(touch_event) 462 463 run.browser_platform.set_file_contents(self._remote_tmp_file, 464 touch_event_cmds) 465 466 # Then run evemu-play with the input redirected from the temp file. 467 run.browser_platform.sh( 468 f"evemu-play --insert-slot0 " 469 f"{shlex.quote(self._touch_device.device_path)} < " 470 f"{self._remote_tmp_file}", 471 shell=True) 472