1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.example.android.vdmdemo.host; 18 19 import android.annotation.SuppressLint; 20 import android.app.ActivityOptions; 21 import android.companion.virtual.VirtualDeviceManager.VirtualDevice; 22 import android.content.Context; 23 import android.content.Intent; 24 import android.graphics.Point; 25 import android.graphics.PointF; 26 import android.hardware.display.DisplayManager; 27 import android.hardware.display.VirtualDisplay; 28 import android.hardware.display.VirtualDisplayConfig; 29 import android.hardware.input.VirtualDpad; 30 import android.hardware.input.VirtualDpadConfig; 31 import android.hardware.input.VirtualKeyEvent; 32 import android.hardware.input.VirtualKeyboard; 33 import android.hardware.input.VirtualKeyboardConfig; 34 import android.hardware.input.VirtualMouse; 35 import android.hardware.input.VirtualMouseButtonEvent; 36 import android.hardware.input.VirtualMouseConfig; 37 import android.hardware.input.VirtualMouseRelativeEvent; 38 import android.hardware.input.VirtualMouseScrollEvent; 39 import android.hardware.input.VirtualNavigationTouchpad; 40 import android.hardware.input.VirtualNavigationTouchpadConfig; 41 import android.hardware.input.VirtualStylus; 42 import android.hardware.input.VirtualStylusButtonEvent; 43 import android.hardware.input.VirtualStylusConfig; 44 import android.hardware.input.VirtualStylusMotionEvent; 45 import android.hardware.input.VirtualTouchEvent; 46 import android.hardware.input.VirtualTouchscreen; 47 import android.hardware.input.VirtualTouchscreenConfig; 48 import android.util.Log; 49 import android.view.Display; 50 import android.view.InputEvent; 51 import android.view.KeyEvent; 52 import android.view.MotionEvent; 53 import android.view.Surface; 54 55 import androidx.annotation.IntDef; 56 57 import com.example.android.vdmdemo.common.RemoteEventProto; 58 import com.example.android.vdmdemo.common.RemoteEventProto.DisplayCapabilities; 59 import com.example.android.vdmdemo.common.RemoteEventProto.DisplayRotation; 60 import com.example.android.vdmdemo.common.RemoteEventProto.RemoteEvent; 61 import com.example.android.vdmdemo.common.RemoteEventProto.RemoteInputEvent; 62 import com.example.android.vdmdemo.common.RemoteEventProto.RemoteKeyEvent; 63 import com.example.android.vdmdemo.common.RemoteEventProto.RemoteMotionEvent; 64 import com.example.android.vdmdemo.common.RemoteEventProto.StopStreaming; 65 import com.example.android.vdmdemo.common.RemoteIo; 66 import com.example.android.vdmdemo.common.VideoManager; 67 68 import java.lang.annotation.Retention; 69 import java.lang.annotation.RetentionPolicy; 70 import java.util.concurrent.atomic.AtomicBoolean; 71 import java.util.function.Consumer; 72 73 @SuppressLint("NewApi") 74 class RemoteDisplay implements AutoCloseable { 75 76 private static final String TAG = "VdmHost"; 77 78 private static final int DISPLAY_FPS = 60; 79 80 private static final int DEFAULT_VIRTUAL_DISPLAY_FLAGS = 81 DisplayManager.VIRTUAL_DISPLAY_FLAG_TRUSTED 82 | DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC 83 | DisplayManager.VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY; 84 85 static final int DISPLAY_TYPE_APP = 0; 86 static final int DISPLAY_TYPE_HOME = 1; 87 static final int DISPLAY_TYPE_MIRROR = 2; 88 @IntDef(value = {DISPLAY_TYPE_APP, DISPLAY_TYPE_HOME, DISPLAY_TYPE_MIRROR}) 89 @Retention(RetentionPolicy.SOURCE) 90 public @interface DisplayType {} 91 92 private final Context mContext; 93 private final RemoteIo mRemoteIo; 94 private final PreferenceController mPreferenceController; 95 private final Consumer<RemoteEvent> mRemoteEventConsumer = this::processRemoteEvent; 96 private final VirtualDisplay mVirtualDisplay; 97 private final VirtualDpad mDpad; 98 private final int mRemoteDisplayId; 99 private final VirtualDevice mVirtualDevice; 100 private final @DisplayType int mDisplayType; 101 private final AtomicBoolean mClosed = new AtomicBoolean(false); 102 private int mRotation; 103 private int mWidth; 104 private int mHeight; 105 private int mDpi; 106 107 private VideoManager mVideoManager; 108 private VirtualTouchscreen mTouchscreen; 109 private VirtualMouse mMouse; 110 private VirtualNavigationTouchpad mNavigationTouchpad; 111 private VirtualKeyboard mKeyboard; 112 private VirtualStylus mStylus; 113 114 @SuppressLint("WrongConstant") RemoteDisplay( Context context, RemoteEvent event, VirtualDevice virtualDevice, RemoteIo remoteIo, @DisplayType int displayType, PreferenceController preferenceController)115 RemoteDisplay( 116 Context context, 117 RemoteEvent event, 118 VirtualDevice virtualDevice, 119 RemoteIo remoteIo, 120 @DisplayType int displayType, 121 PreferenceController preferenceController) { 122 mContext = context; 123 mRemoteIo = remoteIo; 124 mRemoteDisplayId = event.getDisplayId(); 125 mVirtualDevice = virtualDevice; 126 mDisplayType = displayType; 127 mPreferenceController = preferenceController; 128 129 setCapabilities(event.getDisplayCapabilities()); 130 131 int flags = DEFAULT_VIRTUAL_DISPLAY_FLAGS; 132 if (mPreferenceController.getBoolean(R.string.pref_enable_display_rotation)) { 133 flags |= DisplayManager.VIRTUAL_DISPLAY_FLAG_ROTATES_WITH_CONTENT; 134 } 135 if (mDisplayType == DISPLAY_TYPE_MIRROR) { 136 flags &= ~DisplayManager.VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY; 137 } 138 139 VirtualDisplayConfig.Builder virtualDisplayBuilder = 140 new VirtualDisplayConfig.Builder( 141 "VirtualDisplay" + mRemoteDisplayId, mWidth, mHeight, mDpi) 142 .setFlags(flags); 143 144 if (mDisplayType == DISPLAY_TYPE_HOME) { 145 virtualDisplayBuilder = VdmCompat.setHomeSupported(virtualDisplayBuilder, flags); 146 } 147 148 mVirtualDisplay = 149 virtualDevice.createVirtualDisplay( 150 virtualDisplayBuilder.build(), 151 /* executor= */ Runnable::run, 152 /* callback= */ null); 153 154 VdmCompat.setDisplayImePolicy( 155 mVirtualDevice, 156 getDisplayId(), 157 mPreferenceController.getInt(R.string.pref_display_ime_policy)); 158 159 mDpad = 160 virtualDevice.createVirtualDpad( 161 new VirtualDpadConfig.Builder() 162 .setAssociatedDisplayId(mVirtualDisplay.getDisplay().getDisplayId()) 163 .setInputDeviceName("vdmdemo-dpad" + mRemoteDisplayId) 164 .build()); 165 mKeyboard = 166 mVirtualDevice.createVirtualKeyboard( 167 new VirtualKeyboardConfig.Builder() 168 .setInputDeviceName( 169 "vdmdemo-keyboard" + mRemoteDisplayId) 170 .setAssociatedDisplayId(getDisplayId()) 171 .build()); 172 173 remoteIo.addMessageConsumer(mRemoteEventConsumer); 174 175 reset(); 176 } 177 reset(DisplayCapabilities capabilities)178 void reset(DisplayCapabilities capabilities) { 179 setCapabilities(capabilities); 180 mVirtualDisplay.resize(mWidth, mHeight, mDpi); 181 reset(); 182 } 183 reset()184 private void reset() { 185 if (mVideoManager != null) { 186 mVideoManager.stop(); 187 } 188 mVideoManager = VideoManager.createDisplayEncoder(mRemoteDisplayId, mRemoteIo, 189 mPreferenceController.getBoolean(R.string.pref_record_encoder_output)); 190 Surface surface = mVideoManager.createInputSurface(mWidth, mHeight, DISPLAY_FPS); 191 mVirtualDisplay.setSurface(surface); 192 193 mRotation = mVirtualDisplay.getDisplay().getRotation(); 194 195 if (mTouchscreen != null) { 196 mTouchscreen.close(); 197 } 198 if (mStylus != null) { 199 mStylus.close(); 200 } 201 mTouchscreen = 202 mVirtualDevice.createVirtualTouchscreen( 203 new VirtualTouchscreenConfig.Builder(mWidth, mHeight) 204 .setAssociatedDisplayId(mVirtualDisplay.getDisplay().getDisplayId()) 205 .setInputDeviceName("vdmdemo-touchscreen" + mRemoteDisplayId) 206 .build()); 207 208 mVideoManager.startEncoding(); 209 } 210 setCapabilities(DisplayCapabilities capabilities)211 private void setCapabilities(DisplayCapabilities capabilities) { 212 mWidth = capabilities.getViewportWidth(); 213 mHeight = capabilities.getViewportHeight(); 214 mDpi = capabilities.getDensityDpi(); 215 216 // Video encoder needs round dimensions... 217 mHeight -= mHeight % 10; 218 mWidth -= mWidth % 10; 219 } 220 launchIntent(Intent intent)221 void launchIntent(Intent intent) { 222 mContext.startActivity( 223 intent, ActivityOptions.makeBasic().setLaunchDisplayId(getDisplayId()).toBundle()); 224 } 225 getRemoteDisplayId()226 int getRemoteDisplayId() { 227 return mRemoteDisplayId; 228 } 229 getDisplayId()230 int getDisplayId() { 231 return mVirtualDisplay.getDisplay().getDisplayId(); 232 } 233 getDisplaySize()234 PointF getDisplaySize() { 235 return new PointF(mWidth, mHeight); 236 } 237 onDisplayChanged()238 void onDisplayChanged() { 239 if (mRotation != mVirtualDisplay.getDisplay().getRotation()) { 240 mRotation = mVirtualDisplay.getDisplay().getRotation(); 241 int rotationDegrees = displayRotationToDegrees(mRotation); 242 Log.v(TAG, "Notify client for rotation event: " + rotationDegrees); 243 mRemoteIo.sendMessage( 244 RemoteEvent.newBuilder() 245 .setDisplayId(getRemoteDisplayId()) 246 .setDisplayRotation( 247 DisplayRotation.newBuilder() 248 .setRotationDegrees(rotationDegrees)) 249 .build()); 250 } 251 } 252 processRemoteEvent(RemoteEvent event)253 void processRemoteEvent(RemoteEvent event) { 254 if (event.getDisplayId() != mRemoteDisplayId) { 255 return; 256 } 257 if (event.hasHomeEvent()) { 258 goHome(); 259 } else if (event.hasInputEvent()) { 260 processInputEvent(event.getInputEvent()); 261 } else if (event.hasStopStreaming() && event.getStopStreaming().getPause()) { 262 if (mVideoManager != null) { 263 mVideoManager.stop(); 264 mVideoManager = null; 265 } 266 } 267 } 268 goHome()269 void goHome() { 270 if (mDisplayType != DISPLAY_TYPE_HOME && mDisplayType != DISPLAY_TYPE_MIRROR) { 271 return; 272 } 273 Intent homeIntent = new Intent(Intent.ACTION_MAIN); 274 homeIntent.addCategory(Intent.CATEGORY_HOME); 275 homeIntent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); 276 int targetDisplayId = 277 mDisplayType == DISPLAY_TYPE_MIRROR ? Display.DEFAULT_DISPLAY : getDisplayId(); 278 mContext.startActivity( 279 homeIntent, 280 ActivityOptions.makeBasic().setLaunchDisplayId(targetDisplayId).toBundle()); 281 } 282 processInputEvent(RemoteInputEvent inputEvent)283 private void processInputEvent(RemoteInputEvent inputEvent) { 284 switch (inputEvent.getDeviceType()) { 285 case DEVICE_TYPE_NONE: 286 Log.e(TAG, "Received no input device type"); 287 break; 288 case DEVICE_TYPE_DPAD: 289 mDpad.sendKeyEvent(remoteEventToVirtualKeyEvent(inputEvent)); 290 break; 291 case DEVICE_TYPE_NAVIGATION_TOUCHPAD: 292 processNavigationTouchpadEvent(remoteEventToVirtualTouchEvent(inputEvent)); 293 break; 294 case DEVICE_TYPE_MOUSE: 295 processMouseEvent(inputEvent); 296 break; 297 case DEVICE_TYPE_TOUCHSCREEN: 298 mTouchscreen.sendTouchEvent(remoteEventToVirtualTouchEvent(inputEvent)); 299 break; 300 case DEVICE_TYPE_KEYBOARD: 301 mKeyboard.sendKeyEvent(remoteEventToVirtualKeyEvent(inputEvent)); 302 break; 303 default: 304 Log.e( 305 TAG, 306 "processInputEvent got an invalid input device type: " 307 + inputEvent.getDeviceType().getNumber()); 308 break; 309 } 310 } 311 processInputEvent(RemoteEventProto.InputDeviceType deviceType, InputEvent event)312 void processInputEvent(RemoteEventProto.InputDeviceType deviceType, InputEvent event) { 313 switch (deviceType) { 314 case DEVICE_TYPE_DPAD: 315 mDpad.sendKeyEvent(keyEventToVirtualKeyEvent((KeyEvent) event)); 316 break; 317 case DEVICE_TYPE_NAVIGATION_TOUCHPAD: 318 processNavigationTouchpadEvent(motionEventToVirtualTouchEvent((MotionEvent) event)); 319 break; 320 case DEVICE_TYPE_KEYBOARD: 321 mKeyboard.sendKeyEvent(keyEventToVirtualKeyEvent((KeyEvent) event)); 322 break; 323 default: 324 Log.e( 325 TAG, 326 "processInputEvent got an invalid input device type: " 327 + deviceType.getNumber()); 328 break; 329 } 330 } 331 processNavigationTouchpadEvent(VirtualTouchEvent event)332 private void processNavigationTouchpadEvent(VirtualTouchEvent event) { 333 if (mNavigationTouchpad == null) { 334 // Any arbitrarily big enough nav touchpad would work. 335 Point displaySize = new Point(5000, 5000); 336 mNavigationTouchpad = 337 mVirtualDevice.createVirtualNavigationTouchpad( 338 new VirtualNavigationTouchpadConfig.Builder( 339 displaySize.x, displaySize.y) 340 .setAssociatedDisplayId(getDisplayId()) 341 .setInputDeviceName( 342 "vdmdemo-navtouchpad" + mRemoteDisplayId) 343 .build()); 344 } 345 mNavigationTouchpad.sendTouchEvent(event); 346 347 } 348 processVirtualMouseEvent(Object mouseEvent)349 void processVirtualMouseEvent(Object mouseEvent) { 350 if (!createMouseIfNeeded()) { 351 return; 352 } 353 if (mouseEvent instanceof VirtualMouseButtonEvent) { 354 mMouse.sendButtonEvent((VirtualMouseButtonEvent) mouseEvent); 355 } else if (mouseEvent instanceof VirtualMouseScrollEvent) { 356 mMouse.sendScrollEvent((VirtualMouseScrollEvent) mouseEvent); 357 } else if (mouseEvent instanceof VirtualMouseRelativeEvent) { 358 mMouse.sendRelativeEvent((VirtualMouseRelativeEvent) mouseEvent); 359 } 360 } 361 processVirtualStylusEvent(Object stylusEvent)362 void processVirtualStylusEvent(Object stylusEvent) { 363 if (mStylus == null) { 364 mStylus = mVirtualDevice.createVirtualStylus( 365 new VirtualStylusConfig.Builder(mWidth, mHeight) 366 .setAssociatedDisplayId(getDisplayId()) 367 .setInputDeviceName("vdmdemo-stylus" + mRemoteDisplayId) 368 .build()); 369 } 370 if (stylusEvent instanceof VirtualStylusMotionEvent) { 371 mStylus.sendMotionEvent((VirtualStylusMotionEvent) stylusEvent); 372 } else if (stylusEvent instanceof VirtualStylusButtonEvent) { 373 mStylus.sendButtonEvent((VirtualStylusButtonEvent) stylusEvent); 374 } 375 } 376 processMouseEvent(RemoteInputEvent inputEvent)377 private void processMouseEvent(RemoteInputEvent inputEvent) { 378 if (!createMouseIfNeeded()) { 379 return; 380 } 381 if (inputEvent.hasMouseButtonEvent()) { 382 mMouse.sendButtonEvent( 383 new VirtualMouseButtonEvent.Builder() 384 .setButtonCode(inputEvent.getMouseButtonEvent().getKeyCode()) 385 .setAction(inputEvent.getMouseButtonEvent().getAction()) 386 .build()); 387 } else if (inputEvent.hasMouseScrollEvent()) { 388 mMouse.sendScrollEvent( 389 new VirtualMouseScrollEvent.Builder() 390 .setXAxisMovement(inputEvent.getMouseScrollEvent().getX()) 391 .setYAxisMovement(inputEvent.getMouseScrollEvent().getY()) 392 .build()); 393 } else if (inputEvent.hasMouseRelativeEvent()) { 394 PointF cursorPosition = mMouse.getCursorPosition(); 395 mMouse.sendRelativeEvent( 396 new VirtualMouseRelativeEvent.Builder() 397 .setRelativeX( 398 inputEvent.getMouseRelativeEvent().getX() - cursorPosition.x) 399 .setRelativeY( 400 inputEvent.getMouseRelativeEvent().getY() - cursorPosition.y) 401 .build()); 402 } else { 403 Log.e(TAG, "Received an invalid mouse event"); 404 } 405 } 406 createMouseIfNeeded()407 private boolean createMouseIfNeeded() { 408 if (mMouse == null && VdmCompat.canCreateVirtualMouse(mContext)) { 409 mMouse = 410 mVirtualDevice.createVirtualMouse( 411 new VirtualMouseConfig.Builder() 412 .setAssociatedDisplayId(getDisplayId()) 413 .setInputDeviceName("vdmdemo-mouse" + mRemoteDisplayId) 414 .build()); 415 } 416 return mMouse != null; 417 } 418 getVirtualTouchEventAction(int action)419 private static int getVirtualTouchEventAction(int action) { 420 return switch (action) { 421 case MotionEvent.ACTION_POINTER_DOWN -> VirtualTouchEvent.ACTION_DOWN; 422 case MotionEvent.ACTION_POINTER_UP -> VirtualTouchEvent.ACTION_UP; 423 default -> action; 424 }; 425 } 426 getVirtualTouchEventToolType(int action)427 private static int getVirtualTouchEventToolType(int action) { 428 return switch (action) { 429 case MotionEvent.ACTION_CANCEL -> VirtualTouchEvent.TOOL_TYPE_PALM; 430 default -> VirtualTouchEvent.TOOL_TYPE_FINGER; 431 }; 432 } 433 434 // Surface rotation is in opposite direction to display rotation. 435 // See https://developer.android.com/reference/android/view/Display?hl=en#getRotation() 436 private static int displayRotationToDegrees(int displayRotation) { 437 return switch (displayRotation) { 438 case Surface.ROTATION_90 -> -90; 439 case Surface.ROTATION_180 -> 180; 440 case Surface.ROTATION_270 -> 90; 441 default -> 0; 442 }; 443 } 444 445 private static VirtualKeyEvent remoteEventToVirtualKeyEvent(RemoteInputEvent event) { 446 RemoteKeyEvent keyEvent = event.getKeyEvent(); 447 return new VirtualKeyEvent.Builder() 448 .setEventTimeNanos((long) (event.getTimestampMs() * 1e6)) 449 .setKeyCode(keyEvent.getKeyCode()) 450 .setAction(keyEvent.getAction()) 451 .build(); 452 } 453 454 private static VirtualKeyEvent keyEventToVirtualKeyEvent(KeyEvent keyEvent) { 455 return new VirtualKeyEvent.Builder() 456 .setEventTimeNanos((long) (keyEvent.getEventTime() * 1e6)) 457 .setKeyCode(keyEvent.getKeyCode()) 458 .setAction(keyEvent.getAction()) 459 .build(); 460 } 461 462 private static VirtualTouchEvent remoteEventToVirtualTouchEvent(RemoteInputEvent event) { 463 RemoteMotionEvent motionEvent = event.getTouchEvent(); 464 return new VirtualTouchEvent.Builder() 465 .setEventTimeNanos((long) (event.getTimestampMs() * 1e6)) 466 .setPointerId(motionEvent.getPointerId()) 467 .setAction(getVirtualTouchEventAction(motionEvent.getAction())) 468 .setPressure(motionEvent.getPressure() * 255f) 469 .setToolType(getVirtualTouchEventToolType(motionEvent.getAction())) 470 .setX(motionEvent.getX()) 471 .setY(motionEvent.getY()) 472 .build(); 473 } 474 475 private static VirtualTouchEvent motionEventToVirtualTouchEvent(MotionEvent motionEvent) { 476 return new VirtualTouchEvent.Builder() 477 .setEventTimeNanos((long) (motionEvent.getEventTime() * 1e6)) 478 .setPointerId(1) 479 .setAction(getVirtualTouchEventAction(motionEvent.getAction())) 480 .setPressure(motionEvent.getPressure() * 255f) 481 .setToolType(getVirtualTouchEventToolType(motionEvent.getAction())) 482 .setX(motionEvent.getX()) 483 .setY(motionEvent.getY()) 484 .build(); 485 } 486 487 @Override 488 public void close() { 489 if (mClosed.getAndSet(true)) { // Prevent double closure. 490 return; 491 } 492 mRemoteIo.sendMessage( 493 RemoteEvent.newBuilder() 494 .setDisplayId(getRemoteDisplayId()) 495 .setStopStreaming(StopStreaming.newBuilder().setPause(false)) 496 .build()); 497 mRemoteIo.removeMessageConsumer(mRemoteEventConsumer); 498 mDpad.close(); 499 mTouchscreen.close(); 500 mKeyboard.close(); 501 if (mStylus != null) { 502 mStylus.close(); 503 } 504 if (mMouse != null) { 505 mMouse.close(); 506 } 507 if (mNavigationTouchpad != null) { 508 mNavigationTouchpad.close(); 509 } 510 mVirtualDisplay.release(); 511 if (mVideoManager != null) { 512 mVideoManager.stop(); 513 mVideoManager = null; 514 } 515 } 516 } 517