1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.companion.virtual; 18 19 import android.annotation.CallbackExecutor; 20 import android.annotation.IntDef; 21 import android.annotation.IntRange; 22 import android.annotation.NonNull; 23 import android.annotation.Nullable; 24 import android.annotation.RequiresPermission; 25 import android.annotation.SystemApi; 26 import android.annotation.SystemService; 27 import android.app.PendingIntent; 28 import android.companion.AssociationInfo; 29 import android.companion.virtual.audio.VirtualAudioDevice; 30 import android.companion.virtual.audio.VirtualAudioDevice.AudioConfigurationChangeCallback; 31 import android.content.ComponentName; 32 import android.content.Context; 33 import android.graphics.Point; 34 import android.hardware.display.DisplayManager; 35 import android.hardware.display.DisplayManager.VirtualDisplayFlag; 36 import android.hardware.display.DisplayManagerGlobal; 37 import android.hardware.display.IVirtualDisplayCallback; 38 import android.hardware.display.VirtualDisplay; 39 import android.hardware.display.VirtualDisplayConfig; 40 import android.hardware.input.VirtualKeyboard; 41 import android.hardware.input.VirtualMouse; 42 import android.hardware.input.VirtualTouchscreen; 43 import android.os.Binder; 44 import android.os.Bundle; 45 import android.os.Handler; 46 import android.os.IBinder; 47 import android.os.Looper; 48 import android.os.RemoteException; 49 import android.os.ResultReceiver; 50 import android.util.ArrayMap; 51 import android.view.Surface; 52 53 import java.lang.annotation.ElementType; 54 import java.lang.annotation.Retention; 55 import java.lang.annotation.RetentionPolicy; 56 import java.lang.annotation.Target; 57 import java.util.concurrent.Executor; 58 import java.util.function.IntConsumer; 59 60 /** 61 * System level service for managing virtual devices. 62 * 63 * @hide 64 */ 65 @SystemApi 66 @SystemService(Context.VIRTUAL_DEVICE_SERVICE) 67 public final class VirtualDeviceManager { 68 69 private static final boolean DEBUG = false; 70 private static final String TAG = "VirtualDeviceManager"; 71 72 private static final int DEFAULT_VIRTUAL_DISPLAY_FLAGS = 73 DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC 74 | DisplayManager.VIRTUAL_DISPLAY_FLAG_ROTATES_WITH_CONTENT 75 | DisplayManager.VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY 76 | DisplayManager.VIRTUAL_DISPLAY_FLAG_DESTROY_CONTENT_ON_REMOVAL 77 | DisplayManager.VIRTUAL_DISPLAY_FLAG_SUPPORTS_TOUCH 78 | DisplayManager.VIRTUAL_DISPLAY_FLAG_OWN_DISPLAY_GROUP; 79 80 /** @hide */ 81 @Retention(RetentionPolicy.SOURCE) 82 @IntDef( 83 prefix = "LAUNCH_", 84 value = { 85 LAUNCH_SUCCESS, 86 LAUNCH_FAILURE_PENDING_INTENT_CANCELED, 87 LAUNCH_FAILURE_NO_ACTIVITY}) 88 @Target({ElementType.TYPE_PARAMETER, ElementType.TYPE_USE}) 89 public @interface PendingIntentLaunchStatus {} 90 91 /** 92 * Status for {@link VirtualDevice#launchPendingIntent}, indicating that the launch was 93 * successful. 94 */ 95 public static final int LAUNCH_SUCCESS = 0; 96 97 /** 98 * Status for {@link VirtualDevice#launchPendingIntent}, indicating that the launch failed 99 * because the pending intent was canceled. 100 */ 101 public static final int LAUNCH_FAILURE_PENDING_INTENT_CANCELED = 1; 102 103 /** 104 * Status for {@link VirtualDevice#launchPendingIntent}, indicating that the launch failed 105 * because no activity starts were detected as a result of calling the pending intent. 106 */ 107 public static final int LAUNCH_FAILURE_NO_ACTIVITY = 2; 108 109 private final IVirtualDeviceManager mService; 110 private final Context mContext; 111 112 /** @hide */ VirtualDeviceManager( @ullable IVirtualDeviceManager service, @NonNull Context context)113 public VirtualDeviceManager( 114 @Nullable IVirtualDeviceManager service, @NonNull Context context) { 115 mService = service; 116 mContext = context; 117 } 118 119 /** 120 * Creates a virtual device where applications can launch and receive input events injected by 121 * the creator. 122 * 123 * <p>The {@link android.Manifest.permission#CREATE_VIRTUAL_DEVICE} permission is required to 124 * create virtual devices, which is only available to system apps holding specific roles. 125 * 126 * @param associationId The association ID as returned by {@link AssociationInfo#getId()} from 127 * Companion Device Manager. Virtual devices must have a corresponding association with CDM in 128 * order to be created. 129 * @param params The parameters for creating virtual devices. See {@link VirtualDeviceParams} 130 * for the available options. 131 * @return The created virtual device. 132 */ 133 @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) 134 @NonNull createVirtualDevice( int associationId, @NonNull VirtualDeviceParams params)135 public VirtualDevice createVirtualDevice( 136 int associationId, 137 @NonNull VirtualDeviceParams params) { 138 try { 139 return new VirtualDevice(mService, mContext, associationId, params); 140 } catch (RemoteException e) { 141 throw e.rethrowFromSystemServer(); 142 } 143 } 144 145 /** 146 * A virtual device has its own virtual display, audio output, microphone, and camera etc. The 147 * creator of a virtual device can take the output from the virtual display and stream it over 148 * to another device, and inject input events that are received from the remote device. 149 * 150 * TODO(b/204081582): Consider using a builder pattern for the input APIs. 151 */ 152 public static class VirtualDevice implements AutoCloseable { 153 154 private final Context mContext; 155 private final IVirtualDeviceManager mService; 156 private final IVirtualDevice mVirtualDevice; 157 private final ArrayMap<ActivityListener, ActivityListenerDelegate> mActivityListeners = 158 new ArrayMap<>(); 159 private final IVirtualDeviceActivityListener mActivityListenerBinder = 160 new IVirtualDeviceActivityListener.Stub() { 161 162 @Override 163 public void onTopActivityChanged(int displayId, ComponentName topActivity) { 164 final long token = Binder.clearCallingIdentity(); 165 try { 166 for (int i = 0; i < mActivityListeners.size(); i++) { 167 mActivityListeners.valueAt(i) 168 .onTopActivityChanged(displayId, topActivity); 169 } 170 } finally { 171 Binder.restoreCallingIdentity(token); 172 } 173 } 174 175 @Override 176 public void onDisplayEmpty(int displayId) { 177 final long token = Binder.clearCallingIdentity(); 178 try { 179 for (int i = 0; i < mActivityListeners.size(); i++) { 180 mActivityListeners.valueAt(i).onDisplayEmpty(displayId); 181 } 182 } finally { 183 Binder.restoreCallingIdentity(token); 184 } 185 } 186 }; 187 @Nullable 188 private VirtualAudioDevice mVirtualAudioDevice; 189 VirtualDevice( IVirtualDeviceManager service, Context context, int associationId, VirtualDeviceParams params)190 private VirtualDevice( 191 IVirtualDeviceManager service, 192 Context context, 193 int associationId, 194 VirtualDeviceParams params) throws RemoteException { 195 mService = service; 196 mContext = context.getApplicationContext(); 197 mVirtualDevice = service.createVirtualDevice( 198 new Binder(), 199 mContext.getPackageName(), 200 associationId, 201 params, 202 mActivityListenerBinder); 203 } 204 205 /** 206 * Launches a given pending intent on the give display ID. 207 * 208 * @param displayId The display to launch the pending intent on. This display must be 209 * created from this virtual device. 210 * @param pendingIntent The pending intent to be launched. If the intent is an activity 211 * intent, the activity will be started on the virtual display using 212 * {@link android.app.ActivityOptions#setLaunchDisplayId}. If the intent is a service or 213 * broadcast intent, an attempt will be made to catch activities started as a result of 214 * sending the pending intent and move them to the given display. When it completes, 215 * {@code listener} will be called with the status of whether the launch attempt is 216 * successful or not. 217 * @param executor The executor to run {@code launchCallback} on. 218 * @param listener Listener that is called when the pending intent launching is complete. 219 * The argument is {@link #LAUNCH_SUCCESS} if the launch successfully started an activity 220 * on the virtual display, or one of the {@code LAUNCH_FAILED} status explaining why it 221 * failed. 222 */ launchPendingIntent( int displayId, @NonNull PendingIntent pendingIntent, @NonNull Executor executor, @NonNull IntConsumer listener)223 public void launchPendingIntent( 224 int displayId, 225 @NonNull PendingIntent pendingIntent, 226 @NonNull Executor executor, 227 @NonNull IntConsumer listener) { 228 try { 229 mVirtualDevice.launchPendingIntent( 230 displayId, 231 pendingIntent, 232 new ResultReceiver(new Handler(Looper.getMainLooper())) { 233 @Override 234 protected void onReceiveResult(int resultCode, Bundle resultData) { 235 super.onReceiveResult(resultCode, resultData); 236 executor.execute(() -> listener.accept(resultCode)); 237 } 238 }); 239 } catch (RemoteException e) { 240 e.rethrowFromSystemServer(); 241 } 242 } 243 244 /** 245 * Creates a virtual display for this virtual device. All displays created on the same 246 * device belongs to the same display group. 247 * 248 * @param width The width of the virtual display in pixels, must be greater than 0. 249 * @param height The height of the virtual display in pixels, must be greater than 0. 250 * @param densityDpi The density of the virtual display in dpi, must be greater than 0. 251 * @param surface The surface to which the content of the virtual display should 252 * be rendered, or null if there is none initially. The surface can also be set later using 253 * {@link VirtualDisplay#setSurface(Surface)}. 254 * @param flags A combination of virtual display flags accepted by 255 * {@link DisplayManager#createVirtualDisplay}. In addition, the following flags are 256 * automatically set for all virtual devices: 257 * {@link DisplayManager#VIRTUAL_DISPLAY_FLAG_PUBLIC VIRTUAL_DISPLAY_FLAG_PUBLIC} and 258 * {@link DisplayManager#VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY 259 * VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY}. 260 * @param executor The executor on which {@code callback} will be invoked. This is ignored 261 * if {@code callback} is {@code null}. If {@code callback} is specified, this executor must 262 * not be null. 263 * @param callback Callback to call when the state of the {@link VirtualDisplay} changes 264 * @return The newly created virtual display, or {@code null} if the application could 265 * not create the virtual display. 266 * 267 * @see DisplayManager#createVirtualDisplay 268 */ 269 @Nullable createVirtualDisplay( @ntRangefrom = 1) int width, @IntRange(from = 1) int height, @IntRange(from = 1) int densityDpi, @Nullable Surface surface, @VirtualDisplayFlag int flags, @Nullable @CallbackExecutor Executor executor, @Nullable VirtualDisplay.Callback callback)270 public VirtualDisplay createVirtualDisplay( 271 @IntRange(from = 1) int width, 272 @IntRange(from = 1) int height, 273 @IntRange(from = 1) int densityDpi, 274 @Nullable Surface surface, 275 @VirtualDisplayFlag int flags, 276 @Nullable @CallbackExecutor Executor executor, 277 @Nullable VirtualDisplay.Callback callback) { 278 // TODO(b/205343547): Handle display groups properly instead of creating a new display 279 // group for every new virtual display created using this API. 280 // belongs to the same display group. 281 VirtualDisplayConfig config = new VirtualDisplayConfig.Builder( 282 getVirtualDisplayName(), width, height, densityDpi) 283 .setSurface(surface) 284 .setFlags(getVirtualDisplayFlags(flags)) 285 .build(); 286 IVirtualDisplayCallback callbackWrapper = 287 new DisplayManagerGlobal.VirtualDisplayCallback(callback, executor); 288 final int displayId; 289 try { 290 displayId = mService.createVirtualDisplay(config, callbackWrapper, mVirtualDevice, 291 mContext.getPackageName()); 292 } catch (RemoteException ex) { 293 throw ex.rethrowFromSystemServer(); 294 } 295 DisplayManagerGlobal displayManager = DisplayManagerGlobal.getInstance(); 296 return displayManager.createVirtualDisplayWrapper(config, mContext, callbackWrapper, 297 displayId); 298 } 299 300 /** 301 * Closes the virtual device, stopping and tearing down any virtual displays, associated 302 * virtual audio device, and event injection that's currently in progress. 303 */ 304 @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) close()305 public void close() { 306 try { 307 mVirtualDevice.close(); 308 } catch (RemoteException e) { 309 throw e.rethrowFromSystemServer(); 310 } 311 if (mVirtualAudioDevice != null) { 312 mVirtualAudioDevice.close(); 313 mVirtualAudioDevice = null; 314 } 315 } 316 317 /** 318 * Creates a virtual keyboard. 319 * 320 * @param display the display that the events inputted through this device should target 321 * @param inputDeviceName the name to call this input device 322 * @param vendorId the PCI vendor id 323 * @param productId the product id, as defined by the vendor 324 */ 325 @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) 326 @NonNull createVirtualKeyboard( @onNull VirtualDisplay display, @NonNull String inputDeviceName, int vendorId, int productId)327 public VirtualKeyboard createVirtualKeyboard( 328 @NonNull VirtualDisplay display, 329 @NonNull String inputDeviceName, 330 int vendorId, 331 int productId) { 332 try { 333 final IBinder token = new Binder( 334 "android.hardware.input.VirtualKeyboard:" + inputDeviceName); 335 mVirtualDevice.createVirtualKeyboard(display.getDisplay().getDisplayId(), 336 inputDeviceName, vendorId, productId, token); 337 return new VirtualKeyboard(mVirtualDevice, token); 338 } catch (RemoteException e) { 339 throw e.rethrowFromSystemServer(); 340 } 341 } 342 343 /** 344 * Creates a virtual mouse. 345 * 346 * @param display the display that the events inputted through this device should target 347 * @param inputDeviceName the name to call this input device 348 * @param vendorId the PCI vendor id 349 * @param productId the product id, as defined by the vendor 350 */ 351 @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) 352 @NonNull createVirtualMouse( @onNull VirtualDisplay display, @NonNull String inputDeviceName, int vendorId, int productId)353 public VirtualMouse createVirtualMouse( 354 @NonNull VirtualDisplay display, 355 @NonNull String inputDeviceName, 356 int vendorId, 357 int productId) { 358 try { 359 final IBinder token = new Binder( 360 "android.hardware.input.VirtualMouse:" + inputDeviceName); 361 mVirtualDevice.createVirtualMouse(display.getDisplay().getDisplayId(), 362 inputDeviceName, vendorId, productId, token); 363 return new VirtualMouse(mVirtualDevice, token); 364 } catch (RemoteException e) { 365 throw e.rethrowFromSystemServer(); 366 } 367 } 368 369 /** 370 * Creates a virtual touchscreen. 371 * 372 * @param display the display that the events inputted through this device should target 373 * @param inputDeviceName the name to call this input device 374 * @param vendorId the PCI vendor id 375 * @param productId the product id, as defined by the vendor 376 */ 377 @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) 378 @NonNull createVirtualTouchscreen( @onNull VirtualDisplay display, @NonNull String inputDeviceName, int vendorId, int productId)379 public VirtualTouchscreen createVirtualTouchscreen( 380 @NonNull VirtualDisplay display, 381 @NonNull String inputDeviceName, 382 int vendorId, 383 int productId) { 384 try { 385 final IBinder token = new Binder( 386 "android.hardware.input.VirtualTouchscreen:" + inputDeviceName); 387 final Point size = new Point(); 388 display.getDisplay().getSize(size); 389 mVirtualDevice.createVirtualTouchscreen(display.getDisplay().getDisplayId(), 390 inputDeviceName, vendorId, productId, token, size); 391 return new VirtualTouchscreen(mVirtualDevice, token); 392 } catch (RemoteException e) { 393 throw e.rethrowFromSystemServer(); 394 } 395 } 396 397 /** 398 * Creates a VirtualAudioDevice, capable of recording audio emanating from this device, 399 * or injecting audio from another device. 400 * 401 * <p>Note: One {@link VirtualDevice} can only create one {@link VirtualAudioDevice}, so 402 * calling this method multiple times will return the same instance. When 403 * {@link VirtualDevice#close()} is called, the associated {@link VirtualAudioDevice} will 404 * also be closed automatically. 405 * 406 * @param display The target virtual display to capture from and inject into. 407 * @param executor The {@link Executor} object for the thread on which to execute 408 * the callback. If <code>null</code>, the {@link Executor} associated with 409 * the main {@link Looper} will be used. 410 * @param callback Interface to be notified when playback or recording configuration of 411 * applications running on virtual display is changed. 412 * @return A {@link VirtualAudioDevice} instance. 413 */ 414 @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) 415 @NonNull createVirtualAudioDevice( @onNull VirtualDisplay display, @Nullable Executor executor, @Nullable AudioConfigurationChangeCallback callback)416 public VirtualAudioDevice createVirtualAudioDevice( 417 @NonNull VirtualDisplay display, 418 @Nullable Executor executor, 419 @Nullable AudioConfigurationChangeCallback callback) { 420 if (mVirtualAudioDevice == null) { 421 mVirtualAudioDevice = new VirtualAudioDevice( 422 mContext, mVirtualDevice, display, executor, callback); 423 } 424 return mVirtualAudioDevice; 425 } 426 427 /** 428 * Sets the visibility of the pointer icon for this VirtualDevice's associated displays. 429 * 430 * @param showPointerIcon True if the pointer should be shown; false otherwise. The default 431 * visibility is true. 432 */ 433 @RequiresPermission(android.Manifest.permission.CREATE_VIRTUAL_DEVICE) 434 @NonNull setShowPointerIcon(boolean showPointerIcon)435 public void setShowPointerIcon(boolean showPointerIcon) { 436 try { 437 mVirtualDevice.setShowPointerIcon(showPointerIcon); 438 } catch (RemoteException e) { 439 throw e.rethrowFromSystemServer(); 440 } 441 } 442 443 /** 444 * Returns the display flags that should be added to a particular virtual display. 445 * Additional device-level flags from {@link 446 * com.android.server.companion.virtual.VirtualDeviceImpl#getBaseVirtualDisplayFlags()} will 447 * be added by DisplayManagerService. 448 */ getVirtualDisplayFlags(int flags)449 private int getVirtualDisplayFlags(int flags) { 450 return DEFAULT_VIRTUAL_DISPLAY_FLAGS | flags; 451 } 452 getVirtualDisplayName()453 private String getVirtualDisplayName() { 454 try { 455 // Currently this just use the association ID, which means all of the virtual 456 // displays created using the same virtual device will have the same name. The name 457 // should only be used for informational purposes, and not for identifying the 458 // display in code. 459 return "VirtualDevice_" + mVirtualDevice.getAssociationId(); 460 } catch (RemoteException e) { 461 throw e.rethrowFromSystemServer(); 462 } 463 } 464 465 /** 466 * Adds an activity listener to listen for events such as top activity change or virtual 467 * display task stack became empty. 468 * 469 * @param executor The executor where the listener is executed on. 470 * @param listener The listener to add. 471 * @see #removeActivityListener(ActivityListener) 472 */ addActivityListener( @allbackExecutor @onNull Executor executor, @NonNull ActivityListener listener)473 public void addActivityListener( 474 @CallbackExecutor @NonNull Executor executor, @NonNull ActivityListener listener) { 475 mActivityListeners.put(listener, new ActivityListenerDelegate(listener, executor)); 476 } 477 478 /** 479 * Removes an activity listener previously added with 480 * {@link #addActivityListener}. 481 * 482 * @param listener The listener to remove. 483 * @see #addActivityListener(Executor, ActivityListener) 484 */ removeActivityListener(@onNull ActivityListener listener)485 public void removeActivityListener(@NonNull ActivityListener listener) { 486 mActivityListeners.remove(listener); 487 } 488 } 489 490 /** 491 * Listener for activity changes in this virtual device. 492 */ 493 public interface ActivityListener { 494 495 /** 496 * Called when the top activity is changed. 497 * 498 * <p>Note: When there are no activities running on the virtual display, the 499 * {@link #onDisplayEmpty(int)} will be called. If the value topActivity is cached, it 500 * should be cleared when {@link #onDisplayEmpty(int)} is called. 501 * 502 * @param displayId The display ID on which the activity change happened. 503 * @param topActivity The component name of the top activity. 504 */ onTopActivityChanged(int displayId, @NonNull ComponentName topActivity)505 void onTopActivityChanged(int displayId, @NonNull ComponentName topActivity); 506 507 /** 508 * Called when the display becomes empty (e.g. if the user hits back on the last 509 * activity of the root task). 510 * 511 * @param displayId The display ID that became empty. 512 */ onDisplayEmpty(int displayId)513 void onDisplayEmpty(int displayId); 514 } 515 516 /** 517 * A wrapper for {@link ActivityListener} that executes callbacks on the given executor. 518 */ 519 private static class ActivityListenerDelegate { 520 @NonNull private final ActivityListener mActivityListener; 521 @NonNull private final Executor mExecutor; 522 ActivityListenerDelegate(@onNull ActivityListener listener, @NonNull Executor executor)523 ActivityListenerDelegate(@NonNull ActivityListener listener, @NonNull Executor executor) { 524 mActivityListener = listener; 525 mExecutor = executor; 526 } 527 onTopActivityChanged(int displayId, ComponentName topActivity)528 public void onTopActivityChanged(int displayId, ComponentName topActivity) { 529 mExecutor.execute(() -> mActivityListener.onTopActivityChanged(displayId, topActivity)); 530 } 531 onDisplayEmpty(int displayId)532 public void onDisplayEmpty(int displayId) { 533 mExecutor.execute(() -> mActivityListener.onDisplayEmpty(displayId)); 534 } 535 } 536 } 537