1 /* 2 * Copyright (C) 2025 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.example.android.vdmdemo.host; 18 19 import android.app.PictureInPictureParams; 20 import android.content.ComponentName; 21 import android.content.Context; 22 import android.content.Intent; 23 import android.content.ServiceConnection; 24 import android.graphics.Bitmap; 25 import android.graphics.Canvas; 26 import android.graphics.PixelFormat; 27 import android.graphics.Rect; 28 import android.graphics.SurfaceTexture; 29 import android.hardware.display.DisplayManager; 30 import android.media.Image; 31 import android.media.ImageReader; 32 import android.os.Bundle; 33 import android.os.IBinder; 34 import android.util.Log; 35 import android.util.Rational; 36 import android.view.Display; 37 import android.view.InputDevice; 38 import android.view.KeyEvent; 39 import android.view.Menu; 40 import android.view.MenuInflater; 41 import android.view.MenuItem; 42 import android.view.Surface; 43 import android.view.TextureView; 44 import android.view.View; 45 46 import androidx.activity.OnBackPressedCallback; 47 import androidx.annotation.GuardedBy; 48 import androidx.annotation.NonNull; 49 import androidx.appcompat.app.AppCompatActivity; 50 import androidx.appcompat.widget.Toolbar; 51 52 import com.example.android.vdmdemo.common.EdgeToEdgeUtils; 53 import com.example.android.vdmdemo.common.RemoteEventProto; 54 55 import dagger.hilt.android.AndroidEntryPoint; 56 57 import java.nio.ByteBuffer; 58 59 import javax.inject.Inject; 60 61 /** 62 * VDM activity, showing an interactive virtual display. 63 */ 64 @AndroidEntryPoint(AppCompatActivity.class) 65 public class DisplayActivity extends Hilt_DisplayActivity 66 implements DisplayManager.DisplayListener { 67 68 public static final String TAG = "VdmHost_DisplayActivity"; 69 70 // Approximately, see 71 // https://developer.android.com/reference/android/util/DisplayMetrics#density 72 private static final float DIP_TO_DPI = 160f; 73 74 /** @see android.app.PictureInPictureParams.Builder#setAspectRatio(android.util.Rational) */ 75 private static final Rational MAX_PIP_RATIO = new Rational(239, 100); 76 private static final Rational MIN_PIP_RATIO = new Rational(100, 239); 77 78 static final String EXTRA_DISPLAY_ID = "displayId"; 79 80 81 @Inject 82 InputController mInputController; 83 84 DisplayManager mDisplayManager; 85 86 private VdmService mVdmService = null; 87 private int mDisplayId; 88 89 private final Object mLock = new Object(); 90 @GuardedBy("mLock") 91 private Surface mSurface; 92 private int mSurfaceWidth; 93 private int mSurfaceHeight; 94 private int mDpi; 95 96 private RemoteDisplay mDisplay; 97 private boolean mPoweredOn = true; 98 private ImageReader mImageReader; 99 100 private final ServiceConnection mServiceConnection = new ServiceConnection() { 101 @Override 102 public void onServiceConnected(ComponentName className, IBinder binder) { 103 synchronized (mLock) { 104 Log.d(TAG, "Connected to VDM Service"); 105 mVdmService = ((VdmService.LocalBinder) binder).getService(); 106 mDisplay = mVdmService.getRemoteDisplay(mDisplayId).orElseGet(() -> 107 mVdmService.createRemoteDisplay( 108 DisplayActivity.this, mDisplayId, 200, 200, mDpi, null)); 109 } 110 if (isInPictureInPictureMode()) { 111 Log.v(TAG, "Initializing copy from display " + mDisplayId + " to PIP window"); 112 mImageReader = ImageReader.newInstance( 113 mDisplay.getWidth(), mDisplay.getHeight(), PixelFormat.RGBA_8888, 2); 114 mDisplay.setSurface(mImageReader.getSurface()); 115 mImageReader.setOnImageAvailableListener((reader) -> { 116 Image image = reader.acquireLatestImage(); 117 synchronized (mLock) { 118 if (image != null && mSurface != null) { 119 copyImageToSurfaceLocked(image); 120 image.close(); 121 } 122 } 123 }, null); 124 } else { 125 synchronized (mLock) { 126 if (mSurface != null) { 127 resetDisplayLocked(); 128 setPictureInPictureParams(buildPictureInPictureParams()); 129 } 130 } 131 } 132 } 133 134 @Override 135 public void onServiceDisconnected(ComponentName className) { 136 Log.d(TAG, "Disconnected from VDM Service"); 137 mVdmService = null; 138 } 139 }; 140 141 @Override onCreate(Bundle savedInstanceState)142 public void onCreate(Bundle savedInstanceState) { 143 super.onCreate(savedInstanceState); 144 145 mDisplayManager = getSystemService(DisplayManager.class); 146 mDisplayManager.registerDisplayListener(this, null); 147 mDisplayId = getIntent().getIntExtra(EXTRA_DISPLAY_ID, Display.INVALID_DISPLAY); 148 mDpi = (int) (getResources().getDisplayMetrics().density * DIP_TO_DPI); 149 150 setContentView(R.layout.activity_display); 151 TextureView textureView = requireViewById(R.id.display_surface_view); 152 Toolbar toolbar = requireViewById(R.id.main_tool_bar); 153 if (isInPictureInPictureMode()) { 154 toolbar.setVisibility(View.GONE); 155 } else { 156 setSupportActionBar(toolbar); 157 setTitle(getTitle() + " " + mDisplayId); 158 EdgeToEdgeUtils.applyTopInsets(toolbar); 159 EdgeToEdgeUtils.applyBottomInsets(textureView); 160 } 161 162 textureView.setSurfaceTextureListener(new TextureView.SurfaceTextureListener() { 163 @Override 164 public void onSurfaceTextureUpdated(@NonNull SurfaceTexture texture) {} 165 166 @Override 167 public void onSurfaceTextureAvailable( 168 @NonNull SurfaceTexture texture, int width, int height) { 169 synchronized (mLock) { 170 Log.d(TAG, "onSurfaceTextureAvailable for local display " + mDisplayId); 171 mSurfaceWidth = width; 172 mSurfaceHeight = height; 173 mSurface = new Surface(texture); 174 if (!isInPictureInPictureMode() && mDisplay != null) { 175 resetDisplayLocked(); 176 } 177 } 178 } 179 180 @Override 181 public boolean onSurfaceTextureDestroyed(@NonNull SurfaceTexture texture) { 182 Log.v(TAG, "onSurfaceTextureDestroyed for local display " + mDisplayId); 183 mSurface = null; 184 return true; 185 } 186 187 @Override 188 public void onSurfaceTextureSizeChanged( 189 @NonNull SurfaceTexture texture, int width, int height) { 190 Log.v(TAG, "onSurfaceTextureSizeChanged for local display " + mDisplayId); 191 synchronized (mLock) { 192 mSurfaceWidth = width; 193 mSurfaceHeight = height; 194 if (!isInPictureInPictureMode() && mDisplay != null) { 195 resetDisplayLocked(); 196 } 197 } 198 } 199 }); 200 201 textureView.setOnTouchListener((v, event) -> { 202 if (event.getDevice().supportsSource(InputDevice.SOURCE_TOUCHSCREEN) 203 && mDisplay != null) { 204 textureView.getParent().requestDisallowInterceptTouchEvent(true); 205 mDisplay.processInputEvent( 206 RemoteEventProto.InputDeviceType.DEVICE_TYPE_TOUCHSCREEN, event); 207 } 208 return true; 209 }); 210 211 textureView.setOnGenericMotionListener((v, event) -> { 212 if (event.getDevice() == null || mDisplay == null 213 || !event.getDevice().supportsSource(InputDevice.SOURCE_MOUSE)) { 214 return false; 215 } 216 mDisplay.processVirtualMouseEvent(event); 217 return true; 218 }); 219 220 OnBackPressedCallback callback = new OnBackPressedCallback(true) { 221 @Override 222 public void handleOnBackPressed() { 223 if (mDisplay != null) { 224 mDisplay.sendBack(); 225 } 226 } 227 }; 228 getOnBackPressedDispatcher().addCallback(this, callback); 229 } 230 231 @Override onStart()232 protected void onStart() { 233 super.onStart(); 234 Intent intent = new Intent(this, VdmService.class); 235 bindService(intent, mServiceConnection, Context.BIND_AUTO_CREATE); 236 } 237 238 @Override onResume()239 protected void onResume() { 240 super.onResume(); 241 mInputController.setFocusedRemoteDisplayId(mDisplayId); 242 } 243 244 @Override onStop()245 protected void onStop() { 246 super.onStop(); 247 unbindService(mServiceConnection); 248 } 249 250 @Override onDestroy()251 protected void onDestroy() { 252 super.onDestroy(); 253 mDisplayManager.unregisterDisplayListener(this); 254 if (mImageReader != null) { 255 mImageReader.close(); 256 mImageReader = null; 257 } 258 } 259 260 @Override onCreateOptionsMenu(Menu menu)261 public boolean onCreateOptionsMenu(Menu menu) { 262 MenuInflater inflater = getMenuInflater(); 263 inflater.inflate(R.menu.display, menu); 264 return true; 265 } 266 267 @Override onOptionsItemSelected(MenuItem item)268 public boolean onOptionsItemSelected(MenuItem item) { 269 switch (item.getItemId()) { 270 case R.id.close: 271 if (mVdmService != null) { 272 mVdmService.closeRemoteDisplay(mDisplayId); 273 } 274 return true; 275 case R.id.pip: 276 enterPictureInPictureMode(buildPictureInPictureParams()); 277 return true; 278 case R.id.power: 279 if (mDisplay != null) { 280 mPoweredOn = !mPoweredOn; 281 mVdmService.setPowerState(mPoweredOn); 282 } 283 return true; 284 case R.id.home: 285 if (mDisplay != null) { 286 mDisplay.goHome(); 287 } 288 return true; 289 case R.id.back: 290 if (mDisplay != null) { 291 mDisplay.sendBack(); 292 } 293 return true; 294 default: 295 return super.onOptionsItemSelected(item); 296 } 297 } 298 299 @Override dispatchKeyEvent(KeyEvent event)300 public boolean dispatchKeyEvent(KeyEvent event) { 301 mDisplay.processInputEvent(RemoteEventProto.InputDeviceType.DEVICE_TYPE_KEYBOARD, event); 302 return true; 303 } 304 buildPictureInPictureParams()305 private PictureInPictureParams buildPictureInPictureParams() { 306 Rational ratio = new Rational(mDisplay.getWidth(), mDisplay.getHeight()); 307 if (ratio.compareTo(MAX_PIP_RATIO) > 0) { 308 ratio = MAX_PIP_RATIO; 309 } else if (ratio.compareTo(MIN_PIP_RATIO) < 0) { 310 ratio = MIN_PIP_RATIO; 311 } 312 Rect rect = new Rect(); 313 View textureView = requireViewById(R.id.display_surface_view); 314 textureView.getGlobalVisibleRect(rect); 315 return new PictureInPictureParams.Builder() 316 .setAutoEnterEnabled(true) 317 .setAspectRatio(ratio) 318 .setExpandedAspectRatio(ratio) 319 .setSourceRectHint(rect) 320 .setSeamlessResizeEnabled(false) 321 .build(); 322 } 323 324 @GuardedBy("mLock") resetDisplayLocked()325 private void resetDisplayLocked() { 326 if (mDisplay.getWidth() != mSurfaceWidth || mDisplay.getHeight() != mSurfaceHeight) { 327 Log.v(TAG, "Resizing display " + mDisplayId + " to " + mSurfaceWidth 328 + "/" + mSurfaceHeight); 329 mDisplay.reset(mSurfaceWidth, mSurfaceHeight, mDpi); 330 } 331 mDisplay.setSurface(mSurface); 332 } 333 334 @GuardedBy("mLock") copyImageToSurfaceLocked(Image image)335 private void copyImageToSurfaceLocked(Image image) { 336 ByteBuffer buffer = image.getPlanes()[0].getBuffer(); 337 int pixelStride = image.getPlanes()[0].getPixelStride(); 338 int rowStride = image.getPlanes()[0].getRowStride(); 339 int pixelBytesPerRow = pixelStride * image.getWidth(); 340 int rowPadding = rowStride - pixelBytesPerRow; 341 342 // Remove the row padding bytes from the buffer before converting to a Bitmap 343 ByteBuffer trimmedBuffer = ByteBuffer.allocate(buffer.remaining()); 344 buffer.rewind(); 345 while (buffer.hasRemaining()) { 346 for (int i = 0; i < pixelBytesPerRow; ++i) { 347 trimmedBuffer.put(buffer.get()); 348 } 349 buffer.position(buffer.position() + rowPadding); // Skip the padding bytes 350 } 351 trimmedBuffer.flip(); // Prepare the trimmed buffer for reading 352 353 Canvas canvas = mSurface.lockCanvas(null); 354 Bitmap bitmap = 355 Bitmap.createBitmap(image.getWidth(), image.getHeight(), Bitmap.Config.ARGB_8888); 356 bitmap.copyPixelsFromBuffer(trimmedBuffer); 357 Bitmap scaled = Bitmap.createScaledBitmap(bitmap, mSurfaceWidth, mSurfaceHeight, false); 358 // Draw the Bitmap onto the Canvas 359 canvas.drawBitmap(scaled, 0f, 0f, null); 360 361 bitmap.recycle(); 362 mSurface.unlockCanvasAndPost(canvas); 363 } 364 365 @Override onDisplayAdded(int displayId)366 public void onDisplayAdded(int displayId) {} 367 368 @Override onDisplayRemoved(int displayId)369 public void onDisplayRemoved(int displayId) { 370 if (mDisplay != null && displayId == mDisplay.getDisplayId()) { 371 finishAndRemoveTask(); 372 } 373 } 374 375 @Override onDisplayChanged(int displayId)376 public void onDisplayChanged(int displayId) {} 377 } 378