1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.systemui.screenrecord; 18 19 import static android.content.Context.MEDIA_PROJECTION_SERVICE; 20 21 import static com.android.systemui.screenrecord.ScreenRecordingAudioSource.INTERNAL; 22 import static com.android.systemui.screenrecord.ScreenRecordingAudioSource.MIC; 23 import static com.android.systemui.screenrecord.ScreenRecordingAudioSource.MIC_AND_INTERNAL; 24 25 import android.annotation.Nullable; 26 import android.app.ActivityManager; 27 import android.content.ContentResolver; 28 import android.content.ContentValues; 29 import android.content.Context; 30 import android.graphics.Bitmap; 31 import android.graphics.drawable.Icon; 32 import android.hardware.display.DisplayManager; 33 import android.hardware.display.VirtualDisplay; 34 import android.media.MediaCodec; 35 import android.media.MediaCodecInfo; 36 import android.media.MediaFormat; 37 import android.media.MediaMuxer; 38 import android.media.MediaRecorder; 39 import android.media.ThumbnailUtils; 40 import android.media.projection.IMediaProjection; 41 import android.media.projection.IMediaProjectionManager; 42 import android.media.projection.MediaProjection; 43 import android.media.projection.MediaProjectionManager; 44 import android.media.projection.StopReason; 45 import android.net.Uri; 46 import android.os.Handler; 47 import android.os.IBinder; 48 import android.os.RemoteException; 49 import android.os.ServiceManager; 50 import android.provider.MediaStore; 51 import android.util.DisplayMetrics; 52 import android.util.Log; 53 import android.util.Size; 54 import android.view.Display; 55 import android.view.Surface; 56 57 import com.android.internal.R; 58 import com.android.systemui.mediaprojection.MediaProjectionCaptureTarget; 59 import com.android.systemui.recordissue.ScreenRecordingStartTimeStore; 60 61 import java.io.Closeable; 62 import java.io.File; 63 import java.io.IOException; 64 import java.io.OutputStream; 65 import java.nio.file.Files; 66 import java.text.SimpleDateFormat; 67 import java.util.ArrayList; 68 import java.util.Date; 69 import java.util.List; 70 71 /** 72 * Recording screen and mic/internal audio 73 */ 74 public class ScreenMediaRecorder extends MediaProjection.Callback { 75 private static final int TOTAL_NUM_TRACKS = 1; 76 private static final int VIDEO_FRAME_RATE = 30; 77 private static final int VIDEO_FRAME_RATE_TO_RESOLUTION_RATIO = 6; 78 private static final int AUDIO_BIT_RATE = 196000; 79 private static final int AUDIO_SAMPLE_RATE = 44100; 80 private static final int MAX_DURATION_MS = 60 * 60 * 1000; 81 private static final long MAX_FILESIZE_BYTES = 5000000000L; 82 private static final String TAG = "ScreenMediaRecorder"; 83 84 85 private File mTempVideoFile; 86 private File mTempAudioFile; 87 private MediaProjection mMediaProjection; 88 private Surface mInputSurface; 89 private VirtualDisplay mVirtualDisplay; 90 private MediaRecorder mMediaRecorder; 91 private int mUid; 92 private ScreenRecordingMuxer mMuxer; 93 private ScreenInternalAudioRecorder mAudio; 94 private ScreenRecordingAudioSource mAudioSource; 95 private final MediaProjectionCaptureTarget mCaptureRegion; 96 private final ScreenRecordingStartTimeStore mScreenRecordingStartTimeStore; 97 private final Handler mHandler; 98 private final int mDisplayId; 99 100 private Context mContext; 101 ScreenMediaRecorderListener mListener; 102 ScreenMediaRecorder( Context context, Handler handler, int uid, ScreenRecordingAudioSource audioSource, MediaProjectionCaptureTarget captureRegion, int displayId, ScreenMediaRecorderListener listener, ScreenRecordingStartTimeStore screenRecordingStartTimeStore)103 public ScreenMediaRecorder( 104 Context context, 105 Handler handler, 106 int uid, 107 ScreenRecordingAudioSource audioSource, 108 MediaProjectionCaptureTarget captureRegion, 109 int displayId, 110 ScreenMediaRecorderListener listener, 111 ScreenRecordingStartTimeStore screenRecordingStartTimeStore) { 112 mContext = context; 113 mHandler = handler; 114 mUid = uid; 115 mCaptureRegion = captureRegion; 116 mListener = listener; 117 mAudioSource = audioSource; 118 mDisplayId = displayId; 119 mScreenRecordingStartTimeStore = screenRecordingStartTimeStore; 120 } 121 prepare()122 private void prepare() throws IOException, RemoteException, RuntimeException { 123 //Setup media projection 124 IBinder b = ServiceManager.getService(MEDIA_PROJECTION_SERVICE); 125 IMediaProjectionManager mediaService = 126 IMediaProjectionManager.Stub.asInterface(b); 127 IMediaProjection proj = 128 mediaService.createProjection( 129 mUid, 130 mContext.getPackageName(), 131 MediaProjectionManager.TYPE_SCREEN_CAPTURE, 132 false, 133 mDisplayId); 134 IMediaProjection projection = IMediaProjection.Stub.asInterface(proj.asBinder()); 135 if (mCaptureRegion != null) { 136 projection.setLaunchCookie(mCaptureRegion.getLaunchCookie()); 137 projection.setTaskId(mCaptureRegion.getTaskId()); 138 } 139 mMediaProjection = new MediaProjection(mContext, projection); 140 mMediaProjection.registerCallback(this, mHandler); 141 142 File cacheDir = mContext.getCacheDir(); 143 cacheDir.mkdirs(); 144 mTempVideoFile = File.createTempFile("temp", ".mp4", cacheDir); 145 146 // Set up media recorder 147 mMediaRecorder = new MediaRecorder(); 148 149 // Set up audio source 150 if (mAudioSource == MIC) { 151 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.DEFAULT); 152 } 153 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 154 155 mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4); 156 157 158 // Set up video 159 DisplayMetrics metrics = new DisplayMetrics(); 160 DisplayManager dm = mContext.getSystemService(DisplayManager.class); 161 Display display = dm.getDisplay(mDisplayId); 162 display.getRealMetrics(metrics); 163 int refreshRate = (int) display.getRefreshRate(); 164 int[] dimens = getSupportedSize(metrics.widthPixels, metrics.heightPixels, refreshRate); 165 int width = dimens[0]; 166 int height = dimens[1]; 167 refreshRate = dimens[2]; 168 int vidBitRate = width * height * refreshRate / VIDEO_FRAME_RATE 169 * VIDEO_FRAME_RATE_TO_RESOLUTION_RATIO; 170 mMediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264); 171 mMediaRecorder.setVideoEncodingProfileLevel( 172 MediaCodecInfo.CodecProfileLevel.AVCProfileHigh, 173 MediaCodecInfo.CodecProfileLevel.AVCLevel3); 174 mMediaRecorder.setVideoSize(width, height); 175 mMediaRecorder.setVideoFrameRate(refreshRate); 176 mMediaRecorder.setVideoEncodingBitRate(vidBitRate); 177 mMediaRecorder.setMaxDuration(MAX_DURATION_MS); 178 mMediaRecorder.setMaxFileSize(MAX_FILESIZE_BYTES); 179 180 // Set up audio 181 if (mAudioSource == MIC) { 182 mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.HE_AAC); 183 mMediaRecorder.setAudioChannels(TOTAL_NUM_TRACKS); 184 mMediaRecorder.setAudioEncodingBitRate(AUDIO_BIT_RATE); 185 mMediaRecorder.setAudioSamplingRate(AUDIO_SAMPLE_RATE); 186 } 187 188 mMediaRecorder.setOutputFile(mTempVideoFile); 189 mMediaRecorder.prepare(); 190 // Create surface 191 mInputSurface = mMediaRecorder.getSurface(); 192 mVirtualDisplay = mMediaProjection.createVirtualDisplay( 193 "Recording Display", 194 width, 195 height, 196 metrics.densityDpi, 197 DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, 198 mInputSurface, 199 new VirtualDisplay.Callback() { 200 @Override 201 public void onStopped() { 202 onStop(); 203 } 204 }, 205 mHandler); 206 207 mMediaRecorder.setOnInfoListener((mr, what, extra) -> mListener.onInfo(mr, what, extra)); 208 if (mAudioSource == INTERNAL || 209 mAudioSource == MIC_AND_INTERNAL) { 210 mTempAudioFile = File.createTempFile("temp", ".aac", 211 mContext.getCacheDir()); 212 mAudio = new ScreenInternalAudioRecorder(mTempAudioFile.getAbsolutePath(), 213 mMediaProjection, mAudioSource == MIC_AND_INTERNAL); 214 } 215 216 } 217 218 /** 219 * Find the highest supported screen resolution and refresh rate for the given dimensions on 220 * this device, up to actual size and given rate. 221 * If possible this will return the same values as given, but values may be smaller on some 222 * devices. 223 * 224 * @param screenWidth Actual pixel width of screen 225 * @param screenHeight Actual pixel height of screen 226 * @param refreshRate Desired refresh rate 227 * @return array with supported width, height, and refresh rate 228 */ getSupportedSize(final int screenWidth, final int screenHeight, int refreshRate)229 private int[] getSupportedSize(final int screenWidth, final int screenHeight, int refreshRate) 230 throws IOException { 231 String videoType = MediaFormat.MIMETYPE_VIDEO_AVC; 232 233 // Get max size from the decoder, to ensure recordings will be playable on device 234 MediaCodec decoder = MediaCodec.createDecoderByType(videoType); 235 MediaCodecInfo.VideoCapabilities vc = decoder.getCodecInfo() 236 .getCapabilitiesForType(videoType).getVideoCapabilities(); 237 decoder.release(); 238 239 // Check if we can support screen size as-is 240 int width = vc.getSupportedWidths().getUpper(); 241 int height = vc.getSupportedHeights().getUpper(); 242 243 int screenWidthAligned = screenWidth; 244 if (screenWidthAligned % vc.getWidthAlignment() != 0) { 245 screenWidthAligned -= (screenWidthAligned % vc.getWidthAlignment()); 246 } 247 int screenHeightAligned = screenHeight; 248 if (screenHeightAligned % vc.getHeightAlignment() != 0) { 249 screenHeightAligned -= (screenHeightAligned % vc.getHeightAlignment()); 250 } 251 252 if (width >= screenWidthAligned && height >= screenHeightAligned 253 && vc.isSizeSupported(screenWidthAligned, screenHeightAligned)) { 254 // Desired size is supported, now get the rate 255 int maxRate = vc.getSupportedFrameRatesFor(screenWidthAligned, 256 screenHeightAligned).getUpper().intValue(); 257 258 if (maxRate < refreshRate) { 259 refreshRate = maxRate; 260 } 261 Log.d(TAG, "Screen size supported at rate " + refreshRate); 262 return new int[]{screenWidthAligned, screenHeightAligned, refreshRate}; 263 } 264 265 // Otherwise, resize for max supported size 266 double scale = Math.min(((double) width / screenWidth), 267 ((double) height / screenHeight)); 268 269 int scaledWidth = (int) (screenWidth * scale); 270 int scaledHeight = (int) (screenHeight * scale); 271 if (scaledWidth % vc.getWidthAlignment() != 0) { 272 scaledWidth -= (scaledWidth % vc.getWidthAlignment()); 273 } 274 if (scaledHeight % vc.getHeightAlignment() != 0) { 275 scaledHeight -= (scaledHeight % vc.getHeightAlignment()); 276 } 277 278 // Find max supported rate for size 279 int maxRate = vc.getSupportedFrameRatesFor(scaledWidth, scaledHeight) 280 .getUpper().intValue(); 281 if (maxRate < refreshRate) { 282 refreshRate = maxRate; 283 } 284 285 Log.d(TAG, "Resized by " + scale + ": " + scaledWidth + ", " + scaledHeight 286 + ", " + refreshRate); 287 return new int[]{scaledWidth, scaledHeight, refreshRate}; 288 } 289 290 /** 291 * Start screen recording 292 */ start()293 void start() throws IOException, RemoteException, RuntimeException { 294 Log.d(TAG, "start recording"); 295 prepare(); 296 mMediaRecorder.start(); 297 mScreenRecordingStartTimeStore.markStartTime(); 298 recordInternalAudio(); 299 } 300 301 /** 302 * End screen recording, throws an exception if stopping recording failed 303 */ end(@topReason int stopReason)304 void end(@StopReason int stopReason) throws IOException { 305 Closer closer = new Closer(); 306 307 // MediaRecorder might throw RuntimeException if stopped immediately after starting 308 // We should remove the recording in this case as it will be invalid 309 closer.register(mMediaRecorder::stop); 310 closer.register(mMediaRecorder::release); 311 closer.register(mInputSurface::release); 312 closer.register(mVirtualDisplay::release); 313 closer.register(() -> { 314 if (stopReason == StopReason.STOP_UNKNOWN) { 315 // Attempt to call MediaProjection#stop() even if it might have already been called. 316 // If projection has already been stopped, then nothing will happen. Else, stop 317 // will be logged as a manually requested stop from host app. 318 mMediaProjection.stop(); 319 } else { 320 // In any other case, the stop reason is related to the recorder, so pass it on here 321 mMediaProjection.stop(stopReason); 322 } 323 }); 324 closer.register(this::stopInternalAudioRecording); 325 326 closer.close(); 327 328 mMediaRecorder = null; 329 mMediaProjection = null; 330 331 Log.d(TAG, "end recording"); 332 } 333 334 @Override onStop()335 public void onStop() { 336 Log.d(TAG, "The system notified about stopping the projection"); 337 mListener.onStopped(StopReason.STOP_UNKNOWN); 338 } 339 stopInternalAudioRecording()340 private void stopInternalAudioRecording() { 341 if (mAudioSource == INTERNAL || mAudioSource == MIC_AND_INTERNAL) { 342 mAudio.end(); 343 mAudio = null; 344 } 345 } 346 recordInternalAudio()347 private void recordInternalAudio() throws IllegalStateException { 348 if (mAudioSource == INTERNAL || mAudioSource == MIC_AND_INTERNAL) { 349 mAudio.start(); 350 } 351 } 352 353 /** 354 * Store recorded video 355 */ save()356 protected SavedRecording save() throws IOException, IllegalStateException { 357 String fileName = new SimpleDateFormat("'screen-'yyyyMMdd-HHmmss'.mp4'") 358 .format(new Date()); 359 360 ContentValues values = new ContentValues(); 361 values.put(MediaStore.Video.Media.DISPLAY_NAME, fileName); 362 values.put(MediaStore.Video.Media.MIME_TYPE, "video/mp4"); 363 values.put(MediaStore.Video.Media.DATE_ADDED, System.currentTimeMillis()); 364 values.put(MediaStore.Video.Media.DATE_TAKEN, System.currentTimeMillis()); 365 366 ContentResolver resolver = mContext.getContentResolver(); 367 Uri collectionUri = MediaStore.Video.Media.getContentUri( 368 MediaStore.VOLUME_EXTERNAL_PRIMARY); 369 Uri itemUri = resolver.insert(collectionUri, values); 370 371 Log.d(TAG, itemUri.toString()); 372 if (mAudioSource == MIC_AND_INTERNAL || mAudioSource == INTERNAL) { 373 try { 374 Log.d(TAG, "muxing recording"); 375 File file = File.createTempFile("temp", ".mp4", 376 mContext.getCacheDir()); 377 mMuxer = new ScreenRecordingMuxer(MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4, 378 file.getAbsolutePath(), 379 mTempVideoFile.getAbsolutePath(), 380 mTempAudioFile.getAbsolutePath()); 381 mMuxer.mux(); 382 mTempVideoFile.delete(); 383 mTempVideoFile = file; 384 } catch (IOException e) { 385 Log.e(TAG, "muxing recording " + e.getMessage()); 386 e.printStackTrace(); 387 } 388 } 389 390 // Add to the mediastore 391 OutputStream os = resolver.openOutputStream(itemUri, "w"); 392 Files.copy(mTempVideoFile.toPath(), os); 393 os.close(); 394 if (mTempAudioFile != null) mTempAudioFile.delete(); 395 SavedRecording recording = new SavedRecording( 396 itemUri, mTempVideoFile, getRequiredThumbnailSize()); 397 mTempVideoFile.delete(); 398 return recording; 399 } 400 401 /** 402 * Returns the required {@code Size} of the thumbnail. 403 */ getRequiredThumbnailSize()404 private Size getRequiredThumbnailSize() { 405 boolean isLowRam = ActivityManager.isLowRamDeviceStatic(); 406 int thumbnailIconHeight = mContext.getResources().getDimensionPixelSize(isLowRam 407 ? R.dimen.notification_big_picture_max_height_low_ram 408 : R.dimen.notification_big_picture_max_height); 409 int thumbnailIconWidth = mContext.getResources().getDimensionPixelSize(isLowRam 410 ? R.dimen.notification_big_picture_max_width_low_ram 411 : R.dimen.notification_big_picture_max_width); 412 return new Size(thumbnailIconWidth, thumbnailIconHeight); 413 } 414 415 /** 416 * Release the resources without saving the data 417 */ release()418 protected void release() { 419 if (mTempVideoFile != null) { 420 mTempVideoFile.delete(); 421 } 422 if (mTempAudioFile != null) { 423 mTempAudioFile.delete(); 424 } 425 } 426 427 /** 428 * Object representing the recording 429 */ 430 public class SavedRecording { 431 432 private Uri mUri; 433 private Icon mThumbnailIcon; 434 SavedRecording(Uri uri, File file, Size thumbnailSize)435 protected SavedRecording(Uri uri, File file, Size thumbnailSize) { 436 mUri = uri; 437 try { 438 Bitmap thumbnailBitmap = ThumbnailUtils.createVideoThumbnail( 439 file, thumbnailSize, null); 440 mThumbnailIcon = Icon.createWithBitmap(thumbnailBitmap); 441 } catch (IOException e) { 442 Log.e(TAG, "Error creating thumbnail", e); 443 } 444 } 445 getUri()446 public Uri getUri() { 447 return mUri; 448 } 449 getThumbnail()450 public @Nullable Icon getThumbnail() { 451 return mThumbnailIcon; 452 } 453 } 454 455 interface ScreenMediaRecorderListener { 456 /** 457 * Called to indicate an info or a warning during recording. 458 * See {@link MediaRecorder.OnInfoListener} for the full description. 459 */ onInfo(MediaRecorder mr, int what, int extra)460 void onInfo(MediaRecorder mr, int what, int extra); 461 462 /** 463 * Called when the recording stopped by the system. 464 * For example, this might happen when doing partial screen sharing of an app 465 * and the app that is being captured is closed. 466 */ onStopped(@topReason int stopReason)467 void onStopped(@StopReason int stopReason); 468 } 469 470 /** 471 * Allows to register multiple {@link Closeable} objects and close them all by calling 472 * {@link Closer#close}. If there is an exception thrown during closing of one 473 * of the registered closeables it will continue trying closing the rest closeables. 474 * If there are one or more exceptions thrown they will be re-thrown at the end. 475 * In case of multiple exceptions only the first one will be thrown and all the rest 476 * will be printed. 477 */ 478 private static class Closer implements Closeable { 479 private final List<Closeable> mCloseables = new ArrayList<>(); 480 register(Closeable closeable)481 void register(Closeable closeable) { 482 mCloseables.add(closeable); 483 } 484 485 @Override close()486 public void close() throws IOException { 487 Throwable throwable = null; 488 489 for (int i = 0; i < mCloseables.size(); i++) { 490 Closeable closeable = mCloseables.get(i); 491 492 try { 493 closeable.close(); 494 } catch (Throwable e) { 495 if (throwable == null) { 496 throwable = e; 497 } else { 498 e.printStackTrace(); 499 } 500 } 501 } 502 503 if (throwable != null) { 504 if (throwable instanceof IOException) { 505 throw (IOException) throwable; 506 } 507 508 if (throwable instanceof RuntimeException) { 509 throw (RuntimeException) throwable; 510 } 511 512 throw (Error) throwable; 513 } 514 } 515 } 516 } 517