1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.systemui.screenrecord; 18 19 import static android.content.Context.MEDIA_PROJECTION_SERVICE; 20 21 import static com.android.systemui.screenrecord.ScreenRecordingAudioSource.INTERNAL; 22 import static com.android.systemui.screenrecord.ScreenRecordingAudioSource.MIC; 23 import static com.android.systemui.screenrecord.ScreenRecordingAudioSource.MIC_AND_INTERNAL; 24 25 import android.annotation.Nullable; 26 import android.content.ContentResolver; 27 import android.content.ContentValues; 28 import android.content.Context; 29 import android.graphics.Bitmap; 30 import android.hardware.display.DisplayManager; 31 import android.hardware.display.VirtualDisplay; 32 import android.media.MediaCodec; 33 import android.media.MediaCodecInfo; 34 import android.media.MediaFormat; 35 import android.media.MediaMuxer; 36 import android.media.MediaRecorder; 37 import android.media.ThumbnailUtils; 38 import android.media.projection.IMediaProjection; 39 import android.media.projection.IMediaProjectionManager; 40 import android.media.projection.MediaProjection; 41 import android.media.projection.MediaProjectionManager; 42 import android.net.Uri; 43 import android.os.Handler; 44 import android.os.IBinder; 45 import android.os.RemoteException; 46 import android.os.ServiceManager; 47 import android.provider.MediaStore; 48 import android.util.DisplayMetrics; 49 import android.util.Log; 50 import android.util.Size; 51 import android.view.Surface; 52 import android.view.WindowManager; 53 54 import com.android.systemui.media.MediaProjectionCaptureTarget; 55 import java.io.File; 56 import java.io.Closeable; 57 import java.io.IOException; 58 import java.io.OutputStream; 59 import java.nio.file.Files; 60 import java.text.SimpleDateFormat; 61 import java.util.ArrayList; 62 import java.util.Date; 63 import java.util.List; 64 65 /** 66 * Recording screen and mic/internal audio 67 */ 68 public class ScreenMediaRecorder extends MediaProjection.Callback { 69 private static final int TOTAL_NUM_TRACKS = 1; 70 private static final int VIDEO_FRAME_RATE = 30; 71 private static final int VIDEO_FRAME_RATE_TO_RESOLUTION_RATIO = 6; 72 private static final int AUDIO_BIT_RATE = 196000; 73 private static final int AUDIO_SAMPLE_RATE = 44100; 74 private static final int MAX_DURATION_MS = 60 * 60 * 1000; 75 private static final long MAX_FILESIZE_BYTES = 5000000000L; 76 private static final String TAG = "ScreenMediaRecorder"; 77 78 79 private File mTempVideoFile; 80 private File mTempAudioFile; 81 private MediaProjection mMediaProjection; 82 private Surface mInputSurface; 83 private VirtualDisplay mVirtualDisplay; 84 private MediaRecorder mMediaRecorder; 85 private int mUser; 86 private ScreenRecordingMuxer mMuxer; 87 private ScreenInternalAudioRecorder mAudio; 88 private ScreenRecordingAudioSource mAudioSource; 89 private final MediaProjectionCaptureTarget mCaptureRegion; 90 private final Handler mHandler; 91 92 private Context mContext; 93 ScreenMediaRecorderListener mListener; 94 ScreenMediaRecorder(Context context, Handler handler, int user, ScreenRecordingAudioSource audioSource, MediaProjectionCaptureTarget captureRegion, ScreenMediaRecorderListener listener)95 public ScreenMediaRecorder(Context context, Handler handler, 96 int user, ScreenRecordingAudioSource audioSource, 97 MediaProjectionCaptureTarget captureRegion, 98 ScreenMediaRecorderListener listener) { 99 mContext = context; 100 mHandler = handler; 101 mUser = user; 102 mCaptureRegion = captureRegion; 103 mListener = listener; 104 mAudioSource = audioSource; 105 } 106 prepare()107 private void prepare() throws IOException, RemoteException, RuntimeException { 108 //Setup media projection 109 IBinder b = ServiceManager.getService(MEDIA_PROJECTION_SERVICE); 110 IMediaProjectionManager mediaService = 111 IMediaProjectionManager.Stub.asInterface(b); 112 IMediaProjection proj = null; 113 proj = mediaService.createProjection(mUser, mContext.getPackageName(), 114 MediaProjectionManager.TYPE_SCREEN_CAPTURE, false); 115 IMediaProjection projection = IMediaProjection.Stub.asInterface(proj.asBinder()); 116 if (mCaptureRegion != null) { 117 projection.setLaunchCookie(mCaptureRegion.getLaunchCookie()); 118 } 119 mMediaProjection = new MediaProjection(mContext, projection); 120 mMediaProjection.registerCallback(this, mHandler); 121 122 File cacheDir = mContext.getCacheDir(); 123 cacheDir.mkdirs(); 124 mTempVideoFile = File.createTempFile("temp", ".mp4", cacheDir); 125 126 // Set up media recorder 127 mMediaRecorder = new MediaRecorder(); 128 129 // Set up audio source 130 if (mAudioSource == MIC) { 131 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.DEFAULT); 132 } 133 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 134 135 mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4); 136 137 138 // Set up video 139 DisplayMetrics metrics = new DisplayMetrics(); 140 WindowManager wm = (WindowManager) mContext.getSystemService(Context.WINDOW_SERVICE); 141 wm.getDefaultDisplay().getRealMetrics(metrics); 142 int refreshRate = (int) wm.getDefaultDisplay().getRefreshRate(); 143 int[] dimens = getSupportedSize(metrics.widthPixels, metrics.heightPixels, refreshRate); 144 int width = dimens[0]; 145 int height = dimens[1]; 146 refreshRate = dimens[2]; 147 int vidBitRate = width * height * refreshRate / VIDEO_FRAME_RATE 148 * VIDEO_FRAME_RATE_TO_RESOLUTION_RATIO; 149 mMediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264); 150 mMediaRecorder.setVideoEncodingProfileLevel( 151 MediaCodecInfo.CodecProfileLevel.AVCProfileHigh, 152 MediaCodecInfo.CodecProfileLevel.AVCLevel3); 153 mMediaRecorder.setVideoSize(width, height); 154 mMediaRecorder.setVideoFrameRate(refreshRate); 155 mMediaRecorder.setVideoEncodingBitRate(vidBitRate); 156 mMediaRecorder.setMaxDuration(MAX_DURATION_MS); 157 mMediaRecorder.setMaxFileSize(MAX_FILESIZE_BYTES); 158 159 // Set up audio 160 if (mAudioSource == MIC) { 161 mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.HE_AAC); 162 mMediaRecorder.setAudioChannels(TOTAL_NUM_TRACKS); 163 mMediaRecorder.setAudioEncodingBitRate(AUDIO_BIT_RATE); 164 mMediaRecorder.setAudioSamplingRate(AUDIO_SAMPLE_RATE); 165 } 166 167 mMediaRecorder.setOutputFile(mTempVideoFile); 168 mMediaRecorder.prepare(); 169 // Create surface 170 mInputSurface = mMediaRecorder.getSurface(); 171 mVirtualDisplay = mMediaProjection.createVirtualDisplay( 172 "Recording Display", 173 width, 174 height, 175 metrics.densityDpi, 176 DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, 177 mInputSurface, 178 new VirtualDisplay.Callback() { 179 @Override 180 public void onStopped() { 181 onStop(); 182 } 183 }, 184 mHandler); 185 186 mMediaRecorder.setOnInfoListener((mr, what, extra) -> mListener.onInfo(mr, what, extra)); 187 if (mAudioSource == INTERNAL || 188 mAudioSource == MIC_AND_INTERNAL) { 189 mTempAudioFile = File.createTempFile("temp", ".aac", 190 mContext.getCacheDir()); 191 mAudio = new ScreenInternalAudioRecorder(mTempAudioFile.getAbsolutePath(), 192 mMediaProjection, mAudioSource == MIC_AND_INTERNAL); 193 } 194 195 } 196 197 /** 198 * Find the highest supported screen resolution and refresh rate for the given dimensions on 199 * this device, up to actual size and given rate. 200 * If possible this will return the same values as given, but values may be smaller on some 201 * devices. 202 * 203 * @param screenWidth Actual pixel width of screen 204 * @param screenHeight Actual pixel height of screen 205 * @param refreshRate Desired refresh rate 206 * @return array with supported width, height, and refresh rate 207 */ getSupportedSize(final int screenWidth, final int screenHeight, int refreshRate)208 private int[] getSupportedSize(final int screenWidth, final int screenHeight, int refreshRate) 209 throws IOException { 210 String videoType = MediaFormat.MIMETYPE_VIDEO_AVC; 211 212 // Get max size from the decoder, to ensure recordings will be playable on device 213 MediaCodec decoder = MediaCodec.createDecoderByType(videoType); 214 MediaCodecInfo.VideoCapabilities vc = decoder.getCodecInfo() 215 .getCapabilitiesForType(videoType).getVideoCapabilities(); 216 decoder.release(); 217 218 // Check if we can support screen size as-is 219 int width = vc.getSupportedWidths().getUpper(); 220 int height = vc.getSupportedHeights().getUpper(); 221 222 int screenWidthAligned = screenWidth; 223 if (screenWidthAligned % vc.getWidthAlignment() != 0) { 224 screenWidthAligned -= (screenWidthAligned % vc.getWidthAlignment()); 225 } 226 int screenHeightAligned = screenHeight; 227 if (screenHeightAligned % vc.getHeightAlignment() != 0) { 228 screenHeightAligned -= (screenHeightAligned % vc.getHeightAlignment()); 229 } 230 231 if (width >= screenWidthAligned && height >= screenHeightAligned 232 && vc.isSizeSupported(screenWidthAligned, screenHeightAligned)) { 233 // Desired size is supported, now get the rate 234 int maxRate = vc.getSupportedFrameRatesFor(screenWidthAligned, 235 screenHeightAligned).getUpper().intValue(); 236 237 if (maxRate < refreshRate) { 238 refreshRate = maxRate; 239 } 240 Log.d(TAG, "Screen size supported at rate " + refreshRate); 241 return new int[]{screenWidthAligned, screenHeightAligned, refreshRate}; 242 } 243 244 // Otherwise, resize for max supported size 245 double scale = Math.min(((double) width / screenWidth), 246 ((double) height / screenHeight)); 247 248 int scaledWidth = (int) (screenWidth * scale); 249 int scaledHeight = (int) (screenHeight * scale); 250 if (scaledWidth % vc.getWidthAlignment() != 0) { 251 scaledWidth -= (scaledWidth % vc.getWidthAlignment()); 252 } 253 if (scaledHeight % vc.getHeightAlignment() != 0) { 254 scaledHeight -= (scaledHeight % vc.getHeightAlignment()); 255 } 256 257 // Find max supported rate for size 258 int maxRate = vc.getSupportedFrameRatesFor(scaledWidth, scaledHeight) 259 .getUpper().intValue(); 260 if (maxRate < refreshRate) { 261 refreshRate = maxRate; 262 } 263 264 Log.d(TAG, "Resized by " + scale + ": " + scaledWidth + ", " + scaledHeight 265 + ", " + refreshRate); 266 return new int[]{scaledWidth, scaledHeight, refreshRate}; 267 } 268 269 /** 270 * Start screen recording 271 */ start()272 void start() throws IOException, RemoteException, RuntimeException { 273 Log.d(TAG, "start recording"); 274 prepare(); 275 mMediaRecorder.start(); 276 recordInternalAudio(); 277 } 278 279 /** 280 * End screen recording, throws an exception if stopping recording failed 281 */ end()282 void end() throws IOException { 283 Closer closer = new Closer(); 284 285 // MediaRecorder might throw RuntimeException if stopped immediately after starting 286 // We should remove the recording in this case as it will be invalid 287 closer.register(mMediaRecorder::stop); 288 closer.register(mMediaRecorder::release); 289 closer.register(mInputSurface::release); 290 closer.register(mVirtualDisplay::release); 291 closer.register(mMediaProjection::stop); 292 closer.register(this::stopInternalAudioRecording); 293 294 closer.close(); 295 296 mMediaRecorder = null; 297 mMediaProjection = null; 298 299 Log.d(TAG, "end recording"); 300 } 301 302 @Override onStop()303 public void onStop() { 304 Log.d(TAG, "The system notified about stopping the projection"); 305 mListener.onStopped(); 306 } 307 stopInternalAudioRecording()308 private void stopInternalAudioRecording() { 309 if (mAudioSource == INTERNAL || mAudioSource == MIC_AND_INTERNAL) { 310 mAudio.end(); 311 mAudio = null; 312 } 313 } 314 recordInternalAudio()315 private void recordInternalAudio() throws IllegalStateException { 316 if (mAudioSource == INTERNAL || mAudioSource == MIC_AND_INTERNAL) { 317 mAudio.start(); 318 } 319 } 320 321 /** 322 * Store recorded video 323 */ save()324 protected SavedRecording save() throws IOException { 325 String fileName = new SimpleDateFormat("'screen-'yyyyMMdd-HHmmss'.mp4'") 326 .format(new Date()); 327 328 ContentValues values = new ContentValues(); 329 values.put(MediaStore.Video.Media.DISPLAY_NAME, fileName); 330 values.put(MediaStore.Video.Media.MIME_TYPE, "video/mp4"); 331 values.put(MediaStore.Video.Media.DATE_ADDED, System.currentTimeMillis()); 332 values.put(MediaStore.Video.Media.DATE_TAKEN, System.currentTimeMillis()); 333 334 ContentResolver resolver = mContext.getContentResolver(); 335 Uri collectionUri = MediaStore.Video.Media.getContentUri( 336 MediaStore.VOLUME_EXTERNAL_PRIMARY); 337 Uri itemUri = resolver.insert(collectionUri, values); 338 339 Log.d(TAG, itemUri.toString()); 340 if (mAudioSource == MIC_AND_INTERNAL || mAudioSource == INTERNAL) { 341 try { 342 Log.d(TAG, "muxing recording"); 343 File file = File.createTempFile("temp", ".mp4", 344 mContext.getCacheDir()); 345 mMuxer = new ScreenRecordingMuxer(MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4, 346 file.getAbsolutePath(), 347 mTempVideoFile.getAbsolutePath(), 348 mTempAudioFile.getAbsolutePath()); 349 mMuxer.mux(); 350 mTempVideoFile.delete(); 351 mTempVideoFile = file; 352 } catch (IOException e) { 353 Log.e(TAG, "muxing recording " + e.getMessage()); 354 e.printStackTrace(); 355 } 356 } 357 358 // Add to the mediastore 359 OutputStream os = resolver.openOutputStream(itemUri, "w"); 360 Files.copy(mTempVideoFile.toPath(), os); 361 os.close(); 362 if (mTempAudioFile != null) mTempAudioFile.delete(); 363 DisplayMetrics metrics = mContext.getResources().getDisplayMetrics(); 364 Size size = new Size(metrics.widthPixels, metrics.heightPixels); 365 SavedRecording recording = new SavedRecording(itemUri, mTempVideoFile, size); 366 mTempVideoFile.delete(); 367 return recording; 368 } 369 370 /** 371 * Release the resources without saving the data 372 */ release()373 protected void release() { 374 if (mTempVideoFile != null) { 375 mTempVideoFile.delete(); 376 } 377 if (mTempAudioFile != null) { 378 mTempAudioFile.delete(); 379 } 380 } 381 382 /** 383 * Object representing the recording 384 */ 385 public class SavedRecording { 386 387 private Uri mUri; 388 private Bitmap mThumbnailBitmap; 389 SavedRecording(Uri uri, File file, Size thumbnailSize)390 protected SavedRecording(Uri uri, File file, Size thumbnailSize) { 391 mUri = uri; 392 try { 393 mThumbnailBitmap = ThumbnailUtils.createVideoThumbnail( 394 file, thumbnailSize, null); 395 } catch (IOException e) { 396 Log.e(TAG, "Error creating thumbnail", e); 397 } 398 } 399 getUri()400 public Uri getUri() { 401 return mUri; 402 } 403 getThumbnail()404 public @Nullable Bitmap getThumbnail() { 405 return mThumbnailBitmap; 406 } 407 } 408 409 interface ScreenMediaRecorderListener { 410 /** 411 * Called to indicate an info or a warning during recording. 412 * See {@link MediaRecorder.OnInfoListener} for the full description. 413 */ onInfo(MediaRecorder mr, int what, int extra)414 void onInfo(MediaRecorder mr, int what, int extra); 415 416 /** 417 * Called when the recording stopped by the system. 418 * For example, this might happen when doing partial screen sharing of an app 419 * and the app that is being captured is closed. 420 */ onStopped()421 void onStopped(); 422 } 423 424 /** 425 * Allows to register multiple {@link Closeable} objects and close them all by calling 426 * {@link Closer#close}. If there is an exception thrown during closing of one 427 * of the registered closeables it will continue trying closing the rest closeables. 428 * If there are one or more exceptions thrown they will be re-thrown at the end. 429 * In case of multiple exceptions only the first one will be thrown and all the rest 430 * will be printed. 431 */ 432 private static class Closer implements Closeable { 433 private final List<Closeable> mCloseables = new ArrayList<>(); 434 register(Closeable closeable)435 void register(Closeable closeable) { 436 mCloseables.add(closeable); 437 } 438 439 @Override close()440 public void close() throws IOException { 441 Throwable throwable = null; 442 443 for (int i = 0; i < mCloseables.size(); i++) { 444 Closeable closeable = mCloseables.get(i); 445 446 try { 447 closeable.close(); 448 } catch (Throwable e) { 449 if (throwable == null) { 450 throwable = e; 451 } else { 452 e.printStackTrace(); 453 } 454 } 455 } 456 457 if (throwable != null) { 458 if (throwable instanceof IOException) { 459 throw (IOException) throwable; 460 } 461 462 if (throwable instanceof RuntimeException) { 463 throw (RuntimeException) throwable; 464 } 465 466 throw (Error) throwable; 467 } 468 } 469 } 470 } 471