1 /* 2 * Copyright 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.google.android.exoplayer2.transformer; 18 19 import static com.google.android.exoplayer2.util.Assertions.checkNotNull; 20 21 import android.content.Context; 22 import android.media.MediaCodec; 23 import android.util.Size; 24 import androidx.annotation.Nullable; 25 import com.google.android.exoplayer2.Format; 26 import com.google.android.exoplayer2.decoder.DecoderInputBuffer; 27 import com.google.android.exoplayer2.util.Util; 28 import com.google.common.collect.ImmutableList; 29 import java.util.List; 30 import org.checkerframework.dataflow.qual.Pure; 31 32 /** 33 * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them. 34 */ 35 /* package */ final class VideoTranscodingSamplePipeline implements SamplePipeline { 36 37 private static final int FRAME_COUNT_UNLIMITED = -1; 38 39 private final int outputRotationDegrees; 40 private final DecoderInputBuffer decoderInputBuffer; 41 private final Codec decoder; 42 private final int maxPendingFrameCount; 43 44 private final FrameProcessorChain frameProcessorChain; 45 46 private final Codec encoder; 47 private final DecoderInputBuffer encoderOutputBuffer; 48 49 private boolean signaledEndOfStreamToEncoder; 50 VideoTranscodingSamplePipeline( Context context, Format inputFormat, TransformationRequest transformationRequest, ImmutableList<GlFrameProcessor> frameProcessors, Codec.DecoderFactory decoderFactory, Codec.EncoderFactory encoderFactory, List<String> allowedOutputMimeTypes, FallbackListener fallbackListener, Transformer.DebugViewProvider debugViewProvider)51 public VideoTranscodingSamplePipeline( 52 Context context, 53 Format inputFormat, 54 TransformationRequest transformationRequest, 55 ImmutableList<GlFrameProcessor> frameProcessors, 56 Codec.DecoderFactory decoderFactory, 57 Codec.EncoderFactory encoderFactory, 58 List<String> allowedOutputMimeTypes, 59 FallbackListener fallbackListener, 60 Transformer.DebugViewProvider debugViewProvider) 61 throws TransformationException { 62 decoderInputBuffer = 63 new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); 64 encoderOutputBuffer = 65 new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); 66 67 // The decoder rotates encoded frames for display by inputFormat.rotationDegrees. 68 int decodedWidth = 69 (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height; 70 int decodedHeight = 71 (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; 72 73 // TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset. 74 ScaleToFitFrameProcessor scaleToFitFrameProcessor = 75 new ScaleToFitFrameProcessor.Builder(context) 76 .setScale(transformationRequest.scaleX, transformationRequest.scaleY) 77 .setRotationDegrees(transformationRequest.rotationDegrees) 78 .build(); 79 PresentationFrameProcessor presentationFrameProcessor = 80 new PresentationFrameProcessor.Builder(context) 81 .setResolution(transformationRequest.outputHeight) 82 .build(); 83 frameProcessorChain = 84 FrameProcessorChain.create( 85 context, 86 inputFormat.pixelWidthHeightRatio, 87 /* inputWidth= */ decodedWidth, 88 /* inputHeight= */ decodedHeight, 89 new ImmutableList.Builder<GlFrameProcessor>() 90 .addAll(frameProcessors) 91 .add(scaleToFitFrameProcessor) 92 .add(presentationFrameProcessor) 93 .build(), 94 transformationRequest.enableHdrEditing); 95 Size requestedEncoderSize = frameProcessorChain.getOutputSize(); 96 outputRotationDegrees = presentationFrameProcessor.getOutputRotationDegrees(); 97 98 Format requestedEncoderFormat = 99 new Format.Builder() 100 .setWidth(requestedEncoderSize.getWidth()) 101 .setHeight(requestedEncoderSize.getHeight()) 102 .setRotationDegrees(0) 103 .setFrameRate(inputFormat.frameRate) 104 .setSampleMimeType( 105 transformationRequest.videoMimeType != null 106 ? transformationRequest.videoMimeType 107 : inputFormat.sampleMimeType) 108 .build(); 109 110 encoder = encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes); 111 Format encoderSupportedFormat = encoder.getConfigurationFormat(); 112 fallbackListener.onTransformationRequestFinalized( 113 createFallbackTransformationRequest( 114 transformationRequest, 115 /* hasOutputFormatRotation= */ outputRotationDegrees == 0, 116 requestedEncoderFormat, 117 encoderSupportedFormat)); 118 119 frameProcessorChain.setOutputSurface( 120 /* outputSurface= */ encoder.getInputSurface(), 121 /* outputWidth= */ encoderSupportedFormat.width, 122 /* outputHeight= */ encoderSupportedFormat.height, 123 debugViewProvider.getDebugPreviewSurfaceView( 124 encoderSupportedFormat.width, encoderSupportedFormat.height)); 125 126 decoder = 127 decoderFactory.createForVideoDecoding( 128 inputFormat, 129 frameProcessorChain.getInputSurface(), 130 transformationRequest.enableRequestSdrToneMapping); 131 maxPendingFrameCount = getMaxPendingFrameCount(); 132 } 133 134 @Override 135 @Nullable dequeueInputBuffer()136 public DecoderInputBuffer dequeueInputBuffer() throws TransformationException { 137 return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null; 138 } 139 140 @Override queueInputBuffer()141 public void queueInputBuffer() throws TransformationException { 142 decoder.queueInputBuffer(decoderInputBuffer); 143 } 144 145 @Override processData()146 public boolean processData() throws TransformationException { 147 frameProcessorChain.getAndRethrowBackgroundExceptions(); 148 if (frameProcessorChain.isEnded()) { 149 if (!signaledEndOfStreamToEncoder) { 150 encoder.signalEndOfInputStream(); 151 signaledEndOfStreamToEncoder = true; 152 } 153 return false; 154 } 155 if (decoder.isEnded()) { 156 return false; 157 } 158 159 boolean processedData = false; 160 while (maybeProcessDecoderOutput()) { 161 processedData = true; 162 } 163 if (decoder.isEnded()) { 164 frameProcessorChain.signalEndOfInputStream(); 165 } 166 // If the decoder produced output, signal that it may be possible to process data again. 167 return processedData; 168 } 169 170 @Override 171 @Nullable getOutputFormat()172 public Format getOutputFormat() throws TransformationException { 173 @Nullable Format format = encoder.getOutputFormat(); 174 return format == null 175 ? null 176 : format.buildUpon().setRotationDegrees(outputRotationDegrees).build(); 177 } 178 179 @Override 180 @Nullable getOutputBuffer()181 public DecoderInputBuffer getOutputBuffer() throws TransformationException { 182 encoderOutputBuffer.data = encoder.getOutputBuffer(); 183 if (encoderOutputBuffer.data == null) { 184 return null; 185 } 186 MediaCodec.BufferInfo bufferInfo = checkNotNull(encoder.getOutputBufferInfo()); 187 encoderOutputBuffer.timeUs = bufferInfo.presentationTimeUs; 188 encoderOutputBuffer.setFlags(bufferInfo.flags); 189 return encoderOutputBuffer; 190 } 191 192 @Override releaseOutputBuffer()193 public void releaseOutputBuffer() throws TransformationException { 194 encoder.releaseOutputBuffer(/* render= */ false); 195 } 196 197 @Override isEnded()198 public boolean isEnded() { 199 return encoder.isEnded(); 200 } 201 202 @Override release()203 public void release() { 204 frameProcessorChain.release(); 205 decoder.release(); 206 encoder.release(); 207 } 208 209 /** 210 * Creates a fallback transformation request to execute, based on device-specific support. 211 * 212 * @param transformationRequest The requested transformation. 213 * @param hasOutputFormatRotation Whether the input video will be rotated to landscape during 214 * processing, with {@link Format#rotationDegrees} of 90 added to the output format. 215 * @param requestedFormat The requested format. 216 * @param supportedFormat A format supported by the device. 217 */ 218 @Pure createFallbackTransformationRequest( TransformationRequest transformationRequest, boolean hasOutputFormatRotation, Format requestedFormat, Format supportedFormat)219 private static TransformationRequest createFallbackTransformationRequest( 220 TransformationRequest transformationRequest, 221 boolean hasOutputFormatRotation, 222 Format requestedFormat, 223 Format supportedFormat) { 224 // TODO(b/210591626): Also update bitrate etc. once encoder configuration and fallback are 225 // implemented. 226 if (Util.areEqual(requestedFormat.sampleMimeType, supportedFormat.sampleMimeType) 227 && (hasOutputFormatRotation 228 ? requestedFormat.width == supportedFormat.width 229 : requestedFormat.height == supportedFormat.height)) { 230 return transformationRequest; 231 } 232 return transformationRequest 233 .buildUpon() 234 .setVideoMimeType(supportedFormat.sampleMimeType) 235 .setResolution(hasOutputFormatRotation ? requestedFormat.width : requestedFormat.height) 236 .build(); 237 } 238 239 /** 240 * Feeds at most one decoder output frame to the next step of the pipeline. 241 * 242 * @return Whether a frame was processed. 243 * @throws TransformationException If a problem occurs while processing the frame. 244 */ maybeProcessDecoderOutput()245 private boolean maybeProcessDecoderOutput() throws TransformationException { 246 if (decoder.getOutputBufferInfo() == null) { 247 return false; 248 } 249 250 if (maxPendingFrameCount != FRAME_COUNT_UNLIMITED 251 && frameProcessorChain.getPendingFrameCount() == maxPendingFrameCount) { 252 return false; 253 } 254 255 frameProcessorChain.registerInputFrame(); 256 decoder.releaseOutputBuffer(/* render= */ true); 257 return true; 258 } 259 260 /** 261 * Returns the maximum number of frames that may be pending in the output {@link 262 * FrameProcessorChain} at a time, or {@link #FRAME_COUNT_UNLIMITED} if it's not necessary to 263 * enforce a limit. 264 */ getMaxPendingFrameCount()265 private static int getMaxPendingFrameCount() { 266 if (Util.SDK_INT < 29) { 267 // Prior to API 29, decoders may drop frames to keep their output surface from growing out of 268 // bounds, while from API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame 269 // dropping even when the surface is full. We never want frame dropping so allow a maximum of 270 // one frame to be pending at a time. 271 // TODO(b/226330223): Investigate increasing this limit. 272 return 1; 273 } 274 if (Util.SDK_INT < 31 275 && ("OnePlus".equals(Util.MANUFACTURER) || "samsung".equals(Util.MANUFACTURER))) { 276 // Some OMX decoders don't correctly track their number of output buffers available, and get 277 // stuck if too many frames are rendered without being processed, so we limit the number of 278 // pending frames to avoid getting stuck. This value is experimentally determined. See also 279 // b/213455700. 280 return 10; 281 } 282 // Otherwise don't limit the number of frames that can be pending at a time, to maximize 283 // throughput. 284 return FRAME_COUNT_UNLIMITED; 285 } 286 } 287