1 /* 2 * Copyright 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.virtualdevice.cts.camera; 18 19 import static android.virtualdevice.cts.camera.util.VirtualCameraUtils.createHandler; 20 21 import static com.google.common.truth.Truth.assertThat; 22 23 import android.media.Image; 24 import android.media.ImageWriter; 25 import android.media.MediaCodec; 26 import android.media.MediaCodecInfo; 27 import android.media.MediaFormat; 28 import android.util.Log; 29 import android.view.Surface; 30 31 import androidx.annotation.NonNull; 32 33 import java.io.IOException; 34 import java.nio.ByteBuffer; 35 import java.util.concurrent.LinkedBlockingDeque; 36 import java.util.concurrent.TimeUnit; 37 import java.util.concurrent.atomic.AtomicReference; 38 39 /** 40 * A fake pair of video encoder/decoder writing mock data 41 * on a surface and incrementing by 1 the provided timestamp for each decoded frame. 42 */ 43 public class SteadyTimestampCodec implements AutoCloseable { 44 45 private static final int VIDEO_BITRATE = 4000000; 46 private static final int FRAME_RATE = 30; 47 private static final int I_FRAME_INTERVAL = 1; 48 private static final String MIMETYPE = MediaFormat.MIMETYPE_VIDEO_AVC; 49 private static final int TIMEOUT_MILLIS = 100; 50 private static final String TAG = "SteadyTimestampCodec"; 51 private static final boolean DEBUG = false; 52 private final AtomicReference<MediaCodec> mDecoderRef; 53 private final AtomicReference<MediaCodec> mEncoderRef; 54 55 private final AtomicReference<Boolean> mCodecRunning = new AtomicReference<>(false); 56 private final LinkedBlockingDeque<byte[]> mBufferQueue = new LinkedBlockingDeque<>(); 57 private final int mWidth; 58 private final int mHeight; 59 private long mRenderTimestampNs; 60 61 private abstract static class MediaCodecCallback extends MediaCodec.Callback { 62 @Override onError(@onNull MediaCodec mediaCodec, @NonNull MediaCodec.CodecException exception)63 public void onError(@NonNull MediaCodec mediaCodec, 64 @NonNull MediaCodec.CodecException exception) { 65 throw exception; 66 } 67 68 @Override onOutputFormatChanged(@onNull MediaCodec mediaCodec, @NonNull MediaFormat mediaFormat)69 public void onOutputFormatChanged(@NonNull MediaCodec mediaCodec, 70 @NonNull MediaFormat mediaFormat) { 71 // Do nothing; 72 } 73 } 74 75 /** 76 * Create a codec with presentation timestamp starting at renderTimestampNs. 77 * 78 * @param width The width of the video to encode/decode 79 * @param height The height of the video to encode/decode 80 * @param renderTimestampNs The timestamp to be associated with the first frame 81 */ SteadyTimestampCodec(int width, int height, long renderTimestampNs)82 public SteadyTimestampCodec(int width, int height, long renderTimestampNs) { 83 mWidth = width; 84 mHeight = height; 85 mRenderTimestampNs = renderTimestampNs; 86 mEncoderRef = new AtomicReference<>(createEncoder()); 87 mDecoderRef = new AtomicReference<>(null); 88 } 89 writeBlankFrame(@onNull Surface surface)90 private static void writeBlankFrame(@NonNull Surface surface) { 91 ImageWriter imageWriter = ImageWriter.newInstance(surface, 1); 92 Image image = imageWriter.dequeueInputImage(); 93 image.setTimestamp(1); 94 imageWriter.queueInputImage(image); 95 imageWriter.close(); 96 } 97 createEncoder()98 private MediaCodec createEncoder() { 99 MediaCodec.Callback encoderCallback = new MediaCodecCallback() { 100 @Override 101 public void onInputBufferAvailable(@NonNull MediaCodec encoder, int i) { 102 if (DEBUG) { 103 Log.d(TAG, 104 "encoder onInputBufferAvailable() called with: codec = [" + encoder 105 + "], i = [" 106 + i + "] mCodecRunning = " + mCodecRunning.get()); 107 } 108 if (!mCodecRunning.get()) { 109 return; 110 } 111 try { 112 if (i >= 0) { 113 ByteBuffer inputBuffer = encoder.getInputBuffer(i); 114 assertThat(inputBuffer).isNotNull(); 115 inputBuffer.clear(); 116 byte[] blackFrameData = generateBlackFrameData(mWidth, mHeight); 117 inputBuffer.put(blackFrameData); 118 if (DEBUG) { 119 Log.d(TAG, "encoder queueInputBuffer() called with: codec = [" 120 + encoder + "], i = [" + i + "]"); 121 } 122 encoder.queueInputBuffer(i, 0, blackFrameData.length, 0, 123 0); // Custom PTS 124 } 125 } catch (IllegalStateException exception) { 126 mCodecRunning.set(false); 127 } 128 } 129 130 @Override 131 public void onOutputBufferAvailable(@NonNull MediaCodec mediaCodec, int i, 132 @NonNull MediaCodec.BufferInfo bufferInfo) { 133 if (DEBUG) { 134 Log.d(TAG, 135 "encoder onOutputBufferAvailable() called with: codec = [" 136 + mediaCodec 137 + "], i = [" 138 + i + "] mCodecRunning = " + mCodecRunning.get()); 139 } 140 if (!mCodecRunning.get()) { 141 return; 142 } 143 ByteBuffer outputBuffer = mediaCodec.getOutputBuffer(i); 144 assertThat(outputBuffer).isNotNull(); 145 byte[] bytes = new byte[outputBuffer.remaining()]; 146 outputBuffer.get(bytes); 147 mBufferQueue.offer(bytes); 148 mediaCodec.releaseOutputBuffer(i, false); 149 } 150 }; 151 152 MediaFormat format = MediaFormat.createVideoFormat(MIMETYPE, mWidth, mHeight); 153 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, 154 MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible); 155 format.setInteger(MediaFormat.KEY_BIT_RATE, VIDEO_BITRATE); 156 format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE); 157 format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, I_FRAME_INTERVAL); 158 159 try { 160 MediaCodec encoder = MediaCodec.createEncoderByType(MIMETYPE); 161 encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 162 encoder.setCallback(encoderCallback, createHandler("encoder-callback")); 163 return encoder; 164 } catch (IOException e) { 165 throw new RuntimeException(e); 166 } 167 } 168 createDecoder(Surface surface)169 private MediaCodec createDecoder(Surface surface) { 170 MediaCodec.Callback decoderCallback = new MediaCodecCallback() { 171 @Override 172 public void onInputBufferAvailable(@NonNull MediaCodec mediaCodec, int i) { 173 if (DEBUG) { 174 Log.d(TAG, 175 "decoder onInputBufferAvailable() called with: codec = [" 176 + mediaCodec 177 + "], i = [" 178 + i + "] mCodecRunning = " + mCodecRunning.get()); 179 } 180 if (!mCodecRunning.get()) { 181 return; 182 } 183 try { 184 byte[] bytes = mBufferQueue.poll(TIMEOUT_MILLIS, 185 TimeUnit.MILLISECONDS); 186 if (!mCodecRunning.get()) { 187 return; 188 } 189 if (bytes == null) { 190 Log.w(TAG, "decoder: onInputBufferAvailable() no data queued"); 191 return; 192 } 193 ByteBuffer inputBuffer = mediaCodec.getInputBuffer(i); 194 assertThat(inputBuffer).isNotNull(); 195 inputBuffer.put(bytes); 196 mediaCodec.queueInputBuffer(i, 0, bytes.length, 0, 0); 197 } catch (InterruptedException e) { 198 throw new RuntimeException("Timeout polling for encoded buffer", e); 199 } catch (IllegalStateException e) { 200 mCodecRunning.set(false); 201 } 202 } 203 204 @Override 205 public void onOutputBufferAvailable(@NonNull MediaCodec mediaCodec, int i, 206 @NonNull MediaCodec.BufferInfo bufferInfo) { 207 if (DEBUG) { 208 Log.d(TAG, 209 "decoder onOutputBufferAvailable() called with: codec = [" + mediaCodec 210 + "], i = [" + i 211 + "] mCodecRunning = \" + mCodecRunning.get());"); 212 } 213 if (!mCodecRunning.get()) { 214 return; 215 } 216 if (DEBUG) { 217 Log.d(TAG, "decoder onOutputBufferAvailable() mRenderTimestampNs:" 218 + mRenderTimestampNs + 1); 219 } 220 mediaCodec.releaseOutputBuffer(i, mRenderTimestampNs++); 221 } 222 }; 223 try { 224 MediaFormat format = MediaFormat.createVideoFormat(MIMETYPE, mWidth, mHeight); 225 MediaCodec decoder = MediaCodec.createDecoderByType(MIMETYPE); 226 decoder.configure(format, surface, null, 0); 227 decoder.setCallback(decoderCallback, createHandler("decoder-callback")); 228 return decoder; 229 } catch (IOException e) { 230 throw new RuntimeException(e); 231 } 232 } 233 generateBlackFrameData(int width, int height)234 private static byte[] generateBlackFrameData(int width, int height) { 235 int ySize = width * height; 236 int uvSize = ySize / 4; 237 byte[] data = new byte[ySize + uvSize * 2]; 238 239 // Y plane (black) 240 for (int i = 0; i < ySize; i++) { 241 data[i] = 0; // Black 242 } 243 244 // U and V planes (neutral gray) 245 for (int i = ySize; i < data.length; i++) { 246 data[i] = (byte) 0xFF; 247 } 248 return data; 249 } 250 251 /** 252 * Set the output surface onto which the decoded data should be written and start the codec. 253 */ setSurfaceAndStart(@onNull Surface surface)254 public void setSurfaceAndStart(@NonNull Surface surface) { 255 writeBlankFrame(surface); 256 MediaCodec decoder = createDecoder(surface); 257 mDecoderRef.set(decoder); 258 mCodecRunning.set(true); 259 mEncoderRef.get().start(); 260 decoder.start(); 261 } 262 263 264 /** Stops and release the codecs */ 265 @Override close()266 public void close() { 267 mCodecRunning.set(false); 268 mDecoderRef.get().stop(); 269 mEncoderRef.get().stop(); 270 mDecoderRef.get().release(); 271 mEncoderRef.get().release(); 272 } 273 } 274