1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.videocodec.cts; 18 19 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; 20 import static android.media.MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR; 21 import static android.media.MediaFormat.KEY_ALLOW_FRAME_DROP; 22 import static android.mediav2.common.cts.CodecEncoderTestBase.getMuxerFormatForMediaType; 23 import static android.mediav2.common.cts.CodecEncoderTestBase.getTempFilePath; 24 import static android.mediav2.common.cts.CodecEncoderTestBase.muxOutput; 25 import static android.mediav2.common.cts.CodecTestBase.ComponentClass.HARDWARE; 26 import static android.mediav2.common.cts.CodecTestBase.Q_DEQ_TIMEOUT_US; 27 28 import static org.junit.Assert.assertEquals; 29 import static org.junit.Assert.assertTrue; 30 import static org.junit.Assert.fail; 31 import static org.junit.Assume.assumeTrue; 32 33 import android.media.MediaCodec; 34 import android.media.MediaExtractor; 35 import android.media.MediaFormat; 36 import android.mediav2.common.cts.CodecTestBase; 37 import android.mediav2.common.cts.CompareStreams; 38 import android.mediav2.common.cts.DecodeStreamToYuv; 39 import android.mediav2.common.cts.EncoderConfigParams; 40 import android.mediav2.common.cts.InputSurface; 41 import android.mediav2.common.cts.OutputSurface; 42 import android.mediav2.common.cts.RawResource; 43 import android.util.Log; 44 45 import androidx.annotation.NonNull; 46 47 import com.android.compatibility.common.util.ApiTest; 48 import com.android.compatibility.common.util.Preconditions; 49 50 import org.junit.After; 51 import org.junit.Test; 52 import org.junit.runner.RunWith; 53 import org.junit.runners.Parameterized; 54 55 import java.io.File; 56 import java.io.FileOutputStream; 57 import java.io.IOException; 58 import java.io.RandomAccessFile; 59 import java.nio.ByteBuffer; 60 import java.util.ArrayList; 61 import java.util.Arrays; 62 import java.util.Collection; 63 import java.util.List; 64 65 /** 66 * This test is similar to {@link android.media.codec.cts.DecodeEditEncodeTest}, except for the 67 * edit part. The DecodeEditEncodeTest does swapping of color planes during editing, this test 68 * performs smoothening and sharpening. Besides this every thing is almost same. 69 * This test additionally validates the output of encoder. As smoothening and sharpening filters 70 * are applied on input, the test compares block level (32x32) between smoothened clip and 71 * sharpened clip and checks if they are as expected. 72 */ 73 @RunWith(Parameterized.class) 74 public class VideoDecodeEditEncodeTest { 75 private static final String LOG_TAG = VideoDecodeEditEncodeTest.class.getSimpleName(); 76 private static final boolean WORK_AROUND_BUGS = false; // avoid fatal codec bugs 77 private static final boolean VERBOSE = false; 78 private static final CodecTestBase.ComponentClass SELECT_SWITCH = HARDWARE; 79 private static final String MEDIA_DIR = WorkDir.getMediaDirString(); 80 private static final String RES_CLIP = 81 MEDIA_DIR + "AVICON-MOBILE-BirthdayHalfway-SI17-CRUW03-L-420-8bit-SDR-1080p-30fps.mp4"; 82 private static final String RES_MEDIA_TYPE = MediaFormat.MIMETYPE_VIDEO_AVC; 83 private static final int TARGET_WIDTH = 1280; 84 private static final int TARGET_HEIGHT = 720; 85 private static final int TARGET_BITRATE = 10000000; 86 private static final float AVG_ACCEPTABLE_QUALITY = 25.0f; // dB 87 88 private static final String[] KERNELS = { 89 // 5x5 Gaussian smoothing filter 90 "float kernel[KERNEL_SIZE];\n" 91 + "kernel[0] = 1.0 / 273.0;\n" 92 + "kernel[1] = 4.0 / 273.0;\n" 93 + "kernel[2] = 7.0 / 273.0;\n" 94 + "kernel[3] = 4.0 / 273.0;\n" 95 + "kernel[4] = 1.0 / 273.0;\n" 96 97 + "kernel[5] = 4.0 / 273.0;\n" 98 + "kernel[6] = 16.0 / 273.0;\n" 99 + "kernel[7] = 26.0 / 273.0;\n" 100 + "kernel[8] = 16.0 / 273.0;\n" 101 + "kernel[9] = 4.0 / 273.0;\n" 102 103 + "kernel[10] = 7.0 / 273.0;\n" 104 + "kernel[11] = 26.0 / 273.0;\n" 105 + "kernel[12] = 41.0 / 273.0;\n" 106 + "kernel[13] = 26.0 / 273.0;\n" 107 + "kernel[14] = 7.0 / 273.0;\n" 108 109 + "kernel[15] = 4.0 / 273.0;\n" 110 + "kernel[16] = 16.0 / 273.0;\n" 111 + "kernel[17] = 26.0 / 273.0;\n" 112 + "kernel[18] = 16.0 / 273.0;\n" 113 + "kernel[19] = 4.0 / 273.0;\n" 114 115 + "kernel[20] = 1.0 / 273.0;\n" 116 + "kernel[21] = 4.0 / 273.0;\n" 117 + "kernel[22] = 7.0 / 273.0;\n" 118 + "kernel[23] = 4.0 / 273.0;\n" 119 + "kernel[24] = 1.0 / 273.0;\n", 120 121 // 5x5 Sharpening filter 122 // Sharpening Kernel = 2 * Identity matrix - Gaussian smoothing kernel 123 "float kernel[KERNEL_SIZE];\n" 124 + "kernel[0] = -1.0 / 273.0;\n" 125 + "kernel[1] = -4.0 / 273.0;\n" 126 + "kernel[2] = -7.0 / 273.0;\n" 127 + "kernel[3] = -4.0 / 273.0;\n" 128 + "kernel[4] = -1.0 / 273.0;\n" 129 130 + "kernel[5] = -4.0 / 273.0;\n" 131 + "kernel[6] = -16.0 / 273.0;\n" 132 + "kernel[7] = -26.0 / 273.0;\n" 133 + "kernel[8] = -16.0 / 273.0;\n" 134 + "kernel[9] = -4.0 / 273.0;\n" 135 136 + "kernel[10] = -7.0 / 273.0;\n" 137 + "kernel[11] = -26.0 / 273.0;\n" 138 + "kernel[12] = 505.0 / 273.0;\n" 139 + "kernel[13] = -26.0 / 273.0;\n" 140 + "kernel[14] = -7.0 / 273.0;\n" 141 142 + "kernel[15] = -4.0 / 273.0;\n" 143 + "kernel[16] = -16.0 / 273.0;\n" 144 + "kernel[17] = -26.0 / 273.0;\n" 145 + "kernel[18] = -16.0 / 273.0;\n" 146 + "kernel[19] = -4.0 / 273.0;\n" 147 148 + "kernel[20] = -1.0 / 273.0;\n" 149 + "kernel[21] = -4.0 / 273.0;\n" 150 + "kernel[22] = -7.0 / 273.0;\n" 151 + "kernel[23] = -4.0 / 273.0;\n" 152 + "kernel[24] = -1.0 / 273.0;\n" 153 }; 154 155 private final String mEncoderName; 156 private final String mMediaType; 157 158 private final ArrayList<String> mTmpFiles = new ArrayList<>(); 159 VideoDecodeEditEncodeTest(String encoderName, String mediaType, @SuppressWarnings("unused") String allTestParams)160 public VideoDecodeEditEncodeTest(String encoderName, String mediaType, 161 @SuppressWarnings("unused") String allTestParams) { 162 mEncoderName = encoderName; 163 mMediaType = mediaType; 164 } 165 166 @Parameterized.Parameters(name = "{index}_{0}_{1}") input()167 public static Collection<Object[]> input() { 168 final boolean isEncoder = true; 169 final boolean needAudio = false; 170 final boolean needVideo = true; 171 // mediaType 172 final List<Object[]> exhaustiveArgsList = Arrays.asList(new Object[][]{ 173 {MediaFormat.MIMETYPE_VIDEO_AVC}, 174 {MediaFormat.MIMETYPE_VIDEO_HEVC}, 175 }); 176 return CodecTestBase.prepareParamList(exhaustiveArgsList, isEncoder, needAudio, needVideo, 177 false, SELECT_SWITCH); 178 } 179 180 @After tearDown()181 public void tearDown() { 182 for (String tmpFile : mTmpFiles) { 183 File tmp = new File(tmpFile); 184 if (tmp.exists()) assertTrue("unable to delete file " + tmpFile, tmp.delete()); 185 } 186 mTmpFiles.clear(); 187 } 188 getShader(String width, String height, String kernel)189 private static String getShader(String width, String height, String kernel) { 190 return "#extension GL_OES_EGL_image_external : require\n" 191 + "#define KERNEL_SIZE 25\n" 192 + "precision mediump float;\n" 193 + "varying vec2 vTextureCoord;\n" 194 + "uniform samplerExternalOES sTexture;\n" 195 + width 196 + height 197 + "const float step_w = 1.0/width;\n" 198 + "const float step_h = 1.0/height;\n" 199 + "void main() {\n" 200 + kernel 201 + " vec2 offset[KERNEL_SIZE];" 202 + " offset[0] = vec2(-2.0 * step_w, -2.0 * step_h);\n" // [-2, -2] // row -2 203 + " offset[1] = vec2(-step_w, -2.0 * step_h);\n" // [-1, -2] 204 + " offset[2] = vec2(0.0, -2.0 * step_h);\n" // [0, -2] 205 + " offset[3] = vec2(step_w, -2.0 * step_h);\n" // [1, -2] 206 + " offset[4] = vec2(2.0 * step_w, -2.0 * step_h);\n" // [2, -2] 207 208 + " offset[5] = vec2(-2.0 * step_w, -step_h);\n" // [-2, -1] // row -1 209 + " offset[6] = vec2(-step_w, -step_h);\n" // [-1, -1] 210 + " offset[7] = vec2(0.0, -step_h);\n" // [0, -1] 211 + " offset[8] = vec2(step_w, -step_h);\n" // [1, -1] 212 + " offset[9] = vec2(2.0 * step_w, -step_h);\n" // [2, -1] 213 214 + " offset[10] = vec2(-2.0 * step_w, 0.0);\n" // [-2, 0] // curr row 215 + " offset[11] = vec2(-step_w, 0.0);\n" // [-1, 0] 216 + " offset[12] = vec2(0.0, 0.0);\n" // [0, 0] 217 + " offset[13] = vec2(step_w, 0.0);\n" // [1, 0] 218 + " offset[14] = vec2(2.0 * step_w, 0.0);\n" // [2, 0] 219 220 + " offset[15] = vec2(-2.0 * step_w, step_h);\n" // [-2, 1] // row +1 221 + " offset[16] = vec2(-step_w, step_h);\n" // [-1, 1] 222 + " offset[17] = vec2(0.0, step_h);\n" // [0, 1] 223 + " offset[18] = vec2(step_w, step_h);\n" // [1, 1] 224 + " offset[19] = vec2(2.0 * step_w, step_h);\n" // [2, 1] 225 226 + " offset[20] = vec2(-2.0 * step_w, 2.0 * step_h);\n" // [-2, 2] // row +2 227 + " offset[21] = vec2(-step_w, 2.0 * step_h);\n" // [-1, 2] 228 + " offset[22] = vec2(0.0, 2.0 * step_h);\n" // [-0, 2] 229 + " offset[23] = vec2(step_w, 2.0 * step_h);\n" // [1, 2] 230 + " offset[24] = vec2(2.0 * step_w, 2.0 * step_h);\n" // [2, 2] 231 232 + " vec4 sum = vec4(0.0);\n" 233 + " vec4 sample;\n" 234 + " for (int i=0; i<KERNEL_SIZE; i++) {\n" 235 + " sample = texture2D(sTexture, vTextureCoord + offset[i]).rgba;\n" 236 + " sum = sum + sample * kernel[i];\n" 237 + " }\n" 238 + " gl_FragColor = sum;\n" 239 + "}\n"; 240 } 241 242 /** 243 * The elementary stream coming out of the encoder needs to be fed back into 244 * the decoder one chunk at a time. If we just wrote the data to a file, we would lose 245 * the information about chunk boundaries. This class stores the encoded data in memory, 246 * retaining the chunk organization. 247 */ 248 private static class VideoChunks { 249 private MediaFormat mMediaFormat; 250 private byte[] mMemory = new byte[1024]; 251 private int mMemIndex = 0; 252 private final ArrayList<MediaCodec.BufferInfo> mBufferInfo = new ArrayList<>(); 253 splitMediaToMuxerParameters(@onNull String srcPath, @NonNull String mediaType, int frameLimit)254 private void splitMediaToMuxerParameters(@NonNull String srcPath, @NonNull String mediaType, 255 int frameLimit) throws IOException { 256 // Set up MediaExtractor to read from the source. 257 MediaExtractor extractor = new MediaExtractor(); 258 Preconditions.assertTestFileExists(srcPath); 259 extractor.setDataSource(srcPath); 260 261 // Set up MediaFormat 262 for (int trackID = 0; trackID < extractor.getTrackCount(); trackID++) { 263 extractor.selectTrack(trackID); 264 MediaFormat format = extractor.getTrackFormat(trackID); 265 if (mediaType.equals(format.getString(MediaFormat.KEY_MIME))) { 266 mMediaFormat = format; 267 break; 268 } else { 269 extractor.unselectTrack(trackID); 270 } 271 } 272 273 if (null == mMediaFormat) { 274 extractor.release(); 275 throw new IllegalArgumentException( 276 "could not find usable track in file " + srcPath); 277 } 278 279 // Set up location for elementary stream 280 File file = new File(srcPath); 281 int bufferSize = (int) file.length(); 282 bufferSize = ((bufferSize + 127) >> 7) << 7; 283 // Ideally, Sum of return values of extractor.readSampleData(...) should not exceed 284 // source file size. But in case of Vorbis, aosp extractor appends an additional 4 285 // bytes to the data at every readSampleData() call. bufferSize <<= 1 empirically 286 // large enough to hold the excess 4 bytes per read call 287 bufferSize <<= 1; 288 mMemory = new byte[bufferSize]; 289 ByteBuffer byteBuffer = ByteBuffer.allocate(bufferSize); 290 mMemIndex = 0; 291 mBufferInfo.clear(); 292 293 // Let MediaExtractor do its thing 294 boolean sawEOS = false; 295 int offset = 0; 296 int frameCount = 0; 297 while (!sawEOS && frameCount < frameLimit) { 298 MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); 299 bufferInfo.offset = offset; 300 bufferInfo.size = extractor.readSampleData(byteBuffer, offset); 301 if (bufferInfo.size < 0) { 302 sawEOS = true; 303 } else { 304 bufferInfo.presentationTimeUs = extractor.getSampleTime(); 305 int flags = extractor.getSampleFlags(); 306 bufferInfo.flags = 0; 307 if ((flags & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) { 308 bufferInfo.flags |= MediaCodec.BUFFER_FLAG_KEY_FRAME; 309 } 310 mBufferInfo.add(bufferInfo); 311 extractor.advance(); 312 } 313 offset += bufferInfo.size; 314 frameCount++; 315 } 316 byteBuffer.rewind(); 317 byteBuffer.get(mMemory, 0, byteBuffer.limit()); 318 mMemIndex = byteBuffer.limit(); 319 extractor.release(); 320 } 321 addChunkData(ByteBuffer buf, MediaCodec.BufferInfo info)322 public void addChunkData(ByteBuffer buf, MediaCodec.BufferInfo info) { 323 MediaCodec.BufferInfo copy = new MediaCodec.BufferInfo(); 324 copy.set(mMemIndex, info.size, info.presentationTimeUs, info.flags); 325 mBufferInfo.add(copy); 326 327 if (mMemIndex + info.size >= mMemory.length) { 328 mMemory = Arrays.copyOf(mMemory, mMemIndex + info.size); 329 } 330 buf.position(info.offset); 331 buf.get(mMemory, mMemIndex, info.size); 332 mMemIndex += info.size; 333 } 334 335 /** 336 * Sets the MediaFormat, for the benefit of a future decoder. 337 */ setMediaFormat(MediaFormat format)338 public void setMediaFormat(MediaFormat format) { 339 mMediaFormat = format; 340 } 341 342 /** 343 * Gets the MediaFormat that was used by the encoder. 344 */ getMediaFormat()345 public MediaFormat getMediaFormat() { 346 return new MediaFormat(mMediaFormat); 347 } 348 349 /** 350 * Returns the number of chunks currently held. 351 */ getNumChunks()352 public int getNumChunks() { 353 return mBufferInfo.size(); 354 } 355 356 /** 357 * Copies the data from chunk N into "dest". Advances dest.position. 358 */ getChunkData(int chunk, ByteBuffer dest)359 public void getChunkData(int chunk, ByteBuffer dest) { 360 int offset = mBufferInfo.get(chunk).offset; 361 int size = mBufferInfo.get(chunk).size; 362 dest.put(mMemory, offset, size); 363 } 364 365 /** 366 * Returns the flags associated with chunk N. 367 */ getChunkFlags(int chunk)368 public int getChunkFlags(int chunk) { 369 return mBufferInfo.get(chunk).flags; 370 } 371 372 /** 373 * Returns the timestamp associated with chunk N. 374 */ getChunkTime(int chunk)375 public long getChunkTime(int chunk) { 376 return mBufferInfo.get(chunk).presentationTimeUs; 377 } 378 getChunkInfos()379 public ArrayList<MediaCodec.BufferInfo> getChunkInfos() { 380 return mBufferInfo; 381 } 382 getBuffer()383 public ByteBuffer getBuffer() { 384 return ByteBuffer.wrap(mMemory); 385 } 386 dumpBuffer()387 public void dumpBuffer() throws IOException { 388 File dump = File.createTempFile(LOG_TAG + "OUT", ".bin"); 389 Log.d(LOG_TAG, "dump file name is " + dump.getAbsolutePath()); 390 try (FileOutputStream outputStream = new FileOutputStream(dump)) { 391 outputStream.write(mMemory, 0, mMemIndex); 392 } 393 } 394 } 395 396 /** 397 * Edits a video file, saving the contents to a new file. This involves decoding and 398 * re-encoding, not to mention conversions between YUV and RGB, and so may be lossy. 399 * <p> 400 * If we recognize the decoded format we can do this in Java code using the ByteBuffer[] 401 * output, but it's not practical to support all OEM formats. By using a SurfaceTexture 402 * for output and a Surface for input, we can avoid issues with obscure formats and can 403 * use a fragment shader to do transformations. 404 */ editVideoFile(VideoChunks inputData, String kernel)405 private VideoChunks editVideoFile(VideoChunks inputData, String kernel) throws IOException { 406 VideoChunks outputData = new VideoChunks(); 407 MediaCodec decoder = null; 408 MediaCodec encoder = null; 409 InputSurface inputSurface = null; 410 OutputSurface outputSurface = null; 411 412 try { 413 // find decoder for the test clip 414 MediaFormat decoderFormat = inputData.getMediaFormat(); 415 decoderFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, COLOR_FormatSurface); 416 ArrayList<MediaFormat> formats = new ArrayList<>(); 417 formats.add(decoderFormat); 418 ArrayList<String> decoders = CodecTestBase.selectCodecs(RES_MEDIA_TYPE, formats, null, 419 false, SELECT_SWITCH); 420 assumeTrue("Could not find decoder for format : " + decoderFormat, decoders.size() > 0); 421 String decoderName = decoders.get(0); 422 423 // build encoder format and check if it is supported by the current component 424 EncoderConfigParams.Builder foreman = 425 new EncoderConfigParams.Builder(mMediaType) 426 .setWidth(TARGET_WIDTH) 427 .setHeight(TARGET_HEIGHT) 428 .setColorFormat(COLOR_FormatSurface) 429 .setInputBitDepth(8) 430 .setFrameRate(30) 431 .setBitRate(TARGET_BITRATE) 432 .setBitRateMode(BITRATE_MODE_VBR); 433 MediaFormat encoderFormat = foreman.build().getFormat(); 434 formats.clear(); 435 formats.add(encoderFormat); 436 assumeTrue("Encoder: " + mEncoderName + " doesn't support format: " + encoderFormat, 437 CodecTestBase.areFormatsSupported(mEncoderName, mMediaType, formats)); 438 439 // configure 440 encoder = MediaCodec.createByCodecName(mEncoderName); 441 encoder.configure(encoderFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 442 inputSurface = new InputSurface(encoder.createInputSurface(), false); 443 if (inputSurface.getWidth() != TARGET_WIDTH 444 || inputSurface.getHeight() != TARGET_HEIGHT) { 445 inputSurface.updateSize(TARGET_WIDTH, TARGET_HEIGHT); 446 } 447 inputSurface.makeCurrent(); 448 encoder.start(); 449 450 // OutputSurface uses the EGL context created by InputSurface. 451 decoder = MediaCodec.createByCodecName(decoderName); 452 outputSurface = new OutputSurface(); 453 outputSurface.changeFragmentShader(getShader( 454 "const float width = " + (float) TARGET_WIDTH + ";\n", 455 "const float height = " + (float) TARGET_HEIGHT + ";\n", kernel)); 456 // do not allow frame drops 457 decoderFormat.setInteger(KEY_ALLOW_FRAME_DROP, 0); 458 459 decoder.configure(decoderFormat, outputSurface.getSurface(), null, 0); 460 decoder.start(); 461 462 // verify that we are not dropping frames 463 MediaFormat format = decoder.getInputFormat(); 464 assertEquals("Could not prevent frame dropping", 0, 465 format.getInteger(KEY_ALLOW_FRAME_DROP)); 466 467 editVideoData(inputData, decoder, outputSurface, inputSurface, encoder, outputData); 468 } finally { 469 if (VERBOSE) { 470 Log.d(LOG_TAG, "shutting down encoder, decoder"); 471 } 472 if (outputSurface != null) { 473 outputSurface.release(); 474 } 475 if (inputSurface != null) { 476 inputSurface.release(); 477 } 478 if (encoder != null) { 479 encoder.stop(); 480 encoder.release(); 481 } 482 if (decoder != null) { 483 decoder.stop(); 484 decoder.release(); 485 } 486 } 487 return outputData; 488 } 489 490 /** 491 * Edits a stream of video data. 492 */ editVideoData(VideoChunks inputData, MediaCodec decoder, OutputSurface outputSurface, InputSurface inputSurface, MediaCodec encoder, VideoChunks outputData)493 private void editVideoData(VideoChunks inputData, MediaCodec decoder, 494 OutputSurface outputSurface, InputSurface inputSurface, MediaCodec encoder, 495 VideoChunks outputData) { 496 ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers(); 497 ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers(); 498 MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); 499 int inputChunk = 0; 500 int outputCount = 0; 501 502 boolean outputDone = false; 503 boolean inputDone = false; 504 boolean decoderDone = false; 505 while (!outputDone) { 506 if (VERBOSE) { 507 Log.d(LOG_TAG, "edit loop"); 508 } 509 510 // Feed more data to the decoder. 511 if (!inputDone) { 512 int inputBufIndex = decoder.dequeueInputBuffer(Q_DEQ_TIMEOUT_US); 513 if (inputBufIndex >= 0) { 514 if (inputChunk == inputData.getNumChunks()) { 515 // End of stream -- send empty frame with EOS flag set. 516 decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L, 517 MediaCodec.BUFFER_FLAG_END_OF_STREAM); 518 inputDone = true; 519 if (VERBOSE) { 520 Log.d(LOG_TAG, "sent input EOS (with zero-length frame)"); 521 } 522 } else { 523 // Copy a chunk of input to the decoder. The first chunk should have 524 // the BUFFER_FLAG_CODEC_CONFIG flag set. 525 ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex]; 526 inputBuf.clear(); 527 inputData.getChunkData(inputChunk, inputBuf); 528 int flags = inputData.getChunkFlags(inputChunk); 529 long time = inputData.getChunkTime(inputChunk); 530 decoder.queueInputBuffer(inputBufIndex, 0, inputBuf.position(), 531 time, flags); 532 if (VERBOSE) { 533 Log.d(LOG_TAG, "submitted frame " + inputChunk + " to dec, size=" 534 + inputBuf.position() + " flags=" + flags); 535 } 536 inputChunk++; 537 } 538 } else { 539 if (VERBOSE) { 540 Log.d(LOG_TAG, "input buffer not available"); 541 } 542 } 543 } 544 545 // Assume output is available. Loop until both assumptions are false. 546 boolean decoderOutputAvailable = !decoderDone; 547 boolean encoderOutputAvailable = true; 548 while (decoderOutputAvailable || encoderOutputAvailable) { 549 // Start by draining any pending output from the encoder. It's important to 550 // do this before we try to stuff any more data in. 551 int encoderStatus = encoder.dequeueOutputBuffer(info, Q_DEQ_TIMEOUT_US); 552 if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) { 553 // no output available yet 554 if (VERBOSE) { 555 Log.d(LOG_TAG, "no output from encoder available"); 556 } 557 encoderOutputAvailable = false; 558 } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 559 encoderOutputBuffers = encoder.getOutputBuffers(); 560 if (VERBOSE) { 561 Log.d(LOG_TAG, "encoder output buffers changed"); 562 } 563 } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 564 MediaFormat newFormat = encoder.getOutputFormat(); 565 outputData.setMediaFormat(newFormat); 566 if (VERBOSE) { 567 Log.d(LOG_TAG, "encoder output format changed: " + newFormat); 568 } 569 } else if (encoderStatus < 0) { 570 fail("unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus); 571 } else { // encoderStatus >= 0 572 ByteBuffer encodedData = encoderOutputBuffers[encoderStatus]; 573 if (encodedData == null) { 574 fail("encoderOutputBuffer " + encoderStatus + " was null"); 575 } 576 577 // Write the data to the output "file". 578 if (info.size != 0) { 579 encodedData.position(info.offset); 580 encodedData.limit(info.offset + info.size); 581 582 outputData.addChunkData(encodedData, info); 583 584 if ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) outputCount++; 585 586 if (VERBOSE) { 587 Log.d(LOG_TAG, "encoder output " + info.size + " bytes"); 588 } 589 } 590 outputDone = (info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0; 591 encoder.releaseOutputBuffer(encoderStatus, false); 592 } 593 if (encoderStatus != MediaCodec.INFO_TRY_AGAIN_LATER) { 594 // Continue attempts to drain output. 595 continue; 596 } 597 598 // Encoder is drained, check to see if we've got a new frame of output from 599 // the decoder. (The output is going to a Surface, rather than a ByteBuffer, 600 // but we still get information through BufferInfo.) 601 if (!decoderDone) { 602 int decoderStatus = decoder.dequeueOutputBuffer(info, Q_DEQ_TIMEOUT_US); 603 if (decoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) { 604 // no output available yet 605 if (VERBOSE) { 606 Log.d(LOG_TAG, "no output from decoder available"); 607 } 608 decoderOutputAvailable = false; 609 } else if (decoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 610 //decoderOutputBuffers = decoder.getOutputBuffers(); 611 if (VERBOSE) { 612 Log.d(LOG_TAG, "decoder output buffers changed (we don't care)"); 613 } 614 } else if (decoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 615 // expected before first buffer of data 616 MediaFormat newFormat = decoder.getOutputFormat(); 617 if (VERBOSE) { 618 Log.d(LOG_TAG, "decoder output format changed: " + newFormat); 619 } 620 } else if (decoderStatus < 0) { 621 fail("unexpected result from decoder.dequeueOutputBuffer: " 622 + decoderStatus); 623 } else { // decoderStatus >= 0 624 if (VERBOSE) { 625 Log.d(LOG_TAG, "surface decoder given buffer " + decoderStatus 626 + " (size=" + info.size + ")"); 627 } 628 // The ByteBuffers are null references, but we still get a nonzero 629 // size for the decoded data. 630 boolean doRender = (info.size != 0); 631 632 // As soon as we call releaseOutputBuffer, the buffer will be forwarded 633 // to SurfaceTexture to convert to a texture. The API doesn't 634 // guarantee that the texture will be available before the call 635 // returns, so we need to wait for the onFrameAvailable callback to 636 // fire. If we don't wait, we risk rendering from the previous frame. 637 decoder.releaseOutputBuffer(decoderStatus, doRender); 638 if (doRender) { 639 // This waits for the image and renders it after it arrives. 640 if (VERBOSE) { 641 Log.d(LOG_TAG, "awaiting frame"); 642 } 643 outputSurface.awaitNewImage(); 644 outputSurface.drawImage(); 645 646 // Send it to the encoder. 647 inputSurface.setPresentationTime(info.presentationTimeUs * 1000); 648 if (VERBOSE) { 649 Log.d(LOG_TAG, "swapBuffers"); 650 } 651 inputSurface.swapBuffers(); 652 } 653 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 654 // forward decoder EOS to encoder 655 if (VERBOSE) { 656 Log.d(LOG_TAG, "signaling input EOS"); 657 } 658 if (WORK_AROUND_BUGS) { 659 // Bail early, possibly dropping a frame. 660 return; 661 } else { 662 encoder.signalEndOfInputStream(); 663 } 664 } 665 } 666 } 667 } 668 } 669 670 if (inputChunk != outputCount) { 671 throw new RuntimeException("frame lost: " + inputChunk + " in, " + outputCount 672 + " out"); 673 } 674 } 675 computeVariance(RawResource yuv)676 private double computeVariance(RawResource yuv) throws IOException { 677 Preconditions.assertTestFileExists(yuv.mFileName); 678 assertEquals("has support for 8 bit clips only", 1, yuv.mBytesPerSample); 679 final int bSize = 16; 680 assertTrue("chosen block size is too large with respect to image dimensions", 681 yuv.mWidth > bSize && yuv.mHeight > bSize); 682 double variance = 0; 683 int blocks = 0; 684 try (RandomAccessFile refStream = new RandomAccessFile(new File(yuv.mFileName), "r")) { 685 int ySize = yuv.mWidth * yuv.mHeight; 686 int uvSize = ySize >> 1; 687 byte[] luma = new byte[ySize]; 688 689 while (true) { 690 int bytesReadRef = refStream.read(luma); 691 if (bytesReadRef == -1) break; 692 assertEquals("bad, reading unaligned frame size", bytesReadRef, ySize); 693 refStream.skipBytes(uvSize); 694 for (int i = 0; i < yuv.mHeight - bSize; i += bSize) { 695 for (int j = 0; j < yuv.mWidth - bSize; j += bSize) { 696 long sse = 0, sum = 0; 697 int offset = i * yuv.mWidth + j; 698 for (int p = 0; p < bSize; p++) { 699 for (int q = 0; q < bSize; q++) { 700 int sample = luma[offset + p * yuv.mWidth + q]; 701 sum += sample; 702 sse += sample * sample; 703 } 704 } 705 double meanOfSquares = ((double) sse) / (bSize * bSize); 706 double mean = ((double) sum) / (bSize * bSize); 707 double squareOfMean = mean * mean; 708 double blockVariance = (meanOfSquares - squareOfMean); 709 assertTrue("variance can't be negative", blockVariance >= 0.0f); 710 variance += blockVariance; 711 assertTrue("caution overflow", variance >= 0.0); 712 blocks++; 713 } 714 } 715 } 716 return variance / blocks; 717 } 718 } 719 720 /** 721 * Extract, Decode, Edit, Encode and Validate. Check description of class 722 * {@link VideoDecodeEditEncodeTest} 723 */ 724 @ApiTest(apis = {"android.opengl.GLES20#GL_FRAGMENT_SHADER", 725 "android.media.format.MediaFormat#KEY_ALLOW_FRAME_DROP", 726 "MediaCodecInfo.CodecCapabilities#COLOR_FormatSurface"}) 727 @Test testVideoEdit()728 public void testVideoEdit() throws IOException, InterruptedException { 729 VideoChunks sourceChunks = new VideoChunks(); 730 sourceChunks.splitMediaToMuxerParameters(RES_CLIP, RES_MEDIA_TYPE, 90); 731 VideoChunks[] outputData = new VideoChunks[2]; 732 for (int i = 0; i < 2; i++) { 733 outputData[i] = editVideoFile(sourceChunks, KERNELS[i]); 734 } 735 String tmpPathA = getTempFilePath(""); 736 mTmpFiles.add(tmpPathA); 737 String tmpPathB = getTempFilePath(""); 738 mTmpFiles.add(tmpPathB); 739 740 int muxerFormat = getMuxerFormatForMediaType(mMediaType); 741 muxOutput(tmpPathA, muxerFormat, outputData[0].getMediaFormat(), outputData[0].getBuffer(), 742 outputData[0].getChunkInfos()); 743 muxOutput(tmpPathB, muxerFormat, outputData[1].getMediaFormat(), outputData[1].getBuffer(), 744 outputData[1].getChunkInfos()); 745 746 CompareStreams cs = null; 747 try { 748 cs = new CompareStreams(mMediaType, tmpPathA, mMediaType, tmpPathB, false, false); 749 double[] avgPSNR = cs.getAvgPSNR(); 750 final double weightedAvgPSNR = (4 * avgPSNR[0] + avgPSNR[1] + avgPSNR[2]) / 6; 751 if (weightedAvgPSNR < AVG_ACCEPTABLE_QUALITY) { 752 fail(String.format("Average PSNR of the sequence: %f is < threshold : %f\n", 753 weightedAvgPSNR, AVG_ACCEPTABLE_QUALITY)); 754 } 755 } finally { 756 if (cs != null) cs.cleanUp(); 757 } 758 DecodeStreamToYuv yuvRes = new DecodeStreamToYuv(mMediaType, tmpPathA, Integer.MAX_VALUE, 759 LOG_TAG); 760 RawResource yuv = yuvRes.getDecodedYuv(); 761 mTmpFiles.add(yuv.mFileName); 762 double varA = computeVariance(yuv); 763 764 yuvRes = new DecodeStreamToYuv(mMediaType, tmpPathB, Integer.MAX_VALUE, LOG_TAG); 765 yuv = yuvRes.getDecodedYuv(); 766 mTmpFiles.add(yuv.mFileName); 767 double varB = computeVariance(yuv); 768 769 Log.d(LOG_TAG, "variance is " + varA + " " + varB); 770 assertTrue(String.format("Blurred clip variance is not less than sharpened clip. Variance" 771 + " of blurred clip is %f, variance of sharpened clip is %f", varA, varB), 772 varA < varB); 773 } 774 } 775