• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2023 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.mediav2.common.cts;
18 
19 import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUVP010;
20 import static android.mediav2.common.cts.DecodeStreamToYuv.findDecoderForStream;
21 import static android.mediav2.common.cts.DecodeStreamToYuv.getFormatInStream;
22 import static android.mediav2.common.cts.DecodeStreamToYuv.getImage;
23 import static android.mediav2.common.cts.VideoErrorManager.computeMSE;
24 import static android.mediav2.common.cts.VideoErrorManager.computePSNR;
25 
26 import static org.junit.Assert.assertEquals;
27 import static org.junit.Assert.assertNotNull;
28 
29 import android.graphics.ImageFormat;
30 import android.media.Image;
31 import android.media.MediaCodec;
32 import android.media.MediaExtractor;
33 import android.media.MediaFormat;
34 import android.util.Log;
35 
36 import com.android.compatibility.common.util.MediaUtils;
37 
38 import org.junit.Assume;
39 
40 import java.io.File;
41 import java.io.FileInputStream;
42 import java.io.IOException;
43 import java.nio.ByteBuffer;
44 import java.util.ArrayList;
45 import java.util.Arrays;
46 
47 /**
48  * Wrapper class for storing YUV Planes of an image
49  */
50 class YUVImage {
51     public ArrayList<byte[]> mData = new ArrayList<>();
52 }
53 
54 /**
55  * Utility class for video encoder tests to validate the encoded output.
56  * <p>
57  * The class computes the PSNR between encoders output and input. As the input to an encoder can
58  * be raw yuv buffer or the output of a decoder that is connected to the encoder, the test
59  * accepts YUV as well as compressed streams for validation.
60  * <p>
61  * Before validation, the class checks if the input and output have same width, height and bitdepth.
62  */
63 public class CompareStreams extends CodecDecoderTestBase {
64     private static final String LOG_TAG = CompareStreams.class.getSimpleName();
65 
66     private final RawResource mRefYuv;
67     private final MediaFormat mStreamFormat;
68     private final ByteBuffer mStreamBuffer;
69     private final ArrayList<MediaCodec.BufferInfo> mStreamBufferInfos;
70     private final boolean mAllowRefResize;
71     private final boolean mAllowRefLoopBack;
72     private final double[] mGlobalMSE;
73     private final double[] mMinimumMSE;
74     private final double[] mGlobalPSNR;
75     private final double[] mMinimumPSNR;
76     private final double[] mAvgPSNR;
77     private final ArrayList<double[]> mFramesPSNR;
78 
79     private final ArrayList<String> mTmpFiles = new ArrayList<>();
80     private boolean mGenerateStats;
81     private int mFileOffset;
82     private int mFileSize;
83     private int mFrameSize;
84     private byte[] mInputData;
85 
CompareStreams(RawResource refYuv, String testMediaType, String testFile, MediaFormat testFormat, ByteBuffer testBuffer, ArrayList<MediaCodec.BufferInfo> testBufferInfos, boolean allowRefResize, boolean allowRefLoopBack)86     private CompareStreams(RawResource refYuv, String testMediaType, String testFile,
87             MediaFormat testFormat, ByteBuffer testBuffer,
88             ArrayList<MediaCodec.BufferInfo> testBufferInfos, boolean allowRefResize,
89             boolean allowRefLoopBack) throws IOException {
90         super(findDecoderForStream(testMediaType, testFile), testMediaType, testFile, LOG_TAG);
91         mRefYuv = refYuv;
92         mStreamFormat = testFormat;
93         mStreamBuffer = testBuffer;
94         mStreamBufferInfos = testBufferInfos;
95         mAllowRefResize = allowRefResize;
96         mAllowRefLoopBack = allowRefLoopBack;
97         mMinimumMSE = new double[3];
98         Arrays.fill(mMinimumMSE, Float.MAX_VALUE);
99         mGlobalMSE = new double[3];
100         Arrays.fill(mGlobalMSE, 0.0);
101         mGlobalPSNR = new double[3];
102         mMinimumPSNR = new double[3];
103         mAvgPSNR = new double[3];
104         Arrays.fill(mAvgPSNR, 0.0);
105         mFramesPSNR = new ArrayList<>();
106     }
107 
CompareStreams(RawResource refYuv, String testMediaType, String testFile, boolean allowRefResize, boolean allowRefLoopBack)108     public CompareStreams(RawResource refYuv, String testMediaType, String testFile,
109             boolean allowRefResize, boolean allowRefLoopBack) throws IOException {
110         this(refYuv, testMediaType, testFile, null, null, null, allowRefResize, allowRefLoopBack);
111     }
112 
CompareStreams(MediaFormat refFormat, ByteBuffer refBuffer, ArrayList<MediaCodec.BufferInfo> refBufferInfos, MediaFormat testFormat, ByteBuffer testBuffer, ArrayList<MediaCodec.BufferInfo> testBufferInfos, boolean allowRefResize, boolean allowRefLoopBack)113     public CompareStreams(MediaFormat refFormat, ByteBuffer refBuffer,
114             ArrayList<MediaCodec.BufferInfo> refBufferInfos, MediaFormat testFormat,
115             ByteBuffer testBuffer, ArrayList<MediaCodec.BufferInfo> testBufferInfos,
116             boolean allowRefResize, boolean allowRefLoopBack) throws IOException {
117         this(new DecodeStreamToYuv(refFormat, refBuffer, refBufferInfos).getDecodedYuv(), null,
118                 null, testFormat, testBuffer, testBufferInfos, allowRefResize, allowRefLoopBack);
119         mTmpFiles.add(mRefYuv.mFileName);
120     }
121 
CompareStreams(String refMediaType, String refFile, String testMediaType, String testFile, boolean allowRefResize, boolean allowRefLoopBack)122     public CompareStreams(String refMediaType, String refFile, String testMediaType,
123             String testFile, boolean allowRefResize, boolean allowRefLoopBack) throws IOException {
124         this(new DecodeStreamToYuv(refMediaType, refFile).getDecodedYuv(), testMediaType, testFile,
125                 allowRefResize, allowRefLoopBack);
126         mTmpFiles.add(mRefYuv.mFileName);
127     }
128 
fillByteArray(int tgtFrameWidth, int tgtFrameHeight, int bytesPerSample, int inpFrameWidth, int inpFrameHeight, byte[] inputData)129     static YUVImage fillByteArray(int tgtFrameWidth, int tgtFrameHeight,
130             int bytesPerSample, int inpFrameWidth, int inpFrameHeight, byte[] inputData) {
131         YUVImage yuvImage = new YUVImage();
132         int inOffset = 0;
133         for (int plane = 0; plane < 3; plane++) {
134             int width, height, tileWidth, tileHeight;
135             if (plane != 0) {
136                 width = tgtFrameWidth / 2;
137                 height = tgtFrameHeight / 2;
138                 tileWidth = inpFrameWidth / 2;
139                 tileHeight = inpFrameHeight / 2;
140             } else {
141                 width = tgtFrameWidth;
142                 height = tgtFrameHeight;
143                 tileWidth = inpFrameWidth;
144                 tileHeight = inpFrameHeight;
145             }
146             byte[] outputData = new byte[width * height * bytesPerSample];
147             for (int k = 0; k < height; k += tileHeight) {
148                 int rowsToCopy = Math.min(height - k, tileHeight);
149                 for (int j = 0; j < rowsToCopy; j++) {
150                     for (int i = 0; i < width; i += tileWidth) {
151                         int colsToCopy = Math.min(width - i, tileWidth);
152                         System.arraycopy(inputData,
153                                 inOffset + j * tileWidth * bytesPerSample,
154                                 outputData,
155                                 (k + j) * width * bytesPerSample + i * bytesPerSample,
156                                 colsToCopy * bytesPerSample);
157                     }
158                 }
159             }
160             inOffset += tileWidth * tileHeight * bytesPerSample;
161             yuvImage.mData.add(outputData);
162         }
163         return yuvImage;
164     }
165 
dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info)166     protected void dequeueOutput(int bufferIndex, MediaCodec.BufferInfo info) {
167         if (info.size > 0) {
168             Image img = mCodec.getOutputImage(bufferIndex);
169             assertNotNull(img);
170             YUVImage yuvImage = getImage(img);
171             MediaFormat format = mCodec.getOutputFormat();
172             int width = getWidth(format);
173             int height = getHeight(format);
174             if (mOutputCount == 0) {
175                 int imgFormat = img.getFormat();
176                 int bytesPerSample = (ImageFormat.getBitsPerPixel(imgFormat) * 2) / (8 * 3);
177                 if (mRefYuv.mBytesPerSample != bytesPerSample) {
178                     String msg = String.format(
179                             "Reference file bytesPerSample and Test file bytesPerSample are not "
180                                     + "same. Reference bytesPerSample : %d, Test bytesPerSample :"
181                                     + " %d", mRefYuv.mBytesPerSample, bytesPerSample);
182                     throw new IllegalArgumentException(msg);
183                 }
184                 if (!mAllowRefResize && (mRefYuv.mWidth != width || mRefYuv.mHeight != height)) {
185                     String msg = String.format(
186                             "Reference file attributes and Test file attributes are not same. "
187                                     + "Reference width : %d, height : %d, bytesPerSample : %d, "
188                                     + "Test width : %d, height : %d, bytesPerSample : %d",
189                             mRefYuv.mWidth, mRefYuv.mHeight, mRefYuv.mBytesPerSample, width,
190                             height, bytesPerSample);
191                     throw new IllegalArgumentException(msg);
192                 }
193                 mFileOffset = 0;
194                 mFileSize = (int) new File(mRefYuv.mFileName).length();
195                 mFrameSize = mRefYuv.mWidth * mRefYuv.mHeight * mRefYuv.mBytesPerSample * 3 / 2;
196                 mInputData = new byte[mFrameSize];
197             }
198             try (FileInputStream fInp = new FileInputStream(mRefYuv.mFileName)) {
199                 assertEquals(mFileOffset, fInp.skip(mFileOffset));
200                 assertEquals(mFrameSize, fInp.read(mInputData));
201                 mFileOffset += mFrameSize;
202                 if (mAllowRefLoopBack && mFileOffset == mFileSize) mFileOffset = 0;
203                 YUVImage yuvRefImage = fillByteArray(width, height, mRefYuv.mBytesPerSample,
204                         mRefYuv.mWidth, mRefYuv.mHeight, mInputData);
205                 updateErrorStats(yuvRefImage.mData.get(0), yuvRefImage.mData.get(1),
206                         yuvRefImage.mData.get(2), yuvImage.mData.get(0), yuvImage.mData.get(1),
207                         yuvImage.mData.get(2));
208 
209             } catch (IOException e) {
210                 throw new RuntimeException(e);
211             }
212             mOutputCount++;
213         }
214         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
215             mSawOutputEOS = true;
216             mGenerateStats = true;
217             finalizerErrorStats();
218         }
219         if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
220             mOutputBuff.saveOutPTS(info.presentationTimeUs);
221             mOutputCount++;
222         }
223         mCodec.releaseOutputBuffer(bufferIndex, false);
224     }
225 
updateErrorStats(byte[] yRef, byte[] uRef, byte[] vRef, byte[] yTest, byte[] uTest, byte[] vTest)226     private void updateErrorStats(byte[] yRef, byte[] uRef, byte[] vRef, byte[] yTest,
227             byte[] uTest, byte[] vTest) {
228         double curYMSE = computeMSE(yRef, yTest, mRefYuv.mBytesPerSample);
229         mGlobalMSE[0] += curYMSE;
230         mMinimumMSE[0] = Math.min(mMinimumMSE[0], curYMSE);
231 
232         double curUMSE = computeMSE(uRef, uTest, mRefYuv.mBytesPerSample);
233         mGlobalMSE[1] += curUMSE;
234         mMinimumMSE[1] = Math.min(mMinimumMSE[1], curUMSE);
235 
236         double curVMSE = computeMSE(vRef, vTest, mRefYuv.mBytesPerSample);
237         mGlobalMSE[2] += curVMSE;
238         mMinimumMSE[2] = Math.min(mMinimumMSE[2], curVMSE);
239 
240         double yFramePSNR = computePSNR(curYMSE, mRefYuv.mBytesPerSample);
241         double uFramePSNR = computePSNR(curUMSE, mRefYuv.mBytesPerSample);
242         double vFramePSNR = computePSNR(curVMSE, mRefYuv.mBytesPerSample);
243         mAvgPSNR[0] += yFramePSNR;
244         mAvgPSNR[1] += uFramePSNR;
245         mAvgPSNR[2] += vFramePSNR;
246         mFramesPSNR.add(new double[]{yFramePSNR, uFramePSNR, vFramePSNR});
247     }
248 
finalizerErrorStats()249     private void finalizerErrorStats() {
250         for (int i = 0; i < mGlobalPSNR.length; i++) {
251             mGlobalMSE[i] /= mFramesPSNR.size();
252             mGlobalPSNR[i] = computePSNR(mGlobalMSE[i], mRefYuv.mBytesPerSample);
253             mMinimumPSNR[i] = computePSNR(mMinimumMSE[i], mRefYuv.mBytesPerSample);
254             mAvgPSNR[i] /= mFramesPSNR.size();
255         }
256         if (ENABLE_LOGS) {
257             String msg = String.format(
258                     "global_psnr_y:%.2f, global_psnr_u:%.2f, global_psnr_v:%.2f, min_psnr_y:%"
259                             + ".2f, min_psnr_u:%.2f, min_psnr_v:%.2f avg_psnr_y:%.2f, "
260                             + "avg_psnr_u:%.2f, avg_psnr_v:%.2f",
261                     mGlobalPSNR[0], mGlobalPSNR[1], mGlobalPSNR[2], mMinimumPSNR[0],
262                     mMinimumPSNR[1], mMinimumPSNR[2], mAvgPSNR[0], mAvgPSNR[1], mAvgPSNR[2]);
263             Log.v(LOG_TAG, msg);
264         }
265     }
266 
generateErrorStats()267     private void generateErrorStats() throws IOException, InterruptedException {
268         if (!mGenerateStats) {
269             if (MediaUtils.isTv()) {
270                 // Some TV devices support HDR10 display with VO instead of GPU. In this case,
271                 // COLOR_FormatYUVP010 may not be supported.
272                 MediaFormat format = mStreamFormat != null ? mStreamFormat :
273                         getFormatInStream(mMediaType, mTestFile);
274                 ArrayList<MediaFormat> formatList = new ArrayList<>();
275                 formatList.add(format);
276                 boolean isHBD = doesAnyFormatHaveHDRProfile(mMediaType, formatList);
277                 if (isHBD || mTestFile.contains("10bit")) {
278                     if (!hasSupportForColorFormat(mCodecName, mMediaType, COLOR_FormatYUVP010)) {
279                         Assume.assumeTrue("Could not validate the encoded output as"
280                                 + " COLOR_FormatYUVP010 is not supported by the decoder", false);
281                     }
282                 }
283             }
284             if (mStreamFormat != null) {
285                 decodeToMemory(mStreamBuffer, mStreamBufferInfos, mStreamFormat, mCodecName);
286             } else {
287                 decodeToMemory(mTestFile, mCodecName, 0, MediaExtractor.SEEK_TO_CLOSEST_SYNC,
288                         Integer.MAX_VALUE);
289             }
290         }
291     }
292 
293     /**
294      * @see VideoErrorManager#getGlobalPSNR()
295      */
getGlobalPSNR()296     public double[] getGlobalPSNR() throws IOException, InterruptedException {
297         generateErrorStats();
298         return mGlobalPSNR;
299     }
300 
301     /**
302      * @see VideoErrorManager#getMinimumPSNR()
303      */
getMinimumPSNR()304     public double[] getMinimumPSNR() throws IOException, InterruptedException {
305         generateErrorStats();
306         return mMinimumPSNR;
307     }
308 
309     /**
310      * @see VideoErrorManager#getFramesPSNR()
311      */
getFramesPSNR()312     public ArrayList<double[]> getFramesPSNR() throws IOException, InterruptedException {
313         generateErrorStats();
314         return mFramesPSNR;
315     }
316 
317     /**
318      * @see VideoErrorManager#getAvgPSNR()
319      */
getAvgPSNR()320     public double[] getAvgPSNR() throws IOException, InterruptedException {
321         generateErrorStats();
322         return mAvgPSNR;
323     }
324 
cleanUp()325     public void cleanUp() {
326         for (String tmpFile : mTmpFiles) {
327             File tmp = new File(tmpFile);
328             if (tmp.exists()) tmp.delete();
329         }
330         mTmpFiles.clear();
331     }
332 }
333