• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1/*
2 *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#import "RTCCVPixelBuffer.h"
12
13#import "api/video_frame_buffer/RTCNativeMutableI420Buffer.h"
14
15#include "common_video/libyuv/include/webrtc_libyuv.h"
16#include "rtc_base/checks.h"
17#include "rtc_base/logging.h"
18#include "third_party/libyuv/include/libyuv.h"
19
20#if !defined(NDEBUG) && defined(WEBRTC_IOS)
21#import <UIKit/UIKit.h>
22#import <VideoToolbox/VideoToolbox.h>
23#endif
24
25@implementation RTC_OBJC_TYPE (RTCCVPixelBuffer) {
26  int _width;
27  int _height;
28  int _bufferWidth;
29  int _bufferHeight;
30  int _cropWidth;
31  int _cropHeight;
32}
33
34@synthesize pixelBuffer = _pixelBuffer;
35@synthesize cropX = _cropX;
36@synthesize cropY = _cropY;
37@synthesize cropWidth = _cropWidth;
38@synthesize cropHeight = _cropHeight;
39
40+ (NSSet<NSNumber*>*)supportedPixelFormats {
41  return [NSSet setWithObjects:@(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange),
42                               @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange),
43                               @(kCVPixelFormatType_32BGRA),
44                               @(kCVPixelFormatType_32ARGB),
45                               nil];
46}
47
48- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer {
49  return [self initWithPixelBuffer:pixelBuffer
50                      adaptedWidth:CVPixelBufferGetWidth(pixelBuffer)
51                     adaptedHeight:CVPixelBufferGetHeight(pixelBuffer)
52                         cropWidth:CVPixelBufferGetWidth(pixelBuffer)
53                        cropHeight:CVPixelBufferGetHeight(pixelBuffer)
54                             cropX:0
55                             cropY:0];
56}
57
58- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer
59                       adaptedWidth:(int)adaptedWidth
60                      adaptedHeight:(int)adaptedHeight
61                          cropWidth:(int)cropWidth
62                         cropHeight:(int)cropHeight
63                              cropX:(int)cropX
64                              cropY:(int)cropY {
65  if (self = [super init]) {
66    _width = adaptedWidth;
67    _height = adaptedHeight;
68    _pixelBuffer = pixelBuffer;
69    _bufferWidth = CVPixelBufferGetWidth(_pixelBuffer);
70    _bufferHeight = CVPixelBufferGetHeight(_pixelBuffer);
71    _cropWidth = cropWidth;
72    _cropHeight = cropHeight;
73    // Can only crop at even pixels.
74    _cropX = cropX & ~1;
75    _cropY = cropY & ~1;
76    CVBufferRetain(_pixelBuffer);
77  }
78
79  return self;
80}
81
82- (void)dealloc {
83  CVBufferRelease(_pixelBuffer);
84}
85
86- (int)width {
87  return _width;
88}
89
90- (int)height {
91  return _height;
92}
93
94- (BOOL)requiresCropping {
95  return _cropWidth != _bufferWidth || _cropHeight != _bufferHeight;
96}
97
98- (BOOL)requiresScalingToWidth:(int)width height:(int)height {
99  return _cropWidth != width || _cropHeight != height;
100}
101
102- (int)bufferSizeForCroppingAndScalingToWidth:(int)width height:(int)height {
103  const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer);
104  switch (srcPixelFormat) {
105    case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
106    case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: {
107      int srcChromaWidth = (_cropWidth + 1) / 2;
108      int srcChromaHeight = (_cropHeight + 1) / 2;
109      int dstChromaWidth = (width + 1) / 2;
110      int dstChromaHeight = (height + 1) / 2;
111
112      return srcChromaWidth * srcChromaHeight * 2 + dstChromaWidth * dstChromaHeight * 2;
113    }
114    case kCVPixelFormatType_32BGRA:
115    case kCVPixelFormatType_32ARGB: {
116      return 0;  // Scaling RGBA frames does not require a temporary buffer.
117    }
118  }
119  RTC_DCHECK_NOTREACHED() << "Unsupported pixel format.";
120  return 0;
121}
122
123- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer
124        withTempBuffer:(nullable uint8_t*)tmpBuffer {
125  const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer);
126  const OSType dstPixelFormat = CVPixelBufferGetPixelFormatType(outputPixelBuffer);
127
128  switch (srcPixelFormat) {
129    case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
130    case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: {
131      size_t dstWidth = CVPixelBufferGetWidth(outputPixelBuffer);
132      size_t dstHeight = CVPixelBufferGetHeight(outputPixelBuffer);
133      if (dstWidth > 0 && dstHeight > 0) {
134        RTC_DCHECK(dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ||
135                   dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
136        if ([self requiresScalingToWidth:dstWidth height:dstHeight]) {
137          RTC_DCHECK(tmpBuffer);
138        }
139        [self cropAndScaleNV12To:outputPixelBuffer withTempBuffer:tmpBuffer];
140      }
141      break;
142    }
143    case kCVPixelFormatType_32BGRA:
144    case kCVPixelFormatType_32ARGB: {
145      RTC_DCHECK(srcPixelFormat == dstPixelFormat);
146      [self cropAndScaleARGBTo:outputPixelBuffer];
147      break;
148    }
149    default: {
150      RTC_DCHECK_NOTREACHED() << "Unsupported pixel format.";
151    }
152  }
153
154  return YES;
155}
156- (id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>)cropAndScaleWith:(int)offsetX
157                                                   offsetY:(int)offsetY
158                                                 cropWidth:(int)cropWidth
159                                                cropHeight:(int)cropHeight
160                                                scaleWidth:(int)scaleWidth
161                                               scaleHeight:(int)scaleHeight {
162  return [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc]
163      initWithPixelBuffer:_pixelBuffer
164             adaptedWidth:scaleWidth
165            adaptedHeight:scaleHeight
166                cropWidth:cropWidth * _cropWidth / _width
167               cropHeight:cropHeight * _cropHeight / _height
168                    cropX:_cropX + offsetX * _cropWidth / _width
169                    cropY:_cropY + offsetY * _cropHeight / _height];
170}
171
172- (id<RTC_OBJC_TYPE(RTCI420Buffer)>)toI420 {
173  const OSType pixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer);
174
175  CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly);
176
177  RTC_OBJC_TYPE(RTCMutableI420Buffer)* i420Buffer =
178      [[RTC_OBJC_TYPE(RTCMutableI420Buffer) alloc] initWithWidth:[self width] height:[self height]];
179
180  switch (pixelFormat) {
181    case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
182    case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: {
183      const uint8_t* srcY =
184          static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0));
185      const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0);
186      const uint8_t* srcUV =
187          static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1));
188      const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1);
189
190      // Crop just by modifying pointers.
191      srcY += srcYStride * _cropY + _cropX;
192      srcUV += srcUVStride * (_cropY / 2) + _cropX;
193
194      // TODO(magjed): Use a frame buffer pool.
195      webrtc::NV12ToI420Scaler nv12ToI420Scaler;
196      nv12ToI420Scaler.NV12ToI420Scale(srcY,
197                                       srcYStride,
198                                       srcUV,
199                                       srcUVStride,
200                                       _cropWidth,
201                                       _cropHeight,
202                                       i420Buffer.mutableDataY,
203                                       i420Buffer.strideY,
204                                       i420Buffer.mutableDataU,
205                                       i420Buffer.strideU,
206                                       i420Buffer.mutableDataV,
207                                       i420Buffer.strideV,
208                                       i420Buffer.width,
209                                       i420Buffer.height);
210      break;
211    }
212    case kCVPixelFormatType_32BGRA:
213    case kCVPixelFormatType_32ARGB: {
214      CVPixelBufferRef scaledPixelBuffer = NULL;
215      CVPixelBufferRef sourcePixelBuffer = NULL;
216      if ([self requiresCropping] ||
217          [self requiresScalingToWidth:i420Buffer.width height:i420Buffer.height]) {
218        CVPixelBufferCreate(
219            NULL, i420Buffer.width, i420Buffer.height, pixelFormat, NULL, &scaledPixelBuffer);
220        [self cropAndScaleTo:scaledPixelBuffer withTempBuffer:NULL];
221
222        CVPixelBufferLockBaseAddress(scaledPixelBuffer, kCVPixelBufferLock_ReadOnly);
223        sourcePixelBuffer = scaledPixelBuffer;
224      } else {
225        sourcePixelBuffer = _pixelBuffer;
226      }
227      const uint8_t* src = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(sourcePixelBuffer));
228      const size_t bytesPerRow = CVPixelBufferGetBytesPerRow(sourcePixelBuffer);
229
230      if (pixelFormat == kCVPixelFormatType_32BGRA) {
231        // Corresponds to libyuv::FOURCC_ARGB
232        libyuv::ARGBToI420(src,
233                           bytesPerRow,
234                           i420Buffer.mutableDataY,
235                           i420Buffer.strideY,
236                           i420Buffer.mutableDataU,
237                           i420Buffer.strideU,
238                           i420Buffer.mutableDataV,
239                           i420Buffer.strideV,
240                           i420Buffer.width,
241                           i420Buffer.height);
242      } else if (pixelFormat == kCVPixelFormatType_32ARGB) {
243        // Corresponds to libyuv::FOURCC_BGRA
244        libyuv::BGRAToI420(src,
245                           bytesPerRow,
246                           i420Buffer.mutableDataY,
247                           i420Buffer.strideY,
248                           i420Buffer.mutableDataU,
249                           i420Buffer.strideU,
250                           i420Buffer.mutableDataV,
251                           i420Buffer.strideV,
252                           i420Buffer.width,
253                           i420Buffer.height);
254      }
255
256      if (scaledPixelBuffer) {
257        CVPixelBufferUnlockBaseAddress(scaledPixelBuffer, kCVPixelBufferLock_ReadOnly);
258        CVBufferRelease(scaledPixelBuffer);
259      }
260      break;
261    }
262    default: {
263      RTC_DCHECK_NOTREACHED() << "Unsupported pixel format.";
264    }
265  }
266
267  CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly);
268
269  return i420Buffer;
270}
271
272#pragma mark - Debugging
273
274#if !defined(NDEBUG) && defined(WEBRTC_IOS)
275- (id)debugQuickLookObject {
276  CGImageRef cgImage;
277  VTCreateCGImageFromCVPixelBuffer(_pixelBuffer, NULL, &cgImage);
278  UIImage *image = [UIImage imageWithCGImage:cgImage scale:1.0 orientation:UIImageOrientationUp];
279  CGImageRelease(cgImage);
280  return image;
281}
282#endif
283
284#pragma mark - Private
285
286- (void)cropAndScaleNV12To:(CVPixelBufferRef)outputPixelBuffer withTempBuffer:(uint8_t*)tmpBuffer {
287  // Prepare output pointers.
288  CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0);
289  if (cvRet != kCVReturnSuccess) {
290    RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
291  }
292  const int dstWidth = CVPixelBufferGetWidth(outputPixelBuffer);
293  const int dstHeight = CVPixelBufferGetHeight(outputPixelBuffer);
294  uint8_t* dstY =
295      reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0));
296  const int dstYStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0);
297  uint8_t* dstUV =
298      reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 1));
299  const int dstUVStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 1);
300
301  // Prepare source pointers.
302  CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly);
303  const uint8_t* srcY = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0));
304  const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0);
305  const uint8_t* srcUV = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1));
306  const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1);
307
308  // Crop just by modifying pointers.
309  srcY += srcYStride * _cropY + _cropX;
310  srcUV += srcUVStride * (_cropY / 2) + _cropX;
311
312  webrtc::NV12Scale(tmpBuffer,
313                    srcY,
314                    srcYStride,
315                    srcUV,
316                    srcUVStride,
317                    _cropWidth,
318                    _cropHeight,
319                    dstY,
320                    dstYStride,
321                    dstUV,
322                    dstUVStride,
323                    dstWidth,
324                    dstHeight);
325
326  CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly);
327  CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0);
328}
329
330- (void)cropAndScaleARGBTo:(CVPixelBufferRef)outputPixelBuffer {
331  // Prepare output pointers.
332  CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0);
333  if (cvRet != kCVReturnSuccess) {
334    RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
335  }
336  const int dstWidth = CVPixelBufferGetWidth(outputPixelBuffer);
337  const int dstHeight = CVPixelBufferGetHeight(outputPixelBuffer);
338
339  uint8_t* dst = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddress(outputPixelBuffer));
340  const int dstStride = CVPixelBufferGetBytesPerRow(outputPixelBuffer);
341
342  // Prepare source pointers.
343  CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly);
344  const uint8_t* src = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(_pixelBuffer));
345  const int srcStride = CVPixelBufferGetBytesPerRow(_pixelBuffer);
346
347  // Crop just by modifying pointers. Need to ensure that src pointer points to a byte corresponding
348  // to the start of a new pixel (byte with B for BGRA) so that libyuv scales correctly.
349  const int bytesPerPixel = 4;
350  src += srcStride * _cropY + (_cropX * bytesPerPixel);
351
352  // kCVPixelFormatType_32BGRA corresponds to libyuv::FOURCC_ARGB
353  libyuv::ARGBScale(src,
354                    srcStride,
355                    _cropWidth,
356                    _cropHeight,
357                    dst,
358                    dstStride,
359                    dstWidth,
360                    dstHeight,
361                    libyuv::kFilterBox);
362
363  CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly);
364  CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0);
365}
366
367@end
368