1/* 2 * Copyright 2017 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#import "RTCCVPixelBuffer.h" 12 13#import "api/video_frame_buffer/RTCNativeMutableI420Buffer.h" 14 15#include "common_video/libyuv/include/webrtc_libyuv.h" 16#include "rtc_base/checks.h" 17#include "rtc_base/logging.h" 18#include "third_party/libyuv/include/libyuv.h" 19 20#if !defined(NDEBUG) && defined(WEBRTC_IOS) 21#import <UIKit/UIKit.h> 22#import <VideoToolbox/VideoToolbox.h> 23#endif 24 25@implementation RTC_OBJC_TYPE (RTCCVPixelBuffer) { 26 int _width; 27 int _height; 28 int _bufferWidth; 29 int _bufferHeight; 30 int _cropWidth; 31 int _cropHeight; 32} 33 34@synthesize pixelBuffer = _pixelBuffer; 35@synthesize cropX = _cropX; 36@synthesize cropY = _cropY; 37@synthesize cropWidth = _cropWidth; 38@synthesize cropHeight = _cropHeight; 39 40+ (NSSet<NSNumber*>*)supportedPixelFormats { 41 return [NSSet setWithObjects:@(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange), 42 @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange), 43 @(kCVPixelFormatType_32BGRA), 44 @(kCVPixelFormatType_32ARGB), 45 nil]; 46} 47 48- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer { 49 return [self initWithPixelBuffer:pixelBuffer 50 adaptedWidth:CVPixelBufferGetWidth(pixelBuffer) 51 adaptedHeight:CVPixelBufferGetHeight(pixelBuffer) 52 cropWidth:CVPixelBufferGetWidth(pixelBuffer) 53 cropHeight:CVPixelBufferGetHeight(pixelBuffer) 54 cropX:0 55 cropY:0]; 56} 57 58- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer 59 adaptedWidth:(int)adaptedWidth 60 adaptedHeight:(int)adaptedHeight 61 cropWidth:(int)cropWidth 62 cropHeight:(int)cropHeight 63 cropX:(int)cropX 64 cropY:(int)cropY { 65 if (self = [super init]) { 66 _width = adaptedWidth; 67 _height = adaptedHeight; 68 _pixelBuffer = pixelBuffer; 69 _bufferWidth = CVPixelBufferGetWidth(_pixelBuffer); 70 _bufferHeight = CVPixelBufferGetHeight(_pixelBuffer); 71 _cropWidth = cropWidth; 72 _cropHeight = cropHeight; 73 // Can only crop at even pixels. 74 _cropX = cropX & ~1; 75 _cropY = cropY & ~1; 76 CVBufferRetain(_pixelBuffer); 77 } 78 79 return self; 80} 81 82- (void)dealloc { 83 CVBufferRelease(_pixelBuffer); 84} 85 86- (int)width { 87 return _width; 88} 89 90- (int)height { 91 return _height; 92} 93 94- (BOOL)requiresCropping { 95 return _cropWidth != _bufferWidth || _cropHeight != _bufferHeight; 96} 97 98- (BOOL)requiresScalingToWidth:(int)width height:(int)height { 99 return _cropWidth != width || _cropHeight != height; 100} 101 102- (int)bufferSizeForCroppingAndScalingToWidth:(int)width height:(int)height { 103 const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); 104 switch (srcPixelFormat) { 105 case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: 106 case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: { 107 int srcChromaWidth = (_cropWidth + 1) / 2; 108 int srcChromaHeight = (_cropHeight + 1) / 2; 109 int dstChromaWidth = (width + 1) / 2; 110 int dstChromaHeight = (height + 1) / 2; 111 112 return srcChromaWidth * srcChromaHeight * 2 + dstChromaWidth * dstChromaHeight * 2; 113 } 114 case kCVPixelFormatType_32BGRA: 115 case kCVPixelFormatType_32ARGB: { 116 return 0; // Scaling RGBA frames does not require a temporary buffer. 117 } 118 } 119 RTC_NOTREACHED() << "Unsupported pixel format."; 120 return 0; 121} 122 123- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer 124 withTempBuffer:(nullable uint8_t*)tmpBuffer { 125 const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); 126 const OSType dstPixelFormat = CVPixelBufferGetPixelFormatType(outputPixelBuffer); 127 128 switch (srcPixelFormat) { 129 case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: 130 case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: { 131 size_t dstWidth = CVPixelBufferGetWidth(outputPixelBuffer); 132 size_t dstHeight = CVPixelBufferGetHeight(outputPixelBuffer); 133 if (dstWidth > 0 && dstHeight > 0) { 134 RTC_DCHECK(dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange || 135 dstPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); 136 if ([self requiresScalingToWidth:dstWidth height:dstHeight]) { 137 RTC_DCHECK(tmpBuffer); 138 } 139 [self cropAndScaleNV12To:outputPixelBuffer withTempBuffer:tmpBuffer]; 140 } 141 break; 142 } 143 case kCVPixelFormatType_32BGRA: 144 case kCVPixelFormatType_32ARGB: { 145 RTC_DCHECK(srcPixelFormat == dstPixelFormat); 146 [self cropAndScaleARGBTo:outputPixelBuffer]; 147 break; 148 } 149 default: { RTC_NOTREACHED() << "Unsupported pixel format."; } 150 } 151 152 return YES; 153} 154 155- (id<RTC_OBJC_TYPE(RTCI420Buffer)>)toI420 { 156 const OSType pixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); 157 158 CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); 159 160 RTC_OBJC_TYPE(RTCMutableI420Buffer)* i420Buffer = 161 [[RTC_OBJC_TYPE(RTCMutableI420Buffer) alloc] initWithWidth:[self width] height:[self height]]; 162 163 switch (pixelFormat) { 164 case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: 165 case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: { 166 const uint8_t* srcY = 167 static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); 168 const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0); 169 const uint8_t* srcUV = 170 static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); 171 const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1); 172 173 // Crop just by modifying pointers. 174 srcY += srcYStride * _cropY + _cropX; 175 srcUV += srcUVStride * (_cropY / 2) + _cropX; 176 177 // TODO(magjed): Use a frame buffer pool. 178 webrtc::NV12ToI420Scaler nv12ToI420Scaler; 179 nv12ToI420Scaler.NV12ToI420Scale(srcY, 180 srcYStride, 181 srcUV, 182 srcUVStride, 183 _cropWidth, 184 _cropHeight, 185 i420Buffer.mutableDataY, 186 i420Buffer.strideY, 187 i420Buffer.mutableDataU, 188 i420Buffer.strideU, 189 i420Buffer.mutableDataV, 190 i420Buffer.strideV, 191 i420Buffer.width, 192 i420Buffer.height); 193 break; 194 } 195 case kCVPixelFormatType_32BGRA: 196 case kCVPixelFormatType_32ARGB: { 197 CVPixelBufferRef scaledPixelBuffer = NULL; 198 CVPixelBufferRef sourcePixelBuffer = NULL; 199 if ([self requiresCropping] || 200 [self requiresScalingToWidth:i420Buffer.width height:i420Buffer.height]) { 201 CVPixelBufferCreate( 202 NULL, i420Buffer.width, i420Buffer.height, pixelFormat, NULL, &scaledPixelBuffer); 203 [self cropAndScaleTo:scaledPixelBuffer withTempBuffer:NULL]; 204 205 CVPixelBufferLockBaseAddress(scaledPixelBuffer, kCVPixelBufferLock_ReadOnly); 206 sourcePixelBuffer = scaledPixelBuffer; 207 } else { 208 sourcePixelBuffer = _pixelBuffer; 209 } 210 const uint8_t* src = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(sourcePixelBuffer)); 211 const size_t bytesPerRow = CVPixelBufferGetBytesPerRow(sourcePixelBuffer); 212 213 if (pixelFormat == kCVPixelFormatType_32BGRA) { 214 // Corresponds to libyuv::FOURCC_ARGB 215 libyuv::ARGBToI420(src, 216 bytesPerRow, 217 i420Buffer.mutableDataY, 218 i420Buffer.strideY, 219 i420Buffer.mutableDataU, 220 i420Buffer.strideU, 221 i420Buffer.mutableDataV, 222 i420Buffer.strideV, 223 i420Buffer.width, 224 i420Buffer.height); 225 } else if (pixelFormat == kCVPixelFormatType_32ARGB) { 226 // Corresponds to libyuv::FOURCC_BGRA 227 libyuv::BGRAToI420(src, 228 bytesPerRow, 229 i420Buffer.mutableDataY, 230 i420Buffer.strideY, 231 i420Buffer.mutableDataU, 232 i420Buffer.strideU, 233 i420Buffer.mutableDataV, 234 i420Buffer.strideV, 235 i420Buffer.width, 236 i420Buffer.height); 237 } 238 239 if (scaledPixelBuffer) { 240 CVPixelBufferUnlockBaseAddress(scaledPixelBuffer, kCVPixelBufferLock_ReadOnly); 241 CVBufferRelease(scaledPixelBuffer); 242 } 243 break; 244 } 245 default: { RTC_NOTREACHED() << "Unsupported pixel format."; } 246 } 247 248 CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); 249 250 return i420Buffer; 251} 252 253#pragma mark - Debugging 254 255#if !defined(NDEBUG) && defined(WEBRTC_IOS) 256- (id)debugQuickLookObject { 257 CGImageRef cgImage; 258 VTCreateCGImageFromCVPixelBuffer(_pixelBuffer, NULL, &cgImage); 259 UIImage *image = [UIImage imageWithCGImage:cgImage scale:1.0 orientation:UIImageOrientationUp]; 260 CGImageRelease(cgImage); 261 return image; 262} 263#endif 264 265#pragma mark - Private 266 267- (void)cropAndScaleNV12To:(CVPixelBufferRef)outputPixelBuffer withTempBuffer:(uint8_t*)tmpBuffer { 268 // Prepare output pointers. 269 CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0); 270 if (cvRet != kCVReturnSuccess) { 271 RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; 272 } 273 const int dstWidth = CVPixelBufferGetWidth(outputPixelBuffer); 274 const int dstHeight = CVPixelBufferGetHeight(outputPixelBuffer); 275 uint8_t* dstY = 276 reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0)); 277 const int dstYStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0); 278 uint8_t* dstUV = 279 reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 1)); 280 const int dstUVStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 1); 281 282 // Prepare source pointers. 283 CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); 284 const uint8_t* srcY = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); 285 const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0); 286 const uint8_t* srcUV = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); 287 const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1); 288 289 // Crop just by modifying pointers. 290 srcY += srcYStride * _cropY + _cropX; 291 srcUV += srcUVStride * (_cropY / 2) + _cropX; 292 293 webrtc::NV12Scale(tmpBuffer, 294 srcY, 295 srcYStride, 296 srcUV, 297 srcUVStride, 298 _cropWidth, 299 _cropHeight, 300 dstY, 301 dstYStride, 302 dstUV, 303 dstUVStride, 304 dstWidth, 305 dstHeight); 306 307 CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); 308 CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0); 309} 310 311- (void)cropAndScaleARGBTo:(CVPixelBufferRef)outputPixelBuffer { 312 // Prepare output pointers. 313 CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0); 314 if (cvRet != kCVReturnSuccess) { 315 RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; 316 } 317 const int dstWidth = CVPixelBufferGetWidth(outputPixelBuffer); 318 const int dstHeight = CVPixelBufferGetHeight(outputPixelBuffer); 319 320 uint8_t* dst = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddress(outputPixelBuffer)); 321 const int dstStride = CVPixelBufferGetBytesPerRow(outputPixelBuffer); 322 323 // Prepare source pointers. 324 CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); 325 const uint8_t* src = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(_pixelBuffer)); 326 const int srcStride = CVPixelBufferGetBytesPerRow(_pixelBuffer); 327 328 // Crop just by modifying pointers. Need to ensure that src pointer points to a byte corresponding 329 // to the start of a new pixel (byte with B for BGRA) so that libyuv scales correctly. 330 const int bytesPerPixel = 4; 331 src += srcStride * _cropY + (_cropX * bytesPerPixel); 332 333 // kCVPixelFormatType_32BGRA corresponds to libyuv::FOURCC_ARGB 334 libyuv::ARGBScale(src, 335 srcStride, 336 _cropWidth, 337 _cropHeight, 338 dst, 339 dstStride, 340 dstWidth, 341 dstHeight, 342 libyuv::kFilterBox); 343 344 CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); 345 CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0); 346} 347 348@end 349