1/* 2 * Copyright 2017 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#import "RTCMTLVideoView.h" 12 13#import <Metal/Metal.h> 14#import <MetalKit/MetalKit.h> 15 16#import "base/RTCLogging.h" 17#import "base/RTCVideoFrame.h" 18#import "base/RTCVideoFrameBuffer.h" 19#import "components/video_frame_buffer/RTCCVPixelBuffer.h" 20 21#import "RTCMTLI420Renderer.h" 22#import "RTCMTLNV12Renderer.h" 23#import "RTCMTLRGBRenderer.h" 24 25// To avoid unreconized symbol linker errors, we're taking advantage of the objc runtime. 26// Linking errors occur when compiling for architectures that don't support Metal. 27#define MTKViewClass NSClassFromString(@"MTKView") 28#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer") 29#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer") 30#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer") 31 32@interface RTC_OBJC_TYPE (RTCMTLVideoView) 33()<MTKViewDelegate> @property(nonatomic) RTCMTLI420Renderer *rendererI420; 34@property(nonatomic) RTCMTLNV12Renderer *rendererNV12; 35@property(nonatomic) RTCMTLRGBRenderer *rendererRGB; 36@property(nonatomic) MTKView *metalView; 37@property(atomic) RTC_OBJC_TYPE(RTCVideoFrame) * videoFrame; 38@property(nonatomic) CGSize videoFrameSize; 39@property(nonatomic) int64_t lastFrameTimeNs; 40@end 41 42@implementation RTC_OBJC_TYPE (RTCMTLVideoView) 43 44@synthesize delegate = _delegate; 45@synthesize rendererI420 = _rendererI420; 46@synthesize rendererNV12 = _rendererNV12; 47@synthesize rendererRGB = _rendererRGB; 48@synthesize metalView = _metalView; 49@synthesize videoFrame = _videoFrame; 50@synthesize videoFrameSize = _videoFrameSize; 51@synthesize lastFrameTimeNs = _lastFrameTimeNs; 52@synthesize rotationOverride = _rotationOverride; 53 54- (instancetype)initWithFrame:(CGRect)frameRect { 55 self = [super initWithFrame:frameRect]; 56 if (self) { 57 [self configure]; 58 } 59 return self; 60} 61 62- (instancetype)initWithCoder:(NSCoder *)aCoder { 63 self = [super initWithCoder:aCoder]; 64 if (self) { 65 [self configure]; 66 } 67 return self; 68} 69 70- (BOOL)isEnabled { 71 return !self.metalView.paused; 72} 73 74- (void)setEnabled:(BOOL)enabled { 75 self.metalView.paused = !enabled; 76} 77 78- (UIViewContentMode)videoContentMode { 79 return self.metalView.contentMode; 80} 81 82- (void)setVideoContentMode:(UIViewContentMode)mode { 83 self.metalView.contentMode = mode; 84} 85 86#pragma mark - Private 87 88+ (BOOL)isMetalAvailable { 89#if defined(RTC_SUPPORTS_METAL) 90 return MTLCreateSystemDefaultDevice() != nil; 91#else 92 return NO; 93#endif 94} 95 96+ (MTKView *)createMetalView:(CGRect)frame { 97 return [[MTKViewClass alloc] initWithFrame:frame]; 98} 99 100+ (RTCMTLNV12Renderer *)createNV12Renderer { 101 return [[RTCMTLNV12RendererClass alloc] init]; 102} 103 104+ (RTCMTLI420Renderer *)createI420Renderer { 105 return [[RTCMTLI420RendererClass alloc] init]; 106} 107 108+ (RTCMTLRGBRenderer *)createRGBRenderer { 109 return [[RTCMTLRGBRenderer alloc] init]; 110} 111 112- (void)configure { 113 NSAssert([RTC_OBJC_TYPE(RTCMTLVideoView) isMetalAvailable], 114 @"Metal not availiable on this device"); 115 116 self.metalView = [RTC_OBJC_TYPE(RTCMTLVideoView) createMetalView:self.bounds]; 117 self.metalView.delegate = self; 118 self.metalView.contentMode = UIViewContentModeScaleAspectFill; 119 [self addSubview:self.metalView]; 120 self.videoFrameSize = CGSizeZero; 121} 122 123- (void)setMultipleTouchEnabled:(BOOL)multipleTouchEnabled { 124 [super setMultipleTouchEnabled:multipleTouchEnabled]; 125 self.metalView.multipleTouchEnabled = multipleTouchEnabled; 126} 127 128- (void)layoutSubviews { 129 [super layoutSubviews]; 130 131 CGRect bounds = self.bounds; 132 self.metalView.frame = bounds; 133 if (!CGSizeEqualToSize(self.videoFrameSize, CGSizeZero)) { 134 self.metalView.drawableSize = [self drawableSize]; 135 } else { 136 self.metalView.drawableSize = bounds.size; 137 } 138} 139 140#pragma mark - MTKViewDelegate methods 141 142- (void)drawInMTKView:(nonnull MTKView *)view { 143 NSAssert(view == self.metalView, @"Receiving draw callbacks from foreign instance."); 144 RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame = self.videoFrame; 145 // Skip rendering if we've already rendered this frame. 146 if (!videoFrame || videoFrame.timeStampNs == self.lastFrameTimeNs) { 147 return; 148 } 149 150 if (CGRectIsEmpty(view.bounds)) { 151 return; 152 } 153 154 RTCMTLRenderer *renderer; 155 if ([videoFrame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { 156 RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)videoFrame.buffer; 157 const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer); 158 if (pixelFormat == kCVPixelFormatType_32BGRA || pixelFormat == kCVPixelFormatType_32ARGB) { 159 if (!self.rendererRGB) { 160 self.rendererRGB = [RTC_OBJC_TYPE(RTCMTLVideoView) createRGBRenderer]; 161 if (![self.rendererRGB addRenderingDestination:self.metalView]) { 162 self.rendererRGB = nil; 163 RTCLogError(@"Failed to create RGB renderer"); 164 return; 165 } 166 } 167 renderer = self.rendererRGB; 168 } else { 169 if (!self.rendererNV12) { 170 self.rendererNV12 = [RTC_OBJC_TYPE(RTCMTLVideoView) createNV12Renderer]; 171 if (![self.rendererNV12 addRenderingDestination:self.metalView]) { 172 self.rendererNV12 = nil; 173 RTCLogError(@"Failed to create NV12 renderer"); 174 return; 175 } 176 } 177 renderer = self.rendererNV12; 178 } 179 } else { 180 if (!self.rendererI420) { 181 self.rendererI420 = [RTC_OBJC_TYPE(RTCMTLVideoView) createI420Renderer]; 182 if (![self.rendererI420 addRenderingDestination:self.metalView]) { 183 self.rendererI420 = nil; 184 RTCLogError(@"Failed to create I420 renderer"); 185 return; 186 } 187 } 188 renderer = self.rendererI420; 189 } 190 191 renderer.rotationOverride = self.rotationOverride; 192 193 [renderer drawFrame:videoFrame]; 194 self.lastFrameTimeNs = videoFrame.timeStampNs; 195} 196 197- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size { 198} 199 200#pragma mark - 201 202- (void)setRotationOverride:(NSValue *)rotationOverride { 203 _rotationOverride = rotationOverride; 204 205 self.metalView.drawableSize = [self drawableSize]; 206 [self setNeedsLayout]; 207} 208 209- (RTCVideoRotation)frameRotation { 210 if (self.rotationOverride) { 211 RTCVideoRotation rotation; 212 if (@available(iOS 11, *)) { 213 [self.rotationOverride getValue:&rotation size:sizeof(rotation)]; 214 } else { 215 [self.rotationOverride getValue:&rotation]; 216 } 217 return rotation; 218 } 219 220 return self.videoFrame.rotation; 221} 222 223- (CGSize)drawableSize { 224 // Flip width/height if the rotations are not the same. 225 CGSize videoFrameSize = self.videoFrameSize; 226 RTCVideoRotation frameRotation = [self frameRotation]; 227 228 BOOL useLandscape = 229 (frameRotation == RTCVideoRotation_0) || (frameRotation == RTCVideoRotation_180); 230 BOOL sizeIsLandscape = (self.videoFrame.rotation == RTCVideoRotation_0) || 231 (self.videoFrame.rotation == RTCVideoRotation_180); 232 233 if (useLandscape == sizeIsLandscape) { 234 return videoFrameSize; 235 } else { 236 return CGSizeMake(videoFrameSize.height, videoFrameSize.width); 237 } 238} 239 240#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer) 241 242- (void)setSize:(CGSize)size { 243 __weak RTC_OBJC_TYPE(RTCMTLVideoView) *weakSelf = self; 244 dispatch_async(dispatch_get_main_queue(), ^{ 245 RTC_OBJC_TYPE(RTCMTLVideoView) *strongSelf = weakSelf; 246 247 strongSelf.videoFrameSize = size; 248 CGSize drawableSize = [strongSelf drawableSize]; 249 250 strongSelf.metalView.drawableSize = drawableSize; 251 [strongSelf setNeedsLayout]; 252 [strongSelf.delegate videoView:self didChangeVideoSize:size]; 253 }); 254} 255 256- (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame { 257 if (!self.isEnabled) { 258 return; 259 } 260 261 if (frame == nil) { 262 RTCLogInfo(@"Incoming frame is nil. Exiting render callback."); 263 return; 264 } 265 self.videoFrame = frame; 266} 267 268@end 269