1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef API_VIDEO_VIDEO_FRAME_H_ 12 #define API_VIDEO_VIDEO_FRAME_H_ 13 14 #include <stdint.h> 15 16 #include <utility> 17 18 #include "absl/types/optional.h" 19 #include "api/rtp_packet_infos.h" 20 #include "api/scoped_refptr.h" 21 #include "api/video/color_space.h" 22 #include "api/video/hdr_metadata.h" 23 #include "api/video/video_frame_buffer.h" 24 #include "api/video/video_rotation.h" 25 #include "rtc_base/checks.h" 26 #include "rtc_base/system/rtc_export.h" 27 28 namespace webrtc { 29 30 class RTC_EXPORT VideoFrame { 31 public: 32 struct RTC_EXPORT UpdateRect { 33 int offset_x; 34 int offset_y; 35 int width; 36 int height; 37 38 // Makes this UpdateRect a bounding box of this and other rect. 39 void Union(const UpdateRect& other); 40 41 // Makes this UpdateRect an intersection of this and other rect. 42 void Intersect(const UpdateRect& other); 43 44 // Sets everything to 0, making this UpdateRect a zero-size (empty) update. 45 void MakeEmptyUpdate(); 46 47 bool IsEmpty() const; 48 49 // Per-member equality check. Empty rectangles with different offsets would 50 // be considered different. 51 bool operator==(const UpdateRect& other) const { 52 return other.offset_x == offset_x && other.offset_y == offset_y && 53 other.width == width && other.height == height; 54 } 55 56 bool operator!=(const UpdateRect& other) const { return !(*this == other); } 57 58 // Scales update_rect given original frame dimensions. 59 // Cropping is applied first, then rect is scaled down. 60 // Update rect is snapped to 2x2 grid due to possible UV subsampling and 61 // then expanded by additional 2 pixels in each direction to accommodate any 62 // possible scaling artifacts. 63 // Note, close but not equal update_rects on original frame may result in 64 // the same scaled update rects. 65 UpdateRect ScaleWithFrame(int frame_width, 66 int frame_height, 67 int crop_x, 68 int crop_y, 69 int crop_width, 70 int crop_height, 71 int scaled_width, 72 int scaled_height) const; 73 }; 74 75 struct RTC_EXPORT ProcessingTime { ElapsedProcessingTime76 TimeDelta Elapsed() const { return finish - start; } 77 Timestamp start; 78 Timestamp finish; 79 }; 80 81 // Preferred way of building VideoFrame objects. 82 class RTC_EXPORT Builder { 83 public: 84 Builder(); 85 ~Builder(); 86 87 VideoFrame build(); 88 Builder& set_video_frame_buffer( 89 const rtc::scoped_refptr<VideoFrameBuffer>& buffer); 90 Builder& set_timestamp_ms(int64_t timestamp_ms); 91 Builder& set_timestamp_us(int64_t timestamp_us); 92 Builder& set_timestamp_rtp(uint32_t timestamp_rtp); 93 Builder& set_ntp_time_ms(int64_t ntp_time_ms); 94 Builder& set_rotation(VideoRotation rotation); 95 Builder& set_color_space(const absl::optional<ColorSpace>& color_space); 96 Builder& set_color_space(const ColorSpace* color_space); 97 Builder& set_id(uint16_t id); 98 Builder& set_update_rect(const absl::optional<UpdateRect>& update_rect); 99 Builder& set_packet_infos(RtpPacketInfos packet_infos); 100 101 private: 102 uint16_t id_ = 0; 103 rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_; 104 int64_t timestamp_us_ = 0; 105 uint32_t timestamp_rtp_ = 0; 106 int64_t ntp_time_ms_ = 0; 107 VideoRotation rotation_ = kVideoRotation_0; 108 absl::optional<ColorSpace> color_space_; 109 absl::optional<UpdateRect> update_rect_; 110 RtpPacketInfos packet_infos_; 111 }; 112 113 // To be deprecated. Migrate all use to Builder. 114 VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer, 115 webrtc::VideoRotation rotation, 116 int64_t timestamp_us); 117 VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer, 118 uint32_t timestamp_rtp, 119 int64_t render_time_ms, 120 VideoRotation rotation); 121 122 ~VideoFrame(); 123 124 // Support move and copy. 125 VideoFrame(const VideoFrame&); 126 VideoFrame(VideoFrame&&); 127 VideoFrame& operator=(const VideoFrame&); 128 VideoFrame& operator=(VideoFrame&&); 129 130 // Get frame width. 131 int width() const; 132 // Get frame height. 133 int height() const; 134 // Get frame size in pixels. 135 uint32_t size() const; 136 137 // Get frame ID. Returns 0 if ID is not set. Not guarantee to be transferred 138 // from the sender to the receiver, but preserved on single side. The id 139 // should be propagated between all frame modifications during its lifetime 140 // from capturing to sending as encoded image. It is intended to be unique 141 // over a time window of a few minutes for peer connection, to which 142 // corresponding video stream belongs to. id()143 uint16_t id() const { return id_; } set_id(uint16_t id)144 void set_id(uint16_t id) { id_ = id; } 145 146 // System monotonic clock, same timebase as rtc::TimeMicros(). timestamp_us()147 int64_t timestamp_us() const { return timestamp_us_; } set_timestamp_us(int64_t timestamp_us)148 void set_timestamp_us(int64_t timestamp_us) { timestamp_us_ = timestamp_us; } 149 150 // TODO(nisse): After the cricket::VideoFrame and webrtc::VideoFrame 151 // merge, timestamps other than timestamp_us will likely be 152 // deprecated. 153 154 // Set frame timestamp (90kHz). set_timestamp(uint32_t timestamp)155 void set_timestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; } 156 157 // Get frame timestamp (90kHz). timestamp()158 uint32_t timestamp() const { return timestamp_rtp_; } 159 160 // For now, transport_frame_id and rtp timestamp are the same. 161 // TODO(nisse): Must be handled differently for QUIC. transport_frame_id()162 uint32_t transport_frame_id() const { return timestamp(); } 163 164 // Set capture ntp time in milliseconds. set_ntp_time_ms(int64_t ntp_time_ms)165 void set_ntp_time_ms(int64_t ntp_time_ms) { ntp_time_ms_ = ntp_time_ms; } 166 167 // Get capture ntp time in milliseconds. ntp_time_ms()168 int64_t ntp_time_ms() const { return ntp_time_ms_; } 169 170 // Naming convention for Coordination of Video Orientation. Please see 171 // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ts_126114v120700p.pdf 172 // 173 // "pending rotation" or "pending" = a frame that has a VideoRotation > 0. 174 // 175 // "not pending" = a frame that has a VideoRotation == 0. 176 // 177 // "apply rotation" = modify a frame from being "pending" to being "not 178 // pending" rotation (a no-op for "unrotated"). 179 // rotation()180 VideoRotation rotation() const { return rotation_; } set_rotation(VideoRotation rotation)181 void set_rotation(VideoRotation rotation) { rotation_ = rotation; } 182 183 // Get color space when available. color_space()184 const absl::optional<ColorSpace>& color_space() const { return color_space_; } set_color_space(const absl::optional<ColorSpace> & color_space)185 void set_color_space(const absl::optional<ColorSpace>& color_space) { 186 color_space_ = color_space; 187 } 188 189 // Get render time in milliseconds. 190 // TODO(nisse): Deprecated. Migrate all users to timestamp_us(). 191 int64_t render_time_ms() const; 192 193 // Return the underlying buffer. Never nullptr for a properly 194 // initialized VideoFrame. 195 rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer() const; 196 197 void set_video_frame_buffer( 198 const rtc::scoped_refptr<VideoFrameBuffer>& buffer); 199 200 // TODO(nisse): Deprecated. 201 // Return true if the frame is stored in a texture. is_texture()202 bool is_texture() const { 203 return video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative; 204 } 205 has_update_rect()206 bool has_update_rect() const { return update_rect_.has_value(); } 207 208 // Returns update_rect set by the builder or set_update_rect() or whole frame 209 // rect if no update rect is available. update_rect()210 UpdateRect update_rect() const { 211 return update_rect_.value_or(UpdateRect{0, 0, width(), height()}); 212 } 213 214 // Rectangle must be within the frame dimensions. set_update_rect(const VideoFrame::UpdateRect & update_rect)215 void set_update_rect(const VideoFrame::UpdateRect& update_rect) { 216 RTC_DCHECK_GE(update_rect.offset_x, 0); 217 RTC_DCHECK_GE(update_rect.offset_y, 0); 218 RTC_DCHECK_LE(update_rect.offset_x + update_rect.width, width()); 219 RTC_DCHECK_LE(update_rect.offset_y + update_rect.height, height()); 220 update_rect_ = update_rect; 221 } 222 clear_update_rect()223 void clear_update_rect() { update_rect_ = absl::nullopt; } 224 225 // Get information about packets used to assemble this video frame. Might be 226 // empty if the information isn't available. packet_infos()227 const RtpPacketInfos& packet_infos() const { return packet_infos_; } set_packet_infos(RtpPacketInfos value)228 void set_packet_infos(RtpPacketInfos value) { 229 packet_infos_ = std::move(value); 230 } 231 processing_time()232 const absl::optional<ProcessingTime> processing_time() const { 233 return processing_time_; 234 } set_processing_time(const ProcessingTime & processing_time)235 void set_processing_time(const ProcessingTime& processing_time) { 236 processing_time_ = processing_time; 237 } 238 239 private: 240 VideoFrame(uint16_t id, 241 const rtc::scoped_refptr<VideoFrameBuffer>& buffer, 242 int64_t timestamp_us, 243 uint32_t timestamp_rtp, 244 int64_t ntp_time_ms, 245 VideoRotation rotation, 246 const absl::optional<ColorSpace>& color_space, 247 const absl::optional<UpdateRect>& update_rect, 248 RtpPacketInfos packet_infos); 249 250 uint16_t id_; 251 // An opaque reference counted handle that stores the pixel data. 252 rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_; 253 uint32_t timestamp_rtp_; 254 int64_t ntp_time_ms_; 255 int64_t timestamp_us_; 256 VideoRotation rotation_; 257 absl::optional<ColorSpace> color_space_; 258 // Updated since the last frame area. If present it means that the bounding 259 // box of all the changes is within the rectangular area and is close to it. 260 // If absent, it means that there's no information about the change at all and 261 // update_rect() will return a rectangle corresponding to the entire frame. 262 absl::optional<UpdateRect> update_rect_; 263 // Information about packets used to assemble this video frame. This is needed 264 // by |SourceTracker| when the frame is delivered to the RTCRtpReceiver's 265 // MediaStreamTrack, in order to implement getContributingSources(). See: 266 // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources 267 RtpPacketInfos packet_infos_; 268 // Processing timestamps of the frame. For received video frames these are the 269 // timestamps when the frame is sent to the decoder and the decoded image 270 // returned from the decoder. 271 // Currently, not set for locally captured video frames. 272 absl::optional<ProcessingTime> processing_time_; 273 }; 274 275 } // namespace webrtc 276 277 #endif // API_VIDEO_VIDEO_FRAME_H_ 278