• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef API_VIDEO_VIDEO_FRAME_H_
12 #define API_VIDEO_VIDEO_FRAME_H_
13 
14 #include <stdint.h>
15 
16 #include <utility>
17 
18 #include "absl/types/optional.h"
19 #include "api/rtp_packet_infos.h"
20 #include "api/scoped_refptr.h"
21 #include "api/video/color_space.h"
22 #include "api/video/hdr_metadata.h"
23 #include "api/video/video_frame_buffer.h"
24 #include "api/video/video_rotation.h"
25 #include "rtc_base/checks.h"
26 #include "rtc_base/system/rtc_export.h"
27 
28 namespace webrtc {
29 
30 class RTC_EXPORT VideoFrame {
31  public:
32   struct RTC_EXPORT UpdateRect {
33     int offset_x;
34     int offset_y;
35     int width;
36     int height;
37 
38     // Makes this UpdateRect a bounding box of this and other rect.
39     void Union(const UpdateRect& other);
40 
41     // Makes this UpdateRect an intersection of this and other rect.
42     void Intersect(const UpdateRect& other);
43 
44     // Sets everything to 0, making this UpdateRect a zero-size (empty) update.
45     void MakeEmptyUpdate();
46 
47     bool IsEmpty() const;
48 
49     // Per-member equality check. Empty rectangles with different offsets would
50     // be considered different.
51     bool operator==(const UpdateRect& other) const {
52       return other.offset_x == offset_x && other.offset_y == offset_y &&
53              other.width == width && other.height == height;
54     }
55 
56     bool operator!=(const UpdateRect& other) const { return !(*this == other); }
57 
58     // Scales update_rect given original frame dimensions.
59     // Cropping is applied first, then rect is scaled down.
60     // Update rect is snapped to 2x2 grid due to possible UV subsampling and
61     // then expanded by additional 2 pixels in each direction to accommodate any
62     // possible scaling artifacts.
63     // Note, close but not equal update_rects on original frame may result in
64     // the same scaled update rects.
65     UpdateRect ScaleWithFrame(int frame_width,
66                               int frame_height,
67                               int crop_x,
68                               int crop_y,
69                               int crop_width,
70                               int crop_height,
71                               int scaled_width,
72                               int scaled_height) const;
73   };
74 
75   struct RTC_EXPORT ProcessingTime {
ElapsedProcessingTime76     TimeDelta Elapsed() const { return finish - start; }
77     Timestamp start;
78     Timestamp finish;
79   };
80 
81   // Preferred way of building VideoFrame objects.
82   class RTC_EXPORT Builder {
83    public:
84     Builder();
85     ~Builder();
86 
87     VideoFrame build();
88     Builder& set_video_frame_buffer(
89         const rtc::scoped_refptr<VideoFrameBuffer>& buffer);
90     Builder& set_timestamp_ms(int64_t timestamp_ms);
91     Builder& set_timestamp_us(int64_t timestamp_us);
92     Builder& set_timestamp_rtp(uint32_t timestamp_rtp);
93     Builder& set_ntp_time_ms(int64_t ntp_time_ms);
94     Builder& set_rotation(VideoRotation rotation);
95     Builder& set_color_space(const absl::optional<ColorSpace>& color_space);
96     Builder& set_color_space(const ColorSpace* color_space);
97     Builder& set_id(uint16_t id);
98     Builder& set_update_rect(const absl::optional<UpdateRect>& update_rect);
99     Builder& set_packet_infos(RtpPacketInfos packet_infos);
100 
101    private:
102     uint16_t id_ = 0;
103     rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
104     int64_t timestamp_us_ = 0;
105     uint32_t timestamp_rtp_ = 0;
106     int64_t ntp_time_ms_ = 0;
107     VideoRotation rotation_ = kVideoRotation_0;
108     absl::optional<ColorSpace> color_space_;
109     absl::optional<UpdateRect> update_rect_;
110     RtpPacketInfos packet_infos_;
111   };
112 
113   // To be deprecated. Migrate all use to Builder.
114   VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
115              webrtc::VideoRotation rotation,
116              int64_t timestamp_us);
117   VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
118              uint32_t timestamp_rtp,
119              int64_t render_time_ms,
120              VideoRotation rotation);
121 
122   ~VideoFrame();
123 
124   // Support move and copy.
125   VideoFrame(const VideoFrame&);
126   VideoFrame(VideoFrame&&);
127   VideoFrame& operator=(const VideoFrame&);
128   VideoFrame& operator=(VideoFrame&&);
129 
130   // Get frame width.
131   int width() const;
132   // Get frame height.
133   int height() const;
134   // Get frame size in pixels.
135   uint32_t size() const;
136 
137   // Get frame ID. Returns 0 if ID is not set. Not guarantee to be transferred
138   // from the sender to the receiver, but preserved on single side. The id
139   // should be propagated between all frame modifications during its lifetime
140   // from capturing to sending as encoded image. It is intended to be unique
141   // over a time window of a few minutes for peer connection, to which
142   // corresponding video stream belongs to.
id()143   uint16_t id() const { return id_; }
set_id(uint16_t id)144   void set_id(uint16_t id) { id_ = id; }
145 
146   // System monotonic clock, same timebase as rtc::TimeMicros().
timestamp_us()147   int64_t timestamp_us() const { return timestamp_us_; }
set_timestamp_us(int64_t timestamp_us)148   void set_timestamp_us(int64_t timestamp_us) { timestamp_us_ = timestamp_us; }
149 
150   // TODO(nisse): After the cricket::VideoFrame and webrtc::VideoFrame
151   // merge, timestamps other than timestamp_us will likely be
152   // deprecated.
153 
154   // Set frame timestamp (90kHz).
set_timestamp(uint32_t timestamp)155   void set_timestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; }
156 
157   // Get frame timestamp (90kHz).
timestamp()158   uint32_t timestamp() const { return timestamp_rtp_; }
159 
160   // For now, transport_frame_id and rtp timestamp are the same.
161   // TODO(nisse): Must be handled differently for QUIC.
transport_frame_id()162   uint32_t transport_frame_id() const { return timestamp(); }
163 
164   // Set capture ntp time in milliseconds.
set_ntp_time_ms(int64_t ntp_time_ms)165   void set_ntp_time_ms(int64_t ntp_time_ms) { ntp_time_ms_ = ntp_time_ms; }
166 
167   // Get capture ntp time in milliseconds.
ntp_time_ms()168   int64_t ntp_time_ms() const { return ntp_time_ms_; }
169 
170   // Naming convention for Coordination of Video Orientation. Please see
171   // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ts_126114v120700p.pdf
172   //
173   // "pending rotation" or "pending" = a frame that has a VideoRotation > 0.
174   //
175   // "not pending" = a frame that has a VideoRotation == 0.
176   //
177   // "apply rotation" = modify a frame from being "pending" to being "not
178   //                    pending" rotation (a no-op for "unrotated").
179   //
rotation()180   VideoRotation rotation() const { return rotation_; }
set_rotation(VideoRotation rotation)181   void set_rotation(VideoRotation rotation) { rotation_ = rotation; }
182 
183   // Get color space when available.
color_space()184   const absl::optional<ColorSpace>& color_space() const { return color_space_; }
set_color_space(const absl::optional<ColorSpace> & color_space)185   void set_color_space(const absl::optional<ColorSpace>& color_space) {
186     color_space_ = color_space;
187   }
188 
189   // Get render time in milliseconds.
190   // TODO(nisse): Deprecated. Migrate all users to timestamp_us().
191   int64_t render_time_ms() const;
192 
193   // Return the underlying buffer. Never nullptr for a properly
194   // initialized VideoFrame.
195   rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer() const;
196 
197   void set_video_frame_buffer(
198       const rtc::scoped_refptr<VideoFrameBuffer>& buffer);
199 
200   // TODO(nisse): Deprecated.
201   // Return true if the frame is stored in a texture.
is_texture()202   bool is_texture() const {
203     return video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative;
204   }
205 
has_update_rect()206   bool has_update_rect() const { return update_rect_.has_value(); }
207 
208   // Returns update_rect set by the builder or set_update_rect() or whole frame
209   // rect if no update rect is available.
update_rect()210   UpdateRect update_rect() const {
211     return update_rect_.value_or(UpdateRect{0, 0, width(), height()});
212   }
213 
214   // Rectangle must be within the frame dimensions.
set_update_rect(const VideoFrame::UpdateRect & update_rect)215   void set_update_rect(const VideoFrame::UpdateRect& update_rect) {
216     RTC_DCHECK_GE(update_rect.offset_x, 0);
217     RTC_DCHECK_GE(update_rect.offset_y, 0);
218     RTC_DCHECK_LE(update_rect.offset_x + update_rect.width, width());
219     RTC_DCHECK_LE(update_rect.offset_y + update_rect.height, height());
220     update_rect_ = update_rect;
221   }
222 
clear_update_rect()223   void clear_update_rect() { update_rect_ = absl::nullopt; }
224 
225   // Get information about packets used to assemble this video frame. Might be
226   // empty if the information isn't available.
packet_infos()227   const RtpPacketInfos& packet_infos() const { return packet_infos_; }
set_packet_infos(RtpPacketInfos value)228   void set_packet_infos(RtpPacketInfos value) {
229     packet_infos_ = std::move(value);
230   }
231 
processing_time()232   const absl::optional<ProcessingTime> processing_time() const {
233     return processing_time_;
234   }
set_processing_time(const ProcessingTime & processing_time)235   void set_processing_time(const ProcessingTime& processing_time) {
236     processing_time_ = processing_time;
237   }
238 
239  private:
240   VideoFrame(uint16_t id,
241              const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
242              int64_t timestamp_us,
243              uint32_t timestamp_rtp,
244              int64_t ntp_time_ms,
245              VideoRotation rotation,
246              const absl::optional<ColorSpace>& color_space,
247              const absl::optional<UpdateRect>& update_rect,
248              RtpPacketInfos packet_infos);
249 
250   uint16_t id_;
251   // An opaque reference counted handle that stores the pixel data.
252   rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
253   uint32_t timestamp_rtp_;
254   int64_t ntp_time_ms_;
255   int64_t timestamp_us_;
256   VideoRotation rotation_;
257   absl::optional<ColorSpace> color_space_;
258   // Updated since the last frame area. If present it means that the bounding
259   // box of all the changes is within the rectangular area and is close to it.
260   // If absent, it means that there's no information about the change at all and
261   // update_rect() will return a rectangle corresponding to the entire frame.
262   absl::optional<UpdateRect> update_rect_;
263   // Information about packets used to assemble this video frame. This is needed
264   // by |SourceTracker| when the frame is delivered to the RTCRtpReceiver's
265   // MediaStreamTrack, in order to implement getContributingSources(). See:
266   // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources
267   RtpPacketInfos packet_infos_;
268   // Processing timestamps of the frame. For received video frames these are the
269   // timestamps when the frame is sent to the decoder and the decoded image
270   // returned from the decoder.
271   // Currently, not set for locally captured video frames.
272   absl::optional<ProcessingTime> processing_time_;
273 };
274 
275 }  // namespace webrtc
276 
277 #endif  // API_VIDEO_VIDEO_FRAME_H_
278