• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef API_VIDEO_ENCODED_IMAGE_H_
12 #define API_VIDEO_ENCODED_IMAGE_H_
13 
14 #include <stdint.h>
15 
16 #include <map>
17 #include <utility>
18 
19 #include "absl/types/optional.h"
20 #include "api/rtp_packet_infos.h"
21 #include "api/scoped_refptr.h"
22 #include "api/video/color_space.h"
23 #include "api/video/video_codec_constants.h"
24 #include "api/video/video_content_type.h"
25 #include "api/video/video_frame_type.h"
26 #include "api/video/video_rotation.h"
27 #include "api/video/video_timing.h"
28 #include "common_types.h"  // NOLINT(build/include_directory)
29 #include "rtc_base/checks.h"
30 #include "rtc_base/deprecation.h"
31 #include "rtc_base/ref_count.h"
32 #include "rtc_base/system/rtc_export.h"
33 
34 namespace webrtc {
35 
36 // Abstract interface for buffer storage. Intended to support buffers owned by
37 // external encoders with special release requirements, e.g, java encoders with
38 // releaseOutputBuffer.
39 class EncodedImageBufferInterface : public rtc::RefCountInterface {
40  public:
41   virtual const uint8_t* data() const = 0;
42   // TODO(bugs.webrtc.org/9378): Make interface essentially read-only, delete
43   // this non-const data method.
44   virtual uint8_t* data() = 0;
45   virtual size_t size() const = 0;
46 };
47 
48 // Basic implementation of EncodedImageBufferInterface.
49 class RTC_EXPORT EncodedImageBuffer : public EncodedImageBufferInterface {
50  public:
Create()51   static rtc::scoped_refptr<EncodedImageBuffer> Create() { return Create(0); }
52   static rtc::scoped_refptr<EncodedImageBuffer> Create(size_t size);
53   static rtc::scoped_refptr<EncodedImageBuffer> Create(const uint8_t* data,
54                                                        size_t size);
55 
56   const uint8_t* data() const override;
57   uint8_t* data() override;
58   size_t size() const override;
59   void Realloc(size_t t);
60 
61  protected:
62   explicit EncodedImageBuffer(size_t size);
63   EncodedImageBuffer(const uint8_t* data, size_t size);
64   ~EncodedImageBuffer();
65 
66   size_t size_;
67   uint8_t* buffer_;
68 };
69 
70 // TODO(bug.webrtc.org/9378): This is a legacy api class, which is slowly being
71 // cleaned up. Direct use of its members is strongly discouraged.
72 class RTC_EXPORT EncodedImage {
73  public:
74   EncodedImage();
75   EncodedImage(EncodedImage&&);
76   // Discouraged: potentially expensive.
77   EncodedImage(const EncodedImage&);
78   EncodedImage(uint8_t* buffer, size_t length, size_t capacity);
79 
80   ~EncodedImage();
81 
82   EncodedImage& operator=(EncodedImage&&);
83   // Discouraged: potentially expensive.
84   EncodedImage& operator=(const EncodedImage&);
85 
86   // TODO(nisse): Change style to timestamp(), set_timestamp(), for consistency
87   // with the VideoFrame class.
88   // Set frame timestamp (90kHz).
SetTimestamp(uint32_t timestamp)89   void SetTimestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; }
90 
91   // Get frame timestamp (90kHz).
Timestamp()92   uint32_t Timestamp() const { return timestamp_rtp_; }
93 
94   void SetEncodeTime(int64_t encode_start_ms, int64_t encode_finish_ms);
95 
NtpTimeMs()96   int64_t NtpTimeMs() const { return ntp_time_ms_; }
97 
SpatialIndex()98   absl::optional<int> SpatialIndex() const { return spatial_index_; }
SetSpatialIndex(absl::optional<int> spatial_index)99   void SetSpatialIndex(absl::optional<int> spatial_index) {
100     RTC_DCHECK_GE(spatial_index.value_or(0), 0);
101     RTC_DCHECK_LT(spatial_index.value_or(0), kMaxSpatialLayers);
102     spatial_index_ = spatial_index;
103   }
104 
105   // These methods can be used to set/get size of subframe with spatial index
106   // |spatial_index| on encoded frames that consist of multiple spatial layers.
107   absl::optional<size_t> SpatialLayerFrameSize(int spatial_index) const;
108   void SetSpatialLayerFrameSize(int spatial_index, size_t size_bytes);
109 
ColorSpace()110   const webrtc::ColorSpace* ColorSpace() const {
111     return color_space_ ? &*color_space_ : nullptr;
112   }
SetColorSpace(const absl::optional<webrtc::ColorSpace> & color_space)113   void SetColorSpace(const absl::optional<webrtc::ColorSpace>& color_space) {
114     color_space_ = color_space;
115   }
116 
PacketInfos()117   const RtpPacketInfos& PacketInfos() const { return packet_infos_; }
SetPacketInfos(RtpPacketInfos packet_infos)118   void SetPacketInfos(RtpPacketInfos packet_infos) {
119     packet_infos_ = std::move(packet_infos);
120   }
121 
RetransmissionAllowed()122   bool RetransmissionAllowed() const { return retransmission_allowed_; }
SetRetransmissionAllowed(bool retransmission_allowed)123   void SetRetransmissionAllowed(bool retransmission_allowed) {
124     retransmission_allowed_ = retransmission_allowed;
125   }
126 
size()127   size_t size() const { return size_; }
set_size(size_t new_size)128   void set_size(size_t new_size) {
129     // Allow set_size(0) even if we have no buffer.
130     RTC_DCHECK_LE(new_size, new_size == 0 ? 0 : capacity());
131     size_ = new_size;
132   }
133   // TODO(nisse): Delete, provide only read-only access to the buffer.
capacity()134   size_t capacity() const {
135     return buffer_ ? capacity_ : (encoded_data_ ? encoded_data_->size() : 0);
136   }
137 
SetEncodedData(rtc::scoped_refptr<EncodedImageBufferInterface> encoded_data)138   void SetEncodedData(
139       rtc::scoped_refptr<EncodedImageBufferInterface> encoded_data) {
140     encoded_data_ = encoded_data;
141     size_ = encoded_data->size();
142     buffer_ = nullptr;
143   }
144 
ClearEncodedData()145   void ClearEncodedData() {
146     encoded_data_ = nullptr;
147     size_ = 0;
148     buffer_ = nullptr;
149     capacity_ = 0;
150   }
151 
GetEncodedData()152   rtc::scoped_refptr<EncodedImageBufferInterface> GetEncodedData() const {
153     RTC_DCHECK(buffer_ == nullptr);
154     return encoded_data_;
155   }
156 
157   // TODO(nisse): Delete, provide only read-only access to the buffer.
data()158   uint8_t* data() {
159     return buffer_ ? buffer_
160                    : (encoded_data_ ? encoded_data_->data() : nullptr);
161   }
data()162   const uint8_t* data() const {
163     return buffer_ ? buffer_
164                    : (encoded_data_ ? encoded_data_->data() : nullptr);
165   }
166 
167   // Hack to workaround lack of ownership of the encoded data. If we don't
168   // already own the underlying data, make an owned copy.
169   void Retain();
170 
171   uint32_t _encodedWidth = 0;
172   uint32_t _encodedHeight = 0;
173   // NTP time of the capture time in local timebase in milliseconds.
174   // TODO(minyue): make this member private.
175   int64_t ntp_time_ms_ = 0;
176   int64_t capture_time_ms_ = 0;
177   VideoFrameType _frameType = VideoFrameType::kVideoFrameDelta;
178   VideoRotation rotation_ = kVideoRotation_0;
179   VideoContentType content_type_ = VideoContentType::UNSPECIFIED;
180   bool _completeFrame = false;
181   int qp_ = -1;  // Quantizer value.
182 
183   // When an application indicates non-zero values here, it is taken as an
184   // indication that all future frames will be constrained with those limits
185   // until the application indicates a change again.
186   PlayoutDelay playout_delay_ = {-1, -1};
187 
188   struct Timing {
189     uint8_t flags = VideoSendTiming::kInvalid;
190     int64_t encode_start_ms = 0;
191     int64_t encode_finish_ms = 0;
192     int64_t packetization_finish_ms = 0;
193     int64_t pacer_exit_ms = 0;
194     int64_t network_timestamp_ms = 0;
195     int64_t network2_timestamp_ms = 0;
196     int64_t receive_start_ms = 0;
197     int64_t receive_finish_ms = 0;
198   } timing_;
199 
200  private:
201   // TODO(bugs.webrtc.org/9378): We're transitioning to always owning the
202   // encoded data.
203   rtc::scoped_refptr<EncodedImageBufferInterface> encoded_data_;
204   size_t size_;  // Size of encoded frame data.
205   // Non-null when used with an un-owned buffer.
206   uint8_t* buffer_;
207   // Allocated size of _buffer; relevant only if it's non-null.
208   size_t capacity_;
209   uint32_t timestamp_rtp_ = 0;
210   absl::optional<int> spatial_index_;
211   std::map<int, size_t> spatial_layer_frame_size_bytes_;
212   absl::optional<webrtc::ColorSpace> color_space_;
213   // Information about packets used to assemble this video frame. This is needed
214   // by |SourceTracker| when the frame is delivered to the RTCRtpReceiver's
215   // MediaStreamTrack, in order to implement getContributingSources(). See:
216   // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources
217   RtpPacketInfos packet_infos_;
218   bool retransmission_allowed_ = true;
219 };
220 
221 }  // namespace webrtc
222 
223 #endif  // API_VIDEO_ENCODED_IMAGE_H_
224