1 /* 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef API_VIDEO_ENCODED_IMAGE_H_ 12 #define API_VIDEO_ENCODED_IMAGE_H_ 13 14 #include <stdint.h> 15 16 #include <map> 17 #include <utility> 18 19 #include "absl/types/optional.h" 20 #include "api/rtp_packet_infos.h" 21 #include "api/scoped_refptr.h" 22 #include "api/video/color_space.h" 23 #include "api/video/video_codec_constants.h" 24 #include "api/video/video_content_type.h" 25 #include "api/video/video_frame_type.h" 26 #include "api/video/video_rotation.h" 27 #include "api/video/video_timing.h" 28 #include "common_types.h" // NOLINT(build/include_directory) 29 #include "rtc_base/checks.h" 30 #include "rtc_base/deprecation.h" 31 #include "rtc_base/ref_count.h" 32 #include "rtc_base/system/rtc_export.h" 33 34 namespace webrtc { 35 36 // Abstract interface for buffer storage. Intended to support buffers owned by 37 // external encoders with special release requirements, e.g, java encoders with 38 // releaseOutputBuffer. 39 class EncodedImageBufferInterface : public rtc::RefCountInterface { 40 public: 41 virtual const uint8_t* data() const = 0; 42 // TODO(bugs.webrtc.org/9378): Make interface essentially read-only, delete 43 // this non-const data method. 44 virtual uint8_t* data() = 0; 45 virtual size_t size() const = 0; 46 }; 47 48 // Basic implementation of EncodedImageBufferInterface. 49 class RTC_EXPORT EncodedImageBuffer : public EncodedImageBufferInterface { 50 public: Create()51 static rtc::scoped_refptr<EncodedImageBuffer> Create() { return Create(0); } 52 static rtc::scoped_refptr<EncodedImageBuffer> Create(size_t size); 53 static rtc::scoped_refptr<EncodedImageBuffer> Create(const uint8_t* data, 54 size_t size); 55 56 const uint8_t* data() const override; 57 uint8_t* data() override; 58 size_t size() const override; 59 void Realloc(size_t t); 60 61 protected: 62 explicit EncodedImageBuffer(size_t size); 63 EncodedImageBuffer(const uint8_t* data, size_t size); 64 ~EncodedImageBuffer(); 65 66 size_t size_; 67 uint8_t* buffer_; 68 }; 69 70 // TODO(bug.webrtc.org/9378): This is a legacy api class, which is slowly being 71 // cleaned up. Direct use of its members is strongly discouraged. 72 class RTC_EXPORT EncodedImage { 73 public: 74 EncodedImage(); 75 EncodedImage(EncodedImage&&); 76 // Discouraged: potentially expensive. 77 EncodedImage(const EncodedImage&); 78 EncodedImage(uint8_t* buffer, size_t length, size_t capacity); 79 80 ~EncodedImage(); 81 82 EncodedImage& operator=(EncodedImage&&); 83 // Discouraged: potentially expensive. 84 EncodedImage& operator=(const EncodedImage&); 85 86 // TODO(nisse): Change style to timestamp(), set_timestamp(), for consistency 87 // with the VideoFrame class. 88 // Set frame timestamp (90kHz). SetTimestamp(uint32_t timestamp)89 void SetTimestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; } 90 91 // Get frame timestamp (90kHz). Timestamp()92 uint32_t Timestamp() const { return timestamp_rtp_; } 93 94 void SetEncodeTime(int64_t encode_start_ms, int64_t encode_finish_ms); 95 NtpTimeMs()96 int64_t NtpTimeMs() const { return ntp_time_ms_; } 97 SpatialIndex()98 absl::optional<int> SpatialIndex() const { return spatial_index_; } SetSpatialIndex(absl::optional<int> spatial_index)99 void SetSpatialIndex(absl::optional<int> spatial_index) { 100 RTC_DCHECK_GE(spatial_index.value_or(0), 0); 101 RTC_DCHECK_LT(spatial_index.value_or(0), kMaxSpatialLayers); 102 spatial_index_ = spatial_index; 103 } 104 105 // These methods can be used to set/get size of subframe with spatial index 106 // |spatial_index| on encoded frames that consist of multiple spatial layers. 107 absl::optional<size_t> SpatialLayerFrameSize(int spatial_index) const; 108 void SetSpatialLayerFrameSize(int spatial_index, size_t size_bytes); 109 ColorSpace()110 const webrtc::ColorSpace* ColorSpace() const { 111 return color_space_ ? &*color_space_ : nullptr; 112 } SetColorSpace(const absl::optional<webrtc::ColorSpace> & color_space)113 void SetColorSpace(const absl::optional<webrtc::ColorSpace>& color_space) { 114 color_space_ = color_space; 115 } 116 PacketInfos()117 const RtpPacketInfos& PacketInfos() const { return packet_infos_; } SetPacketInfos(RtpPacketInfos packet_infos)118 void SetPacketInfos(RtpPacketInfos packet_infos) { 119 packet_infos_ = std::move(packet_infos); 120 } 121 RetransmissionAllowed()122 bool RetransmissionAllowed() const { return retransmission_allowed_; } SetRetransmissionAllowed(bool retransmission_allowed)123 void SetRetransmissionAllowed(bool retransmission_allowed) { 124 retransmission_allowed_ = retransmission_allowed; 125 } 126 size()127 size_t size() const { return size_; } set_size(size_t new_size)128 void set_size(size_t new_size) { 129 // Allow set_size(0) even if we have no buffer. 130 RTC_DCHECK_LE(new_size, new_size == 0 ? 0 : capacity()); 131 size_ = new_size; 132 } 133 // TODO(nisse): Delete, provide only read-only access to the buffer. capacity()134 size_t capacity() const { 135 return buffer_ ? capacity_ : (encoded_data_ ? encoded_data_->size() : 0); 136 } 137 SetEncodedData(rtc::scoped_refptr<EncodedImageBufferInterface> encoded_data)138 void SetEncodedData( 139 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_data) { 140 encoded_data_ = encoded_data; 141 size_ = encoded_data->size(); 142 buffer_ = nullptr; 143 } 144 ClearEncodedData()145 void ClearEncodedData() { 146 encoded_data_ = nullptr; 147 size_ = 0; 148 buffer_ = nullptr; 149 capacity_ = 0; 150 } 151 GetEncodedData()152 rtc::scoped_refptr<EncodedImageBufferInterface> GetEncodedData() const { 153 RTC_DCHECK(buffer_ == nullptr); 154 return encoded_data_; 155 } 156 157 // TODO(nisse): Delete, provide only read-only access to the buffer. data()158 uint8_t* data() { 159 return buffer_ ? buffer_ 160 : (encoded_data_ ? encoded_data_->data() : nullptr); 161 } data()162 const uint8_t* data() const { 163 return buffer_ ? buffer_ 164 : (encoded_data_ ? encoded_data_->data() : nullptr); 165 } 166 167 // Hack to workaround lack of ownership of the encoded data. If we don't 168 // already own the underlying data, make an owned copy. 169 void Retain(); 170 171 uint32_t _encodedWidth = 0; 172 uint32_t _encodedHeight = 0; 173 // NTP time of the capture time in local timebase in milliseconds. 174 // TODO(minyue): make this member private. 175 int64_t ntp_time_ms_ = 0; 176 int64_t capture_time_ms_ = 0; 177 VideoFrameType _frameType = VideoFrameType::kVideoFrameDelta; 178 VideoRotation rotation_ = kVideoRotation_0; 179 VideoContentType content_type_ = VideoContentType::UNSPECIFIED; 180 bool _completeFrame = false; 181 int qp_ = -1; // Quantizer value. 182 183 // When an application indicates non-zero values here, it is taken as an 184 // indication that all future frames will be constrained with those limits 185 // until the application indicates a change again. 186 PlayoutDelay playout_delay_ = {-1, -1}; 187 188 struct Timing { 189 uint8_t flags = VideoSendTiming::kInvalid; 190 int64_t encode_start_ms = 0; 191 int64_t encode_finish_ms = 0; 192 int64_t packetization_finish_ms = 0; 193 int64_t pacer_exit_ms = 0; 194 int64_t network_timestamp_ms = 0; 195 int64_t network2_timestamp_ms = 0; 196 int64_t receive_start_ms = 0; 197 int64_t receive_finish_ms = 0; 198 } timing_; 199 200 private: 201 // TODO(bugs.webrtc.org/9378): We're transitioning to always owning the 202 // encoded data. 203 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_data_; 204 size_t size_; // Size of encoded frame data. 205 // Non-null when used with an un-owned buffer. 206 uint8_t* buffer_; 207 // Allocated size of _buffer; relevant only if it's non-null. 208 size_t capacity_; 209 uint32_t timestamp_rtp_ = 0; 210 absl::optional<int> spatial_index_; 211 std::map<int, size_t> spatial_layer_frame_size_bytes_; 212 absl::optional<webrtc::ColorSpace> color_space_; 213 // Information about packets used to assemble this video frame. This is needed 214 // by |SourceTracker| when the frame is delivered to the RTCRtpReceiver's 215 // MediaStreamTrack, in order to implement getContributingSources(). See: 216 // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources 217 RtpPacketInfos packet_infos_; 218 bool retransmission_allowed_ = true; 219 }; 220 221 } // namespace webrtc 222 223 #endif // API_VIDEO_ENCODED_IMAGE_H_ 224