1 /* 2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef API_AUDIO_AUDIO_FRAME_H_ 12 #define API_AUDIO_AUDIO_FRAME_H_ 13 14 #include <stddef.h> 15 #include <stdint.h> 16 17 #include "api/audio/channel_layout.h" 18 #include "api/rtp_packet_infos.h" 19 20 namespace webrtc { 21 22 /* This class holds up to 120 ms of super-wideband (32 kHz) stereo audio. It 23 * allows for adding and subtracting frames while keeping track of the resulting 24 * states. 25 * 26 * Notes 27 * - This is a de-facto api, not designed for external use. The AudioFrame class 28 * is in need of overhaul or even replacement, and anyone depending on it 29 * should be prepared for that. 30 * - The total number of samples is samples_per_channel_ * num_channels_. 31 * - Stereo data is interleaved starting with the left channel. 32 */ 33 class AudioFrame { 34 public: 35 // Using constexpr here causes linker errors unless the variable also has an 36 // out-of-class definition, which is impractical in this header-only class. 37 // (This makes no sense because it compiles as an enum value, which we most 38 // certainly cannot take the address of, just fine.) C++17 introduces inline 39 // variables which should allow us to switch to constexpr and keep this a 40 // header-only class. 41 enum : size_t { 42 // Stereo, 32 kHz, 120 ms (2 * 32 * 120) 43 // Stereo, 192 kHz, 20 ms (2 * 192 * 20) 44 kMaxDataSizeSamples = 7680, 45 kMaxDataSizeBytes = kMaxDataSizeSamples * sizeof(int16_t), 46 }; 47 48 enum VADActivity { kVadActive = 0, kVadPassive = 1, kVadUnknown = 2 }; 49 enum SpeechType { 50 kNormalSpeech = 0, 51 kPLC = 1, 52 kCNG = 2, 53 kPLCCNG = 3, 54 kCodecPLC = 5, 55 kUndefined = 4 56 }; 57 58 AudioFrame(); 59 60 AudioFrame(const AudioFrame&) = delete; 61 AudioFrame& operator=(const AudioFrame&) = delete; 62 63 // Resets all members to their default state. 64 void Reset(); 65 // Same as Reset(), but leaves mute state unchanged. Muting a frame requires 66 // the buffer to be zeroed on the next call to mutable_data(). Callers 67 // intending to write to the buffer immediately after Reset() can instead use 68 // ResetWithoutMuting() to skip this wasteful zeroing. 69 void ResetWithoutMuting(); 70 71 void UpdateFrame(uint32_t timestamp, 72 const int16_t* data, 73 size_t samples_per_channel, 74 int sample_rate_hz, 75 SpeechType speech_type, 76 VADActivity vad_activity, 77 size_t num_channels = 1); 78 79 void CopyFrom(const AudioFrame& src); 80 81 // Sets a wall-time clock timestamp in milliseconds to be used for profiling 82 // of time between two points in the audio chain. 83 // Example: 84 // t0: UpdateProfileTimeStamp() 85 // t1: ElapsedProfileTimeMs() => t1 - t0 [msec] 86 void UpdateProfileTimeStamp(); 87 // Returns the time difference between now and when UpdateProfileTimeStamp() 88 // was last called. Returns -1 if UpdateProfileTimeStamp() has not yet been 89 // called. 90 int64_t ElapsedProfileTimeMs() const; 91 92 // data() returns a zeroed static buffer if the frame is muted. 93 // mutable_frame() always returns a non-static buffer; the first call to 94 // mutable_frame() zeros the non-static buffer and marks the frame unmuted. 95 const int16_t* data() const; 96 int16_t* mutable_data(); 97 98 // Prefer to mute frames using AudioFrameOperations::Mute. 99 void Mute(); 100 // Frame is muted by default. 101 bool muted() const; 102 max_16bit_samples()103 size_t max_16bit_samples() const { return kMaxDataSizeSamples; } samples_per_channel()104 size_t samples_per_channel() const { return samples_per_channel_; } num_channels()105 size_t num_channels() const { return num_channels_; } channel_layout()106 ChannelLayout channel_layout() const { return channel_layout_; } sample_rate_hz()107 int sample_rate_hz() const { return sample_rate_hz_; } 108 set_absolute_capture_timestamp_ms(int64_t absolute_capture_time_stamp_ms)109 void set_absolute_capture_timestamp_ms( 110 int64_t absolute_capture_time_stamp_ms) { 111 absolute_capture_timestamp_ms_ = absolute_capture_time_stamp_ms; 112 } 113 absolute_capture_timestamp_ms()114 absl::optional<int64_t> absolute_capture_timestamp_ms() const { 115 return absolute_capture_timestamp_ms_; 116 } 117 118 // RTP timestamp of the first sample in the AudioFrame. 119 uint32_t timestamp_ = 0; 120 // Time since the first frame in milliseconds. 121 // -1 represents an uninitialized value. 122 int64_t elapsed_time_ms_ = -1; 123 // NTP time of the estimated capture time in local timebase in milliseconds. 124 // -1 represents an uninitialized value. 125 int64_t ntp_time_ms_ = -1; 126 size_t samples_per_channel_ = 0; 127 int sample_rate_hz_ = 0; 128 size_t num_channels_ = 0; 129 ChannelLayout channel_layout_ = CHANNEL_LAYOUT_NONE; 130 SpeechType speech_type_ = kUndefined; 131 VADActivity vad_activity_ = kVadUnknown; 132 // Monotonically increasing timestamp intended for profiling of audio frames. 133 // Typically used for measuring elapsed time between two different points in 134 // the audio path. No lock is used to save resources and we are thread safe 135 // by design. 136 // TODO(nisse@webrtc.org): consider using absl::optional. 137 int64_t profile_timestamp_ms_ = 0; 138 139 // Information about packets used to assemble this audio frame. This is needed 140 // by `SourceTracker` when the frame is delivered to the RTCRtpReceiver's 141 // MediaStreamTrack, in order to implement getContributingSources(). See: 142 // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources 143 // 144 // TODO(bugs.webrtc.org/10757): 145 // Note that this information might not be fully accurate since we currently 146 // don't have a proper way to track it across the audio sync buffer. The 147 // sync buffer is the small sample-holding buffer located after the audio 148 // decoder and before where samples are assembled into output frames. 149 // 150 // `RtpPacketInfos` may also be empty if the audio samples did not come from 151 // RTP packets. E.g. if the audio were locally generated by packet loss 152 // concealment, comfort noise generation, etc. 153 RtpPacketInfos packet_infos_; 154 155 private: 156 // A permanently zeroed out buffer to represent muted frames. This is a 157 // header-only class, so the only way to avoid creating a separate empty 158 // buffer per translation unit is to wrap a static in an inline function. 159 static const int16_t* empty_data(); 160 161 int16_t data_[kMaxDataSizeSamples]; 162 bool muted_ = true; 163 164 // Absolute capture timestamp when this audio frame was originally captured. 165 // This is only valid for audio frames captured on this machine. The absolute 166 // capture timestamp of a received frame is found in `packet_infos_`. 167 // This timestamp MUST be based on the same clock as rtc::TimeMillis(). 168 absl::optional<int64_t> absolute_capture_timestamp_ms_; 169 }; 170 171 } // namespace webrtc 172 173 #endif // API_AUDIO_AUDIO_FRAME_H_ 174