1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/video_engine/stream_synchronization.h"
12
13 #include <assert.h>
14 #include <math.h>
15 #include <stdlib.h>
16
17 #include <algorithm>
18
19 #include "webrtc/system_wrappers/interface/logging.h"
20
21 namespace webrtc {
22
23 static const int kMaxChangeMs = 80;
24 static const int kMaxDeltaDelayMs = 10000;
25 static const int kFilterLength = 4;
26 // Minimum difference between audio and video to warrant a change.
27 static const int kMinDeltaMs = 30;
28
29 struct ViESyncDelay {
ViESyncDelaywebrtc::ViESyncDelay30 ViESyncDelay() {
31 extra_video_delay_ms = 0;
32 last_video_delay_ms = 0;
33 extra_audio_delay_ms = 0;
34 last_audio_delay_ms = 0;
35 network_delay = 120;
36 }
37
38 int extra_video_delay_ms;
39 int last_video_delay_ms;
40 int extra_audio_delay_ms;
41 int last_audio_delay_ms;
42 int network_delay;
43 };
44
StreamSynchronization(int audio_channel_id,int video_channel_id)45 StreamSynchronization::StreamSynchronization(int audio_channel_id,
46 int video_channel_id)
47 : channel_delay_(new ViESyncDelay),
48 audio_channel_id_(audio_channel_id),
49 video_channel_id_(video_channel_id),
50 base_target_delay_ms_(0),
51 avg_diff_ms_(0) {}
52
~StreamSynchronization()53 StreamSynchronization::~StreamSynchronization() {
54 delete channel_delay_;
55 }
56
ComputeRelativeDelay(const Measurements & audio_measurement,const Measurements & video_measurement,int * relative_delay_ms)57 bool StreamSynchronization::ComputeRelativeDelay(
58 const Measurements& audio_measurement,
59 const Measurements& video_measurement,
60 int* relative_delay_ms) {
61 assert(relative_delay_ms);
62 if (audio_measurement.rtcp.size() < 2 || video_measurement.rtcp.size() < 2) {
63 // We need two RTCP SR reports per stream to do synchronization.
64 return false;
65 }
66 int64_t audio_last_capture_time_ms;
67 if (!RtpToNtpMs(audio_measurement.latest_timestamp,
68 audio_measurement.rtcp,
69 &audio_last_capture_time_ms)) {
70 return false;
71 }
72 int64_t video_last_capture_time_ms;
73 if (!RtpToNtpMs(video_measurement.latest_timestamp,
74 video_measurement.rtcp,
75 &video_last_capture_time_ms)) {
76 return false;
77 }
78 if (video_last_capture_time_ms < 0) {
79 return false;
80 }
81 // Positive diff means that video_measurement is behind audio_measurement.
82 *relative_delay_ms = video_measurement.latest_receive_time_ms -
83 audio_measurement.latest_receive_time_ms -
84 (video_last_capture_time_ms - audio_last_capture_time_ms);
85 if (*relative_delay_ms > kMaxDeltaDelayMs ||
86 *relative_delay_ms < -kMaxDeltaDelayMs) {
87 return false;
88 }
89 return true;
90 }
91
ComputeDelays(int relative_delay_ms,int current_audio_delay_ms,int * total_audio_delay_target_ms,int * total_video_delay_target_ms)92 bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
93 int current_audio_delay_ms,
94 int* total_audio_delay_target_ms,
95 int* total_video_delay_target_ms) {
96 assert(total_audio_delay_target_ms && total_video_delay_target_ms);
97
98 int current_video_delay_ms = *total_video_delay_target_ms;
99 LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms
100 << ", network delay diff: " << channel_delay_->network_delay
101 << " current diff: " << relative_delay_ms
102 << " for channel " << audio_channel_id_;
103 // Calculate the difference between the lowest possible video delay and
104 // the current audio delay.
105 int current_diff_ms = current_video_delay_ms - current_audio_delay_ms +
106 relative_delay_ms;
107
108 avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ +
109 current_diff_ms) / kFilterLength;
110 if (abs(avg_diff_ms_) < kMinDeltaMs) {
111 // Don't adjust if the diff is within our margin.
112 return false;
113 }
114
115 // Make sure we don't move too fast.
116 int diff_ms = avg_diff_ms_ / 2;
117 diff_ms = std::min(diff_ms, kMaxChangeMs);
118 diff_ms = std::max(diff_ms, -kMaxChangeMs);
119
120 // Reset the average after a move to prevent overshooting reaction.
121 avg_diff_ms_ = 0;
122
123 if (diff_ms > 0) {
124 // The minimum video delay is longer than the current audio delay.
125 // We need to decrease extra video delay, or add extra audio delay.
126 if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) {
127 // We have extra delay added to ViE. Reduce this delay before adding
128 // extra delay to VoE.
129 channel_delay_->extra_video_delay_ms -= diff_ms;
130 channel_delay_->extra_audio_delay_ms = base_target_delay_ms_;
131 } else { // channel_delay_->extra_video_delay_ms > 0
132 // We have no extra video delay to remove, increase the audio delay.
133 channel_delay_->extra_audio_delay_ms += diff_ms;
134 channel_delay_->extra_video_delay_ms = base_target_delay_ms_;
135 }
136 } else { // if (diff_ms > 0)
137 // The video delay is lower than the current audio delay.
138 // We need to decrease extra audio delay, or add extra video delay.
139 if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) {
140 // We have extra delay in VoiceEngine.
141 // Start with decreasing the voice delay.
142 // Note: diff_ms is negative; add the negative difference.
143 channel_delay_->extra_audio_delay_ms += diff_ms;
144 channel_delay_->extra_video_delay_ms = base_target_delay_ms_;
145 } else { // channel_delay_->extra_audio_delay_ms > base_target_delay_ms_
146 // We have no extra delay in VoiceEngine, increase the video delay.
147 // Note: diff_ms is negative; subtract the negative difference.
148 channel_delay_->extra_video_delay_ms -= diff_ms; // X - (-Y) = X + Y.
149 channel_delay_->extra_audio_delay_ms = base_target_delay_ms_;
150 }
151 }
152
153 // Make sure that video is never below our target.
154 channel_delay_->extra_video_delay_ms = std::max(
155 channel_delay_->extra_video_delay_ms, base_target_delay_ms_);
156
157 int new_video_delay_ms;
158 if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) {
159 new_video_delay_ms = channel_delay_->extra_video_delay_ms;
160 } else {
161 // No change to the extra video delay. We are changing audio and we only
162 // allow to change one at the time.
163 new_video_delay_ms = channel_delay_->last_video_delay_ms;
164 }
165
166 // Make sure that we don't go below the extra video delay.
167 new_video_delay_ms = std::max(
168 new_video_delay_ms, channel_delay_->extra_video_delay_ms);
169
170 // Verify we don't go above the maximum allowed video delay.
171 new_video_delay_ms =
172 std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
173
174 int new_audio_delay_ms;
175 if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) {
176 new_audio_delay_ms = channel_delay_->extra_audio_delay_ms;
177 } else {
178 // No change to the audio delay. We are changing video and we only
179 // allow to change one at the time.
180 new_audio_delay_ms = channel_delay_->last_audio_delay_ms;
181 }
182
183 // Make sure that we don't go below the extra audio delay.
184 new_audio_delay_ms = std::max(
185 new_audio_delay_ms, channel_delay_->extra_audio_delay_ms);
186
187 // Verify we don't go above the maximum allowed audio delay.
188 new_audio_delay_ms =
189 std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
190
191 // Remember our last audio and video delays.
192 channel_delay_->last_video_delay_ms = new_video_delay_ms;
193 channel_delay_->last_audio_delay_ms = new_audio_delay_ms;
194
195 LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms
196 << " and audio delay " << channel_delay_->extra_audio_delay_ms
197 << " for video channel " << video_channel_id_
198 << " for audio channel " << audio_channel_id_;
199
200 // Return values.
201 *total_video_delay_target_ms = new_video_delay_ms;
202 *total_audio_delay_target_ms = new_audio_delay_ms;
203 return true;
204 }
205
SetTargetBufferingDelay(int target_delay_ms)206 void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) {
207 // Initial extra delay for audio (accounting for existing extra delay).
208 channel_delay_->extra_audio_delay_ms +=
209 target_delay_ms - base_target_delay_ms_;
210 channel_delay_->last_audio_delay_ms +=
211 target_delay_ms - base_target_delay_ms_;
212
213 // The video delay is compared to the last value (and how much we can update
214 // is limited by that as well).
215 channel_delay_->last_video_delay_ms +=
216 target_delay_ms - base_target_delay_ms_;
217
218 channel_delay_->extra_video_delay_ms +=
219 target_delay_ms - base_target_delay_ms_;
220
221 // Video is already delayed by the desired amount.
222 base_target_delay_ms_ = target_delay_ms;
223 }
224
225 } // namespace webrtc
226