1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "webrtc/video/stream_synchronization.h"
12
13 #include <assert.h>
14 #include <math.h>
15 #include <stdlib.h>
16
17 #include <algorithm>
18
19 #include "webrtc/base/logging.h"
20
21 namespace webrtc {
22
23 static const int kMaxChangeMs = 80;
24 static const int kMaxDeltaDelayMs = 10000;
25 static const int kFilterLength = 4;
26 // Minimum difference between audio and video to warrant a change.
27 static const int kMinDeltaMs = 30;
28
29 struct ViESyncDelay {
ViESyncDelaywebrtc::ViESyncDelay30 ViESyncDelay() {
31 extra_video_delay_ms = 0;
32 last_video_delay_ms = 0;
33 extra_audio_delay_ms = 0;
34 last_audio_delay_ms = 0;
35 network_delay = 120;
36 }
37
38 int extra_video_delay_ms;
39 int last_video_delay_ms;
40 int extra_audio_delay_ms;
41 int last_audio_delay_ms;
42 int network_delay;
43 };
44
StreamSynchronization(uint32_t video_primary_ssrc,int audio_channel_id)45 StreamSynchronization::StreamSynchronization(uint32_t video_primary_ssrc,
46 int audio_channel_id)
47 : channel_delay_(new ViESyncDelay),
48 video_primary_ssrc_(video_primary_ssrc),
49 audio_channel_id_(audio_channel_id),
50 base_target_delay_ms_(0),
51 avg_diff_ms_(0) {
52 }
53
~StreamSynchronization()54 StreamSynchronization::~StreamSynchronization() {
55 delete channel_delay_;
56 }
57
ComputeRelativeDelay(const Measurements & audio_measurement,const Measurements & video_measurement,int * relative_delay_ms)58 bool StreamSynchronization::ComputeRelativeDelay(
59 const Measurements& audio_measurement,
60 const Measurements& video_measurement,
61 int* relative_delay_ms) {
62 assert(relative_delay_ms);
63 if (audio_measurement.rtcp.size() < 2 || video_measurement.rtcp.size() < 2) {
64 // We need two RTCP SR reports per stream to do synchronization.
65 return false;
66 }
67 int64_t audio_last_capture_time_ms;
68 if (!RtpToNtpMs(audio_measurement.latest_timestamp,
69 audio_measurement.rtcp,
70 &audio_last_capture_time_ms)) {
71 return false;
72 }
73 int64_t video_last_capture_time_ms;
74 if (!RtpToNtpMs(video_measurement.latest_timestamp,
75 video_measurement.rtcp,
76 &video_last_capture_time_ms)) {
77 return false;
78 }
79 if (video_last_capture_time_ms < 0) {
80 return false;
81 }
82 // Positive diff means that video_measurement is behind audio_measurement.
83 *relative_delay_ms = video_measurement.latest_receive_time_ms -
84 audio_measurement.latest_receive_time_ms -
85 (video_last_capture_time_ms - audio_last_capture_time_ms);
86 if (*relative_delay_ms > kMaxDeltaDelayMs ||
87 *relative_delay_ms < -kMaxDeltaDelayMs) {
88 return false;
89 }
90 return true;
91 }
92
ComputeDelays(int relative_delay_ms,int current_audio_delay_ms,int * total_audio_delay_target_ms,int * total_video_delay_target_ms)93 bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
94 int current_audio_delay_ms,
95 int* total_audio_delay_target_ms,
96 int* total_video_delay_target_ms) {
97 assert(total_audio_delay_target_ms && total_video_delay_target_ms);
98
99 int current_video_delay_ms = *total_video_delay_target_ms;
100 LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms
101 << ", network delay diff: " << channel_delay_->network_delay
102 << " current diff: " << relative_delay_ms
103 << " for channel " << audio_channel_id_;
104 // Calculate the difference between the lowest possible video delay and
105 // the current audio delay.
106 int current_diff_ms = current_video_delay_ms - current_audio_delay_ms +
107 relative_delay_ms;
108
109 avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ +
110 current_diff_ms) / kFilterLength;
111 if (abs(avg_diff_ms_) < kMinDeltaMs) {
112 // Don't adjust if the diff is within our margin.
113 return false;
114 }
115
116 // Make sure we don't move too fast.
117 int diff_ms = avg_diff_ms_ / 2;
118 diff_ms = std::min(diff_ms, kMaxChangeMs);
119 diff_ms = std::max(diff_ms, -kMaxChangeMs);
120
121 // Reset the average after a move to prevent overshooting reaction.
122 avg_diff_ms_ = 0;
123
124 if (diff_ms > 0) {
125 // The minimum video delay is longer than the current audio delay.
126 // We need to decrease extra video delay, or add extra audio delay.
127 if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) {
128 // We have extra delay added to ViE. Reduce this delay before adding
129 // extra delay to VoE.
130 channel_delay_->extra_video_delay_ms -= diff_ms;
131 channel_delay_->extra_audio_delay_ms = base_target_delay_ms_;
132 } else { // channel_delay_->extra_video_delay_ms > 0
133 // We have no extra video delay to remove, increase the audio delay.
134 channel_delay_->extra_audio_delay_ms += diff_ms;
135 channel_delay_->extra_video_delay_ms = base_target_delay_ms_;
136 }
137 } else { // if (diff_ms > 0)
138 // The video delay is lower than the current audio delay.
139 // We need to decrease extra audio delay, or add extra video delay.
140 if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) {
141 // We have extra delay in VoiceEngine.
142 // Start with decreasing the voice delay.
143 // Note: diff_ms is negative; add the negative difference.
144 channel_delay_->extra_audio_delay_ms += diff_ms;
145 channel_delay_->extra_video_delay_ms = base_target_delay_ms_;
146 } else { // channel_delay_->extra_audio_delay_ms > base_target_delay_ms_
147 // We have no extra delay in VoiceEngine, increase the video delay.
148 // Note: diff_ms is negative; subtract the negative difference.
149 channel_delay_->extra_video_delay_ms -= diff_ms; // X - (-Y) = X + Y.
150 channel_delay_->extra_audio_delay_ms = base_target_delay_ms_;
151 }
152 }
153
154 // Make sure that video is never below our target.
155 channel_delay_->extra_video_delay_ms = std::max(
156 channel_delay_->extra_video_delay_ms, base_target_delay_ms_);
157
158 int new_video_delay_ms;
159 if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) {
160 new_video_delay_ms = channel_delay_->extra_video_delay_ms;
161 } else {
162 // No change to the extra video delay. We are changing audio and we only
163 // allow to change one at the time.
164 new_video_delay_ms = channel_delay_->last_video_delay_ms;
165 }
166
167 // Make sure that we don't go below the extra video delay.
168 new_video_delay_ms = std::max(
169 new_video_delay_ms, channel_delay_->extra_video_delay_ms);
170
171 // Verify we don't go above the maximum allowed video delay.
172 new_video_delay_ms =
173 std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
174
175 int new_audio_delay_ms;
176 if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) {
177 new_audio_delay_ms = channel_delay_->extra_audio_delay_ms;
178 } else {
179 // No change to the audio delay. We are changing video and we only
180 // allow to change one at the time.
181 new_audio_delay_ms = channel_delay_->last_audio_delay_ms;
182 }
183
184 // Make sure that we don't go below the extra audio delay.
185 new_audio_delay_ms = std::max(
186 new_audio_delay_ms, channel_delay_->extra_audio_delay_ms);
187
188 // Verify we don't go above the maximum allowed audio delay.
189 new_audio_delay_ms =
190 std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
191
192 // Remember our last audio and video delays.
193 channel_delay_->last_video_delay_ms = new_video_delay_ms;
194 channel_delay_->last_audio_delay_ms = new_audio_delay_ms;
195
196 LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms
197 << " for video primary SSRC " << video_primary_ssrc_
198 << " and audio delay " << channel_delay_->extra_audio_delay_ms
199 << " for audio channel " << audio_channel_id_;
200
201 // Return values.
202 *total_video_delay_target_ms = new_video_delay_ms;
203 *total_audio_delay_target_ms = new_audio_delay_ms;
204 return true;
205 }
206
SetTargetBufferingDelay(int target_delay_ms)207 void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) {
208 // Initial extra delay for audio (accounting for existing extra delay).
209 channel_delay_->extra_audio_delay_ms +=
210 target_delay_ms - base_target_delay_ms_;
211 channel_delay_->last_audio_delay_ms +=
212 target_delay_ms - base_target_delay_ms_;
213
214 // The video delay is compared to the last value (and how much we can update
215 // is limited by that as well).
216 channel_delay_->last_video_delay_ms +=
217 target_delay_ms - base_target_delay_ms_;
218
219 channel_delay_->extra_video_delay_ms +=
220 target_delay_ms - base_target_delay_ms_;
221
222 // Video is already delayed by the desired amount.
223 base_target_delay_ms_ = target_delay_ms;
224 }
225
226 } // namespace webrtc
227