• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 #ifndef API_TEST_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_
11 #define API_TEST_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_
12 
13 #include <map>
14 #include <memory>
15 #include <string>
16 #include <utility>
17 #include <vector>
18 
19 #include "absl/memory/memory.h"
20 #include "absl/strings/string_view.h"
21 #include "absl/types/optional.h"
22 #include "api/async_resolver_factory.h"
23 #include "api/call/call_factory_interface.h"
24 #include "api/fec_controller.h"
25 #include "api/function_view.h"
26 #include "api/media_stream_interface.h"
27 #include "api/peer_connection_interface.h"
28 #include "api/rtc_event_log/rtc_event_log_factory_interface.h"
29 #include "api/rtp_parameters.h"
30 #include "api/task_queue/task_queue_factory.h"
31 #include "api/test/audio_quality_analyzer_interface.h"
32 #include "api/test/frame_generator_interface.h"
33 #include "api/test/simulated_network.h"
34 #include "api/test/stats_observer_interface.h"
35 #include "api/test/track_id_stream_info_map.h"
36 #include "api/test/video_quality_analyzer_interface.h"
37 #include "api/transport/network_control.h"
38 #include "api/units/time_delta.h"
39 #include "api/video_codecs/video_decoder_factory.h"
40 #include "api/video_codecs/video_encoder.h"
41 #include "api/video_codecs/video_encoder_factory.h"
42 #include "media/base/media_constants.h"
43 #include "rtc_base/network.h"
44 #include "rtc_base/rtc_certificate_generator.h"
45 #include "rtc_base/ssl_certificate.h"
46 #include "rtc_base/thread.h"
47 
48 namespace webrtc {
49 namespace webrtc_pc_e2e {
50 
51 constexpr size_t kDefaultSlidesWidth = 1850;
52 constexpr size_t kDefaultSlidesHeight = 1110;
53 
54 // API is in development. Can be changed/removed without notice.
55 class PeerConnectionE2EQualityTestFixture {
56  public:
57   // The index of required capturing device in OS provided list of video
58   // devices. On Linux and Windows the list will be obtained via
59   // webrtc::VideoCaptureModule::DeviceInfo, on Mac OS via
60   // [RTCCameraVideoCapturer captureDevices].
61   enum class CapturingDeviceIndex : size_t {};
62 
63   // Contains parameters for screen share scrolling.
64   //
65   // If scrolling is enabled, then it will be done by putting sliding window
66   // on source video and moving this window from top left corner to the
67   // bottom right corner of the picture.
68   //
69   // In such case source dimensions must be greater or equal to the sliding
70   // window dimensions. So |source_width| and |source_height| are the dimensions
71   // of the source frame, while |VideoConfig::width| and |VideoConfig::height|
72   // are the dimensions of the sliding window.
73   //
74   // Because |source_width| and |source_height| are dimensions of the source
75   // frame, they have to be width and height of videos from
76   // |ScreenShareConfig::slides_yuv_file_names|.
77   //
78   // Because scrolling have to be done on single slide it also requires, that
79   // |duration| must be less or equal to
80   // |ScreenShareConfig::slide_change_interval|.
81   struct ScrollingParams {
ScrollingParamsScrollingParams82     ScrollingParams(TimeDelta duration,
83                     size_t source_width,
84                     size_t source_height)
85         : duration(duration),
86           source_width(source_width),
87           source_height(source_height) {
88       RTC_CHECK_GT(duration.ms(), 0);
89     }
90 
91     // Duration of scrolling.
92     TimeDelta duration;
93     // Width of source slides video.
94     size_t source_width;
95     // Height of source slides video.
96     size_t source_height;
97   };
98 
99   // Contains screen share video stream properties.
100   struct ScreenShareConfig {
ScreenShareConfigScreenShareConfig101     explicit ScreenShareConfig(TimeDelta slide_change_interval)
102         : slide_change_interval(slide_change_interval) {
103       RTC_CHECK_GT(slide_change_interval.ms(), 0);
104     }
105 
106     // Shows how long one slide should be presented on the screen during
107     // slide generation.
108     TimeDelta slide_change_interval;
109     // If true, slides will be generated programmatically. No scrolling params
110     // will be applied in such case.
111     bool generate_slides = false;
112     // If present scrolling will be applied. Please read extra requirement on
113     // |slides_yuv_file_names| for scrolling.
114     absl::optional<ScrollingParams> scrolling_params;
115     // Contains list of yuv files with slides.
116     //
117     // If empty, default set of slides will be used. In such case
118     // |VideoConfig::width| must be equal to |kDefaultSlidesWidth| and
119     // |VideoConfig::height| must be equal to |kDefaultSlidesHeight| or if
120     // |scrolling_params| are specified, then |ScrollingParams::source_width|
121     // must be equal to |kDefaultSlidesWidth| and
122     // |ScrollingParams::source_height| must be equal to |kDefaultSlidesHeight|.
123     std::vector<std::string> slides_yuv_file_names;
124   };
125 
126   // Config for Vp8 simulcast or Vp9 SVC testing.
127   //
128   // SVC support is limited:
129   // During SVC testing there is no SFU, so framework will try to emulate SFU
130   // behavior in regular p2p call. Because of it there are such limitations:
131   //  * if |target_spatial_index| is not equal to the highest spatial layer
132   //    then no packet/frame drops are allowed.
133   //
134   //    If there will be any drops, that will affect requested layer, then
135   //    WebRTC SVC implementation will continue decoding only the highest
136   //    available layer and won't restore lower layers, so analyzer won't
137   //    receive required data which will cause wrong results or test failures.
138   struct VideoSimulcastConfig {
VideoSimulcastConfigVideoSimulcastConfig139     explicit VideoSimulcastConfig(int simulcast_streams_count)
140         : simulcast_streams_count(simulcast_streams_count) {
141       RTC_CHECK_GT(simulcast_streams_count, 1);
142     }
VideoSimulcastConfigVideoSimulcastConfig143     VideoSimulcastConfig(int simulcast_streams_count, int target_spatial_index)
144         : simulcast_streams_count(simulcast_streams_count),
145           target_spatial_index(target_spatial_index) {
146       RTC_CHECK_GT(simulcast_streams_count, 1);
147       RTC_CHECK_GE(target_spatial_index, 0);
148       RTC_CHECK_LT(target_spatial_index, simulcast_streams_count);
149     }
150 
151     // Specified amount of simulcast streams/SVC layers, depending on which
152     // encoder is used.
153     int simulcast_streams_count;
154     // Specifies spatial index of the video stream to analyze.
155     // There are 2 cases:
156     // 1. simulcast encoder is used:
157     //    in such case |target_spatial_index| will specify the index of
158     //    simulcast stream, that should be analyzed. Other streams will be
159     //    dropped.
160     // 2. SVC encoder is used:
161     //    in such case |target_spatial_index| will specify the top interesting
162     //    spatial layer and all layers below, including target one will be
163     //    processed. All layers above target one will be dropped.
164     // If not specified than whatever stream will be received will be analyzed.
165     // It requires Selective Forwarding Unit (SFU) to be configured in the
166     // network.
167     absl::optional<int> target_spatial_index;
168 
169     // Encoding parameters per simulcast layer. If not empty, |encoding_params|
170     // size have to be equal to |simulcast_streams_count|. Will be used to set
171     // transceiver send encoding params for simulcast layers. Applicable only
172     // for codecs that support simulcast (ex. Vp8) and will be ignored
173     // otherwise. RtpEncodingParameters::rid may be changed by fixture
174     // implementation to ensure signaling correctness.
175     std::vector<RtpEncodingParameters> encoding_params;
176   };
177 
178   // Contains properties of single video stream.
179   struct VideoConfig {
VideoConfigVideoConfig180     VideoConfig(size_t width, size_t height, int32_t fps)
181         : width(width), height(height), fps(fps) {}
182 
183     // Video stream width.
184     const size_t width;
185     // Video stream height.
186     const size_t height;
187     const int32_t fps;
188     // Have to be unique among all specified configs for all peers in the call.
189     // Will be auto generated if omitted.
190     absl::optional<std::string> stream_label;
191     // Will be set for current video track. If equals to kText or kDetailed -
192     // screencast in on.
193     absl::optional<VideoTrackInterface::ContentHint> content_hint;
194     // If presented video will be transfered in simulcast/SVC mode depending on
195     // which encoder is used.
196     //
197     // Simulcast is supported only from 1st added peer. For VP8 simulcast only
198     // without RTX is supported so it will be automatically disabled for all
199     // simulcast tracks. For VP9 simulcast enables VP9 SVC mode and support RTX,
200     // but only on non-lossy networks. See more in documentation to
201     // VideoSimulcastConfig.
202     absl::optional<VideoSimulcastConfig> simulcast_config;
203     // Count of temporal layers for video stream. This value will be set into
204     // each RtpEncodingParameters of RtpParameters of corresponding
205     // RtpSenderInterface for this video stream.
206     absl::optional<int> temporal_layers_count;
207     // Sets the maximum encode bitrate in bps. If this value is not set, the
208     // encoder will be capped at an internal maximum value around 2 Mbps
209     // depending on the resolution. This means that it will never be able to
210     // utilize a high bandwidth link.
211     absl::optional<int> max_encode_bitrate_bps;
212     // Sets the minimum encode bitrate in bps. If this value is not set, the
213     // encoder will use an internal minimum value. Please note that if this
214     // value is set higher than the bandwidth of the link, the encoder will
215     // generate more data than the link can handle regardless of the bandwidth
216     // estimation.
217     absl::optional<int> min_encode_bitrate_bps;
218     // If specified the input stream will be also copied to specified file.
219     // It is actually one of the test's output file, which contains copy of what
220     // was captured during the test for this video stream on sender side.
221     // It is useful when generator is used as input.
222     absl::optional<std::string> input_dump_file_name;
223     // If specified this file will be used as output on the receiver side for
224     // this stream. If multiple streams will be produced by input stream,
225     // output files will be appended with indexes. The produced files contains
226     // what was rendered for this video stream on receiver side.
227     absl::optional<std::string> output_dump_file_name;
228     // If true will display input and output video on the user's screen.
229     bool show_on_screen = false;
230     // If specified, determines a sync group to which this video stream belongs.
231     // According to bugs.webrtc.org/4762 WebRTC supports synchronization only
232     // for pair of single audio and single video stream.
233     absl::optional<std::string> sync_group;
234   };
235 
236   // Contains properties for audio in the call.
237   struct AudioConfig {
238     enum Mode {
239       kGenerated,
240       kFile,
241     };
242     // Have to be unique among all specified configs for all peers in the call.
243     // Will be auto generated if omitted.
244     absl::optional<std::string> stream_label;
245     Mode mode = kGenerated;
246     // Have to be specified only if mode = kFile
247     absl::optional<std::string> input_file_name;
248     // If specified the input stream will be also copied to specified file.
249     absl::optional<std::string> input_dump_file_name;
250     // If specified the output stream will be copied to specified file.
251     absl::optional<std::string> output_dump_file_name;
252 
253     // Audio options to use.
254     cricket::AudioOptions audio_options;
255     // Sampling frequency of input audio data (from file or generated).
256     int sampling_frequency_in_hz = 48000;
257     // If specified, determines a sync group to which this audio stream belongs.
258     // According to bugs.webrtc.org/4762 WebRTC supports synchronization only
259     // for pair of single audio and single video stream.
260     absl::optional<std::string> sync_group;
261   };
262 
263   // This class is used to fully configure one peer inside the call.
264   class PeerConfigurer {
265    public:
266     virtual ~PeerConfigurer() = default;
267 
268     // Sets peer name that will be used to report metrics related to this peer.
269     // If not set, some default name will be assigned. All names have to be
270     // unique.
271     virtual PeerConfigurer* SetName(absl::string_view name) = 0;
272 
273     // The parameters of the following 9 methods will be passed to the
274     // PeerConnectionFactoryInterface implementation that will be created for
275     // this peer.
276     virtual PeerConfigurer* SetTaskQueueFactory(
277         std::unique_ptr<TaskQueueFactory> task_queue_factory) = 0;
278     virtual PeerConfigurer* SetCallFactory(
279         std::unique_ptr<CallFactoryInterface> call_factory) = 0;
280     virtual PeerConfigurer* SetEventLogFactory(
281         std::unique_ptr<RtcEventLogFactoryInterface> event_log_factory) = 0;
282     virtual PeerConfigurer* SetFecControllerFactory(
283         std::unique_ptr<FecControllerFactoryInterface>
284             fec_controller_factory) = 0;
285     virtual PeerConfigurer* SetNetworkControllerFactory(
286         std::unique_ptr<NetworkControllerFactoryInterface>
287             network_controller_factory) = 0;
288     virtual PeerConfigurer* SetVideoEncoderFactory(
289         std::unique_ptr<VideoEncoderFactory> video_encoder_factory) = 0;
290     virtual PeerConfigurer* SetVideoDecoderFactory(
291         std::unique_ptr<VideoDecoderFactory> video_decoder_factory) = 0;
292     // Set a custom NetEqFactory to be used in the call.
293     virtual PeerConfigurer* SetNetEqFactory(
294         std::unique_ptr<NetEqFactory> neteq_factory) = 0;
295 
296     // The parameters of the following 4 methods will be passed to the
297     // PeerConnectionInterface implementation that will be created for this
298     // peer.
299     virtual PeerConfigurer* SetAsyncResolverFactory(
300         std::unique_ptr<webrtc::AsyncResolverFactory>
301             async_resolver_factory) = 0;
302     virtual PeerConfigurer* SetRTCCertificateGenerator(
303         std::unique_ptr<rtc::RTCCertificateGeneratorInterface>
304             cert_generator) = 0;
305     virtual PeerConfigurer* SetSSLCertificateVerifier(
306         std::unique_ptr<rtc::SSLCertificateVerifier> tls_cert_verifier) = 0;
307     virtual PeerConfigurer* SetIceTransportFactory(
308         std::unique_ptr<IceTransportFactory> factory) = 0;
309 
310     // Add new video stream to the call that will be sent from this peer.
311     // Default implementation of video frames generator will be used.
312     virtual PeerConfigurer* AddVideoConfig(VideoConfig config) = 0;
313     // Add new video stream to the call that will be sent from this peer with
314     // provided own implementation of video frames generator.
315     virtual PeerConfigurer* AddVideoConfig(
316         VideoConfig config,
317         std::unique_ptr<test::FrameGeneratorInterface> generator) = 0;
318     // Add new video stream to the call that will be sent from this peer.
319     // Capturing device with specified index will be used to get input video.
320     virtual PeerConfigurer* AddVideoConfig(
321         VideoConfig config,
322         CapturingDeviceIndex capturing_device_index) = 0;
323     // Set the audio stream for the call from this peer. If this method won't
324     // be invoked, this peer will send no audio.
325     virtual PeerConfigurer* SetAudioConfig(AudioConfig config) = 0;
326     // If is set, an RTCEventLog will be saved in that location and it will be
327     // available for further analysis.
328     virtual PeerConfigurer* SetRtcEventLogPath(std::string path) = 0;
329     // If is set, an AEC dump will be saved in that location and it will be
330     // available for further analysis.
331     virtual PeerConfigurer* SetAecDumpPath(std::string path) = 0;
332     virtual PeerConfigurer* SetRTCConfiguration(
333         PeerConnectionInterface::RTCConfiguration configuration) = 0;
334     // Set bitrate parameters on PeerConnection. This constraints will be
335     // applied to all summed RTP streams for this peer.
336     virtual PeerConfigurer* SetBitrateSettings(
337         BitrateSettings bitrate_settings) = 0;
338   };
339 
340   // Contains configuration for echo emulator.
341   struct EchoEmulationConfig {
342     // Delay which represents the echo path delay, i.e. how soon rendered signal
343     // should reach capturer.
344     TimeDelta echo_delay = TimeDelta::Millis(50);
345   };
346 
347   struct VideoCodecConfig {
VideoCodecConfigVideoCodecConfig348     explicit VideoCodecConfig(std::string name)
349         : name(std::move(name)), required_params() {}
VideoCodecConfigVideoCodecConfig350     VideoCodecConfig(std::string name,
351                      std::map<std::string, std::string> required_params)
352         : name(std::move(name)), required_params(std::move(required_params)) {}
353     // Next two fields are used to specify concrete video codec, that should be
354     // used in the test. Video code will be negotiated in SDP during offer/
355     // answer exchange.
356     // Video codec name. You can find valid names in
357     // media/base/media_constants.h
358     std::string name = cricket::kVp8CodecName;
359     // Map of parameters, that have to be specified on SDP codec. Each parameter
360     // is described by key and value. Codec parameters will match the specified
361     // map if and only if for each key from |required_params| there will be
362     // a parameter with name equal to this key and parameter value will be equal
363     // to the value from |required_params| for this key.
364     // If empty then only name will be used to match the codec.
365     std::map<std::string, std::string> required_params;
366   };
367 
368   // Contains parameters, that describe how long framework should run quality
369   // test.
370   struct RunParams {
RunParamsRunParams371     explicit RunParams(TimeDelta run_duration) : run_duration(run_duration) {}
372 
373     // Specifies how long the test should be run. This time shows how long
374     // the media should flow after connection was established and before
375     // it will be shut downed.
376     TimeDelta run_duration;
377 
378     // List of video codecs to use during the test. These codecs will be
379     // negotiated in SDP during offer/answer exchange. The order of these codecs
380     // during negotiation will be the same as in |video_codecs|. Codecs have
381     // to be available in codecs list provided by peer connection to be
382     // negotiated. If some of specified codecs won't be found, the test will
383     // crash.
384     // If list is empty Vp8 with no required_params will be used.
385     std::vector<VideoCodecConfig> video_codecs;
386     bool use_ulp_fec = false;
387     bool use_flex_fec = false;
388     // Specifies how much video encoder target bitrate should be different than
389     // target bitrate, provided by WebRTC stack. Must be greater then 0. Can be
390     // used to emulate overshooting of video encoders. This multiplier will
391     // be applied for all video encoder on both sides for all layers. Bitrate
392     // estimated by WebRTC stack will be multiplied on this multiplier and then
393     // provided into VideoEncoder::SetRates(...).
394     double video_encoder_bitrate_multiplier = 1.0;
395     // If true will set conference mode in SDP media section for all video
396     // tracks for all peers.
397     bool use_conference_mode = false;
398     // If specified echo emulation will be done, by mixing the render audio into
399     // the capture signal. In such case input signal will be reduced by half to
400     // avoid saturation or compression in the echo path simulation.
401     absl::optional<EchoEmulationConfig> echo_emulation_config;
402   };
403 
404   // Represent an entity that will report quality metrics after test.
405   class QualityMetricsReporter : public StatsObserverInterface {
406    public:
407     virtual ~QualityMetricsReporter() = default;
408 
409     // Invoked by framework after peer connection factory and peer connection
410     // itself will be created but before offer/answer exchange will be started.
411     // |test_case_name| is name of test case, that should be used to report all
412     // metrics.
413     // |reporter_helper| is a pointer to a class that will allow track_id to
414     // stream_id matching. The caller is responsible for ensuring the
415     // TrackIdStreamInfoMap will be valid from Start() to
416     // StopAndReportResults().
417     virtual void Start(absl::string_view test_case_name,
418                        const TrackIdStreamInfoMap* reporter_helper) = 0;
419 
420     // Invoked by framework after call is ended and peer connection factory and
421     // peer connection are destroyed.
422     virtual void StopAndReportResults() = 0;
423   };
424 
425   virtual ~PeerConnectionE2EQualityTestFixture() = default;
426 
427   // Add activity that will be executed on the best effort at least after
428   // |target_time_since_start| after call will be set up (after offer/answer
429   // exchange, ICE gathering will be done and ICE candidates will passed to
430   // remote side). |func| param is amount of time spent from the call set up.
431   virtual void ExecuteAt(TimeDelta target_time_since_start,
432                          std::function<void(TimeDelta)> func) = 0;
433   // Add activity that will be executed every |interval| with first execution
434   // on the best effort at least after |initial_delay_since_start| after call
435   // will be set up (after all participants will be connected). |func| param is
436   // amount of time spent from the call set up.
437   virtual void ExecuteEvery(TimeDelta initial_delay_since_start,
438                             TimeDelta interval,
439                             std::function<void(TimeDelta)> func) = 0;
440 
441   // Add stats reporter entity to observe the test.
442   virtual void AddQualityMetricsReporter(
443       std::unique_ptr<QualityMetricsReporter> quality_metrics_reporter) = 0;
444 
445   // Add a new peer to the call and return an object through which caller
446   // can configure peer's behavior.
447   // |network_thread| will be used as network thread for peer's peer connection
448   // |network_manager| will be used to provide network interfaces for peer's
449   // peer connection.
450   // |configurer| function will be used to configure peer in the call.
451   virtual void AddPeer(rtc::Thread* network_thread,
452                        rtc::NetworkManager* network_manager,
453                        rtc::FunctionView<void(PeerConfigurer*)> configurer) = 0;
454   // Runs the media quality test, which includes setting up the call with
455   // configured participants, running it according to provided |run_params| and
456   // terminating it properly at the end. During call duration media quality
457   // metrics are gathered, which are then reported to stdout and (if configured)
458   // to the json/protobuf output file through the WebRTC perf test results
459   // reporting system.
460   virtual void Run(RunParams run_params) = 0;
461 
462   // Returns real test duration - the time of test execution measured during
463   // test. Client must call this method only after test is finished (after
464   // Run(...) method returned). Test execution time is time from end of call
465   // setup (offer/answer, ICE candidates exchange done and ICE connected) to
466   // start of call tear down (PeerConnection closed).
467   virtual TimeDelta GetRealTestDuration() const = 0;
468 };
469 
470 }  // namespace webrtc_pc_e2e
471 }  // namespace webrtc
472 
473 #endif  // API_TEST_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_
474