1 /* 2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 #ifndef API_TEST_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ 11 #define API_TEST_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ 12 13 #include <map> 14 #include <memory> 15 #include <string> 16 #include <utility> 17 #include <vector> 18 19 #include "absl/memory/memory.h" 20 #include "absl/strings/string_view.h" 21 #include "absl/types/optional.h" 22 #include "api/async_resolver_factory.h" 23 #include "api/call/call_factory_interface.h" 24 #include "api/fec_controller.h" 25 #include "api/function_view.h" 26 #include "api/media_stream_interface.h" 27 #include "api/peer_connection_interface.h" 28 #include "api/rtc_event_log/rtc_event_log_factory_interface.h" 29 #include "api/rtp_parameters.h" 30 #include "api/task_queue/task_queue_factory.h" 31 #include "api/test/audio_quality_analyzer_interface.h" 32 #include "api/test/frame_generator_interface.h" 33 #include "api/test/simulated_network.h" 34 #include "api/test/stats_observer_interface.h" 35 #include "api/test/track_id_stream_info_map.h" 36 #include "api/test/video_quality_analyzer_interface.h" 37 #include "api/transport/network_control.h" 38 #include "api/units/time_delta.h" 39 #include "api/video_codecs/video_decoder_factory.h" 40 #include "api/video_codecs/video_encoder.h" 41 #include "api/video_codecs/video_encoder_factory.h" 42 #include "media/base/media_constants.h" 43 #include "rtc_base/network.h" 44 #include "rtc_base/rtc_certificate_generator.h" 45 #include "rtc_base/ssl_certificate.h" 46 #include "rtc_base/thread.h" 47 48 namespace webrtc { 49 namespace webrtc_pc_e2e { 50 51 constexpr size_t kDefaultSlidesWidth = 1850; 52 constexpr size_t kDefaultSlidesHeight = 1110; 53 54 // API is in development. Can be changed/removed without notice. 55 class PeerConnectionE2EQualityTestFixture { 56 public: 57 // The index of required capturing device in OS provided list of video 58 // devices. On Linux and Windows the list will be obtained via 59 // webrtc::VideoCaptureModule::DeviceInfo, on Mac OS via 60 // [RTCCameraVideoCapturer captureDevices]. 61 enum class CapturingDeviceIndex : size_t {}; 62 63 // Contains parameters for screen share scrolling. 64 // 65 // If scrolling is enabled, then it will be done by putting sliding window 66 // on source video and moving this window from top left corner to the 67 // bottom right corner of the picture. 68 // 69 // In such case source dimensions must be greater or equal to the sliding 70 // window dimensions. So |source_width| and |source_height| are the dimensions 71 // of the source frame, while |VideoConfig::width| and |VideoConfig::height| 72 // are the dimensions of the sliding window. 73 // 74 // Because |source_width| and |source_height| are dimensions of the source 75 // frame, they have to be width and height of videos from 76 // |ScreenShareConfig::slides_yuv_file_names|. 77 // 78 // Because scrolling have to be done on single slide it also requires, that 79 // |duration| must be less or equal to 80 // |ScreenShareConfig::slide_change_interval|. 81 struct ScrollingParams { ScrollingParamsScrollingParams82 ScrollingParams(TimeDelta duration, 83 size_t source_width, 84 size_t source_height) 85 : duration(duration), 86 source_width(source_width), 87 source_height(source_height) { 88 RTC_CHECK_GT(duration.ms(), 0); 89 } 90 91 // Duration of scrolling. 92 TimeDelta duration; 93 // Width of source slides video. 94 size_t source_width; 95 // Height of source slides video. 96 size_t source_height; 97 }; 98 99 // Contains screen share video stream properties. 100 struct ScreenShareConfig { ScreenShareConfigScreenShareConfig101 explicit ScreenShareConfig(TimeDelta slide_change_interval) 102 : slide_change_interval(slide_change_interval) { 103 RTC_CHECK_GT(slide_change_interval.ms(), 0); 104 } 105 106 // Shows how long one slide should be presented on the screen during 107 // slide generation. 108 TimeDelta slide_change_interval; 109 // If true, slides will be generated programmatically. No scrolling params 110 // will be applied in such case. 111 bool generate_slides = false; 112 // If present scrolling will be applied. Please read extra requirement on 113 // |slides_yuv_file_names| for scrolling. 114 absl::optional<ScrollingParams> scrolling_params; 115 // Contains list of yuv files with slides. 116 // 117 // If empty, default set of slides will be used. In such case 118 // |VideoConfig::width| must be equal to |kDefaultSlidesWidth| and 119 // |VideoConfig::height| must be equal to |kDefaultSlidesHeight| or if 120 // |scrolling_params| are specified, then |ScrollingParams::source_width| 121 // must be equal to |kDefaultSlidesWidth| and 122 // |ScrollingParams::source_height| must be equal to |kDefaultSlidesHeight|. 123 std::vector<std::string> slides_yuv_file_names; 124 }; 125 126 // Config for Vp8 simulcast or Vp9 SVC testing. 127 // 128 // SVC support is limited: 129 // During SVC testing there is no SFU, so framework will try to emulate SFU 130 // behavior in regular p2p call. Because of it there are such limitations: 131 // * if |target_spatial_index| is not equal to the highest spatial layer 132 // then no packet/frame drops are allowed. 133 // 134 // If there will be any drops, that will affect requested layer, then 135 // WebRTC SVC implementation will continue decoding only the highest 136 // available layer and won't restore lower layers, so analyzer won't 137 // receive required data which will cause wrong results or test failures. 138 struct VideoSimulcastConfig { VideoSimulcastConfigVideoSimulcastConfig139 explicit VideoSimulcastConfig(int simulcast_streams_count) 140 : simulcast_streams_count(simulcast_streams_count) { 141 RTC_CHECK_GT(simulcast_streams_count, 1); 142 } VideoSimulcastConfigVideoSimulcastConfig143 VideoSimulcastConfig(int simulcast_streams_count, int target_spatial_index) 144 : simulcast_streams_count(simulcast_streams_count), 145 target_spatial_index(target_spatial_index) { 146 RTC_CHECK_GT(simulcast_streams_count, 1); 147 RTC_CHECK_GE(target_spatial_index, 0); 148 RTC_CHECK_LT(target_spatial_index, simulcast_streams_count); 149 } 150 151 // Specified amount of simulcast streams/SVC layers, depending on which 152 // encoder is used. 153 int simulcast_streams_count; 154 // Specifies spatial index of the video stream to analyze. 155 // There are 2 cases: 156 // 1. simulcast encoder is used: 157 // in such case |target_spatial_index| will specify the index of 158 // simulcast stream, that should be analyzed. Other streams will be 159 // dropped. 160 // 2. SVC encoder is used: 161 // in such case |target_spatial_index| will specify the top interesting 162 // spatial layer and all layers below, including target one will be 163 // processed. All layers above target one will be dropped. 164 // If not specified than whatever stream will be received will be analyzed. 165 // It requires Selective Forwarding Unit (SFU) to be configured in the 166 // network. 167 absl::optional<int> target_spatial_index; 168 169 // Encoding parameters per simulcast layer. If not empty, |encoding_params| 170 // size have to be equal to |simulcast_streams_count|. Will be used to set 171 // transceiver send encoding params for simulcast layers. Applicable only 172 // for codecs that support simulcast (ex. Vp8) and will be ignored 173 // otherwise. RtpEncodingParameters::rid may be changed by fixture 174 // implementation to ensure signaling correctness. 175 std::vector<RtpEncodingParameters> encoding_params; 176 }; 177 178 // Contains properties of single video stream. 179 struct VideoConfig { VideoConfigVideoConfig180 VideoConfig(size_t width, size_t height, int32_t fps) 181 : width(width), height(height), fps(fps) {} 182 183 // Video stream width. 184 const size_t width; 185 // Video stream height. 186 const size_t height; 187 const int32_t fps; 188 // Have to be unique among all specified configs for all peers in the call. 189 // Will be auto generated if omitted. 190 absl::optional<std::string> stream_label; 191 // Will be set for current video track. If equals to kText or kDetailed - 192 // screencast in on. 193 absl::optional<VideoTrackInterface::ContentHint> content_hint; 194 // If presented video will be transfered in simulcast/SVC mode depending on 195 // which encoder is used. 196 // 197 // Simulcast is supported only from 1st added peer. For VP8 simulcast only 198 // without RTX is supported so it will be automatically disabled for all 199 // simulcast tracks. For VP9 simulcast enables VP9 SVC mode and support RTX, 200 // but only on non-lossy networks. See more in documentation to 201 // VideoSimulcastConfig. 202 absl::optional<VideoSimulcastConfig> simulcast_config; 203 // Count of temporal layers for video stream. This value will be set into 204 // each RtpEncodingParameters of RtpParameters of corresponding 205 // RtpSenderInterface for this video stream. 206 absl::optional<int> temporal_layers_count; 207 // Sets the maximum encode bitrate in bps. If this value is not set, the 208 // encoder will be capped at an internal maximum value around 2 Mbps 209 // depending on the resolution. This means that it will never be able to 210 // utilize a high bandwidth link. 211 absl::optional<int> max_encode_bitrate_bps; 212 // Sets the minimum encode bitrate in bps. If this value is not set, the 213 // encoder will use an internal minimum value. Please note that if this 214 // value is set higher than the bandwidth of the link, the encoder will 215 // generate more data than the link can handle regardless of the bandwidth 216 // estimation. 217 absl::optional<int> min_encode_bitrate_bps; 218 // If specified the input stream will be also copied to specified file. 219 // It is actually one of the test's output file, which contains copy of what 220 // was captured during the test for this video stream on sender side. 221 // It is useful when generator is used as input. 222 absl::optional<std::string> input_dump_file_name; 223 // If specified this file will be used as output on the receiver side for 224 // this stream. If multiple streams will be produced by input stream, 225 // output files will be appended with indexes. The produced files contains 226 // what was rendered for this video stream on receiver side. 227 absl::optional<std::string> output_dump_file_name; 228 // If true will display input and output video on the user's screen. 229 bool show_on_screen = false; 230 // If specified, determines a sync group to which this video stream belongs. 231 // According to bugs.webrtc.org/4762 WebRTC supports synchronization only 232 // for pair of single audio and single video stream. 233 absl::optional<std::string> sync_group; 234 }; 235 236 // Contains properties for audio in the call. 237 struct AudioConfig { 238 enum Mode { 239 kGenerated, 240 kFile, 241 }; 242 // Have to be unique among all specified configs for all peers in the call. 243 // Will be auto generated if omitted. 244 absl::optional<std::string> stream_label; 245 Mode mode = kGenerated; 246 // Have to be specified only if mode = kFile 247 absl::optional<std::string> input_file_name; 248 // If specified the input stream will be also copied to specified file. 249 absl::optional<std::string> input_dump_file_name; 250 // If specified the output stream will be copied to specified file. 251 absl::optional<std::string> output_dump_file_name; 252 253 // Audio options to use. 254 cricket::AudioOptions audio_options; 255 // Sampling frequency of input audio data (from file or generated). 256 int sampling_frequency_in_hz = 48000; 257 // If specified, determines a sync group to which this audio stream belongs. 258 // According to bugs.webrtc.org/4762 WebRTC supports synchronization only 259 // for pair of single audio and single video stream. 260 absl::optional<std::string> sync_group; 261 }; 262 263 // This class is used to fully configure one peer inside the call. 264 class PeerConfigurer { 265 public: 266 virtual ~PeerConfigurer() = default; 267 268 // Sets peer name that will be used to report metrics related to this peer. 269 // If not set, some default name will be assigned. All names have to be 270 // unique. 271 virtual PeerConfigurer* SetName(absl::string_view name) = 0; 272 273 // The parameters of the following 9 methods will be passed to the 274 // PeerConnectionFactoryInterface implementation that will be created for 275 // this peer. 276 virtual PeerConfigurer* SetTaskQueueFactory( 277 std::unique_ptr<TaskQueueFactory> task_queue_factory) = 0; 278 virtual PeerConfigurer* SetCallFactory( 279 std::unique_ptr<CallFactoryInterface> call_factory) = 0; 280 virtual PeerConfigurer* SetEventLogFactory( 281 std::unique_ptr<RtcEventLogFactoryInterface> event_log_factory) = 0; 282 virtual PeerConfigurer* SetFecControllerFactory( 283 std::unique_ptr<FecControllerFactoryInterface> 284 fec_controller_factory) = 0; 285 virtual PeerConfigurer* SetNetworkControllerFactory( 286 std::unique_ptr<NetworkControllerFactoryInterface> 287 network_controller_factory) = 0; 288 virtual PeerConfigurer* SetVideoEncoderFactory( 289 std::unique_ptr<VideoEncoderFactory> video_encoder_factory) = 0; 290 virtual PeerConfigurer* SetVideoDecoderFactory( 291 std::unique_ptr<VideoDecoderFactory> video_decoder_factory) = 0; 292 // Set a custom NetEqFactory to be used in the call. 293 virtual PeerConfigurer* SetNetEqFactory( 294 std::unique_ptr<NetEqFactory> neteq_factory) = 0; 295 296 // The parameters of the following 4 methods will be passed to the 297 // PeerConnectionInterface implementation that will be created for this 298 // peer. 299 virtual PeerConfigurer* SetAsyncResolverFactory( 300 std::unique_ptr<webrtc::AsyncResolverFactory> 301 async_resolver_factory) = 0; 302 virtual PeerConfigurer* SetRTCCertificateGenerator( 303 std::unique_ptr<rtc::RTCCertificateGeneratorInterface> 304 cert_generator) = 0; 305 virtual PeerConfigurer* SetSSLCertificateVerifier( 306 std::unique_ptr<rtc::SSLCertificateVerifier> tls_cert_verifier) = 0; 307 virtual PeerConfigurer* SetIceTransportFactory( 308 std::unique_ptr<IceTransportFactory> factory) = 0; 309 310 // Add new video stream to the call that will be sent from this peer. 311 // Default implementation of video frames generator will be used. 312 virtual PeerConfigurer* AddVideoConfig(VideoConfig config) = 0; 313 // Add new video stream to the call that will be sent from this peer with 314 // provided own implementation of video frames generator. 315 virtual PeerConfigurer* AddVideoConfig( 316 VideoConfig config, 317 std::unique_ptr<test::FrameGeneratorInterface> generator) = 0; 318 // Add new video stream to the call that will be sent from this peer. 319 // Capturing device with specified index will be used to get input video. 320 virtual PeerConfigurer* AddVideoConfig( 321 VideoConfig config, 322 CapturingDeviceIndex capturing_device_index) = 0; 323 // Set the audio stream for the call from this peer. If this method won't 324 // be invoked, this peer will send no audio. 325 virtual PeerConfigurer* SetAudioConfig(AudioConfig config) = 0; 326 // If is set, an RTCEventLog will be saved in that location and it will be 327 // available for further analysis. 328 virtual PeerConfigurer* SetRtcEventLogPath(std::string path) = 0; 329 // If is set, an AEC dump will be saved in that location and it will be 330 // available for further analysis. 331 virtual PeerConfigurer* SetAecDumpPath(std::string path) = 0; 332 virtual PeerConfigurer* SetRTCConfiguration( 333 PeerConnectionInterface::RTCConfiguration configuration) = 0; 334 // Set bitrate parameters on PeerConnection. This constraints will be 335 // applied to all summed RTP streams for this peer. 336 virtual PeerConfigurer* SetBitrateSettings( 337 BitrateSettings bitrate_settings) = 0; 338 }; 339 340 // Contains configuration for echo emulator. 341 struct EchoEmulationConfig { 342 // Delay which represents the echo path delay, i.e. how soon rendered signal 343 // should reach capturer. 344 TimeDelta echo_delay = TimeDelta::Millis(50); 345 }; 346 347 struct VideoCodecConfig { VideoCodecConfigVideoCodecConfig348 explicit VideoCodecConfig(std::string name) 349 : name(std::move(name)), required_params() {} VideoCodecConfigVideoCodecConfig350 VideoCodecConfig(std::string name, 351 std::map<std::string, std::string> required_params) 352 : name(std::move(name)), required_params(std::move(required_params)) {} 353 // Next two fields are used to specify concrete video codec, that should be 354 // used in the test. Video code will be negotiated in SDP during offer/ 355 // answer exchange. 356 // Video codec name. You can find valid names in 357 // media/base/media_constants.h 358 std::string name = cricket::kVp8CodecName; 359 // Map of parameters, that have to be specified on SDP codec. Each parameter 360 // is described by key and value. Codec parameters will match the specified 361 // map if and only if for each key from |required_params| there will be 362 // a parameter with name equal to this key and parameter value will be equal 363 // to the value from |required_params| for this key. 364 // If empty then only name will be used to match the codec. 365 std::map<std::string, std::string> required_params; 366 }; 367 368 // Contains parameters, that describe how long framework should run quality 369 // test. 370 struct RunParams { RunParamsRunParams371 explicit RunParams(TimeDelta run_duration) : run_duration(run_duration) {} 372 373 // Specifies how long the test should be run. This time shows how long 374 // the media should flow after connection was established and before 375 // it will be shut downed. 376 TimeDelta run_duration; 377 378 // List of video codecs to use during the test. These codecs will be 379 // negotiated in SDP during offer/answer exchange. The order of these codecs 380 // during negotiation will be the same as in |video_codecs|. Codecs have 381 // to be available in codecs list provided by peer connection to be 382 // negotiated. If some of specified codecs won't be found, the test will 383 // crash. 384 // If list is empty Vp8 with no required_params will be used. 385 std::vector<VideoCodecConfig> video_codecs; 386 bool use_ulp_fec = false; 387 bool use_flex_fec = false; 388 // Specifies how much video encoder target bitrate should be different than 389 // target bitrate, provided by WebRTC stack. Must be greater then 0. Can be 390 // used to emulate overshooting of video encoders. This multiplier will 391 // be applied for all video encoder on both sides for all layers. Bitrate 392 // estimated by WebRTC stack will be multiplied on this multiplier and then 393 // provided into VideoEncoder::SetRates(...). 394 double video_encoder_bitrate_multiplier = 1.0; 395 // If true will set conference mode in SDP media section for all video 396 // tracks for all peers. 397 bool use_conference_mode = false; 398 // If specified echo emulation will be done, by mixing the render audio into 399 // the capture signal. In such case input signal will be reduced by half to 400 // avoid saturation or compression in the echo path simulation. 401 absl::optional<EchoEmulationConfig> echo_emulation_config; 402 }; 403 404 // Represent an entity that will report quality metrics after test. 405 class QualityMetricsReporter : public StatsObserverInterface { 406 public: 407 virtual ~QualityMetricsReporter() = default; 408 409 // Invoked by framework after peer connection factory and peer connection 410 // itself will be created but before offer/answer exchange will be started. 411 // |test_case_name| is name of test case, that should be used to report all 412 // metrics. 413 // |reporter_helper| is a pointer to a class that will allow track_id to 414 // stream_id matching. The caller is responsible for ensuring the 415 // TrackIdStreamInfoMap will be valid from Start() to 416 // StopAndReportResults(). 417 virtual void Start(absl::string_view test_case_name, 418 const TrackIdStreamInfoMap* reporter_helper) = 0; 419 420 // Invoked by framework after call is ended and peer connection factory and 421 // peer connection are destroyed. 422 virtual void StopAndReportResults() = 0; 423 }; 424 425 virtual ~PeerConnectionE2EQualityTestFixture() = default; 426 427 // Add activity that will be executed on the best effort at least after 428 // |target_time_since_start| after call will be set up (after offer/answer 429 // exchange, ICE gathering will be done and ICE candidates will passed to 430 // remote side). |func| param is amount of time spent from the call set up. 431 virtual void ExecuteAt(TimeDelta target_time_since_start, 432 std::function<void(TimeDelta)> func) = 0; 433 // Add activity that will be executed every |interval| with first execution 434 // on the best effort at least after |initial_delay_since_start| after call 435 // will be set up (after all participants will be connected). |func| param is 436 // amount of time spent from the call set up. 437 virtual void ExecuteEvery(TimeDelta initial_delay_since_start, 438 TimeDelta interval, 439 std::function<void(TimeDelta)> func) = 0; 440 441 // Add stats reporter entity to observe the test. 442 virtual void AddQualityMetricsReporter( 443 std::unique_ptr<QualityMetricsReporter> quality_metrics_reporter) = 0; 444 445 // Add a new peer to the call and return an object through which caller 446 // can configure peer's behavior. 447 // |network_thread| will be used as network thread for peer's peer connection 448 // |network_manager| will be used to provide network interfaces for peer's 449 // peer connection. 450 // |configurer| function will be used to configure peer in the call. 451 virtual void AddPeer(rtc::Thread* network_thread, 452 rtc::NetworkManager* network_manager, 453 rtc::FunctionView<void(PeerConfigurer*)> configurer) = 0; 454 // Runs the media quality test, which includes setting up the call with 455 // configured participants, running it according to provided |run_params| and 456 // terminating it properly at the end. During call duration media quality 457 // metrics are gathered, which are then reported to stdout and (if configured) 458 // to the json/protobuf output file through the WebRTC perf test results 459 // reporting system. 460 virtual void Run(RunParams run_params) = 0; 461 462 // Returns real test duration - the time of test execution measured during 463 // test. Client must call this method only after test is finished (after 464 // Run(...) method returned). Test execution time is time from end of call 465 // setup (offer/answer, ICE candidates exchange done and ICE connected) to 466 // start of call tear down (PeerConnection closed). 467 virtual TimeDelta GetRealTestDuration() const = 0; 468 }; 469 470 } // namespace webrtc_pc_e2e 471 } // namespace webrtc 472 473 #endif // API_TEST_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ 474