1 /* 2 * libjingle 3 * Copyright 2010 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 // Declaration of abstract class VideoCapturer 29 30 #ifndef TALK_MEDIA_BASE_VIDEOCAPTURER_H_ 31 #define TALK_MEDIA_BASE_VIDEOCAPTURER_H_ 32 33 #include <algorithm> 34 #include <string> 35 #include <vector> 36 37 #include "talk/media/base/mediachannel.h" 38 #include "talk/media/base/videoadapter.h" 39 #include "talk/media/base/videocommon.h" 40 #include "talk/media/base/videoframefactory.h" 41 #include "talk/media/devices/devicemanager.h" 42 #include "webrtc/base/basictypes.h" 43 #include "webrtc/base/criticalsection.h" 44 #include "webrtc/base/messagehandler.h" 45 #include "webrtc/base/rollingaccumulator.h" 46 #include "webrtc/base/scoped_ptr.h" 47 #include "webrtc/base/sigslot.h" 48 #include "webrtc/base/thread.h" 49 #include "webrtc/base/timing.h" 50 51 52 namespace cricket { 53 54 // Current state of the capturer. 55 // TODO(hellner): CS_NO_DEVICE is an error code not a capture state. Separate 56 // error codes and states. 57 enum CaptureState { 58 CS_STOPPED, // The capturer has been stopped or hasn't started yet. 59 CS_STARTING, // The capturer is in the process of starting. Note, it may 60 // still fail to start. 61 CS_RUNNING, // The capturer has been started successfully and is now 62 // capturing. 63 CS_PAUSED, // The capturer has been paused. 64 CS_FAILED, // The capturer failed to start. 65 CS_NO_DEVICE, // The capturer has no device and consequently failed to start. 66 }; 67 68 class VideoFrame; 69 70 struct CapturedFrame { 71 static const uint32_t kFrameHeaderSize = 40; // Size from width to data_size. 72 static const uint32_t kUnknownDataSize = 0xFFFFFFFF; 73 74 CapturedFrame(); 75 76 // Get the number of bytes of the frame data. If data_size is known, return 77 // it directly. Otherwise, calculate the size based on width, height, and 78 // fourcc. Return true if succeeded. 79 bool GetDataSize(uint32_t* size) const; 80 81 // The width and height of the captured frame could be different from those 82 // of VideoFormat. Once the first frame is captured, the width, height, 83 // fourcc, pixel_width, and pixel_height should keep the same over frames. 84 int width; // in number of pixels 85 int height; // in number of pixels 86 uint32_t fourcc; // compression 87 uint32_t pixel_width; // width of a pixel, default is 1 88 uint32_t pixel_height; // height of a pixel, default is 1 89 int64_t time_stamp; // timestamp of when the frame was captured, in unix 90 // time with nanosecond units. 91 uint32_t data_size; // number of bytes of the frame data 92 93 webrtc::VideoRotation rotation; // rotation in degrees of the frame. 94 95 void* data; // pointer to the frame data. This object allocates the 96 // memory or points to an existing memory. 97 98 private: 99 RTC_DISALLOW_COPY_AND_ASSIGN(CapturedFrame); 100 }; 101 102 // VideoCapturer is an abstract class that defines the interfaces for video 103 // capturing. The subclasses implement the video capturer for various types of 104 // capturers and various platforms. 105 // 106 // The captured frames may need to be adapted (for example, cropping). 107 // Video adaptation is built into and enabled by default. After a frame has 108 // been captured from the device, it is sent to the video adapter, then out to 109 // the encoder. 110 // 111 // Programming model: 112 // Create an object of a subclass of VideoCapturer 113 // Initialize 114 // SignalStateChange.connect() 115 // SignalFrameCaptured.connect() 116 // Find the capture format for Start() by either calling GetSupportedFormats() 117 // and selecting one of the supported or calling GetBestCaptureFormat(). 118 // video_adapter()->OnOutputFormatRequest(desired_encoding_format) 119 // Start() 120 // GetCaptureFormat() optionally 121 // Stop() 122 // 123 // Assumption: 124 // The Start() and Stop() methods are called by a single thread (E.g., the 125 // media engine thread). Hence, the VideoCapture subclasses dont need to be 126 // thread safe. 127 // 128 class VideoCapturer 129 : public sigslot::has_slots<>, 130 public rtc::MessageHandler { 131 public: 132 // All signals are marshalled to |thread| or the creating thread if 133 // none is provided. 134 VideoCapturer(); 135 explicit VideoCapturer(rtc::Thread* thread); ~VideoCapturer()136 virtual ~VideoCapturer() {} 137 138 // Gets the id of the underlying device, which is available after the capturer 139 // is initialized. Can be used to determine if two capturers reference the 140 // same device. GetId()141 const std::string& GetId() const { return id_; } 142 143 // Get the capture formats supported by the video capturer. The supported 144 // formats are non empty after the device has been opened successfully. 145 const std::vector<VideoFormat>* GetSupportedFormats() const; 146 147 // Get the best capture format for the desired format. The best format is the 148 // same as one of the supported formats except that the frame interval may be 149 // different. If the application asks for 16x9 and the camera does not support 150 // 16x9 HD or the application asks for 16x10, we find the closest 4x3 and then 151 // crop; Otherwise, we find what the application asks for. Note that we assume 152 // that for HD, the desired format is always 16x9. The subclasses can override 153 // the default implementation. 154 // Parameters 155 // desired: the input desired format. If desired.fourcc is not kAnyFourcc, 156 // the best capture format has the exactly same fourcc. Otherwise, 157 // the best capture format uses a fourcc in GetPreferredFourccs(). 158 // best_format: the output of the best capture format. 159 // Return false if there is no such a best format, that is, the desired format 160 // is not supported. 161 virtual bool GetBestCaptureFormat(const VideoFormat& desired, 162 VideoFormat* best_format); 163 164 // TODO(hellner): deprecate (make private) the Start API in favor of this one. 165 // Also remove CS_STARTING as it is implied by the return 166 // value of StartCapturing(). 167 bool StartCapturing(const VideoFormat& capture_format); 168 // Start the video capturer with the specified capture format. 169 // Parameter 170 // capture_format: The caller got this parameter by either calling 171 // GetSupportedFormats() and selecting one of the supported 172 // or calling GetBestCaptureFormat(). 173 // Return 174 // CS_STARTING: The capturer is trying to start. Success or failure will 175 // be notified via the |SignalStateChange| callback. 176 // CS_RUNNING: if the capturer is started and capturing. 177 // CS_PAUSED: Will never be returned. 178 // CS_FAILED: if the capturer failes to start.. 179 // CS_NO_DEVICE: if the capturer has no device and fails to start. 180 virtual CaptureState Start(const VideoFormat& capture_format) = 0; 181 // Sets the desired aspect ratio. If the capturer is capturing at another 182 // aspect ratio it will crop the width or the height so that asked for 183 // aspect ratio is acheived. Note that ratio_w and ratio_h do not need to be 184 // relatively prime. 185 void UpdateAspectRatio(int ratio_w, int ratio_h); 186 void ClearAspectRatio(); 187 188 // Get the current capture format, which is set by the Start() call. 189 // Note that the width and height of the captured frames may differ from the 190 // capture format. For example, the capture format is HD but the captured 191 // frames may be smaller than HD. GetCaptureFormat()192 const VideoFormat* GetCaptureFormat() const { 193 return capture_format_.get(); 194 } 195 196 // Pause the video capturer. 197 virtual bool Pause(bool paused); 198 // Stop the video capturer. 199 virtual void Stop() = 0; 200 // Check if the video capturer is running. 201 virtual bool IsRunning() = 0; 202 // Restart the video capturer with the new |capture_format|. 203 // Default implementation stops and starts the capturer. 204 virtual bool Restart(const VideoFormat& capture_format); 205 // TODO(thorcarpenter): This behavior of keeping the camera open just to emit 206 // black frames is a total hack and should be fixed. 207 // When muting, produce black frames then pause the camera. 208 // When unmuting, start the camera. Camera starts unmuted. 209 virtual bool MuteToBlackThenPause(bool muted); IsMuted()210 virtual bool IsMuted() const { 211 return muted_; 212 } capture_state()213 CaptureState capture_state() const { 214 return capture_state_; 215 } 216 217 // Tells videocapturer whether to apply the pending rotation. By default, the 218 // rotation is applied and the generated frame is up right. When set to false, 219 // generated frames will carry the rotation information from 220 // SetCaptureRotation. Return value indicates whether this operation succeeds. 221 virtual bool SetApplyRotation(bool enable); GetApplyRotation()222 virtual bool GetApplyRotation() { return apply_rotation_; } 223 224 // Returns true if the capturer is screencasting. This can be used to 225 // implement screencast specific behavior. 226 virtual bool IsScreencast() const = 0; 227 228 // Caps the VideoCapturer's format according to max_format. It can e.g. be 229 // used to prevent cameras from capturing at a resolution or framerate that 230 // the capturer is capable of but not performing satisfactorily at. 231 // The capping is an upper bound for each component of the capturing format. 232 // The fourcc component is ignored. 233 void ConstrainSupportedFormats(const VideoFormat& max_format); 234 set_enable_camera_list(bool enable_camera_list)235 void set_enable_camera_list(bool enable_camera_list) { 236 enable_camera_list_ = enable_camera_list; 237 } enable_camera_list()238 bool enable_camera_list() { 239 return enable_camera_list_; 240 } 241 242 // Enable scaling to ensure square pixels. set_square_pixel_aspect_ratio(bool square_pixel_aspect_ratio)243 void set_square_pixel_aspect_ratio(bool square_pixel_aspect_ratio) { 244 square_pixel_aspect_ratio_ = square_pixel_aspect_ratio; 245 } square_pixel_aspect_ratio()246 bool square_pixel_aspect_ratio() { 247 return square_pixel_aspect_ratio_; 248 } 249 250 // Signal all capture state changes that are not a direct result of calling 251 // Start(). 252 sigslot::signal2<VideoCapturer*, CaptureState> SignalStateChange; 253 // Frame callbacks are multithreaded to allow disconnect and connect to be 254 // called concurrently. It also ensures that it is safe to call disconnect 255 // at any time which is needed since the signal may be called from an 256 // unmarshalled thread owned by the VideoCapturer. 257 // Signal the captured frame to downstream. 258 sigslot::signal2<VideoCapturer*, const CapturedFrame*, 259 sigslot::multi_threaded_local> SignalFrameCaptured; 260 // Signal the captured and possibly adapted frame to downstream consumers 261 // such as the encoder. 262 sigslot::signal2<VideoCapturer*, const VideoFrame*, 263 sigslot::multi_threaded_local> SignalVideoFrame; 264 265 // If true, run video adaptation. By default, video adaptation is enabled 266 // and users must call video_adapter()->OnOutputFormatRequest() 267 // to receive frames. enable_video_adapter()268 bool enable_video_adapter() const { return enable_video_adapter_; } set_enable_video_adapter(bool enable_video_adapter)269 void set_enable_video_adapter(bool enable_video_adapter) { 270 enable_video_adapter_ = enable_video_adapter; 271 } 272 video_adapter()273 CoordinatedVideoAdapter* video_adapter() { return &video_adapter_; } video_adapter()274 const CoordinatedVideoAdapter* video_adapter() const { 275 return &video_adapter_; 276 } 277 278 // Takes ownership. 279 void set_frame_factory(VideoFrameFactory* frame_factory); 280 281 // Gets statistics for tracked variables recorded since the last call to 282 // GetStats. Note that calling GetStats resets any gathered data so it 283 // should be called only periodically to log statistics. 284 void GetStats(VariableInfo<int>* adapt_drop_stats, 285 VariableInfo<int>* effect_drop_stats, 286 VariableInfo<double>* frame_time_stats, 287 VideoFormat* last_captured_frame_format); 288 289 protected: 290 // Callback attached to SignalFrameCaptured where SignalVideoFrames is called. 291 void OnFrameCaptured(VideoCapturer* video_capturer, 292 const CapturedFrame* captured_frame); 293 void SetCaptureState(CaptureState state); 294 295 // Marshals SignalStateChange onto thread_. 296 void OnMessage(rtc::Message* message); 297 298 // subclasses override this virtual method to provide a vector of fourccs, in 299 // order of preference, that are expected by the media engine. 300 virtual bool GetPreferredFourccs(std::vector<uint32_t>* fourccs) = 0; 301 302 // mutators to set private attributes SetId(const std::string & id)303 void SetId(const std::string& id) { 304 id_ = id; 305 } 306 SetCaptureFormat(const VideoFormat * format)307 void SetCaptureFormat(const VideoFormat* format) { 308 capture_format_.reset(format ? new VideoFormat(*format) : NULL); 309 if (capture_format_) { 310 ASSERT(capture_format_->interval > 0 && 311 "Capture format expected to have positive interval."); 312 // Video adapter really only cares about capture format interval. 313 video_adapter_.SetInputFormat(*capture_format_); 314 } 315 } 316 317 void SetSupportedFormats(const std::vector<VideoFormat>& formats); frame_factory()318 VideoFrameFactory* frame_factory() { return frame_factory_.get(); } 319 320 private: 321 void Construct(); 322 // Get the distance between the desired format and the supported format. 323 // Return the max distance if they mismatch. See the implementation for 324 // details. 325 int64_t GetFormatDistance(const VideoFormat& desired, 326 const VideoFormat& supported); 327 328 // Convert captured frame to readable string for LOG messages. 329 std::string ToString(const CapturedFrame* frame) const; 330 331 // Updates filtered_supported_formats_ so that it contains the formats in 332 // supported_formats_ that fulfill all applied restrictions. 333 void UpdateFilteredSupportedFormats(); 334 // Returns true if format doesn't fulfill all applied restrictions. 335 bool ShouldFilterFormat(const VideoFormat& format) const; 336 337 void UpdateStats(const CapturedFrame* captured_frame); 338 339 // Helper function to save statistics on the current data from a 340 // RollingAccumulator into stats. 341 template<class T> 342 static void GetVariableSnapshot( 343 const rtc::RollingAccumulator<T>& data, 344 VariableInfo<T>* stats); 345 346 rtc::Thread* thread_; 347 std::string id_; 348 CaptureState capture_state_; 349 rtc::scoped_ptr<VideoFrameFactory> frame_factory_; 350 rtc::scoped_ptr<VideoFormat> capture_format_; 351 std::vector<VideoFormat> supported_formats_; 352 rtc::scoped_ptr<VideoFormat> max_format_; 353 std::vector<VideoFormat> filtered_supported_formats_; 354 355 int ratio_w_; // View resolution. e.g. 1280 x 720. 356 int ratio_h_; 357 bool enable_camera_list_; 358 bool square_pixel_aspect_ratio_; // Enable scaling to square pixels. 359 int scaled_width_; // Current output size from ComputeScale. 360 int scaled_height_; 361 bool muted_; 362 int black_frame_count_down_; 363 364 bool enable_video_adapter_; 365 CoordinatedVideoAdapter video_adapter_; 366 367 rtc::Timing frame_length_time_reporter_; 368 rtc::CriticalSection frame_stats_crit_; 369 370 int adapt_frame_drops_; 371 rtc::RollingAccumulator<int> adapt_frame_drops_data_; 372 double previous_frame_time_; 373 rtc::RollingAccumulator<double> frame_time_data_; 374 // The captured frame format before potential adapation. 375 VideoFormat last_captured_frame_format_; 376 377 // Whether capturer should apply rotation to the frame before signaling it. 378 bool apply_rotation_; 379 380 RTC_DISALLOW_COPY_AND_ASSIGN(VideoCapturer); 381 }; 382 383 } // namespace cricket 384 385 #endif // TALK_MEDIA_BASE_VIDEOCAPTURER_H_ 386