3 * Copyright 2004 Google Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #ifndef TALK_MEDIA_WEBRTCVIDEOENGINE_H_
29 #define TALK_MEDIA_WEBRTCVIDEOENGINE_H_
34 #include "talk/base/scoped_ptr.h"
35 #include "talk/media/base/codec.h"
36 #include "talk/media/base/videocommon.h"
37 #include "talk/media/webrtc/webrtccommon.h"
38 #include "talk/media/webrtc/webrtcexport.h"
39 #include "talk/media/webrtc/webrtcvideoencoderfactory.h"
40 #include "talk/session/media/channel.h"
41 #include "webrtc/video_engine/include/vie_base.h"
43 #if !defined(LIBPEERCONNECTION_LIB) && \
44 !defined(LIBPEERCONNECTION_IMPLEMENTATION)
45 #error "Bogus include."
49 class VideoCaptureModule;
53 class ViEExternalCapture;
59 } // namespace talk_base
63 class CoordinatedVideoAdapter;
64 class ViETraceWrapper;
70 class VoiceMediaChannel;
71 class WebRtcDecoderObserver;
72 class WebRtcEncoderObserver;
73 class WebRtcLocalStreamInfo;
74 class WebRtcRenderAdapter;
75 class WebRtcVideoChannelRecvInfo;
76 class WebRtcVideoChannelSendInfo;
77 class WebRtcVideoDecoderFactory;
78 class WebRtcVideoEncoderFactory;
79 class WebRtcVideoMediaChannel;
80 class WebRtcVoiceEngine;
85 class WebRtcVideoEngine : public sigslot::has_slots<>,
86 public webrtc::TraceCallback,
87 public WebRtcVideoEncoderFactory::Observer {
89 // Creates the WebRtcVideoEngine with internal VideoCaptureModule.
91 // For testing purposes. Allows the WebRtcVoiceEngine,
92 // ViEWrapper and CpuMonitor to be mocks.
93 // TODO(juberti): Remove the 3-arg ctor once fake tracing is implemented.
94 WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine,
95 ViEWrapper* vie_wrapper,
96 talk_base::CpuMonitor* cpu_monitor);
97 WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine,
98 ViEWrapper* vie_wrapper,
99 ViETraceWrapper* tracing,
100 talk_base::CpuMonitor* cpu_monitor);
101 ~WebRtcVideoEngine();
103 // Basic video engine implementation.
104 bool Init(talk_base::Thread* worker_thread);
107 int GetCapabilities();
108 bool SetOptions(const VideoOptions &options);
109 bool SetDefaultEncoderConfig(const VideoEncoderConfig& config);
110 VideoEncoderConfig GetDefaultEncoderConfig() const;
112 WebRtcVideoMediaChannel* CreateChannel(VoiceMediaChannel* voice_channel);
114 const std::vector<VideoCodec>& codecs() const;
115 const std::vector<RtpHeaderExtension>& rtp_header_extensions() const;
116 void SetLogging(int min_sev, const char* filter);
118 bool SetLocalRenderer(VideoRenderer* renderer);
119 sigslot::repeater2<VideoCapturer*, CaptureState> SignalCaptureStateChange;
121 // Set the VoiceEngine for A/V sync. This can only be called before Init.
122 bool SetVoiceEngine(WebRtcVoiceEngine* voice_engine);
123 // Set a WebRtcVideoDecoderFactory for external decoding. Video engine does
124 // not take the ownership of |decoder_factory|. The caller needs to make sure
125 // that |decoder_factory| outlives the video engine.
126 void SetExternalDecoderFactory(WebRtcVideoDecoderFactory* decoder_factory);
127 // Set a WebRtcVideoEncoderFactory for external encoding. Video engine does
128 // not take the ownership of |encoder_factory|. The caller needs to make sure
129 // that |encoder_factory| outlives the video engine.
130 void SetExternalEncoderFactory(WebRtcVideoEncoderFactory* encoder_factory);
131 // Enable the render module with timing control.
132 bool EnableTimedRender();
134 // Returns an external decoder for the given codec type. The return value
135 // can be NULL if decoder factory is not given or it does not support the
136 // codec type. The caller takes the ownership of the returned object.
137 webrtc::VideoDecoder* CreateExternalDecoder(webrtc::VideoCodecType type);
138 // Releases the decoder instance created by CreateExternalDecoder().
139 void DestroyExternalDecoder(webrtc::VideoDecoder* decoder);
141 // Returns an external encoder for the given codec type. The return value
142 // can be NULL if encoder factory is not given or it does not support the
143 // codec type. The caller takes the ownership of the returned object.
144 webrtc::VideoEncoder* CreateExternalEncoder(webrtc::VideoCodecType type);
145 // Releases the encoder instance created by CreateExternalEncoder().
146 void DestroyExternalEncoder(webrtc::VideoEncoder* encoder);
148 // Returns true if the codec type is supported by the external encoder.
149 bool IsExternalEncoderCodecType(webrtc::VideoCodecType type) const;
151 // Functions called by WebRtcVideoMediaChannel.
152 talk_base::Thread* worker_thread() { return worker_thread_; }
153 ViEWrapper* vie() { return vie_wrapper_.get(); }
154 const VideoFormat& default_codec_format() const {
155 return default_codec_format_;
157 int GetLastEngineError();
158 bool FindCodec(const VideoCodec& in);
159 bool CanSendCodec(const VideoCodec& in, const VideoCodec& current,
161 void RegisterChannel(WebRtcVideoMediaChannel* channel);
162 void UnregisterChannel(WebRtcVideoMediaChannel* channel);
163 bool ConvertFromCricketVideoCodec(const VideoCodec& in_codec,
164 webrtc::VideoCodec* out_codec);
165 // Check whether the supplied trace should be ignored.
166 bool ShouldIgnoreTrace(const std::string& trace);
167 int GetNumOfChannels();
169 VideoFormat GetStartCaptureFormat() const { return default_codec_format_; }
171 talk_base::CpuMonitor* cpu_monitor() { return cpu_monitor_.get(); }
174 // When a video processor registers with the engine.
175 // SignalMediaFrame will be invoked for every video frame.
176 // See videoprocessor.h for param reference.
177 sigslot::signal3<uint32, VideoFrame*, bool*> SignalMediaFrame;
180 typedef std::vector<WebRtcVideoMediaChannel*> VideoChannels;
181 struct VideoCodecPref {
184 // For RTX, this field is the payload-type that RTX applies to.
185 // For other codecs, it should be set to -1.
186 int associated_payload_type;
190 static const VideoCodecPref kVideoCodecPrefs[];
191 static const VideoFormatPod kVideoFormats[];
192 static const VideoFormatPod kDefaultVideoFormat;
194 void Construct(ViEWrapper* vie_wrapper,
195 ViETraceWrapper* tracing,
196 WebRtcVoiceEngine* voice_engine,
197 talk_base::CpuMonitor* cpu_monitor);
198 bool SetDefaultCodec(const VideoCodec& codec);
199 bool RebuildCodecList(const VideoCodec& max_codec);
200 void SetTraceFilter(int filter);
201 void SetTraceOptions(const std::string& options);
202 bool InitVideoEngine();
204 // webrtc::TraceCallback implementation.
205 virtual void Print(webrtc::TraceLevel level, const char* trace, int length);
207 // WebRtcVideoEncoderFactory::Observer implementation.
208 virtual void OnCodecsAvailable();
210 talk_base::Thread* worker_thread_;
211 talk_base::scoped_ptr<ViEWrapper> vie_wrapper_;
212 bool vie_wrapper_base_initialized_;
213 talk_base::scoped_ptr<ViETraceWrapper> tracing_;
214 WebRtcVoiceEngine* voice_engine_;
215 talk_base::scoped_ptr<webrtc::VideoRender> render_module_;
216 WebRtcVideoEncoderFactory* encoder_factory_;
217 WebRtcVideoDecoderFactory* decoder_factory_;
218 std::vector<VideoCodec> video_codecs_;
219 std::vector<RtpHeaderExtension> rtp_header_extensions_;
220 VideoFormat default_codec_format_;
223 talk_base::CriticalSection channels_crit_;
224 VideoChannels channels_;
226 bool capture_started_;
227 int local_renderer_w_;
228 int local_renderer_h_;
229 VideoRenderer* local_renderer_;
231 talk_base::scoped_ptr<talk_base::CpuMonitor> cpu_monitor_;
234 class WebRtcVideoMediaChannel : public talk_base::MessageHandler,
235 public VideoMediaChannel,
236 public webrtc::Transport {
238 WebRtcVideoMediaChannel(WebRtcVideoEngine* engine,
239 VoiceMediaChannel* voice_channel);
240 ~WebRtcVideoMediaChannel();
243 WebRtcVideoEngine* engine() { return engine_; }
244 VoiceMediaChannel* voice_channel() { return voice_channel_; }
245 int video_channel() const { return vie_channel_; }
246 bool sending() const { return sending_; }
248 // VideoMediaChannel implementation
249 virtual bool SetRecvCodecs(const std::vector<VideoCodec> &codecs);
250 virtual bool SetSendCodecs(const std::vector<VideoCodec> &codecs);
251 virtual bool GetSendCodec(VideoCodec* send_codec);
252 virtual bool SetSendStreamFormat(uint32 ssrc, const VideoFormat& format);
253 virtual bool SetRender(bool render);
254 virtual bool SetSend(bool send);
256 virtual bool AddSendStream(const StreamParams& sp);
257 virtual bool RemoveSendStream(uint32 ssrc);
258 virtual bool AddRecvStream(const StreamParams& sp);
259 virtual bool RemoveRecvStream(uint32 ssrc);
260 virtual bool SetRenderer(uint32 ssrc, VideoRenderer* renderer);
261 virtual bool GetStats(const StatsOptions& options, VideoMediaInfo* info);
262 virtual bool SetCapturer(uint32 ssrc, VideoCapturer* capturer);
263 virtual bool SendIntraFrame();
264 virtual bool RequestIntraFrame();
266 virtual void OnPacketReceived(talk_base::Buffer* packet,
267 const talk_base::PacketTime& packet_time);
268 virtual void OnRtcpReceived(talk_base::Buffer* packet,
269 const talk_base::PacketTime& packet_time);
270 virtual void OnReadyToSend(bool ready);
271 virtual bool MuteStream(uint32 ssrc, bool on);
272 virtual bool SetRecvRtpHeaderExtensions(
273 const std::vector<RtpHeaderExtension>& extensions);
274 virtual bool SetSendRtpHeaderExtensions(
275 const std::vector<RtpHeaderExtension>& extensions);
276 virtual int GetRtpSendTimeExtnId() const;
277 virtual bool SetStartSendBandwidth(int bps);
278 virtual bool SetMaxSendBandwidth(int bps);
279 virtual bool SetOptions(const VideoOptions &options);
280 virtual bool GetOptions(VideoOptions *options) const {
284 virtual void SetInterface(NetworkInterface* iface);
285 virtual void UpdateAspectRatio(int ratio_w, int ratio_h);
287 // Public functions for use by tests and other specialized code.
288 uint32 send_ssrc() const { return 0; }
289 bool GetRenderer(uint32 ssrc, VideoRenderer** renderer);
290 bool GetVideoAdapter(uint32 ssrc, CoordinatedVideoAdapter** video_adapter);
291 void SendFrame(VideoCapturer* capturer, const VideoFrame* frame);
292 bool SendFrame(WebRtcVideoChannelSendInfo* channel_info,
293 const VideoFrame* frame, bool is_screencast);
295 // Thunk functions for use with HybridVideoEngine
296 void OnLocalFrame(VideoCapturer* capturer, const VideoFrame* frame) {
297 SendFrame(0u, frame, capturer->IsScreencast());
299 void OnLocalFrameFormat(VideoCapturer* capturer, const VideoFormat* format) {
302 virtual void OnMessage(talk_base::Message* msg);
305 int GetLastEngineError() { return engine()->GetLastEngineError(); }
306 virtual int SendPacket(int channel, const void* data, int len);
307 virtual int SendRTCPPacket(int channel, const void* data, int len);
310 typedef std::map<uint32, WebRtcVideoChannelRecvInfo*> RecvChannelMap;
311 typedef std::map<uint32, WebRtcVideoChannelSendInfo*> SendChannelMap;
312 typedef int (webrtc::ViERTP_RTCP::* ExtensionSetterFunction)(int, bool, int);
314 enum MediaDirection { MD_RECV, MD_SEND, MD_SENDRECV };
316 // Creates and initializes a ViE channel. When successful |channel_id| will
317 // contain the new channel's ID. If |receiving| is true |ssrc| is the
318 // remote ssrc. If |sending| is true the ssrc is local ssrc. If both
319 // |receiving| and |sending| is true the ssrc must be 0 and the channel will
320 // be created as a default channel. The ssrc must be different for receive
321 // channels and it must be different for send channels. If the same SSRC is
322 // being used for creating channel more than once, this function will fail
324 bool CreateChannel(uint32 ssrc_key, MediaDirection direction,
326 bool ConfigureChannel(int channel_id, MediaDirection direction,
328 bool ConfigureReceiving(int channel_id, uint32 remote_ssrc_key);
329 bool ConfigureSending(int channel_id, uint32 local_ssrc_key);
330 bool SetNackFec(int channel_id, int red_payload_type, int fec_payload_type,
332 bool SetSendCodec(const webrtc::VideoCodec& codec, int min_bitrate,
333 int start_bitrate, int max_bitrate);
334 bool SetSendCodec(WebRtcVideoChannelSendInfo* send_channel,
335 const webrtc::VideoCodec& codec, int min_bitrate,
336 int start_bitrate, int max_bitrate);
337 void LogSendCodecChange(const std::string& reason);
338 // Prepares the channel with channel id |info->channel_id()| to receive all
339 // codecs in |receive_codecs_| and start receive packets.
340 bool SetReceiveCodecs(WebRtcVideoChannelRecvInfo* info);
341 // Returns the channel number that receives the stream with SSRC |ssrc|.
342 int GetRecvChannelNum(uint32 ssrc);
343 // Given captured video frame size, checks if we need to reset vie send codec.
344 // |reset| is set to whether resetting has happened on vie or not.
345 // Returns false on error.
346 bool MaybeResetVieSendCodec(WebRtcVideoChannelSendInfo* send_channel,
347 int new_width, int new_height, bool is_screencast,
349 // Checks the current bitrate estimate and modifies the start bitrate
351 void MaybeChangeStartBitrate(int channel_id, webrtc::VideoCodec* video_codec);
352 // Helper function for starting the sending of media on all channels or
353 // |channel_id|. Note that these two function do not change |sending_|.
355 bool StartSend(WebRtcVideoChannelSendInfo* send_channel);
356 // Helper function for stop the sending of media on all channels or
357 // |channel_id|. Note that these two function do not change |sending_|.
359 bool StopSend(WebRtcVideoChannelSendInfo* send_channel);
360 bool SendIntraFrame(int channel_id);
362 bool HasReadySendChannels();
364 // Send channel key returns the key corresponding to the provided local SSRC
365 // in |key|. The return value is true upon success.
366 // If the local ssrc correspond to that of the default channel the key is 0.
367 // For all other channels the returned key will be the same as the local ssrc.
368 bool GetSendChannelKey(uint32 local_ssrc, uint32* key);
369 WebRtcVideoChannelSendInfo* GetSendChannel(uint32 local_ssrc);
370 // Creates a new unique key that can be used for inserting a new send channel
371 // into |send_channels_|
372 bool CreateSendChannelKey(uint32 local_ssrc, uint32* key);
373 // Get the number of the send channels |capturer| registered with.
374 int GetSendChannelNum(VideoCapturer* capturer);
376 bool IsDefaultChannel(int channel_id) const {
377 return channel_id == vie_channel_;
379 uint32 GetDefaultChannelSsrc();
381 bool DeleteSendChannel(uint32 ssrc_key);
383 bool InConferenceMode() const {
384 return options_.conference_mode.GetWithDefaultIfUnset(false);
386 bool RemoveCapturer(uint32 ssrc);
389 talk_base::MessageQueue* worker_thread() { return engine_->worker_thread(); }
390 void QueueBlackFrame(uint32 ssrc, int64 timestamp, int framerate);
391 void FlushBlackFrame(uint32 ssrc, int64 timestamp);
393 void SetNetworkTransmissionState(bool is_transmitting);
395 bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id,
396 const RtpHeaderExtension* extension);
397 bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id,
398 const std::vector<RtpHeaderExtension>& extensions,
399 const char header_extension_uri[]);
401 // Signal when cpu adaptation has no further scope to adapt.
402 void OnCpuAdaptationUnable();
404 // Set the local (send-side) RTX SSRC corresponding to primary_ssrc.
405 bool SetLocalRtxSsrc(int channel_id, const StreamParams& send_params,
406 uint32 primary_ssrc, int stream_idx);
408 // Connect |capturer| to WebRtcVideoMediaChannel if it is only registered
409 // to one send channel, i.e. the first send channel.
410 void MaybeConnectCapturer(VideoCapturer* capturer);
411 // Disconnect |capturer| from WebRtcVideoMediaChannel if it is only registered
412 // to one send channel, i.e. the last send channel.
413 void MaybeDisconnectCapturer(VideoCapturer* capturer);
416 WebRtcVideoEngine* engine_;
417 VoiceMediaChannel* voice_channel_;
420 // Receiver Estimated Max Bitrate
422 VideoOptions options_;
424 // Global recv side state.
425 // Note the default channel (vie_channel_), i.e. the send channel
426 // corresponding to all the receive channels (this must be done for REMB to
427 // work properly), resides in both recv_channels_ and send_channels_ with the
429 RecvChannelMap recv_channels_; // Contains all receive channels.
430 std::vector<webrtc::VideoCodec> receive_codecs_;
431 bool render_started_;
432 uint32 first_receive_ssrc_;
433 std::vector<RtpHeaderExtension> receive_extensions_;
435 // Global send side state.
436 SendChannelMap send_channels_;
437 talk_base::scoped_ptr<webrtc::VideoCodec> send_codec_;
441 int send_min_bitrate_;
442 int send_start_bitrate_;
443 int send_max_bitrate_;
445 std::vector<RtpHeaderExtension> send_extensions_;
447 // The aspect ratio that the channel desires. 0 means there is no desired
453 } // namespace cricket
455 #endif // TALK_MEDIA_WEBRTCVIDEOENGINE_H_