2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
11 #ifndef WEBRTC_VOICE_ENGINE_CHANNEL_H_
12 #define WEBRTC_VOICE_ENGINE_CHANNEL_H_
14 #include "webrtc/common_audio/resampler/include/push_resampler.h"
15 #include "webrtc/common_types.h"
16 #include "webrtc/modules/audio_coding/main/interface/audio_coding_module.h"
17 #include "webrtc/modules/audio_conference_mixer/interface/audio_conference_mixer_defines.h"
18 #include "webrtc/modules/audio_processing/rms_level.h"
19 #include "webrtc/modules/bitrate_controller/include/bitrate_controller.h"
20 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
21 #include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h"
22 #include "webrtc/modules/utility/interface/file_player.h"
23 #include "webrtc/modules/utility/interface/file_recorder.h"
24 #include "webrtc/system_wrappers/interface/scoped_ptr.h"
25 #include "webrtc/voice_engine/dtmf_inband.h"
26 #include "webrtc/voice_engine/dtmf_inband_queue.h"
27 #include "webrtc/voice_engine/include/voe_audio_processing.h"
28 #include "webrtc/voice_engine/include/voe_network.h"
29 #include "webrtc/voice_engine/level_indicator.h"
30 #include "webrtc/voice_engine/network_predictor.h"
31 #include "webrtc/voice_engine/shared_data.h"
32 #include "webrtc/voice_engine/voice_engine_defines.h"
34 #ifdef WEBRTC_DTMF_DETECTION
35 // TelephoneEventDetectionMethods, TelephoneEventObserver
36 #include "webrtc/voice_engine/include/voe_dtmf.h"
41 class TimestampWrapAroundHandler;
46 class AudioDeviceModule;
48 class CriticalSectionWrapper;
51 class ReceiveStatistics;
52 class RemoteNtpTimeEstimator;
54 class RTPPayloadRegistry;
56 class RTPReceiverAudio;
58 class TelephoneEventHandler;
60 class VoEMediaProcess;
61 class VoERTCPObserver;
63 class VoiceEngineObserver;
65 struct CallStatistics;
72 class StatisticsProxy;
76 // Helper class to simplify locking scheme for members that are accessed from
78 // Example: a member can be set on thread T1 and read by an internal audio
79 // thread T2. Accessing the member via this class ensures that we are
80 // safe and also avoid TSan v2 warnings.
84 State() : rx_apm_is_enabled(false),
85 input_external_media(false),
86 output_file_playing(false),
87 input_file_playing(false),
92 bool rx_apm_is_enabled;
93 bool input_external_media;
94 bool output_file_playing;
95 bool input_file_playing;
101 ChannelState() : lock_(CriticalSectionWrapper::CreateCriticalSection()) {
103 virtual ~ChannelState() {}
106 CriticalSectionScoped lock(lock_.get());
111 CriticalSectionScoped lock(lock_.get());
115 void SetRxApmIsEnabled(bool enable) {
116 CriticalSectionScoped lock(lock_.get());
117 state_.rx_apm_is_enabled = enable;
120 void SetInputExternalMedia(bool enable) {
121 CriticalSectionScoped lock(lock_.get());
122 state_.input_external_media = enable;
125 void SetOutputFilePlaying(bool enable) {
126 CriticalSectionScoped lock(lock_.get());
127 state_.output_file_playing = enable;
130 void SetInputFilePlaying(bool enable) {
131 CriticalSectionScoped lock(lock_.get());
132 state_.input_file_playing = enable;
135 void SetPlaying(bool enable) {
136 CriticalSectionScoped lock(lock_.get());
137 state_.playing = enable;
140 void SetSending(bool enable) {
141 CriticalSectionScoped lock(lock_.get());
142 state_.sending = enable;
145 void SetReceiving(bool enable) {
146 CriticalSectionScoped lock(lock_.get());
147 state_.receiving = enable;
151 scoped_ptr<CriticalSectionWrapper> lock_;
159 public FileCallback, // receiving notification from file player & recorder
161 public RtpAudioFeedback,
162 public AudioPacketizationCallback, // receive encoded packets from the ACM
163 public ACMVADCallback, // receive voice activity from the ACM
164 public MixerParticipant // supplies output mixer with audio frames
167 enum {KNumSocketThreads = 1};
168 enum {KNumberOfSocketBuffers = 8};
170 static int32_t CreateChannel(Channel*& channel,
173 const Config& config);
174 Channel(int32_t channelId, uint32_t instanceId, const Config& config);
176 int32_t SetEngineInformation(
177 Statistics& engineStatistics,
178 OutputMixer& outputMixer,
179 TransmitMixer& transmitMixer,
180 ProcessThread& moduleProcessThread,
181 AudioDeviceModule& audioDeviceModule,
182 VoiceEngineObserver* voiceEngineObserver,
183 CriticalSectionWrapper* callbackCritSect);
184 int32_t UpdateLocalTimeStamp();
189 int32_t StartPlayout();
190 int32_t StopPlayout();
193 int32_t StartReceiving();
194 int32_t StopReceiving();
196 int32_t SetNetEQPlayoutMode(NetEqModes mode);
197 int32_t GetNetEQPlayoutMode(NetEqModes& mode);
198 int32_t RegisterVoiceEngineObserver(VoiceEngineObserver& observer);
199 int32_t DeRegisterVoiceEngineObserver();
202 int32_t GetSendCodec(CodecInst& codec);
203 int32_t GetRecCodec(CodecInst& codec);
204 int32_t SetSendCodec(const CodecInst& codec);
205 int32_t SetVADStatus(bool enableVAD, ACMVADMode mode, bool disableDTX);
206 int32_t GetVADStatus(bool& enabledVAD, ACMVADMode& mode, bool& disabledDTX);
207 int32_t SetRecPayloadType(const CodecInst& codec);
208 int32_t GetRecPayloadType(CodecInst& codec);
209 int32_t SetSendCNPayloadType(int type, PayloadFrequencies frequency);
210 int SetOpusMaxBandwidth(int bandwidth_hz);
212 // VoE dual-streaming.
213 int SetSecondarySendCodec(const CodecInst& codec, int red_payload_type);
214 void RemoveSecondarySendCodec();
215 int GetSecondarySendCodec(CodecInst* codec);
218 int32_t RegisterExternalTransport(Transport& transport);
219 int32_t DeRegisterExternalTransport();
220 int32_t ReceivedRTPPacket(const int8_t* data, int32_t length,
221 const PacketTime& packet_time);
222 int32_t ReceivedRTCPPacket(const int8_t* data, int32_t length);
225 int StartPlayingFileLocally(const char* fileName, bool loop,
230 const CodecInst* codecInst);
231 int StartPlayingFileLocally(InStream* stream, FileFormats format,
235 const CodecInst* codecInst);
236 int StopPlayingFileLocally();
237 int IsPlayingFileLocally() const;
238 int RegisterFilePlayingToMixer();
239 int StartPlayingFileAsMicrophone(const char* fileName, bool loop,
244 const CodecInst* codecInst);
245 int StartPlayingFileAsMicrophone(InStream* stream,
250 const CodecInst* codecInst);
251 int StopPlayingFileAsMicrophone();
252 int IsPlayingFileAsMicrophone() const;
253 int StartRecordingPlayout(const char* fileName, const CodecInst* codecInst);
254 int StartRecordingPlayout(OutStream* stream, const CodecInst* codecInst);
255 int StopRecordingPlayout();
257 void SetMixWithMicStatus(bool mix);
259 // VoEExternalMediaProcessing
260 int RegisterExternalMediaProcessing(ProcessingTypes type,
261 VoEMediaProcess& processObject);
262 int DeRegisterExternalMediaProcessing(ProcessingTypes type);
263 int SetExternalMixing(bool enabled);
266 int GetSpeechOutputLevel(uint32_t& level) const;
267 int GetSpeechOutputLevelFullRange(uint32_t& level) const;
268 int SetMute(bool enable);
270 int SetOutputVolumePan(float left, float right);
271 int GetOutputVolumePan(float& left, float& right) const;
272 int SetChannelOutputVolumeScaling(float scaling);
273 int GetChannelOutputVolumeScaling(float& scaling) const;
276 int GetNetworkStatistics(NetworkStatistics& stats);
277 void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const;
280 bool GetDelayEstimate(int* jitter_buffer_delay_ms,
281 int* playout_buffer_delay_ms) const;
282 int least_required_delay_ms() const { return least_required_delay_ms_; }
283 int SetInitialPlayoutDelay(int delay_ms);
284 int SetMinimumPlayoutDelay(int delayMs);
285 int GetPlayoutTimestamp(unsigned int& timestamp);
286 void UpdatePlayoutTimestamp(bool rtcp);
287 int SetInitTimestamp(unsigned int timestamp);
288 int SetInitSequenceNumber(short sequenceNumber);
290 // VoEVideoSyncExtended
291 int GetRtpRtcp(RtpRtcp** rtpRtcpModule, RtpReceiver** rtp_receiver) const;
294 int SendTelephoneEventOutband(unsigned char eventCode, int lengthMs,
295 int attenuationDb, bool playDtmfEvent);
296 int SendTelephoneEventInband(unsigned char eventCode, int lengthMs,
297 int attenuationDb, bool playDtmfEvent);
298 int SetDtmfPlayoutStatus(bool enable);
299 bool DtmfPlayoutStatus() const;
300 int SetSendTelephoneEventPayloadType(unsigned char type);
301 int GetSendTelephoneEventPayloadType(unsigned char& type);
303 // VoEAudioProcessingImpl
304 int UpdateRxVadDetection(AudioFrame& audioFrame);
305 int RegisterRxVadObserver(VoERxVadCallback &observer);
306 int DeRegisterRxVadObserver();
307 int VoiceActivityIndicator(int &activity);
308 #ifdef WEBRTC_VOICE_ENGINE_AGC
309 int SetRxAgcStatus(bool enable, AgcModes mode);
310 int GetRxAgcStatus(bool& enabled, AgcModes& mode);
311 int SetRxAgcConfig(AgcConfig config);
312 int GetRxAgcConfig(AgcConfig& config);
314 #ifdef WEBRTC_VOICE_ENGINE_NR
315 int SetRxNsStatus(bool enable, NsModes mode);
316 int GetRxNsStatus(bool& enabled, NsModes& mode);
320 int RegisterRTCPObserver(VoERTCPObserver& observer);
321 int DeRegisterRTCPObserver();
322 int SetLocalSSRC(unsigned int ssrc);
323 int GetLocalSSRC(unsigned int& ssrc);
324 int GetRemoteSSRC(unsigned int& ssrc);
325 int SetSendAudioLevelIndicationStatus(bool enable, unsigned char id);
326 int SetReceiveAudioLevelIndicationStatus(bool enable, unsigned char id);
327 int SetSendAbsoluteSenderTimeStatus(bool enable, unsigned char id);
328 int SetReceiveAbsoluteSenderTimeStatus(bool enable, unsigned char id);
329 int SetRTCPStatus(bool enable);
330 int GetRTCPStatus(bool& enabled);
331 int SetRTCP_CNAME(const char cName[256]);
332 int GetRemoteRTCP_CNAME(char cName[256]);
333 int GetRemoteRTCPData(unsigned int& NTPHigh, unsigned int& NTPLow,
334 unsigned int& timestamp,
335 unsigned int& playoutTimestamp, unsigned int* jitter,
336 unsigned short* fractionLost);
337 int SendApplicationDefinedRTCPPacket(unsigned char subType,
338 unsigned int name, const char* data,
339 unsigned short dataLengthInBytes);
340 int GetRTPStatistics(unsigned int& averageJitterMs,
341 unsigned int& maxJitterMs,
342 unsigned int& discardedPackets);
343 int GetRemoteRTCPReportBlocks(std::vector<ReportBlock>* report_blocks);
344 int GetRTPStatistics(CallStatistics& stats);
345 int SetREDStatus(bool enable, int redPayloadtype);
346 int GetREDStatus(bool& enabled, int& redPayloadtype);
347 int SetCodecFECStatus(bool enable);
348 bool GetCodecFECStatus();
349 void SetNACKStatus(bool enable, int maxNumberOfPackets);
350 int StartRTPDump(const char fileNameUTF8[1024], RTPDirections direction);
351 int StopRTPDump(RTPDirections direction);
352 bool RTPDumpIsActive(RTPDirections direction);
353 // Takes ownership of the ViENetwork.
354 void SetVideoEngineBWETarget(ViENetwork* vie_network, int video_channel);
356 // From AudioPacketizationCallback in the ACM
357 int32_t SendData(FrameType frameType,
360 const uint8_t* payloadData,
361 uint16_t payloadSize,
362 const RTPFragmentationHeader* fragmentation);
363 // From ACMVADCallback in the ACM
364 int32_t InFrameType(int16_t frameType);
366 int32_t OnRxVadDetected(int vadDecision);
368 // From RtpData in the RTP/RTCP module
369 int32_t OnReceivedPayloadData(const uint8_t* payloadData,
370 uint16_t payloadSize,
371 const WebRtcRTPHeader* rtpHeader);
373 bool OnRecoveredPacket(const uint8_t* packet, int packet_length);
375 // From RtpFeedback in the RTP/RTCP module
376 int32_t OnInitializeDecoder(
379 const char payloadName[RTP_PAYLOAD_NAME_SIZE],
384 void OnPacketTimeout(int32_t id);
386 void OnReceivedPacket(int32_t id, RtpRtcpPacketType packetType);
388 void OnPeriodicDeadOrAlive(int32_t id,
391 void OnIncomingSSRCChanged(int32_t id,
394 void OnIncomingCSRCChanged(int32_t id,
395 uint32_t CSRC, bool added);
397 void ResetStatistics(uint32_t ssrc);
399 // From RtcpFeedback in the RTP/RTCP module
400 void OnApplicationDataReceived(int32_t id,
404 const uint8_t* data);
406 // From RtpAudioFeedback in the RTP/RTCP module
407 void OnReceivedTelephoneEvent(int32_t id,
411 void OnPlayTelephoneEvent(int32_t id,
416 // From Transport (called by the RTP/RTCP module)
417 int SendPacket(int /*channel*/, const void *data, int len);
418 int SendRTCPPacket(int /*channel*/, const void *data, int len);
420 // From MixerParticipant
421 int32_t GetAudioFrame(int32_t id, AudioFrame& audioFrame);
422 int32_t NeededFrequency(int32_t id);
424 // From MonitorObserver
425 void OnPeriodicProcess();
428 void PlayNotification(int32_t id,
429 uint32_t durationMs);
430 void RecordNotification(int32_t id,
431 uint32_t durationMs);
432 void PlayFileEnded(int32_t id);
433 void RecordFileEnded(int32_t id);
435 uint32_t InstanceId() const
439 int32_t ChannelId() const
445 return channel_state_.Get().playing;
449 return channel_state_.Get().sending;
451 bool Receiving() const
453 return channel_state_.Get().receiving;
455 bool ExternalTransport() const
457 CriticalSectionScoped cs(&_callbackCritSect);
458 return _externalTransport;
460 bool ExternalMixing() const
462 return _externalMixing;
464 RtpRtcp* RtpRtcpModulePtr() const
466 return _rtpRtcpModule.get();
468 int8_t OutputEnergyLevel() const
470 return _outputAudioLevel.Level();
472 uint32_t Demultiplex(const AudioFrame& audioFrame);
473 // Demultiplex the data to the channel's |_audioFrame|. The difference
474 // between this method and the overloaded method above is that |audio_data|
475 // does not go through transmit_mixer and APM.
476 void Demultiplex(const int16_t* audio_data,
478 int number_of_frames,
479 int number_of_channels);
480 uint32_t PrepareEncodeAndSend(int mixingFrequency);
481 uint32_t EncodeAndSend();
483 // From BitrateObserver (called by the RTP/RTCP module).
484 void OnNetworkChanged(const uint32_t bitrate_bps,
485 const uint8_t fraction_lost, // 0 - 255.
489 bool ReceivePacket(const uint8_t* packet, int packet_length,
490 const RTPHeader& header, bool in_order);
491 bool HandleEncapsulation(const uint8_t* packet,
493 const RTPHeader& header);
494 bool IsPacketInOrder(const RTPHeader& header) const;
495 bool IsPacketRetransmitted(const RTPHeader& header, bool in_order) const;
496 int ResendPackets(const uint16_t* sequence_numbers, int length);
497 int InsertInbandDtmfTone();
498 int32_t MixOrReplaceAudioWithFile(int mixingFrequency);
499 int32_t MixAudioWithFile(AudioFrame& audioFrame, int mixingFrequency);
500 int32_t SendPacketRaw(const void *data, int len, bool RTCP);
501 void UpdatePacketDelay(uint32_t timestamp,
502 uint16_t sequenceNumber);
503 void RegisterReceiveCodecsToRTPModule();
505 int SetRedPayloadType(int red_payload_type);
506 int SetSendRtpHeaderExtension(bool enable, RTPExtensionType type,
509 int32_t GetPlayoutFrequency();
511 CriticalSectionWrapper& _fileCritSect;
512 CriticalSectionWrapper& _callbackCritSect;
513 CriticalSectionWrapper& volume_settings_critsect_;
514 uint32_t _instanceId;
517 ChannelState channel_state_;
519 scoped_ptr<RtpHeaderParser> rtp_header_parser_;
520 scoped_ptr<RTPPayloadRegistry> rtp_payload_registry_;
521 scoped_ptr<ReceiveStatistics> rtp_receive_statistics_;
522 scoped_ptr<StatisticsProxy> statistics_proxy_;
523 scoped_ptr<RtpReceiver> rtp_receiver_;
524 TelephoneEventHandler* telephone_event_handler_;
525 scoped_ptr<RtpRtcp> _rtpRtcpModule;
526 scoped_ptr<AudioCodingModule> audio_coding_;
528 RtpDump& _rtpDumpOut;
529 AudioLevel _outputAudioLevel;
530 bool _externalTransport;
531 AudioFrame _audioFrame;
532 scoped_ptr<int16_t[]> mono_recording_audio_;
533 // Downsamples to the codec rate if necessary.
534 PushResampler<int16_t> input_resampler_;
535 uint8_t _audioLevel_dBov;
536 FilePlayer* _inputFilePlayerPtr;
537 FilePlayer* _outputFilePlayerPtr;
538 FileRecorder* _outputFileRecorderPtr;
539 int _inputFilePlayerId;
540 int _outputFilePlayerId;
541 int _outputFileRecorderId;
542 bool _outputFileRecording;
543 DtmfInbandQueue _inbandDtmfQueue;
544 DtmfInband _inbandDtmfGenerator;
545 bool _outputExternalMedia;
546 VoEMediaProcess* _inputExternalMediaCallbackPtr;
547 VoEMediaProcess* _outputExternalMediaCallbackPtr;
549 uint8_t _sendTelephoneEventPayloadType;
551 scoped_ptr<RemoteNtpTimeEstimator> ntp_estimator_;
553 // Timestamp of the audio pulled from NetEq.
554 uint32_t jitter_buffer_playout_timestamp_;
555 uint32_t playout_timestamp_rtp_;
556 uint32_t playout_timestamp_rtcp_;
557 uint32_t playout_delay_ms_;
558 uint32_t _numberOfDiscardedPackets;
559 uint16_t send_sequence_number_;
560 uint8_t restored_packet_[kVoiceEngineMaxIpPacketSizeBytes];
562 scoped_ptr<CriticalSectionWrapper> ts_stats_lock_;
564 scoped_ptr<rtc::TimestampWrapAroundHandler> rtp_ts_wraparound_handler_;
565 // The rtp timestamp of the first played out audio frame.
566 int64_t capture_start_rtp_time_stamp_;
567 // The capture ntp time (in local timebase) of the first played out audio
569 int64_t capture_start_ntp_time_ms_;
572 Statistics* _engineStatisticsPtr;
573 OutputMixer* _outputMixerPtr;
574 TransmitMixer* _transmitMixerPtr;
575 ProcessThread* _moduleProcessThreadPtr;
576 AudioDeviceModule* _audioDeviceModulePtr;
577 VoiceEngineObserver* _voiceEngineObserverPtr; // owned by base
578 CriticalSectionWrapper* _callbackCritSectPtr; // owned by base
579 Transport* _transportPtr; // WebRtc socket or external transport
581 scoped_ptr<AudioProcessing> rx_audioproc_; // far end AudioProcessing
582 VoERxVadCallback* _rxVadObserverPtr;
583 int32_t _oldVadDecision;
584 int32_t _sendFrameType; // Send data is voice, 1-voice, 0-otherwise
585 VoERTCPObserver* _rtcpObserverPtr;
587 bool _externalPlayout;
588 bool _externalMixing;
589 bool _mixFileWithMicrophone;
597 bool _playOutbandDtmfEvent;
598 bool _playInbandDtmfEvent;
600 uint32_t _lastLocalTimeStamp;
601 int8_t _lastPayloadType;
602 bool _includeAudioLevelIndication;
604 bool _rtpPacketTimedOut;
605 bool _rtpPacketTimeOutIsEnabled;
606 uint32_t _rtpTimeOutSeconds;
607 bool _connectionObserver;
608 VoEConnectionObserver* _connectionObserverPtr;
609 AudioFrame::SpeechType _outputSpeechType;
610 ViENetwork* vie_network_;
613 uint32_t _average_jitter_buffer_delay_us;
614 int least_required_delay_ms_;
615 uint32_t _previousTimestamp;
616 uint16_t _recPacketDelayMs;
617 // VoEAudioProcessing
618 bool _RxVadDetection;
619 bool _rxAgcIsEnabled;
621 bool restored_packet_in_use_;
622 // RtcpBandwidthObserver
623 scoped_ptr<BitrateController> bitrate_controller_;
624 scoped_ptr<RtcpBandwidthObserver> rtcp_bandwidth_observer_;
625 scoped_ptr<BitrateObserver> send_bitrate_observer_;
626 scoped_ptr<NetworkPredictor> network_predictor_;
630 } // namespace webrtc
632 #endif // WEBRTC_VOICE_ENGINE_CHANNEL_H_