2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
11 #ifndef WEBRTC_VOICE_ENGINE_CHANNEL_H_
12 #define WEBRTC_VOICE_ENGINE_CHANNEL_H_
14 #include "webrtc/common_audio/resampler/include/push_resampler.h"
15 #include "webrtc/common_types.h"
16 #include "webrtc/modules/audio_coding/main/interface/audio_coding_module.h"
17 #include "webrtc/modules/audio_conference_mixer/interface/audio_conference_mixer_defines.h"
18 #include "webrtc/modules/audio_processing/rms_level.h"
19 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
20 #include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h"
21 #include "webrtc/modules/utility/interface/file_player.h"
22 #include "webrtc/modules/utility/interface/file_recorder.h"
23 #include "webrtc/system_wrappers/interface/scoped_ptr.h"
24 #include "webrtc/voice_engine/dtmf_inband.h"
25 #include "webrtc/voice_engine/dtmf_inband_queue.h"
26 #include "webrtc/voice_engine/include/voe_audio_processing.h"
27 #include "webrtc/voice_engine/include/voe_network.h"
28 #include "webrtc/voice_engine/level_indicator.h"
29 #include "webrtc/voice_engine/shared_data.h"
30 #include "webrtc/voice_engine/voice_engine_defines.h"
32 #ifdef WEBRTC_DTMF_DETECTION
33 // TelephoneEventDetectionMethods, TelephoneEventObserver
34 #include "webrtc/voice_engine/include/voe_dtmf.h"
39 class AudioDeviceModule;
41 class CriticalSectionWrapper;
44 class ReceiveStatistics;
46 class RTPPayloadRegistry;
48 class RTPReceiverAudio;
50 class TelephoneEventHandler;
52 class VoEMediaProcess;
53 class VoERTCPObserver;
55 class VoiceEngineObserver;
57 struct CallStatistics;
64 class StatisticsProxy;
68 // Helper class to simplify locking scheme for members that are accessed from
70 // Example: a member can be set on thread T1 and read by an internal audio
71 // thread T2. Accessing the member via this class ensures that we are
72 // safe and also avoid TSan v2 warnings.
76 State() : rx_apm_is_enabled(false),
77 input_external_media(false),
78 output_file_playing(false),
79 input_file_playing(false),
84 bool rx_apm_is_enabled;
85 bool input_external_media;
86 bool output_file_playing;
87 bool input_file_playing;
93 ChannelState() : lock_(CriticalSectionWrapper::CreateCriticalSection()) {
95 virtual ~ChannelState() {}
98 CriticalSectionScoped lock(lock_.get());
103 CriticalSectionScoped lock(lock_.get());
107 void SetRxApmIsEnabled(bool enable) {
108 CriticalSectionScoped lock(lock_.get());
109 state_.rx_apm_is_enabled = enable;
112 void SetInputExternalMedia(bool enable) {
113 CriticalSectionScoped lock(lock_.get());
114 state_.input_external_media = enable;
117 void SetOutputFilePlaying(bool enable) {
118 CriticalSectionScoped lock(lock_.get());
119 state_.output_file_playing = enable;
122 void SetInputFilePlaying(bool enable) {
123 CriticalSectionScoped lock(lock_.get());
124 state_.input_file_playing = enable;
127 void SetPlaying(bool enable) {
128 CriticalSectionScoped lock(lock_.get());
129 state_.playing = enable;
132 void SetSending(bool enable) {
133 CriticalSectionScoped lock(lock_.get());
134 state_.sending = enable;
137 void SetReceiving(bool enable) {
138 CriticalSectionScoped lock(lock_.get());
139 state_.receiving = enable;
143 scoped_ptr<CriticalSectionWrapper> lock_;
151 public FileCallback, // receiving notification from file player & recorder
153 public RtpAudioFeedback,
154 public AudioPacketizationCallback, // receive encoded packets from the ACM
155 public ACMVADCallback, // receive voice activity from the ACM
156 public MixerParticipant // supplies output mixer with audio frames
159 enum {KNumSocketThreads = 1};
160 enum {KNumberOfSocketBuffers = 8};
162 static int32_t CreateChannel(Channel*& channel,
165 const Config& config);
166 Channel(int32_t channelId, uint32_t instanceId, const Config& config);
168 int32_t SetEngineInformation(
169 Statistics& engineStatistics,
170 OutputMixer& outputMixer,
171 TransmitMixer& transmitMixer,
172 ProcessThread& moduleProcessThread,
173 AudioDeviceModule& audioDeviceModule,
174 VoiceEngineObserver* voiceEngineObserver,
175 CriticalSectionWrapper* callbackCritSect);
176 int32_t UpdateLocalTimeStamp();
181 int32_t StartPlayout();
182 int32_t StopPlayout();
185 int32_t StartReceiving();
186 int32_t StopReceiving();
188 int32_t SetNetEQPlayoutMode(NetEqModes mode);
189 int32_t GetNetEQPlayoutMode(NetEqModes& mode);
190 int32_t RegisterVoiceEngineObserver(VoiceEngineObserver& observer);
191 int32_t DeRegisterVoiceEngineObserver();
194 int32_t GetSendCodec(CodecInst& codec);
195 int32_t GetRecCodec(CodecInst& codec);
196 int32_t SetSendCodec(const CodecInst& codec);
197 int32_t SetVADStatus(bool enableVAD, ACMVADMode mode, bool disableDTX);
198 int32_t GetVADStatus(bool& enabledVAD, ACMVADMode& mode, bool& disabledDTX);
199 int32_t SetRecPayloadType(const CodecInst& codec);
200 int32_t GetRecPayloadType(CodecInst& codec);
201 int32_t SetSendCNPayloadType(int type, PayloadFrequencies frequency);
203 // VoE dual-streaming.
204 int SetSecondarySendCodec(const CodecInst& codec, int red_payload_type);
205 void RemoveSecondarySendCodec();
206 int GetSecondarySendCodec(CodecInst* codec);
209 int32_t RegisterExternalTransport(Transport& transport);
210 int32_t DeRegisterExternalTransport();
211 int32_t ReceivedRTPPacket(const int8_t* data, int32_t length,
212 const PacketTime& packet_time);
213 int32_t ReceivedRTCPPacket(const int8_t* data, int32_t length);
216 int StartPlayingFileLocally(const char* fileName, bool loop,
221 const CodecInst* codecInst);
222 int StartPlayingFileLocally(InStream* stream, FileFormats format,
226 const CodecInst* codecInst);
227 int StopPlayingFileLocally();
228 int IsPlayingFileLocally() const;
229 int RegisterFilePlayingToMixer();
230 int ScaleLocalFilePlayout(float scale);
231 int GetLocalPlayoutPosition(int& positionMs);
232 int StartPlayingFileAsMicrophone(const char* fileName, bool loop,
237 const CodecInst* codecInst);
238 int StartPlayingFileAsMicrophone(InStream* stream,
243 const CodecInst* codecInst);
244 int StopPlayingFileAsMicrophone();
245 int IsPlayingFileAsMicrophone() const;
246 int ScaleFileAsMicrophonePlayout(float scale);
247 int StartRecordingPlayout(const char* fileName, const CodecInst* codecInst);
248 int StartRecordingPlayout(OutStream* stream, const CodecInst* codecInst);
249 int StopRecordingPlayout();
251 void SetMixWithMicStatus(bool mix);
253 // VoEExternalMediaProcessing
254 int RegisterExternalMediaProcessing(ProcessingTypes type,
255 VoEMediaProcess& processObject);
256 int DeRegisterExternalMediaProcessing(ProcessingTypes type);
257 int SetExternalMixing(bool enabled);
260 int GetSpeechOutputLevel(uint32_t& level) const;
261 int GetSpeechOutputLevelFullRange(uint32_t& level) const;
262 int SetMute(bool enable);
264 int SetOutputVolumePan(float left, float right);
265 int GetOutputVolumePan(float& left, float& right) const;
266 int SetChannelOutputVolumeScaling(float scaling);
267 int GetChannelOutputVolumeScaling(float& scaling) const;
270 int GetNetworkStatistics(NetworkStatistics& stats);
271 void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const;
274 bool GetDelayEstimate(int* jitter_buffer_delay_ms,
275 int* playout_buffer_delay_ms) const;
276 int least_required_delay_ms() const { return least_required_delay_ms_; }
277 int SetInitialPlayoutDelay(int delay_ms);
278 int SetMinimumPlayoutDelay(int delayMs);
279 int GetPlayoutTimestamp(unsigned int& timestamp);
280 void UpdatePlayoutTimestamp(bool rtcp);
281 int SetInitTimestamp(unsigned int timestamp);
282 int SetInitSequenceNumber(short sequenceNumber);
284 // VoEVideoSyncExtended
285 int GetRtpRtcp(RtpRtcp** rtpRtcpModule, RtpReceiver** rtp_receiver) const;
288 int SendTelephoneEventOutband(unsigned char eventCode, int lengthMs,
289 int attenuationDb, bool playDtmfEvent);
290 int SendTelephoneEventInband(unsigned char eventCode, int lengthMs,
291 int attenuationDb, bool playDtmfEvent);
292 int SetDtmfPlayoutStatus(bool enable);
293 bool DtmfPlayoutStatus() const;
294 int SetSendTelephoneEventPayloadType(unsigned char type);
295 int GetSendTelephoneEventPayloadType(unsigned char& type);
297 // VoEAudioProcessingImpl
298 int UpdateRxVadDetection(AudioFrame& audioFrame);
299 int RegisterRxVadObserver(VoERxVadCallback &observer);
300 int DeRegisterRxVadObserver();
301 int VoiceActivityIndicator(int &activity);
302 #ifdef WEBRTC_VOICE_ENGINE_AGC
303 int SetRxAgcStatus(bool enable, AgcModes mode);
304 int GetRxAgcStatus(bool& enabled, AgcModes& mode);
305 int SetRxAgcConfig(AgcConfig config);
306 int GetRxAgcConfig(AgcConfig& config);
308 #ifdef WEBRTC_VOICE_ENGINE_NR
309 int SetRxNsStatus(bool enable, NsModes mode);
310 int GetRxNsStatus(bool& enabled, NsModes& mode);
314 int RegisterRTPObserver(VoERTPObserver& observer);
315 int DeRegisterRTPObserver();
316 int RegisterRTCPObserver(VoERTCPObserver& observer);
317 int DeRegisterRTCPObserver();
318 int SetLocalSSRC(unsigned int ssrc);
319 int GetLocalSSRC(unsigned int& ssrc);
320 int GetRemoteSSRC(unsigned int& ssrc);
321 int GetRemoteCSRCs(unsigned int arrCSRC[15]);
322 int SetSendAudioLevelIndicationStatus(bool enable, unsigned char id);
323 int SetReceiveAudioLevelIndicationStatus(bool enable, unsigned char id);
324 int SetSendAbsoluteSenderTimeStatus(bool enable, unsigned char id);
325 int SetReceiveAbsoluteSenderTimeStatus(bool enable, unsigned char id);
326 int SetRTCPStatus(bool enable);
327 int GetRTCPStatus(bool& enabled);
328 int SetRTCP_CNAME(const char cName[256]);
329 int GetRTCP_CNAME(char cName[256]);
330 int GetRemoteRTCP_CNAME(char cName[256]);
331 int GetRemoteRTCPData(unsigned int& NTPHigh, unsigned int& NTPLow,
332 unsigned int& timestamp,
333 unsigned int& playoutTimestamp, unsigned int* jitter,
334 unsigned short* fractionLost);
335 int SendApplicationDefinedRTCPPacket(unsigned char subType,
336 unsigned int name, const char* data,
337 unsigned short dataLengthInBytes);
338 int GetRTPStatistics(unsigned int& averageJitterMs,
339 unsigned int& maxJitterMs,
340 unsigned int& discardedPackets);
341 int GetRemoteRTCPSenderInfo(SenderInfo* sender_info);
342 int GetRemoteRTCPReportBlocks(std::vector<ReportBlock>* report_blocks);
343 int GetRTPStatistics(CallStatistics& stats);
344 int SetFECStatus(bool enable, int redPayloadtype);
345 int GetFECStatus(bool& enabled, int& redPayloadtype);
346 void SetNACKStatus(bool enable, int maxNumberOfPackets);
347 int StartRTPDump(const char fileNameUTF8[1024], RTPDirections direction);
348 int StopRTPDump(RTPDirections direction);
349 bool RTPDumpIsActive(RTPDirections direction);
350 uint32_t LastRemoteTimeStamp() { return _lastRemoteTimeStamp; }
351 // Takes ownership of the ViENetwork.
352 void SetVideoEngineBWETarget(ViENetwork* vie_network, int video_channel);
354 // From AudioPacketizationCallback in the ACM
355 int32_t SendData(FrameType frameType,
358 const uint8_t* payloadData,
359 uint16_t payloadSize,
360 const RTPFragmentationHeader* fragmentation);
361 // From ACMVADCallback in the ACM
362 int32_t InFrameType(int16_t frameType);
364 int32_t OnRxVadDetected(int vadDecision);
366 // From RtpData in the RTP/RTCP module
367 int32_t OnReceivedPayloadData(const uint8_t* payloadData,
368 uint16_t payloadSize,
369 const WebRtcRTPHeader* rtpHeader);
371 bool OnRecoveredPacket(const uint8_t* packet, int packet_length);
373 // From RtpFeedback in the RTP/RTCP module
374 int32_t OnInitializeDecoder(
377 const char payloadName[RTP_PAYLOAD_NAME_SIZE],
382 void OnPacketTimeout(int32_t id);
384 void OnReceivedPacket(int32_t id, RtpRtcpPacketType packetType);
386 void OnPeriodicDeadOrAlive(int32_t id,
389 void OnIncomingSSRCChanged(int32_t id,
392 void OnIncomingCSRCChanged(int32_t id,
393 uint32_t CSRC, bool added);
395 void ResetStatistics(uint32_t ssrc);
397 // From RtcpFeedback in the RTP/RTCP module
398 void OnApplicationDataReceived(int32_t id,
402 const uint8_t* data);
404 // From RtpAudioFeedback in the RTP/RTCP module
405 void OnReceivedTelephoneEvent(int32_t id,
409 void OnPlayTelephoneEvent(int32_t id,
414 // From Transport (called by the RTP/RTCP module)
415 int SendPacket(int /*channel*/, const void *data, int len);
416 int SendRTCPPacket(int /*channel*/, const void *data, int len);
418 // From MixerParticipant
419 int32_t GetAudioFrame(int32_t id, AudioFrame& audioFrame);
420 int32_t NeededFrequency(int32_t id);
422 // From MonitorObserver
423 void OnPeriodicProcess();
426 void PlayNotification(int32_t id,
427 uint32_t durationMs);
428 void RecordNotification(int32_t id,
429 uint32_t durationMs);
430 void PlayFileEnded(int32_t id);
431 void RecordFileEnded(int32_t id);
433 uint32_t InstanceId() const
437 int32_t ChannelId() const
443 return channel_state_.Get().playing;
447 return channel_state_.Get().sending;
449 bool Receiving() const
451 return channel_state_.Get().receiving;
453 bool ExternalTransport() const
455 CriticalSectionScoped cs(&_callbackCritSect);
456 return _externalTransport;
458 bool ExternalMixing() const
460 return _externalMixing;
462 RtpRtcp* RtpRtcpModulePtr() const
464 return _rtpRtcpModule.get();
466 int8_t OutputEnergyLevel() const
468 return _outputAudioLevel.Level();
470 uint32_t Demultiplex(const AudioFrame& audioFrame);
471 // Demultiplex the data to the channel's |_audioFrame|. The difference
472 // between this method and the overloaded method above is that |audio_data|
473 // does not go through transmit_mixer and APM.
474 void Demultiplex(const int16_t* audio_data,
476 int number_of_frames,
477 int number_of_channels);
478 uint32_t PrepareEncodeAndSend(int mixingFrequency);
479 uint32_t EncodeAndSend();
482 bool ReceivePacket(const uint8_t* packet, int packet_length,
483 const RTPHeader& header, bool in_order);
484 bool HandleEncapsulation(const uint8_t* packet,
486 const RTPHeader& header);
487 bool IsPacketInOrder(const RTPHeader& header) const;
488 bool IsPacketRetransmitted(const RTPHeader& header, bool in_order) const;
489 int ResendPackets(const uint16_t* sequence_numbers, int length);
490 int InsertInbandDtmfTone();
491 int32_t MixOrReplaceAudioWithFile(int mixingFrequency);
492 int32_t MixAudioWithFile(AudioFrame& audioFrame, int mixingFrequency);
493 int32_t SendPacketRaw(const void *data, int len, bool RTCP);
494 void UpdatePacketDelay(uint32_t timestamp,
495 uint16_t sequenceNumber);
496 void RegisterReceiveCodecsToRTPModule();
498 int SetRedPayloadType(int red_payload_type);
499 int SetSendRtpHeaderExtension(bool enable, RTPExtensionType type,
502 CriticalSectionWrapper& _fileCritSect;
503 CriticalSectionWrapper& _callbackCritSect;
504 CriticalSectionWrapper& volume_settings_critsect_;
505 uint32_t _instanceId;
508 ChannelState channel_state_;
510 scoped_ptr<RtpHeaderParser> rtp_header_parser_;
511 scoped_ptr<RTPPayloadRegistry> rtp_payload_registry_;
512 scoped_ptr<ReceiveStatistics> rtp_receive_statistics_;
513 scoped_ptr<StatisticsProxy> statistics_proxy_;
514 scoped_ptr<RtpReceiver> rtp_receiver_;
515 TelephoneEventHandler* telephone_event_handler_;
516 scoped_ptr<RtpRtcp> _rtpRtcpModule;
517 scoped_ptr<AudioCodingModule> audio_coding_;
519 RtpDump& _rtpDumpOut;
520 AudioLevel _outputAudioLevel;
521 bool _externalTransport;
522 AudioFrame _audioFrame;
523 scoped_ptr<int16_t[]> mono_recording_audio_;
524 // Downsamples to the codec rate if necessary.
525 PushResampler<int16_t> input_resampler_;
526 uint8_t _audioLevel_dBov;
527 FilePlayer* _inputFilePlayerPtr;
528 FilePlayer* _outputFilePlayerPtr;
529 FileRecorder* _outputFileRecorderPtr;
530 int _inputFilePlayerId;
531 int _outputFilePlayerId;
532 int _outputFileRecorderId;
533 bool _outputFileRecording;
534 DtmfInbandQueue _inbandDtmfQueue;
535 DtmfInband _inbandDtmfGenerator;
536 bool _outputExternalMedia;
537 VoEMediaProcess* _inputExternalMediaCallbackPtr;
538 VoEMediaProcess* _outputExternalMediaCallbackPtr;
540 uint8_t _sendTelephoneEventPayloadType;
542 // Timestamp of the audio pulled from NetEq.
543 uint32_t jitter_buffer_playout_timestamp_;
544 uint32_t playout_timestamp_rtp_;
545 uint32_t playout_timestamp_rtcp_;
546 uint32_t playout_delay_ms_;
547 uint32_t _numberOfDiscardedPackets;
548 uint16_t send_sequence_number_;
549 uint8_t restored_packet_[kVoiceEngineMaxIpPacketSizeBytes];
552 Statistics* _engineStatisticsPtr;
553 OutputMixer* _outputMixerPtr;
554 TransmitMixer* _transmitMixerPtr;
555 ProcessThread* _moduleProcessThreadPtr;
556 AudioDeviceModule* _audioDeviceModulePtr;
557 VoiceEngineObserver* _voiceEngineObserverPtr; // owned by base
558 CriticalSectionWrapper* _callbackCritSectPtr; // owned by base
559 Transport* _transportPtr; // WebRtc socket or external transport
561 scoped_ptr<AudioProcessing> rx_audioproc_; // far end AudioProcessing
562 VoERxVadCallback* _rxVadObserverPtr;
563 int32_t _oldVadDecision;
564 int32_t _sendFrameType; // Send data is voice, 1-voice, 0-otherwise
565 VoERTPObserver* _rtpObserverPtr;
566 VoERTCPObserver* _rtcpObserverPtr;
568 bool _externalPlayout;
569 bool _externalMixing;
570 bool _mixFileWithMicrophone;
579 bool _playOutbandDtmfEvent;
580 bool _playInbandDtmfEvent;
582 uint32_t _lastLocalTimeStamp;
583 uint32_t _lastRemoteTimeStamp;
584 int8_t _lastPayloadType;
585 bool _includeAudioLevelIndication;
587 bool _rtpPacketTimedOut;
588 bool _rtpPacketTimeOutIsEnabled;
589 uint32_t _rtpTimeOutSeconds;
590 bool _connectionObserver;
591 VoEConnectionObserver* _connectionObserverPtr;
592 AudioFrame::SpeechType _outputSpeechType;
593 ViENetwork* vie_network_;
596 uint32_t _average_jitter_buffer_delay_us;
597 int least_required_delay_ms_;
598 uint32_t _previousTimestamp;
599 uint16_t _recPacketDelayMs;
600 // VoEAudioProcessing
601 bool _RxVadDetection;
602 bool _rxAgcIsEnabled;
604 bool restored_packet_in_use_;
608 } // namespace webrtc
610 #endif // WEBRTC_VOICE_ENGINE_CHANNEL_H_