2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
11 #include "webrtc/voice_engine/output_mixer.h"
13 #include "webrtc/modules/audio_processing/include/audio_processing.h"
14 #include "webrtc/modules/utility/interface/audio_frame_operations.h"
15 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
16 #include "webrtc/system_wrappers/interface/file_wrapper.h"
17 #include "webrtc/system_wrappers/interface/trace.h"
18 #include "webrtc/voice_engine/include/voe_external_media.h"
19 #include "webrtc/voice_engine/statistics.h"
20 #include "webrtc/voice_engine/utility.h"
26 OutputMixer::NewMixedAudio(int32_t id,
27 const AudioFrame& generalAudioFrame,
28 const AudioFrame** uniqueAudioFrames,
31 WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
32 "OutputMixer::NewMixedAudio(id=%d, size=%u)", id, size);
34 _audioFrame.CopyFrom(generalAudioFrame);
38 void OutputMixer::MixedParticipants(
40 const ParticipantStatistics* participantStatistics,
43 WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
44 "OutputMixer::MixedParticipants(id=%d, size=%u)", id, size);
47 void OutputMixer::VADPositiveParticipants(int32_t id,
48 const ParticipantStatistics* participantStatistics, uint32_t size)
50 WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
51 "OutputMixer::VADPositiveParticipants(id=%d, size=%u)",
55 void OutputMixer::MixedAudioLevel(int32_t id, uint32_t level)
57 WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
58 "OutputMixer::MixedAudioLevel(id=%d, level=%u)", id, level);
61 void OutputMixer::PlayNotification(int32_t id, uint32_t durationMs)
63 WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
64 "OutputMixer::PlayNotification(id=%d, durationMs=%d)",
69 void OutputMixer::RecordNotification(int32_t id,
72 WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
73 "OutputMixer::RecordNotification(id=%d, durationMs=%d)",
79 void OutputMixer::PlayFileEnded(int32_t id)
81 WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
82 "OutputMixer::PlayFileEnded(id=%d)", id);
87 void OutputMixer::RecordFileEnded(int32_t id)
89 WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
90 "OutputMixer::RecordFileEnded(id=%d)", id);
91 assert(id == _instanceId);
93 CriticalSectionScoped cs(&_fileCritSect);
94 _outputFileRecording = false;
95 WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId,-1),
96 "OutputMixer::RecordFileEnded() =>"
97 "output file recorder module is shutdown");
101 OutputMixer::Create(OutputMixer*& mixer, uint32_t instanceId)
103 WEBRTC_TRACE(kTraceMemory, kTraceVoice, instanceId,
104 "OutputMixer::Create(instanceId=%d)", instanceId);
105 mixer = new OutputMixer(instanceId);
108 WEBRTC_TRACE(kTraceMemory, kTraceVoice, instanceId,
109 "OutputMixer::Create() unable to allocate memory for"
116 OutputMixer::OutputMixer(uint32_t instanceId) :
117 _callbackCritSect(*CriticalSectionWrapper::CreateCriticalSection()),
118 _fileCritSect(*CriticalSectionWrapper::CreateCriticalSection()),
119 _mixerModule(*AudioConferenceMixer::Create(instanceId)),
121 _dtmfGenerator(instanceId),
122 _instanceId(instanceId),
123 _externalMediaCallbackPtr(NULL),
124 _externalMedia(false),
127 _mixingFrequencyHz(8000),
128 _outputFileRecorderPtr(NULL),
129 _outputFileRecording(false)
131 WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(_instanceId,-1),
132 "OutputMixer::OutputMixer() - ctor");
134 if ((_mixerModule.RegisterMixedStreamCallback(*this) == -1) ||
135 (_mixerModule.RegisterMixerStatusCallback(*this, 100) == -1))
137 WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId,-1),
138 "OutputMixer::OutputMixer() failed to register mixer"
142 _dtmfGenerator.Init();
146 OutputMixer::Destroy(OutputMixer*& mixer)
155 OutputMixer::~OutputMixer()
157 WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(_instanceId,-1),
158 "OutputMixer::~OutputMixer() - dtor");
161 DeRegisterExternalMediaProcessing();
164 CriticalSectionScoped cs(&_fileCritSect);
165 if (_outputFileRecorderPtr)
167 _outputFileRecorderPtr->RegisterModuleFileCallback(NULL);
168 _outputFileRecorderPtr->StopRecording();
169 FileRecorder::DestroyFileRecorder(_outputFileRecorderPtr);
170 _outputFileRecorderPtr = NULL;
173 _mixerModule.UnRegisterMixerStatusCallback();
174 _mixerModule.UnRegisterMixedStreamCallback();
175 delete &_mixerModule;
176 delete &_callbackCritSect;
177 delete &_fileCritSect;
181 OutputMixer::SetEngineInformation(voe::Statistics& engineStatistics)
183 WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
184 "OutputMixer::SetEngineInformation()");
185 _engineStatisticsPtr = &engineStatistics;
190 OutputMixer::SetAudioProcessingModule(AudioProcessing* audioProcessingModule)
192 WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
193 "OutputMixer::SetAudioProcessingModule("
194 "audioProcessingModule=0x%x)", audioProcessingModule);
195 _audioProcessingModulePtr = audioProcessingModule;
199 int OutputMixer::RegisterExternalMediaProcessing(
200 VoEMediaProcess& proccess_object)
202 WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
203 "OutputMixer::RegisterExternalMediaProcessing()");
205 CriticalSectionScoped cs(&_callbackCritSect);
206 _externalMediaCallbackPtr = &proccess_object;
207 _externalMedia = true;
212 int OutputMixer::DeRegisterExternalMediaProcessing()
214 WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
215 "OutputMixer::DeRegisterExternalMediaProcessing()");
217 CriticalSectionScoped cs(&_callbackCritSect);
218 _externalMedia = false;
219 _externalMediaCallbackPtr = NULL;
224 int OutputMixer::PlayDtmfTone(uint8_t eventCode, int lengthMs,
227 WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1),
228 "OutputMixer::PlayDtmfTone()");
229 if (_dtmfGenerator.AddTone(eventCode, lengthMs, attenuationDb) != 0)
231 _engineStatisticsPtr->SetLastError(VE_STILL_PLAYING_PREV_DTMF,
233 "OutputMixer::PlayDtmfTone()");
239 int OutputMixer::StartPlayingDtmfTone(uint8_t eventCode,
242 WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1),
243 "OutputMixer::StartPlayingDtmfTone()");
244 if (_dtmfGenerator.StartTone(eventCode, attenuationDb) != 0)
246 _engineStatisticsPtr->SetLastError(
247 VE_STILL_PLAYING_PREV_DTMF,
249 "OutputMixer::StartPlayingDtmfTone())");
255 int OutputMixer::StopPlayingDtmfTone()
257 WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, -1),
258 "OutputMixer::StopPlayingDtmfTone()");
259 return (_dtmfGenerator.StopTone());
263 OutputMixer::SetMixabilityStatus(MixerParticipant& participant,
266 return _mixerModule.SetMixabilityStatus(participant, mixable);
270 OutputMixer::SetAnonymousMixabilityStatus(MixerParticipant& participant,
273 return _mixerModule.SetAnonymousMixabilityStatus(participant,mixable);
277 OutputMixer::MixActiveChannels()
279 return _mixerModule.Process();
283 OutputMixer::GetSpeechOutputLevel(uint32_t& level)
285 int8_t currentLevel = _audioLevel.Level();
286 level = static_cast<uint32_t> (currentLevel);
287 WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId,-1),
288 "GetSpeechOutputLevel() => level=%u", level);
293 OutputMixer::GetSpeechOutputLevelFullRange(uint32_t& level)
295 int16_t currentLevel = _audioLevel.LevelFullRange();
296 level = static_cast<uint32_t> (currentLevel);
297 WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId,-1),
298 "GetSpeechOutputLevelFullRange() => level=%u", level);
303 OutputMixer::SetOutputVolumePan(float left, float right)
305 WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
306 "OutputMixer::SetOutputVolumePan()");
313 OutputMixer::GetOutputVolumePan(float& left, float& right)
317 WEBRTC_TRACE(kTraceStateInfo, kTraceVoice, VoEId(_instanceId,-1),
318 "GetOutputVolumePan() => left=%2.1f, right=%2.1f",
323 int OutputMixer::StartRecordingPlayout(const char* fileName,
324 const CodecInst* codecInst)
326 WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
327 "OutputMixer::StartRecordingPlayout(fileName=%s)", fileName);
329 if (_outputFileRecording)
331 WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId,-1),
332 "StartRecordingPlayout() is already recording");
337 const uint32_t notificationTime(0);
338 CodecInst dummyCodec={100,"L16",16000,320,1,320000};
340 if ((codecInst != NULL) &&
341 ((codecInst->channels < 1) || (codecInst->channels > 2)))
343 _engineStatisticsPtr->SetLastError(
344 VE_BAD_ARGUMENT, kTraceError,
345 "StartRecordingPlayout() invalid compression");
348 if(codecInst == NULL)
350 format = kFileFormatPcm16kHzFile;
351 codecInst=&dummyCodec;
353 else if((STR_CASE_CMP(codecInst->plname,"L16") == 0) ||
354 (STR_CASE_CMP(codecInst->plname,"PCMU") == 0) ||
355 (STR_CASE_CMP(codecInst->plname,"PCMA") == 0))
357 format = kFileFormatWavFile;
361 format = kFileFormatCompressedFile;
364 CriticalSectionScoped cs(&_fileCritSect);
366 // Destroy the old instance
367 if (_outputFileRecorderPtr)
369 _outputFileRecorderPtr->RegisterModuleFileCallback(NULL);
370 FileRecorder::DestroyFileRecorder(_outputFileRecorderPtr);
371 _outputFileRecorderPtr = NULL;
374 _outputFileRecorderPtr = FileRecorder::CreateFileRecorder(
376 (const FileFormats)format);
377 if (_outputFileRecorderPtr == NULL)
379 _engineStatisticsPtr->SetLastError(
380 VE_INVALID_ARGUMENT, kTraceError,
381 "StartRecordingPlayout() fileRecorder format isnot correct");
385 if (_outputFileRecorderPtr->StartRecordingAudioFile(
387 (const CodecInst&)*codecInst,
388 notificationTime) != 0)
390 _engineStatisticsPtr->SetLastError(
391 VE_BAD_FILE, kTraceError,
392 "StartRecordingAudioFile() failed to start file recording");
393 _outputFileRecorderPtr->StopRecording();
394 FileRecorder::DestroyFileRecorder(_outputFileRecorderPtr);
395 _outputFileRecorderPtr = NULL;
398 _outputFileRecorderPtr->RegisterModuleFileCallback(this);
399 _outputFileRecording = true;
404 int OutputMixer::StartRecordingPlayout(OutStream* stream,
405 const CodecInst* codecInst)
407 WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
408 "OutputMixer::StartRecordingPlayout()");
410 if (_outputFileRecording)
412 WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId,-1),
413 "StartRecordingPlayout() is already recording");
418 const uint32_t notificationTime(0);
419 CodecInst dummyCodec={100,"L16",16000,320,1,320000};
421 if (codecInst != NULL && codecInst->channels != 1)
423 _engineStatisticsPtr->SetLastError(
424 VE_BAD_ARGUMENT, kTraceError,
425 "StartRecordingPlayout() invalid compression");
428 if(codecInst == NULL)
430 format = kFileFormatPcm16kHzFile;
431 codecInst=&dummyCodec;
433 else if((STR_CASE_CMP(codecInst->plname,"L16") == 0) ||
434 (STR_CASE_CMP(codecInst->plname,"PCMU") == 0) ||
435 (STR_CASE_CMP(codecInst->plname,"PCMA") == 0))
437 format = kFileFormatWavFile;
441 format = kFileFormatCompressedFile;
444 CriticalSectionScoped cs(&_fileCritSect);
446 // Destroy the old instance
447 if (_outputFileRecorderPtr)
449 _outputFileRecorderPtr->RegisterModuleFileCallback(NULL);
450 FileRecorder::DestroyFileRecorder(_outputFileRecorderPtr);
451 _outputFileRecorderPtr = NULL;
454 _outputFileRecorderPtr = FileRecorder::CreateFileRecorder(
456 (const FileFormats)format);
457 if (_outputFileRecorderPtr == NULL)
459 _engineStatisticsPtr->SetLastError(
460 VE_INVALID_ARGUMENT, kTraceError,
461 "StartRecordingPlayout() fileRecorder format isnot correct");
465 if (_outputFileRecorderPtr->StartRecordingAudioFile(*stream,
467 notificationTime) != 0)
469 _engineStatisticsPtr->SetLastError(VE_BAD_FILE, kTraceError,
470 "StartRecordingAudioFile() failed to start file recording");
471 _outputFileRecorderPtr->StopRecording();
472 FileRecorder::DestroyFileRecorder(_outputFileRecorderPtr);
473 _outputFileRecorderPtr = NULL;
477 _outputFileRecorderPtr->RegisterModuleFileCallback(this);
478 _outputFileRecording = true;
483 int OutputMixer::StopRecordingPlayout()
485 WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId,-1),
486 "OutputMixer::StopRecordingPlayout()");
488 if (!_outputFileRecording)
490 WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId,-1),
491 "StopRecordingPlayout() file isnot recording");
495 CriticalSectionScoped cs(&_fileCritSect);
497 if (_outputFileRecorderPtr->StopRecording() != 0)
499 _engineStatisticsPtr->SetLastError(
500 VE_STOP_RECORDING_FAILED, kTraceError,
501 "StopRecording(), could not stop recording");
504 _outputFileRecorderPtr->RegisterModuleFileCallback(NULL);
505 FileRecorder::DestroyFileRecorder(_outputFileRecorderPtr);
506 _outputFileRecorderPtr = NULL;
507 _outputFileRecording = false;
512 int OutputMixer::GetMixedAudio(int sample_rate_hz,
515 WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
516 "OutputMixer::GetMixedAudio(sample_rate_hz=%d, num_channels=%d)",
517 sample_rate_hz, num_channels);
519 // --- Record playout if enabled
521 CriticalSectionScoped cs(&_fileCritSect);
522 if (_outputFileRecording && _outputFileRecorderPtr)
523 _outputFileRecorderPtr->RecordAudioToFile(_audioFrame);
526 frame->num_channels_ = num_channels;
527 frame->sample_rate_hz_ = sample_rate_hz;
528 // TODO(andrew): Ideally the downmixing would occur much earlier, in
529 // AudioCodingModule.
530 RemixAndResample(_audioFrame, &resampler_, frame);
535 OutputMixer::DoOperationsOnCombinedSignal(bool feed_data_to_apm)
537 if (_audioFrame.sample_rate_hz_ != _mixingFrequencyHz)
539 WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId,-1),
540 "OutputMixer::DoOperationsOnCombinedSignal() => "
541 "mixing frequency = %d", _audioFrame.sample_rate_hz_);
542 _mixingFrequencyHz = _audioFrame.sample_rate_hz_;
545 // --- Insert inband Dtmf tone
546 if (_dtmfGenerator.IsAddingTone())
548 InsertInbandDtmfTone();
551 // Scale left and/or right channel(s) if balance is active
552 if (_panLeft != 1.0 || _panRight != 1.0)
554 if (_audioFrame.num_channels_ == 1)
556 AudioFrameOperations::MonoToStereo(&_audioFrame);
560 // Pure stereo mode (we are receiving a stereo signal).
563 assert(_audioFrame.num_channels_ == 2);
564 AudioFrameOperations::Scale(_panLeft, _panRight, _audioFrame);
567 // --- Far-end Voice Quality Enhancement (AudioProcessing Module)
568 if (feed_data_to_apm)
569 APMAnalyzeReverseStream();
571 // --- External media processing
573 CriticalSectionScoped cs(&_callbackCritSect);
576 const bool is_stereo = (_audioFrame.num_channels_ == 2);
577 if (_externalMediaCallbackPtr)
579 _externalMediaCallbackPtr->Process(
581 kPlaybackAllChannelsMixed,
582 (int16_t*)_audioFrame.data_,
583 _audioFrame.samples_per_channel_,
584 _audioFrame.sample_rate_hz_,
590 // --- Measure audio level (0-9) for the combined signal
591 _audioLevel.ComputeLevel(_audioFrame);
596 // ----------------------------------------------------------------------------
598 // ----------------------------------------------------------------------------
600 void OutputMixer::APMAnalyzeReverseStream() {
601 // Convert from mixing to AudioProcessing sample rate, determined by the send
602 // side. Downmix to mono.
604 frame.num_channels_ = 1;
605 frame.sample_rate_hz_ = _audioProcessingModulePtr->input_sample_rate_hz();
606 RemixAndResample(_audioFrame, &audioproc_resampler_, &frame);
608 if (_audioProcessingModulePtr->AnalyzeReverseStream(&frame) == -1) {
609 WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId,-1),
610 "AudioProcessingModule::AnalyzeReverseStream() => error");
615 OutputMixer::InsertInbandDtmfTone()
617 uint16_t sampleRate(0);
618 _dtmfGenerator.GetSampleRate(sampleRate);
619 if (sampleRate != _audioFrame.sample_rate_hz_)
621 // Update sample rate of Dtmf tone since the mixing frequency changed.
622 _dtmfGenerator.SetSampleRate(
623 (uint16_t)(_audioFrame.sample_rate_hz_));
624 // Reset the tone to be added taking the new sample rate into account.
625 _dtmfGenerator.ResetTone();
628 int16_t toneBuffer[320];
629 uint16_t toneSamples(0);
630 if (_dtmfGenerator.Get10msTone(toneBuffer, toneSamples) == -1)
632 WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, -1),
633 "OutputMixer::InsertInbandDtmfTone() inserting Dtmf"
638 // replace mixed audio with Dtmf tone
639 if (_audioFrame.num_channels_ == 1)
642 memcpy(_audioFrame.data_, toneBuffer, sizeof(int16_t)
647 for (int i = 0; i < _audioFrame.samples_per_channel_; i++)
649 _audioFrame.data_[2 * i] = toneBuffer[i];
650 _audioFrame.data_[2 * i + 1] = 0;
653 assert(_audioFrame.samples_per_channel_ == toneSamples);
659 } // namespace webrtc