src/third_party/webrtc/modules/audio_conference_mixer/source/audio_conference_mixer_impl.cc

   1 /*
   2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include "webrtc/modules/audio_conference_mixer/interface/audio_conference_mixer_defines.h"
  12 #include "webrtc/modules/audio_conference_mixer/source/audio_conference_mixer_impl.h"
  13 #include "webrtc/modules/audio_conference_mixer/source/audio_frame_manipulator.h"
  14 #include "webrtc/modules/audio_processing/include/audio_processing.h"
  15 #include "webrtc/modules/utility/interface/audio_frame_operations.h"
  16 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
  17 #include "webrtc/system_wrappers/interface/trace.h"
  18
  19 namespace webrtc {
  20 namespace {
  21
  22 struct ParticipantFramePair {
  23   MixerParticipant* participant;
  24   AudioFrame* audioFrame;
  25 };
  26
  27 typedef std::list<ParticipantFramePair*> ParticipantFramePairList;
  28
  29 // Mix |frame| into |mixed_frame|, with saturation protection and upmixing.
  30 // These effects are applied to |frame| itself prior to mixing. Assumes that
  31 // |mixed_frame| always has at least as many channels as |frame|. Supports
  32 // stereo at most.
  33 //
  34 // TODO(andrew): consider not modifying |frame| here.
  35 void MixFrames(AudioFrame* mixed_frame, AudioFrame* frame) {
  36   assert(mixed_frame->num_channels_ >= frame->num_channels_);
  37   // Divide by two to avoid saturation in the mixing.
  38   *frame >>= 1;
  39   if (mixed_frame->num_channels_ > frame->num_channels_) {
  40     // We only support mono-to-stereo.
  41     assert(mixed_frame->num_channels_ == 2 &&
  42            frame->num_channels_ == 1);
  43     AudioFrameOperations::MonoToStereo(frame);
  44   }
  45
  46   *mixed_frame += *frame;
  47 }
  48
  49 // Return the max number of channels from a |list| composed of AudioFrames.
  50 int MaxNumChannels(const AudioFrameList* list) {
  51   int max_num_channels = 1;
  52   for (AudioFrameList::const_iterator iter = list->begin();
  53        iter != list->end();
  54        ++iter) {
  55     max_num_channels = std::max(max_num_channels, (*iter)->num_channels_);
  56   }
  57   return max_num_channels;
  58 }
  59
  60 void SetParticipantStatistics(ParticipantStatistics* stats,
  61                               const AudioFrame& frame) {
  62     stats->participant = frame.id_;
  63     stats->level = 0;  // TODO(andrew): to what should this be set?
  64 }
  65
  66 }  // namespace
  67
  68 MixerParticipant::MixerParticipant()
  69     : _mixHistory(new MixHistory()) {
  70 }
  71
  72 MixerParticipant::~MixerParticipant() {
  73     delete _mixHistory;
  74 }
  75
  76 int32_t MixerParticipant::IsMixed(bool& mixed) const {
  77     return _mixHistory->IsMixed(mixed);
  78 }
  79
  80 MixHistory::MixHistory()
  81     : _isMixed(0) {
  82 }
  83
  84 MixHistory::~MixHistory() {
  85 }
  86
  87 int32_t MixHistory::IsMixed(bool& mixed) const {
  88     mixed = _isMixed;
  89     return 0;
  90 }
  91
  92 int32_t MixHistory::WasMixed(bool& wasMixed) const {
  93     // Was mixed is the same as is mixed depending on perspective. This function
  94     // is for the perspective of AudioConferenceMixerImpl.
  95     return IsMixed(wasMixed);
  96 }
  97
  98 int32_t MixHistory::SetIsMixed(const bool mixed) {
  99     _isMixed = mixed;
 100     return 0;
 101 }
 102
 103 void MixHistory::ResetMixedStatus() {
 104     _isMixed = false;
 105 }
 106
 107 AudioConferenceMixer* AudioConferenceMixer::Create(int id) {
 108     AudioConferenceMixerImpl* mixer = new AudioConferenceMixerImpl(id);
 109     if(!mixer->Init()) {
 110         delete mixer;
 111         return NULL;
 112     }
 113     return mixer;
 114 }
 115
 116 AudioConferenceMixerImpl::AudioConferenceMixerImpl(int id)
 117     : _scratchParticipantsToMixAmount(0),
 118       _scratchMixedParticipants(),
 119       _scratchVadPositiveParticipantsAmount(0),
 120       _scratchVadPositiveParticipants(),
 121       _id(id),
 122       _minimumMixingFreq(kLowestPossible),
 123       _mixReceiver(NULL),
 124       _mixerStatusCallback(NULL),
 125       _amountOf10MsBetweenCallbacks(1),
 126       _amountOf10MsUntilNextCallback(0),
 127       _mixerStatusCb(false),
 128       _outputFrequency(kDefaultFrequency),
 129       _sampleSize(0),
 130       _audioFramePool(NULL),
 131       _participantList(),
 132       _additionalParticipantList(),
 133       _numMixedParticipants(0),
 134       _timeStamp(0),
 135       _timeScheduler(kProcessPeriodicityInMs),
 136       _mixedAudioLevel(),
 137       _processCalls(0) {}
 138
 139 bool AudioConferenceMixerImpl::Init() {
 140     _crit.reset(CriticalSectionWrapper::CreateCriticalSection());
 141     if (_crit.get() == NULL)
 142         return false;
 143
 144     _cbCrit.reset(CriticalSectionWrapper::CreateCriticalSection());
 145     if(_cbCrit.get() == NULL)
 146         return false;
 147
 148     Config config;
 149     config.Set<ExperimentalAgc>(new ExperimentalAgc(false));
 150     _limiter.reset(AudioProcessing::Create(config));
 151     if(!_limiter.get())
 152         return false;
 153
 154     MemoryPool<AudioFrame>::CreateMemoryPool(_audioFramePool,
 155                                              DEFAULT_AUDIO_FRAME_POOLSIZE);
 156     if(_audioFramePool == NULL)
 157         return false;
 158
 159     if(SetOutputFrequency(kDefaultFrequency) == -1)
 160         return false;
 161
 162     if(_limiter->gain_control()->set_mode(GainControl::kFixedDigital) !=
 163         _limiter->kNoError)
 164         return false;
 165
 166     // We smoothly limit the mixed frame to -7 dbFS. -6 would correspond to the
 167     // divide-by-2 but -7 is used instead to give a bit of headroom since the
 168     // AGC is not a hard limiter.
 169     if(_limiter->gain_control()->set_target_level_dbfs(7) != _limiter->kNoError)
 170         return false;
 171
 172     if(_limiter->gain_control()->set_compression_gain_db(0)
 173         != _limiter->kNoError)
 174         return false;
 175
 176     if(_limiter->gain_control()->enable_limiter(true) != _limiter->kNoError)
 177         return false;
 178
 179     if(_limiter->gain_control()->Enable(true) != _limiter->kNoError)
 180         return false;
 181
 182     return true;
 183 }
 184
 185 AudioConferenceMixerImpl::~AudioConferenceMixerImpl() {
 186     MemoryPool<AudioFrame>::DeleteMemoryPool(_audioFramePool);
 187     assert(_audioFramePool == NULL);
 188 }
 189
 190 int32_t AudioConferenceMixerImpl::ChangeUniqueId(const int32_t id) {
 191     _id = id;
 192     return 0;
 193 }
 194
 195 // Process should be called every kProcessPeriodicityInMs ms
 196 int32_t AudioConferenceMixerImpl::TimeUntilNextProcess() {
 197     int32_t timeUntilNextProcess = 0;
 198     CriticalSectionScoped cs(_crit.get());
 199     if(_timeScheduler.TimeToNextUpdate(timeUntilNextProcess) != 0) {
 200         WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
 201                      "failed in TimeToNextUpdate() call");
 202         // Sanity check
 203         assert(false);
 204         return -1;
 205     }
 206     return timeUntilNextProcess;
 207 }
 208
 209 int32_t AudioConferenceMixerImpl::Process() {
 210     size_t remainingParticipantsAllowedToMix =
 211         kMaximumAmountOfMixedParticipants;
 212     {
 213         CriticalSectionScoped cs(_crit.get());
 214         assert(_processCalls == 0);
 215         _processCalls++;
 216
 217         // Let the scheduler know that we are running one iteration.
 218         _timeScheduler.UpdateScheduler();
 219     }
 220
 221     AudioFrameList mixList;
 222     AudioFrameList rampOutList;
 223     AudioFrameList additionalFramesList;
 224     std::map<int, MixerParticipant*> mixedParticipantsMap;
 225     {
 226         CriticalSectionScoped cs(_cbCrit.get());
 227
 228         int32_t lowFreq = GetLowestMixingFrequency();
 229         // SILK can run in 12 kHz and 24 kHz. These frequencies are not
 230         // supported so use the closest higher frequency to not lose any
 231         // information.
 232         // TODO(henrike): this is probably more appropriate to do in
 233         //                GetLowestMixingFrequency().
 234         if (lowFreq == 12000) {
 235             lowFreq = 16000;
 236         } else if (lowFreq == 24000) {
 237             lowFreq = 32000;
 238         }
 239         if(lowFreq <= 0) {
 240             CriticalSectionScoped cs(_crit.get());
 241             _processCalls--;
 242             return 0;
 243         } else {
 244             switch(lowFreq) {
 245             case 8000:
 246                 if(OutputFrequency() != kNbInHz) {
 247                     SetOutputFrequency(kNbInHz);
 248                 }
 249                 break;
 250             case 16000:
 251                 if(OutputFrequency() != kWbInHz) {
 252                     SetOutputFrequency(kWbInHz);
 253                 }
 254                 break;
 255             case 32000:
 256                 if(OutputFrequency() != kSwbInHz) {
 257                     SetOutputFrequency(kSwbInHz);
 258                 }
 259                 break;
 260             case 48000:
 261                 if(OutputFrequency() != kFbInHz) {
 262                     SetOutputFrequency(kFbInHz);
 263                 }
 264                 break;
 265             default:
 266                 assert(false);
 267
 268                 CriticalSectionScoped cs(_crit.get());
 269                 _processCalls--;
 270                 return -1;
 271             }
 272         }
 273
 274         UpdateToMix(&mixList, &rampOutList, &mixedParticipantsMap,
 275                     remainingParticipantsAllowedToMix);
 276
 277         GetAdditionalAudio(&additionalFramesList);
 278         UpdateMixedStatus(mixedParticipantsMap);
 279         _scratchParticipantsToMixAmount = mixedParticipantsMap.size();
 280     }
 281
 282     // Get an AudioFrame for mixing from the memory pool.
 283     AudioFrame* mixedAudio = NULL;
 284     if(_audioFramePool->PopMemory(mixedAudio) == -1) {
 285         WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
 286                      "failed PopMemory() call");
 287         assert(false);
 288         return -1;
 289     }
 290
 291     bool timeForMixerCallback = false;
 292     int retval = 0;
 293     int32_t audioLevel = 0;
 294     {
 295         CriticalSectionScoped cs(_crit.get());
 296
 297         // TODO(henrike): it might be better to decide the number of channels
 298         //                with an API instead of dynamically.
 299
 300         // Find the max channels over all mixing lists.
 301         const int num_mixed_channels = std::max(MaxNumChannels(&mixList),
 302             std::max(MaxNumChannels(&additionalFramesList),
 303                      MaxNumChannels(&rampOutList)));
 304
 305         mixedAudio->UpdateFrame(-1, _timeStamp, NULL, 0, _outputFrequency,
 306                                 AudioFrame::kNormalSpeech,
 307                                 AudioFrame::kVadPassive, num_mixed_channels);
 308
 309         _timeStamp += _sampleSize;
 310
 311         MixFromList(*mixedAudio, &mixList);
 312         MixAnonomouslyFromList(*mixedAudio, &additionalFramesList);
 313         MixAnonomouslyFromList(*mixedAudio, &rampOutList);
 314
 315         if(mixedAudio->samples_per_channel_ == 0) {
 316             // Nothing was mixed, set the audio samples to silence.
 317             mixedAudio->samples_per_channel_ = _sampleSize;
 318             mixedAudio->Mute();
 319         } else {
 320             // Only call the limiter if we have something to mix.
 321             if(!LimitMixedAudio(*mixedAudio))
 322                 retval = -1;
 323         }
 324
 325         _mixedAudioLevel.ComputeLevel(mixedAudio->data_,_sampleSize);
 326         audioLevel = _mixedAudioLevel.GetLevel();
 327
 328         if(_mixerStatusCb) {
 329             _scratchVadPositiveParticipantsAmount = 0;
 330             UpdateVADPositiveParticipants(&mixList);
 331             if(_amountOf10MsUntilNextCallback-- == 0) {
 332                 _amountOf10MsUntilNextCallback = _amountOf10MsBetweenCallbacks;
 333                 timeForMixerCallback = true;
 334             }
 335         }
 336     }
 337
 338     {
 339         CriticalSectionScoped cs(_cbCrit.get());
 340         if(_mixReceiver != NULL) {
 341             const AudioFrame** dummy = NULL;
 342             _mixReceiver->NewMixedAudio(
 343                 _id,
 344                 *mixedAudio,
 345                 dummy,
 346                 0);
 347         }
 348
 349         if((_mixerStatusCallback != NULL) &&
 350             timeForMixerCallback) {
 351             _mixerStatusCallback->MixedParticipants(
 352                 _id,
 353                 _scratchMixedParticipants,
 354                 static_cast<uint32_t>(_scratchParticipantsToMixAmount));
 355
 356             _mixerStatusCallback->VADPositiveParticipants(
 357                 _id,
 358                 _scratchVadPositiveParticipants,
 359                 _scratchVadPositiveParticipantsAmount);
 360             _mixerStatusCallback->MixedAudioLevel(_id,audioLevel);
 361         }
 362     }
 363
 364     // Reclaim all outstanding memory.
 365     _audioFramePool->PushMemory(mixedAudio);
 366     ClearAudioFrameList(&mixList);
 367     ClearAudioFrameList(&rampOutList);
 368     ClearAudioFrameList(&additionalFramesList);
 369     {
 370         CriticalSectionScoped cs(_crit.get());
 371         _processCalls--;
 372     }
 373     return retval;
 374 }
 375
 376 int32_t AudioConferenceMixerImpl::RegisterMixedStreamCallback(
 377     AudioMixerOutputReceiver& mixReceiver) {
 378     CriticalSectionScoped cs(_cbCrit.get());
 379     if(_mixReceiver != NULL) {
 380         return -1;
 381     }
 382     _mixReceiver = &mixReceiver;
 383     return 0;
 384 }
 385
 386 int32_t AudioConferenceMixerImpl::UnRegisterMixedStreamCallback() {
 387     CriticalSectionScoped cs(_cbCrit.get());
 388     if(_mixReceiver == NULL) {
 389         return -1;
 390     }
 391     _mixReceiver = NULL;
 392     return 0;
 393 }
 394
 395 int32_t AudioConferenceMixerImpl::SetOutputFrequency(
 396     const Frequency frequency) {
 397     CriticalSectionScoped cs(_crit.get());
 398
 399     _outputFrequency = frequency;
 400     _sampleSize = (_outputFrequency*kProcessPeriodicityInMs) / 1000;
 401
 402     return 0;
 403 }
 404
 405 AudioConferenceMixer::Frequency
 406 AudioConferenceMixerImpl::OutputFrequency() const {
 407     CriticalSectionScoped cs(_crit.get());
 408     return _outputFrequency;
 409 }
 410
 411 int32_t AudioConferenceMixerImpl::RegisterMixerStatusCallback(
 412     AudioMixerStatusReceiver& mixerStatusCallback,
 413     const uint32_t amountOf10MsBetweenCallbacks) {
 414     if(amountOf10MsBetweenCallbacks == 0) {
 415         WEBRTC_TRACE(
 416             kTraceWarning,
 417             kTraceAudioMixerServer,
 418             _id,
 419             "amountOf10MsBetweenCallbacks(%d) needs to be larger than 0");
 420         return -1;
 421     }
 422     {
 423         CriticalSectionScoped cs(_cbCrit.get());
 424         if(_mixerStatusCallback != NULL) {
 425             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
 426                          "Mixer status callback already registered");
 427             return -1;
 428         }
 429         _mixerStatusCallback = &mixerStatusCallback;
 430     }
 431     {
 432         CriticalSectionScoped cs(_crit.get());
 433         _amountOf10MsBetweenCallbacks  = amountOf10MsBetweenCallbacks;
 434         _amountOf10MsUntilNextCallback = 0;
 435         _mixerStatusCb                 = true;
 436     }
 437     return 0;
 438 }
 439
 440 int32_t AudioConferenceMixerImpl::UnRegisterMixerStatusCallback() {
 441     {
 442         CriticalSectionScoped cs(_crit.get());
 443         if(!_mixerStatusCb)
 444         {
 445             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
 446                          "Mixer status callback not registered");
 447             return -1;
 448         }
 449         _mixerStatusCb = false;
 450     }
 451     {
 452         CriticalSectionScoped cs(_cbCrit.get());
 453         _mixerStatusCallback = NULL;
 454     }
 455     return 0;
 456 }
 457
 458 int32_t AudioConferenceMixerImpl::SetMixabilityStatus(
 459     MixerParticipant& participant,
 460     bool mixable) {
 461     if (!mixable) {
 462         // Anonymous participants are in a separate list. Make sure that the
 463         // participant is in the _participantList if it is being mixed.
 464         SetAnonymousMixabilityStatus(participant, false);
 465     }
 466     size_t numMixedParticipants;
 467     {
 468         CriticalSectionScoped cs(_cbCrit.get());
 469         const bool isMixed =
 470             IsParticipantInList(participant, &_participantList);
 471         // API must be called with a new state.
 472         if(!(mixable ^ isMixed)) {
 473             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
 474                          "Mixable is aready %s",
 475                          isMixed ? "ON" : "off");
 476             return -1;
 477         }
 478         bool success = false;
 479         if(mixable) {
 480             success = AddParticipantToList(participant, &_participantList);
 481         } else {
 482             success = RemoveParticipantFromList(participant, &_participantList);
 483         }
 484         if(!success) {
 485             WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
 486                          "failed to %s participant",
 487                          mixable ? "add" : "remove");
 488             assert(false);
 489             return -1;
 490         }
 491
 492         size_t numMixedNonAnonymous = _participantList.size();
 493         if (numMixedNonAnonymous > kMaximumAmountOfMixedParticipants) {
 494             numMixedNonAnonymous = kMaximumAmountOfMixedParticipants;
 495         }
 496         numMixedParticipants =
 497             numMixedNonAnonymous + _additionalParticipantList.size();
 498     }
 499     // A MixerParticipant was added or removed. Make sure the scratch
 500     // buffer is updated if necessary.
 501     // Note: The scratch buffer may only be updated in Process().
 502     CriticalSectionScoped cs(_crit.get());
 503     _numMixedParticipants = numMixedParticipants;
 504     return 0;
 505 }
 506
 507 int32_t AudioConferenceMixerImpl::MixabilityStatus(
 508     MixerParticipant& participant,
 509     bool& mixable) {
 510     CriticalSectionScoped cs(_cbCrit.get());
 511     mixable = IsParticipantInList(participant, &_participantList);
 512     return 0;
 513 }
 514
 515 int32_t AudioConferenceMixerImpl::SetAnonymousMixabilityStatus(
 516     MixerParticipant& participant, const bool anonymous) {
 517     CriticalSectionScoped cs(_cbCrit.get());
 518     if(IsParticipantInList(participant, &_additionalParticipantList)) {
 519         if(anonymous) {
 520             return 0;
 521         }
 522         if(!RemoveParticipantFromList(participant,
 523                                       &_additionalParticipantList)) {
 524             WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
 525                          "unable to remove participant from anonymous list");
 526             assert(false);
 527             return -1;
 528         }
 529         return AddParticipantToList(participant, &_participantList) ? 0 : -1;
 530     }
 531     if(!anonymous) {
 532         return 0;
 533     }
 534     const bool mixable = RemoveParticipantFromList(participant,
 535                                                    &_participantList);
 536     if(!mixable) {
 537         WEBRTC_TRACE(
 538             kTraceWarning,
 539             kTraceAudioMixerServer,
 540             _id,
 541             "participant must be registered before turning it into anonymous");
 542         // Setting anonymous status is only possible if MixerParticipant is
 543         // already registered.
 544         return -1;
 545     }
 546     return AddParticipantToList(participant, &_additionalParticipantList) ?
 547         0 : -1;
 548 }
 549
 550 int32_t AudioConferenceMixerImpl::AnonymousMixabilityStatus(
 551     MixerParticipant& participant, bool& mixable) {
 552     CriticalSectionScoped cs(_cbCrit.get());
 553     mixable = IsParticipantInList(participant,
 554                                   &_additionalParticipantList);
 555     return 0;
 556 }
 557
 558 int32_t AudioConferenceMixerImpl::SetMinimumMixingFrequency(
 559     Frequency freq) {
 560     // Make sure that only allowed sampling frequencies are used. Use closest
 561     // higher sampling frequency to avoid losing information.
 562     if (static_cast<int>(freq) == 12000) {
 563          freq = kWbInHz;
 564     } else if (static_cast<int>(freq) == 24000) {
 565         freq = kSwbInHz;
 566     }
 567
 568     if((freq == kNbInHz) || (freq == kWbInHz) || (freq == kSwbInHz) ||
 569        (freq == kLowestPossible)) {
 570         _minimumMixingFreq=freq;
 571         return 0;
 572     } else {
 573         WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
 574                      "SetMinimumMixingFrequency incorrect frequency: %i",freq);
 575         assert(false);
 576         return -1;
 577     }
 578 }
 579
 580 // Check all AudioFrames that are to be mixed. The highest sampling frequency
 581 // found is the lowest that can be used without losing information.
 582 int32_t AudioConferenceMixerImpl::GetLowestMixingFrequency() {
 583     const int participantListFrequency =
 584         GetLowestMixingFrequencyFromList(&_participantList);
 585     const int anonymousListFrequency =
 586         GetLowestMixingFrequencyFromList(&_additionalParticipantList);
 587     const int highestFreq =
 588         (participantListFrequency > anonymousListFrequency) ?
 589             participantListFrequency : anonymousListFrequency;
 590     // Check if the user specified a lowest mixing frequency.
 591     if(_minimumMixingFreq != kLowestPossible) {
 592         if(_minimumMixingFreq > highestFreq) {
 593             return _minimumMixingFreq;
 594         }
 595     }
 596     return highestFreq;
 597 }
 598
 599 int32_t AudioConferenceMixerImpl::GetLowestMixingFrequencyFromList(
 600     MixerParticipantList* mixList) {
 601     int32_t highestFreq = 8000;
 602     for (MixerParticipantList::iterator iter = mixList->begin();
 603          iter != mixList->end();
 604          ++iter) {
 605         const int32_t neededFrequency = (*iter)->NeededFrequency(_id);
 606         if(neededFrequency > highestFreq) {
 607             highestFreq = neededFrequency;
 608         }
 609     }
 610     return highestFreq;
 611 }
 612
 613 void AudioConferenceMixerImpl::UpdateToMix(
 614     AudioFrameList* mixList,
 615     AudioFrameList* rampOutList,
 616     std::map<int, MixerParticipant*>* mixParticipantList,
 617     size_t& maxAudioFrameCounter) {
 618     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
 619                  "UpdateToMix(mixList,rampOutList,mixParticipantList,%d)",
 620                  maxAudioFrameCounter);
 621     const size_t mixListStartSize = mixList->size();
 622     AudioFrameList activeList;
 623     // Struct needed by the passive lists to keep track of which AudioFrame
 624     // belongs to which MixerParticipant.
 625     ParticipantFramePairList passiveWasNotMixedList;
 626     ParticipantFramePairList passiveWasMixedList;
 627     for (MixerParticipantList::iterator participant = _participantList.begin();
 628          participant != _participantList.end();
 629          ++participant) {
 630         // Stop keeping track of passive participants if there are already
 631         // enough participants available (they wont be mixed anyway).
 632         bool mustAddToPassiveList = (maxAudioFrameCounter >
 633                                     (activeList.size() +
 634                                      passiveWasMixedList.size() +
 635                                      passiveWasNotMixedList.size()));
 636
 637         bool wasMixed = false;
 638         (*participant)->_mixHistory->WasMixed(wasMixed);
 639         AudioFrame* audioFrame = NULL;
 640         if(_audioFramePool->PopMemory(audioFrame) == -1) {
 641             WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
 642                          "failed PopMemory() call");
 643             assert(false);
 644             return;
 645         }
 646         audioFrame->sample_rate_hz_ = _outputFrequency;
 647
 648         if((*participant)->GetAudioFrame(_id,*audioFrame) != 0) {
 649             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
 650                          "failed to GetAudioFrame() from participant");
 651             _audioFramePool->PushMemory(audioFrame);
 652             continue;
 653         }
 654         // TODO(henrike): this assert triggers in some test cases where SRTP is
 655         // used which prevents NetEQ from making a VAD. Temporarily disable this
 656         // assert until the problem is fixed on a higher level.
 657         // assert(audioFrame->vad_activity_ != AudioFrame::kVadUnknown);
 658         if (audioFrame->vad_activity_ == AudioFrame::kVadUnknown) {
 659             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
 660                          "invalid VAD state from participant");
 661         }
 662
 663         if(audioFrame->vad_activity_ == AudioFrame::kVadActive) {
 664             if(!wasMixed) {
 665                 RampIn(*audioFrame);
 666             }
 667
 668             if(activeList.size() >= maxAudioFrameCounter) {
 669                 // There are already more active participants than should be
 670                 // mixed. Only keep the ones with the highest energy.
 671                 AudioFrameList::iterator replaceItem;
 672                 CalculateEnergy(*audioFrame);
 673                 uint32_t lowestEnergy = audioFrame->energy_;
 674
 675                 bool found_replace_item = false;
 676                 for (AudioFrameList::iterator iter = activeList.begin();
 677                      iter != activeList.end();
 678                      ++iter) {
 679                     CalculateEnergy(**iter);
 680                     if((*iter)->energy_ < lowestEnergy) {
 681                         replaceItem = iter;
 682                         lowestEnergy = (*iter)->energy_;
 683                         found_replace_item = true;
 684                     }
 685                 }
 686                 if(found_replace_item) {
 687                     AudioFrame* replaceFrame = *replaceItem;
 688
 689                     bool replaceWasMixed = false;
 690                     std::map<int, MixerParticipant*>::iterator it =
 691                         mixParticipantList->find(replaceFrame->id_);
 692
 693                     // When a frame is pushed to |activeList| it is also pushed
 694                     // to mixParticipantList with the frame's id. This means
 695                     // that the Find call above should never fail.
 696                     assert(it != mixParticipantList->end());
 697                     it->second->_mixHistory->WasMixed(replaceWasMixed);
 698
 699                     mixParticipantList->erase(replaceFrame->id_);
 700                     activeList.erase(replaceItem);
 701
 702                     activeList.push_front(audioFrame);
 703                     (*mixParticipantList)[audioFrame->id_] = *participant;
 704                     assert(mixParticipantList->size() <=
 705                            kMaximumAmountOfMixedParticipants);
 706
 707                     if (replaceWasMixed) {
 708                       RampOut(*replaceFrame);
 709                       rampOutList->push_back(replaceFrame);
 710                       assert(rampOutList->size() <=
 711                              kMaximumAmountOfMixedParticipants);
 712                     } else {
 713                       _audioFramePool->PushMemory(replaceFrame);
 714                     }
 715                 } else {
 716                     if(wasMixed) {
 717                         RampOut(*audioFrame);
 718                         rampOutList->push_back(audioFrame);
 719                         assert(rampOutList->size() <=
 720                                kMaximumAmountOfMixedParticipants);
 721                     } else {
 722                         _audioFramePool->PushMemory(audioFrame);
 723                     }
 724                 }
 725             } else {
 726                 activeList.push_front(audioFrame);
 727                 (*mixParticipantList)[audioFrame->id_] = *participant;
 728                 assert(mixParticipantList->size() <=
 729                        kMaximumAmountOfMixedParticipants);
 730             }
 731         } else {
 732             if(wasMixed) {
 733                 ParticipantFramePair* pair = new ParticipantFramePair;
 734                 pair->audioFrame  = audioFrame;
 735                 pair->participant = *participant;
 736                 passiveWasMixedList.push_back(pair);
 737             } else if(mustAddToPassiveList) {
 738                 RampIn(*audioFrame);
 739                 ParticipantFramePair* pair = new ParticipantFramePair;
 740                 pair->audioFrame  = audioFrame;
 741                 pair->participant = *participant;
 742                 passiveWasNotMixedList.push_back(pair);
 743             } else {
 744                 _audioFramePool->PushMemory(audioFrame);
 745             }
 746         }
 747     }
 748     assert(activeList.size() <= maxAudioFrameCounter);
 749     // At this point it is known which participants should be mixed. Transfer
 750     // this information to this functions output parameters.
 751     for (AudioFrameList::iterator iter = activeList.begin();
 752          iter != activeList.end();
 753          ++iter) {
 754         mixList->push_back(*iter);
 755     }
 756     activeList.clear();
 757     // Always mix a constant number of AudioFrames. If there aren't enough
 758     // active participants mix passive ones. Starting with those that was mixed
 759     // last iteration.
 760     for (ParticipantFramePairList::iterator iter = passiveWasMixedList.begin();
 761          iter != passiveWasMixedList.end();
 762          ++iter) {
 763         if(mixList->size() < maxAudioFrameCounter + mixListStartSize) {
 764             mixList->push_back((*iter)->audioFrame);
 765             (*mixParticipantList)[(*iter)->audioFrame->id_] =
 766                 (*iter)->participant;
 767             assert(mixParticipantList->size() <=
 768                    kMaximumAmountOfMixedParticipants);
 769         } else {
 770             _audioFramePool->PushMemory((*iter)->audioFrame);
 771         }
 772         delete *iter;
 773     }
 774     // And finally the ones that have not been mixed for a while.
 775     for (ParticipantFramePairList::iterator iter =
 776              passiveWasNotMixedList.begin();
 777          iter != passiveWasNotMixedList.end();
 778          ++iter) {
 779         if(mixList->size() <  maxAudioFrameCounter + mixListStartSize) {
 780           mixList->push_back((*iter)->audioFrame);
 781             (*mixParticipantList)[(*iter)->audioFrame->id_] =
 782                 (*iter)->participant;
 783             assert(mixParticipantList->size() <=
 784                    kMaximumAmountOfMixedParticipants);
 785         } else {
 786             _audioFramePool->PushMemory((*iter)->audioFrame);
 787         }
 788         delete *iter;
 789     }
 790     assert(maxAudioFrameCounter + mixListStartSize >= mixList->size());
 791     maxAudioFrameCounter += mixListStartSize - mixList->size();
 792 }
 793
 794 void AudioConferenceMixerImpl::GetAdditionalAudio(
 795     AudioFrameList* additionalFramesList) {
 796     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
 797                  "GetAdditionalAudio(additionalFramesList)");
 798     // The GetAudioFrame() callback may result in the participant being removed
 799     // from additionalParticipantList_. If that happens it will invalidate any
 800     // iterators. Create a copy of the participants list such that the list of
 801     // participants can be traversed safely.
 802     MixerParticipantList additionalParticipantList;
 803     additionalParticipantList.insert(additionalParticipantList.begin(),
 804                                      _additionalParticipantList.begin(),
 805                                      _additionalParticipantList.end());
 806
 807     for (MixerParticipantList::iterator participant =
 808              additionalParticipantList.begin();
 809          participant != additionalParticipantList.end();
 810          ++participant) {
 811         AudioFrame* audioFrame = NULL;
 812         if(_audioFramePool->PopMemory(audioFrame) == -1) {
 813             WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id,
 814                          "failed PopMemory() call");
 815             assert(false);
 816             return;
 817         }
 818         audioFrame->sample_rate_hz_ = _outputFrequency;
 819         if((*participant)->GetAudioFrame(_id, *audioFrame) != 0) {
 820             WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id,
 821                          "failed to GetAudioFrame() from participant");
 822             _audioFramePool->PushMemory(audioFrame);
 823             continue;
 824         }
 825         if(audioFrame->samples_per_channel_ == 0) {
 826             // Empty frame. Don't use it.
 827             _audioFramePool->PushMemory(audioFrame);
 828             continue;
 829         }
 830         additionalFramesList->push_back(audioFrame);
 831     }
 832 }
 833
 834 void AudioConferenceMixerImpl::UpdateMixedStatus(
 835     std::map<int, MixerParticipant*>& mixedParticipantsMap) {
 836     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
 837                  "UpdateMixedStatus(mixedParticipantsMap)");
 838     assert(mixedParticipantsMap.size() <= kMaximumAmountOfMixedParticipants);
 839
 840     // Loop through all participants. If they are in the mix map they
 841     // were mixed.
 842     for (MixerParticipantList::iterator participant = _participantList.begin();
 843          participant != _participantList.end();
 844          ++participant) {
 845         bool isMixed = false;
 846         for (std::map<int, MixerParticipant*>::iterator it =
 847                  mixedParticipantsMap.begin();
 848              it != mixedParticipantsMap.end();
 849              ++it) {
 850           if (it->second == *participant) {
 851             isMixed = true;
 852             break;
 853           }
 854         }
 855         (*participant)->_mixHistory->SetIsMixed(isMixed);
 856     }
 857 }
 858
 859 void AudioConferenceMixerImpl::ClearAudioFrameList(
 860     AudioFrameList* audioFrameList) {
 861     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
 862                  "ClearAudioFrameList(audioFrameList)");
 863     for (AudioFrameList::iterator iter = audioFrameList->begin();
 864          iter != audioFrameList->end();
 865          ++iter) {
 866         _audioFramePool->PushMemory(*iter);
 867     }
 868     audioFrameList->clear();
 869 }
 870
 871 void AudioConferenceMixerImpl::UpdateVADPositiveParticipants(
 872     AudioFrameList* mixList) {
 873     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
 874                  "UpdateVADPositiveParticipants(mixList)");
 875
 876     for (AudioFrameList::iterator iter = mixList->begin();
 877          iter != mixList->end();
 878          ++iter) {
 879         CalculateEnergy(**iter);
 880         if((*iter)->vad_activity_ == AudioFrame::kVadActive) {
 881             _scratchVadPositiveParticipants[
 882                 _scratchVadPositiveParticipantsAmount].participant =
 883                 (*iter)->id_;
 884             // TODO(andrew): to what should this be set?
 885             _scratchVadPositiveParticipants[
 886                 _scratchVadPositiveParticipantsAmount].level = 0;
 887             _scratchVadPositiveParticipantsAmount++;
 888         }
 889     }
 890 }
 891
 892 bool AudioConferenceMixerImpl::IsParticipantInList(
 893     MixerParticipant& participant,
 894     MixerParticipantList* participantList) const {
 895     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
 896                  "IsParticipantInList(participant,participantList)");
 897     for (MixerParticipantList::const_iterator iter = participantList->begin();
 898          iter != participantList->end();
 899          ++iter) {
 900         if(&participant == *iter) {
 901             return true;
 902         }
 903     }
 904     return false;
 905 }
 906
 907 bool AudioConferenceMixerImpl::AddParticipantToList(
 908     MixerParticipant& participant,
 909     MixerParticipantList* participantList) {
 910     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
 911                  "AddParticipantToList(participant, participantList)");
 912     participantList->push_back(&participant);
 913     // Make sure that the mixed status is correct for new MixerParticipant.
 914     participant._mixHistory->ResetMixedStatus();
 915     return true;
 916 }
 917
 918 bool AudioConferenceMixerImpl::RemoveParticipantFromList(
 919     MixerParticipant& participant,
 920     MixerParticipantList* participantList) {
 921     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
 922                  "RemoveParticipantFromList(participant, participantList)");
 923     for (MixerParticipantList::iterator iter = participantList->begin();
 924          iter != participantList->end();
 925          ++iter) {
 926         if(*iter == &participant) {
 927             participantList->erase(iter);
 928             // Participant is no longer mixed, reset to default.
 929             participant._mixHistory->ResetMixedStatus();
 930             return true;
 931         }
 932     }
 933     return false;
 934 }
 935
 936 int32_t AudioConferenceMixerImpl::MixFromList(
 937     AudioFrame& mixedAudio,
 938     const AudioFrameList* audioFrameList) {
 939     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
 940                  "MixFromList(mixedAudio, audioFrameList)");
 941     if(audioFrameList->empty()) return 0;
 942
 943     uint32_t position = 0;
 944     if(_numMixedParticipants == 1) {
 945         // No mixing required here; skip the saturation protection.
 946         AudioFrame* audioFrame = audioFrameList->front();
 947         mixedAudio.CopyFrom(*audioFrame);
 948         SetParticipantStatistics(&_scratchMixedParticipants[position],
 949                                  *audioFrame);
 950         return 0;
 951     }
 952
 953     for (AudioFrameList::const_iterator iter = audioFrameList->begin();
 954          iter != audioFrameList->end();
 955          ++iter) {
 956         if(position >= kMaximumAmountOfMixedParticipants) {
 957             WEBRTC_TRACE(
 958                 kTraceMemory,
 959                 kTraceAudioMixerServer,
 960                 _id,
 961                 "Trying to mix more than max amount of mixed participants:%d!",
 962                 kMaximumAmountOfMixedParticipants);
 963             // Assert and avoid crash
 964             assert(false);
 965             position = 0;
 966         }
 967         MixFrames(&mixedAudio, (*iter));
 968
 969         SetParticipantStatistics(&_scratchMixedParticipants[position],
 970                                  **iter);
 971
 972         position++;
 973     }
 974
 975     return 0;
 976 }
 977
 978 // TODO(andrew): consolidate this function with MixFromList.
 979 int32_t AudioConferenceMixerImpl::MixAnonomouslyFromList(
 980     AudioFrame& mixedAudio,
 981     const AudioFrameList* audioFrameList) {
 982     WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id,
 983                  "MixAnonomouslyFromList(mixedAudio, audioFrameList)");
 984
 985     if(audioFrameList->empty()) return 0;
 986
 987     if(_numMixedParticipants == 1) {
 988         // No mixing required here; skip the saturation protection.
 989         AudioFrame* audioFrame = audioFrameList->front();
 990         mixedAudio.CopyFrom(*audioFrame);
 991         return 0;
 992     }
 993
 994     for (AudioFrameList::const_iterator iter = audioFrameList->begin();
 995          iter != audioFrameList->end();
 996          ++iter) {
 997         MixFrames(&mixedAudio, *iter);
 998     }
 999     return 0;
1000 }
1001
1002 bool AudioConferenceMixerImpl::LimitMixedAudio(AudioFrame& mixedAudio) {
1003     if(_numMixedParticipants == 1) {
1004         return true;
1005     }
1006
1007     // Smoothly limit the mixed frame.
1008     const int error = _limiter->ProcessStream(&mixedAudio);
1009
1010     // And now we can safely restore the level. This procedure results in
1011     // some loss of resolution, deemed acceptable.
1012     //
1013     // It's possible to apply the gain in the AGC (with a target level of 0 dbFS
1014     // and compression gain of 6 dB). However, in the transition frame when this
1015     // is enabled (moving from one to two participants) it has the potential to
1016     // create discontinuities in the mixed frame.
1017     //
1018     // Instead we double the frame (with addition since left-shifting a
1019     // negative value is undefined).
1020     mixedAudio += mixedAudio;
1021
1022     if(error != _limiter->kNoError) {
1023         WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
1024                      "Error from AudioProcessing: %d", error);
1025         assert(false);
1026         return false;
1027     }
1028     return true;
1029 }
1030 }  // namespace webrtc