src/third_party/webrtc/modules/audio_device/android/opensles_output.cc

   1 /*
   2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include "webrtc/modules/audio_device/android/opensles_output.h"
  12
  13 #include <assert.h>
  14
  15 #include "webrtc/modules/audio_device/android/opensles_common.h"
  16 #include "webrtc/modules/audio_device/android/fine_audio_buffer.h"
  17 #include "webrtc/modules/audio_device/android/single_rw_fifo.h"
  18 #include "webrtc/modules/audio_device/audio_device_buffer.h"
  19 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
  20 #include "webrtc/system_wrappers/interface/thread_wrapper.h"
  21 #include "webrtc/system_wrappers/interface/trace.h"
  22
  23 #define VOID_RETURN
  24 #define OPENSL_RETURN_ON_FAILURE(op, ret_val)                    \
  25   do {                                                           \
  26     SLresult err = (op);                                         \
  27     if (err != SL_RESULT_SUCCESS) {                              \
  28       WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_,          \
  29                    "OpenSL error: %d", err);                     \
  30       assert(false);                                             \
  31       return ret_val;                                            \
  32     }                                                            \
  33   } while (0)
  34
  35 static const SLEngineOption kOption[] = {
  36   { SL_ENGINEOPTION_THREADSAFE, static_cast<SLuint32>(SL_BOOLEAN_TRUE) },
  37 };
  38
  39 enum {
  40   kNoUnderrun,
  41   kUnderrun,
  42 };
  43
  44 namespace webrtc {
  45
  46 OpenSlesOutput::OpenSlesOutput(const int32_t id)
  47     : id_(id),
  48       initialized_(false),
  49       speaker_initialized_(false),
  50       play_initialized_(false),
  51       crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
  52       playing_(false),
  53       num_fifo_buffers_needed_(0),
  54       number_underruns_(0),
  55       sles_engine_(NULL),
  56       sles_engine_itf_(NULL),
  57       sles_player_(NULL),
  58       sles_player_itf_(NULL),
  59       sles_player_sbq_itf_(NULL),
  60       sles_output_mixer_(NULL),
  61       audio_buffer_(NULL),
  62       active_queue_(0),
  63       speaker_sampling_rate_(kDefaultSampleRate),
  64       buffer_size_samples_(0),
  65       buffer_size_bytes_(0),
  66       playout_delay_(0) {
  67 }
  68
  69 OpenSlesOutput::~OpenSlesOutput() {
  70 }
  71
  72 int32_t OpenSlesOutput::SetAndroidAudioDeviceObjects(void* javaVM,
  73                                                      void* env,
  74                                                      void* context) {
  75   AudioManagerJni::SetAndroidAudioDeviceObjects(javaVM, env, context);
  76   return 0;
  77 }
  78
  79 void OpenSlesOutput::ClearAndroidAudioDeviceObjects() {
  80   AudioManagerJni::ClearAndroidAudioDeviceObjects();
  81 }
  82
  83 int32_t OpenSlesOutput::Init() {
  84   assert(!initialized_);
  85
  86   // Set up OpenSl engine.
  87   OPENSL_RETURN_ON_FAILURE(slCreateEngine(&sles_engine_, 1, kOption, 0,
  88                                           NULL, NULL),
  89                            -1);
  90   OPENSL_RETURN_ON_FAILURE((*sles_engine_)->Realize(sles_engine_,
  91                                                     SL_BOOLEAN_FALSE),
  92                            -1);
  93   OPENSL_RETURN_ON_FAILURE((*sles_engine_)->GetInterface(sles_engine_,
  94                                                          SL_IID_ENGINE,
  95                                                          &sles_engine_itf_),
  96                            -1);
  97   // Set up OpenSl output mix.
  98   OPENSL_RETURN_ON_FAILURE(
  99       (*sles_engine_itf_)->CreateOutputMix(sles_engine_itf_,
 100                                            &sles_output_mixer_,
 101                                            0,
 102                                            NULL,
 103                                            NULL),
 104       -1);
 105   OPENSL_RETURN_ON_FAILURE(
 106       (*sles_output_mixer_)->Realize(sles_output_mixer_,
 107                                      SL_BOOLEAN_FALSE),
 108       -1);
 109
 110   if (!InitSampleRate()) {
 111     return -1;
 112   }
 113   AllocateBuffers();
 114   initialized_ = true;
 115   return 0;
 116 }
 117
 118 int32_t OpenSlesOutput::Terminate() {
 119   // It is assumed that the caller has stopped recording before terminating.
 120   assert(!playing_);
 121   (*sles_output_mixer_)->Destroy(sles_output_mixer_);
 122   (*sles_engine_)->Destroy(sles_engine_);
 123   initialized_ = false;
 124   speaker_initialized_ = false;
 125   play_initialized_ = false;
 126   return 0;
 127 }
 128
 129 int32_t OpenSlesOutput::PlayoutDeviceName(uint16_t index,
 130                                           char name[kAdmMaxDeviceNameSize],
 131                                           char guid[kAdmMaxGuidSize]) {
 132   assert(index == 0);
 133   // Empty strings.
 134   name[0] = '\0';
 135   guid[0] = '\0';
 136   return 0;
 137 }
 138
 139 int32_t OpenSlesOutput::SetPlayoutDevice(uint16_t index) {
 140   assert(index == 0);
 141   return 0;
 142 }
 143
 144 int32_t OpenSlesOutput::PlayoutIsAvailable(bool& available) {  // NOLINT
 145   available = true;
 146   return 0;
 147 }
 148
 149 int32_t OpenSlesOutput::InitPlayout() {
 150   assert(initialized_);
 151   play_initialized_ = true;
 152   return 0;
 153 }
 154
 155 int32_t OpenSlesOutput::StartPlayout() {
 156   assert(play_initialized_);
 157   assert(!playing_);
 158   if (!CreateAudioPlayer()) {
 159     return -1;
 160   }
 161
 162   // Register callback to receive enqueued buffers.
 163   OPENSL_RETURN_ON_FAILURE(
 164       (*sles_player_sbq_itf_)->RegisterCallback(sles_player_sbq_itf_,
 165                                                 PlayerSimpleBufferQueueCallback,
 166                                                 this),
 167       -1);
 168   if (!EnqueueAllBuffers()) {
 169     return -1;
 170   }
 171
 172   {
 173     // To prevent the compiler from e.g. optimizing the code to
 174     // playing_ = StartCbThreads() which wouldn't have been thread safe.
 175     CriticalSectionScoped lock(crit_sect_.get());
 176     playing_ = true;
 177   }
 178   if (!StartCbThreads()) {
 179     playing_ = false;
 180   }
 181   return 0;
 182 }
 183
 184 int32_t OpenSlesOutput::StopPlayout() {
 185   StopCbThreads();
 186   DestroyAudioPlayer();
 187   playing_ = false;
 188   return 0;
 189 }
 190
 191 int32_t OpenSlesOutput::InitSpeaker() {
 192   assert(!playing_);
 193   speaker_initialized_ = true;
 194   return 0;
 195 }
 196
 197 int32_t OpenSlesOutput::SpeakerVolumeIsAvailable(bool& available) {  // NOLINT
 198   available = true;
 199   return 0;
 200 }
 201
 202 int32_t OpenSlesOutput::SetSpeakerVolume(uint32_t volume) {
 203   assert(speaker_initialized_);
 204   assert(initialized_);
 205   // TODO(hellner): implement.
 206   return 0;
 207 }
 208
 209 int32_t OpenSlesOutput::MaxSpeakerVolume(uint32_t& maxVolume) const {  // NOLINT
 210   assert(speaker_initialized_);
 211   assert(initialized_);
 212   // TODO(hellner): implement.
 213   maxVolume = 0;
 214   return 0;
 215 }
 216
 217 int32_t OpenSlesOutput::MinSpeakerVolume(uint32_t& minVolume) const {  // NOLINT
 218   assert(speaker_initialized_);
 219   assert(initialized_);
 220   // TODO(hellner): implement.
 221   minVolume = 0;
 222   return 0;
 223 }
 224
 225 int32_t OpenSlesOutput::SpeakerVolumeStepSize(
 226     uint16_t& stepSize) const {  // NOLINT
 227   assert(speaker_initialized_);
 228   stepSize = 1;
 229   return 0;
 230 }
 231
 232 int32_t OpenSlesOutput::SpeakerMuteIsAvailable(bool& available) {  // NOLINT
 233   available = false;
 234   return 0;
 235 }
 236
 237 int32_t OpenSlesOutput::StereoPlayoutIsAvailable(bool& available) {  // NOLINT
 238   available = false;
 239   return 0;
 240 }
 241
 242 int32_t OpenSlesOutput::SetStereoPlayout(bool enable) {
 243   if (enable) {
 244     assert(false);
 245     return -1;
 246   }
 247   return 0;
 248 }
 249
 250 int32_t OpenSlesOutput::StereoPlayout(bool& enabled) const {  // NOLINT
 251   enabled = kNumChannels == 2;
 252   return 0;
 253 }
 254
 255 int32_t OpenSlesOutput::PlayoutBuffer(
 256     AudioDeviceModule::BufferType& type,  // NOLINT
 257     uint16_t& sizeMS) const {  // NOLINT
 258   type = AudioDeviceModule::kAdaptiveBufferSize;
 259   sizeMS = playout_delay_;
 260   return 0;
 261 }
 262
 263 int32_t OpenSlesOutput::PlayoutDelay(uint16_t& delayMS) const {  // NOLINT
 264   delayMS = playout_delay_;
 265   return 0;
 266 }
 267
 268 void OpenSlesOutput::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) {
 269   audio_buffer_ = audioBuffer;
 270 }
 271
 272 int32_t OpenSlesOutput::SetLoudspeakerStatus(bool enable) {
 273   return 0;
 274 }
 275
 276 int32_t OpenSlesOutput::GetLoudspeakerStatus(bool& enabled) const {  // NOLINT
 277   enabled = true;
 278   return 0;
 279 }
 280
 281 int OpenSlesOutput::PlayoutDelayMs() {
 282   return playout_delay_;
 283 }
 284
 285 bool OpenSlesOutput::InitSampleRate() {
 286   if (!SetLowLatency()) {
 287     speaker_sampling_rate_ = kDefaultSampleRate;
 288     // Default is to use 10ms buffers.
 289     buffer_size_samples_ = speaker_sampling_rate_ * 10 / 1000;
 290   }
 291   if (audio_buffer_->SetPlayoutSampleRate(speaker_sampling_rate_) < 0) {
 292     return false;
 293   }
 294   if (audio_buffer_->SetPlayoutChannels(kNumChannels) < 0) {
 295     return false;
 296   }
 297   UpdatePlayoutDelay();
 298   return true;
 299 }
 300
 301 void OpenSlesOutput::UpdatePlayoutDelay() {
 302   // TODO(hellner): Add accurate delay estimate.
 303   // On average half the current buffer will have been played out.
 304   int outstanding_samples = (TotalBuffersUsed() - 0.5) * buffer_size_samples_;
 305   playout_delay_ = outstanding_samples / (speaker_sampling_rate_ / 1000);
 306 }
 307
 308 bool OpenSlesOutput::SetLowLatency() {
 309   if (!audio_manager_.low_latency_supported()) {
 310     return false;
 311   }
 312   buffer_size_samples_ = audio_manager_.native_buffer_size();
 313   assert(buffer_size_samples_ > 0);
 314   speaker_sampling_rate_ = audio_manager_.native_output_sample_rate();
 315   assert(speaker_sampling_rate_ > 0);
 316   return true;
 317 }
 318
 319 void OpenSlesOutput::CalculateNumFifoBuffersNeeded() {
 320   int number_of_bytes_needed =
 321       (speaker_sampling_rate_ * kNumChannels * sizeof(int16_t)) * 10 / 1000;
 322
 323   // Ceiling of integer division: 1 + ((x - 1) / y)
 324   int buffers_per_10_ms =
 325       1 + ((number_of_bytes_needed - 1) / buffer_size_bytes_);
 326   // |num_fifo_buffers_needed_| is a multiple of 10ms of buffered up audio.
 327   num_fifo_buffers_needed_ = kNum10MsToBuffer * buffers_per_10_ms;
 328 }
 329
 330 void OpenSlesOutput::AllocateBuffers() {
 331   // Allocate fine buffer to provide frames of the desired size.
 332   buffer_size_bytes_ = buffer_size_samples_ * kNumChannels * sizeof(int16_t);
 333   fine_buffer_.reset(new FineAudioBuffer(audio_buffer_, buffer_size_bytes_,
 334                                          speaker_sampling_rate_));
 335
 336   // Allocate FIFO to handle passing buffers between processing and OpenSl
 337   // threads.
 338   CalculateNumFifoBuffersNeeded();  // Needs |buffer_size_bytes_| to be known
 339   assert(num_fifo_buffers_needed_ > 0);
 340   fifo_.reset(new SingleRwFifo(num_fifo_buffers_needed_));
 341
 342   // Allocate the memory area to be used.
 343   play_buf_.reset(new scoped_ptr<int8_t[]>[TotalBuffersUsed()]);
 344   int required_buffer_size = fine_buffer_->RequiredBufferSizeBytes();
 345   for (int i = 0; i < TotalBuffersUsed(); ++i) {
 346     play_buf_[i].reset(new int8_t[required_buffer_size]);
 347   }
 348 }
 349
 350 int OpenSlesOutput::TotalBuffersUsed() const {
 351   return num_fifo_buffers_needed_ + kNumOpenSlBuffers;
 352 }
 353
 354 bool OpenSlesOutput::EnqueueAllBuffers() {
 355   active_queue_ = 0;
 356   number_underruns_ = 0;
 357   for (int i = 0; i < kNumOpenSlBuffers; ++i) {
 358     memset(play_buf_[i].get(), 0, buffer_size_bytes_);
 359     OPENSL_RETURN_ON_FAILURE(
 360         (*sles_player_sbq_itf_)->Enqueue(
 361             sles_player_sbq_itf_,
 362             reinterpret_cast<void*>(play_buf_[i].get()),
 363             buffer_size_bytes_),
 364         false);
 365   }
 366   // OpenSL playing has been stopped. I.e. only this thread is touching
 367   // |fifo_|.
 368   while (fifo_->size() != 0) {
 369     // Underrun might have happened when pushing new buffers to the FIFO.
 370     fifo_->Pop();
 371   }
 372   for (int i = kNumOpenSlBuffers; i < TotalBuffersUsed(); ++i) {
 373     memset(play_buf_[i].get(), 0, buffer_size_bytes_);
 374     fifo_->Push(play_buf_[i].get());
 375   }
 376   return true;
 377 }
 378
 379 bool OpenSlesOutput::CreateAudioPlayer() {
 380   if (!event_.Start()) {
 381     assert(false);
 382     return false;
 383   }
 384   SLDataLocator_AndroidSimpleBufferQueue simple_buf_queue = {
 385     SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
 386     static_cast<SLuint32>(kNumOpenSlBuffers)
 387   };
 388   SLDataFormat_PCM configuration =
 389       webrtc_opensl::CreatePcmConfiguration(speaker_sampling_rate_);
 390   SLDataSource audio_source = { &simple_buf_queue, &configuration };
 391
 392   SLDataLocator_OutputMix locator_outputmix;
 393   // Setup the data sink structure.
 394   locator_outputmix.locatorType = SL_DATALOCATOR_OUTPUTMIX;
 395   locator_outputmix.outputMix = sles_output_mixer_;
 396   SLDataSink audio_sink = { &locator_outputmix, NULL };
 397
 398   // Interfaces for streaming audio data, setting volume and Android are needed.
 399   // Note the interfaces still need to be initialized. This only tells OpenSl
 400   // that the interfaces will be needed at some point.
 401   SLInterfaceID ids[kNumInterfaces] = {
 402     SL_IID_BUFFERQUEUE, SL_IID_VOLUME, SL_IID_ANDROIDCONFIGURATION };
 403   SLboolean req[kNumInterfaces] = {
 404     SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE };
 405   OPENSL_RETURN_ON_FAILURE(
 406       (*sles_engine_itf_)->CreateAudioPlayer(sles_engine_itf_, &sles_player_,
 407                                              &audio_source, &audio_sink,
 408                                              kNumInterfaces, ids, req),
 409       false);
 410
 411   SLAndroidConfigurationItf player_config;
 412   OPENSL_RETURN_ON_FAILURE(
 413       (*sles_player_)->GetInterface(sles_player_,
 414                                     SL_IID_ANDROIDCONFIGURATION,
 415                                     &player_config),
 416       false);
 417
 418   // Set audio player configuration to SL_ANDROID_STREAM_VOICE which corresponds
 419   // to android.media.AudioManager.STREAM_VOICE_CALL.
 420   SLint32 stream_type = SL_ANDROID_STREAM_VOICE;
 421   OPENSL_RETURN_ON_FAILURE(
 422       (*player_config)->SetConfiguration(player_config,
 423                                          SL_ANDROID_KEY_STREAM_TYPE,
 424                                          &stream_type,
 425                                          sizeof(SLint32)),
 426       false);
 427
 428   // Realize the player in synchronous mode.
 429   OPENSL_RETURN_ON_FAILURE((*sles_player_)->Realize(sles_player_,
 430                                                     SL_BOOLEAN_FALSE),
 431                            false);
 432   OPENSL_RETURN_ON_FAILURE(
 433       (*sles_player_)->GetInterface(sles_player_, SL_IID_PLAY,
 434                                     &sles_player_itf_),
 435       false);
 436   OPENSL_RETURN_ON_FAILURE(
 437       (*sles_player_)->GetInterface(sles_player_, SL_IID_BUFFERQUEUE,
 438                                     &sles_player_sbq_itf_),
 439       false);
 440   return true;
 441 }
 442
 443 void OpenSlesOutput::DestroyAudioPlayer() {
 444   SLAndroidSimpleBufferQueueItf sles_player_sbq_itf = sles_player_sbq_itf_;
 445   {
 446     CriticalSectionScoped lock(crit_sect_.get());
 447     sles_player_sbq_itf_ = NULL;
 448     sles_player_itf_ = NULL;
 449   }
 450   event_.Stop();
 451   if (sles_player_sbq_itf) {
 452     // Release all buffers currently queued up.
 453     OPENSL_RETURN_ON_FAILURE(
 454         (*sles_player_sbq_itf)->Clear(sles_player_sbq_itf),
 455         VOID_RETURN);
 456   }
 457
 458   if (sles_player_) {
 459     (*sles_player_)->Destroy(sles_player_);
 460     sles_player_ = NULL;
 461   }
 462 }
 463
 464 bool OpenSlesOutput::HandleUnderrun(int event_id, int event_msg) {
 465   if (!playing_) {
 466     return false;
 467   }
 468   if (event_id == kNoUnderrun) {
 469     return false;
 470   }
 471   WEBRTC_TRACE(kTraceWarning, kTraceAudioDevice, id_, "Audio underrun");
 472   assert(event_id == kUnderrun);
 473   assert(event_msg > 0);
 474   // Wait for all enqueued buffers to be flushed.
 475   if (event_msg != kNumOpenSlBuffers) {
 476     return true;
 477   }
 478   // All buffers have been flushed. Restart the audio from scratch.
 479   // No need to check sles_player_itf_ as playing_ would be false before it is
 480   // set to NULL.
 481   OPENSL_RETURN_ON_FAILURE(
 482       (*sles_player_itf_)->SetPlayState(sles_player_itf_,
 483                                         SL_PLAYSTATE_STOPPED),
 484       true);
 485   EnqueueAllBuffers();
 486   OPENSL_RETURN_ON_FAILURE(
 487       (*sles_player_itf_)->SetPlayState(sles_player_itf_,
 488                                         SL_PLAYSTATE_PLAYING),
 489       true);
 490   return true;
 491 }
 492
 493 void OpenSlesOutput::PlayerSimpleBufferQueueCallback(
 494     SLAndroidSimpleBufferQueueItf sles_player_sbq_itf,
 495     void* p_context) {
 496   OpenSlesOutput* audio_device = reinterpret_cast<OpenSlesOutput*>(p_context);
 497   audio_device->PlayerSimpleBufferQueueCallbackHandler(sles_player_sbq_itf);
 498 }
 499
 500 void OpenSlesOutput::PlayerSimpleBufferQueueCallbackHandler(
 501     SLAndroidSimpleBufferQueueItf sles_player_sbq_itf) {
 502   if (fifo_->size() <= 0 || number_underruns_ > 0) {
 503     ++number_underruns_;
 504     event_.SignalEvent(kUnderrun, number_underruns_);
 505     return;
 506   }
 507   int8_t* audio = fifo_->Pop();
 508   if (audio)
 509   OPENSL_RETURN_ON_FAILURE(
 510       (*sles_player_sbq_itf)->Enqueue(sles_player_sbq_itf,
 511                                       audio,
 512                                       buffer_size_bytes_),
 513       VOID_RETURN);
 514   event_.SignalEvent(kNoUnderrun, 0);
 515 }
 516
 517 bool OpenSlesOutput::StartCbThreads() {
 518   play_thread_.reset(ThreadWrapper::CreateThread(CbThread,
 519                                                  this,
 520                                                  kRealtimePriority,
 521                                                  "opensl_play_thread"));
 522   assert(play_thread_.get());
 523   OPENSL_RETURN_ON_FAILURE(
 524       (*sles_player_itf_)->SetPlayState(sles_player_itf_,
 525                                         SL_PLAYSTATE_PLAYING),
 526       false);
 527
 528   unsigned int thread_id = 0;
 529   if (!play_thread_->Start(thread_id)) {
 530     assert(false);
 531     return false;
 532   }
 533   return true;
 534 }
 535
 536 void OpenSlesOutput::StopCbThreads() {
 537   {
 538     CriticalSectionScoped lock(crit_sect_.get());
 539     playing_ = false;
 540   }
 541   if (sles_player_itf_) {
 542     OPENSL_RETURN_ON_FAILURE(
 543         (*sles_player_itf_)->SetPlayState(sles_player_itf_,
 544                                           SL_PLAYSTATE_STOPPED),
 545         VOID_RETURN);
 546   }
 547   if (play_thread_.get() == NULL) {
 548     return;
 549   }
 550   event_.Stop();
 551   if (play_thread_->Stop()) {
 552     play_thread_.reset();
 553   } else {
 554     assert(false);
 555   }
 556 }
 557
 558 bool OpenSlesOutput::CbThread(void* context) {
 559   return reinterpret_cast<OpenSlesOutput*>(context)->CbThreadImpl();
 560 }
 561
 562 bool OpenSlesOutput::CbThreadImpl() {
 563   assert(fine_buffer_.get() != NULL);
 564   int event_id;
 565   int event_msg;
 566   // event_ must not be waited on while a lock has been taken.
 567   event_.WaitOnEvent(&event_id, &event_msg);
 568
 569   CriticalSectionScoped lock(crit_sect_.get());
 570   if (HandleUnderrun(event_id, event_msg)) {
 571     return playing_;
 572   }
 573   // if fifo_ is not full it means next item in memory must be free.
 574   while (fifo_->size() < num_fifo_buffers_needed_ && playing_) {
 575     int8_t* audio = play_buf_[active_queue_].get();
 576     fine_buffer_->GetBufferData(audio);
 577     fifo_->Push(audio);
 578     active_queue_ = (active_queue_ + 1) % TotalBuffersUsed();
 579   }
 580   return playing_;
 581 }
 582
 583 }  // namespace webrtc