src/third_party/webrtc/modules/audio_device/android/opensles_input.cc

   1 /*
   2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include "webrtc/modules/audio_device/android/opensles_input.h"
  12
  13 #include <assert.h>
  14
  15 #include "webrtc/modules/audio_device/android/audio_common.h"
  16 #include "webrtc/modules/audio_device/android/opensles_common.h"
  17 #include "webrtc/modules/audio_device/android/single_rw_fifo.h"
  18 #include "webrtc/modules/audio_device/audio_device_buffer.h"
  19 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
  20 #include "webrtc/system_wrappers/interface/thread_wrapper.h"
  21 #include "webrtc/system_wrappers/interface/trace.h"
  22
  23 #define VOID_RETURN
  24 #define OPENSL_RETURN_ON_FAILURE(op, ret_val)                    \
  25   do {                                                           \
  26     SLresult err = (op);                                         \
  27     if (err != SL_RESULT_SUCCESS) {                              \
  28       WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_,          \
  29                    "OpenSL error: %d", err);                     \
  30       assert(false);                                             \
  31       return ret_val;                                            \
  32     }                                                            \
  33   } while (0)
  34
  35 static const SLEngineOption kOption[] = {
  36   { SL_ENGINEOPTION_THREADSAFE, static_cast<SLuint32>(SL_BOOLEAN_TRUE) },
  37 };
  38
  39 enum {
  40   kNoOverrun,
  41   kOverrun,
  42 };
  43
  44 namespace webrtc {
  45
  46 OpenSlesInput::OpenSlesInput(
  47     const int32_t id, PlayoutDelayProvider* delay_provider)
  48     : id_(id),
  49       delay_provider_(delay_provider),
  50       initialized_(false),
  51       mic_initialized_(false),
  52       rec_initialized_(false),
  53       crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
  54       recording_(false),
  55       num_fifo_buffers_needed_(0),
  56       number_overruns_(0),
  57       sles_engine_(NULL),
  58       sles_engine_itf_(NULL),
  59       sles_recorder_(NULL),
  60       sles_recorder_itf_(NULL),
  61       sles_recorder_sbq_itf_(NULL),
  62       audio_buffer_(NULL),
  63       active_queue_(0),
  64       rec_sampling_rate_(0),
  65       agc_enabled_(false),
  66       recording_delay_(0) {
  67 }
  68
  69 OpenSlesInput::~OpenSlesInput() {
  70 }
  71
  72 int32_t OpenSlesInput::SetAndroidAudioDeviceObjects(void* javaVM,
  73                                                     void* env,
  74                                                     void* context) {
  75   return 0;
  76 }
  77
  78 void OpenSlesInput::ClearAndroidAudioDeviceObjects() {
  79 }
  80
  81 int32_t OpenSlesInput::Init() {
  82   assert(!initialized_);
  83
  84   // Set up OpenSL engine.
  85   OPENSL_RETURN_ON_FAILURE(slCreateEngine(&sles_engine_, 1, kOption, 0,
  86                                           NULL, NULL),
  87                            -1);
  88   OPENSL_RETURN_ON_FAILURE((*sles_engine_)->Realize(sles_engine_,
  89                                                     SL_BOOLEAN_FALSE),
  90                            -1);
  91   OPENSL_RETURN_ON_FAILURE((*sles_engine_)->GetInterface(sles_engine_,
  92                                                          SL_IID_ENGINE,
  93                                                          &sles_engine_itf_),
  94                            -1);
  95
  96   if (InitSampleRate() != 0) {
  97     return -1;
  98   }
  99   AllocateBuffers();
 100   initialized_ = true;
 101   return 0;
 102 }
 103
 104 int32_t OpenSlesInput::Terminate() {
 105   // It is assumed that the caller has stopped recording before terminating.
 106   assert(!recording_);
 107   (*sles_engine_)->Destroy(sles_engine_);
 108   initialized_ = false;
 109   mic_initialized_ = false;
 110   rec_initialized_ = false;
 111   return 0;
 112 }
 113
 114 int32_t OpenSlesInput::RecordingDeviceName(uint16_t index,
 115                                            char name[kAdmMaxDeviceNameSize],
 116                                            char guid[kAdmMaxGuidSize]) {
 117   assert(index == 0);
 118   // Empty strings.
 119   name[0] = '\0';
 120   guid[0] = '\0';
 121   return 0;
 122 }
 123
 124 int32_t OpenSlesInput::SetRecordingDevice(uint16_t index) {
 125   assert(index == 0);
 126   return 0;
 127 }
 128
 129 int32_t OpenSlesInput::RecordingIsAvailable(bool& available) {  // NOLINT
 130   available = true;
 131   return 0;
 132 }
 133
 134 int32_t OpenSlesInput::InitRecording() {
 135   assert(initialized_);
 136   rec_initialized_ = true;
 137   return 0;
 138 }
 139
 140 int32_t OpenSlesInput::StartRecording() {
 141   assert(rec_initialized_);
 142   assert(!recording_);
 143   if (!CreateAudioRecorder()) {
 144     return -1;
 145   }
 146   // Setup to receive buffer queue event callbacks.
 147   OPENSL_RETURN_ON_FAILURE(
 148       (*sles_recorder_sbq_itf_)->RegisterCallback(
 149           sles_recorder_sbq_itf_,
 150           RecorderSimpleBufferQueueCallback,
 151           this),
 152       -1);
 153
 154   if (!EnqueueAllBuffers()) {
 155     return -1;
 156   }
 157
 158   {
 159     // To prevent the compiler from e.g. optimizing the code to
 160     // recording_ = StartCbThreads() which wouldn't have been thread safe.
 161     CriticalSectionScoped lock(crit_sect_.get());
 162     recording_ = true;
 163   }
 164   if (!StartCbThreads()) {
 165     recording_ = false;
 166     return -1;
 167   }
 168   return 0;
 169 }
 170
 171 int32_t OpenSlesInput::StopRecording() {
 172   StopCbThreads();
 173   DestroyAudioRecorder();
 174   recording_ = false;
 175   return 0;
 176 }
 177
 178 int32_t OpenSlesInput::SetAGC(bool enable) {
 179   agc_enabled_ = enable;
 180   return 0;
 181 }
 182
 183 int32_t OpenSlesInput::InitMicrophone() {
 184   assert(initialized_);
 185   assert(!recording_);
 186   mic_initialized_ = true;
 187   return 0;
 188 }
 189
 190 int32_t OpenSlesInput::MicrophoneVolumeIsAvailable(bool& available) {  // NOLINT
 191   available = false;
 192   return 0;
 193 }
 194
 195 int32_t OpenSlesInput::MinMicrophoneVolume(
 196     uint32_t& minVolume) const {  // NOLINT
 197   minVolume = 0;
 198   return 0;
 199 }
 200
 201 int32_t OpenSlesInput::MicrophoneVolumeStepSize(
 202     uint16_t& stepSize) const {
 203   stepSize = 1;
 204   return 0;
 205 }
 206
 207 int32_t OpenSlesInput::MicrophoneMuteIsAvailable(bool& available) {  // NOLINT
 208   available = false;  // Mic mute not supported on Android
 209   return 0;
 210 }
 211
 212 int32_t OpenSlesInput::MicrophoneBoostIsAvailable(bool& available) {  // NOLINT
 213   available = false;  // Mic boost not supported on Android.
 214   return 0;
 215 }
 216
 217 int32_t OpenSlesInput::SetMicrophoneBoost(bool enable) {
 218   assert(false);
 219   return -1;  // Not supported
 220 }
 221
 222 int32_t OpenSlesInput::MicrophoneBoost(bool& enabled) const {  // NOLINT
 223   assert(false);
 224   return -1;  // Not supported
 225 }
 226
 227 int32_t OpenSlesInput::StereoRecordingIsAvailable(bool& available) {  // NOLINT
 228   available = false;  // Stereo recording not supported on Android.
 229   return 0;
 230 }
 231
 232 int32_t OpenSlesInput::StereoRecording(bool& enabled) const {  // NOLINT
 233   enabled = false;
 234   return 0;
 235 }
 236
 237 int32_t OpenSlesInput::RecordingDelay(uint16_t& delayMS) const {  // NOLINT
 238   delayMS = recording_delay_;
 239   return 0;
 240 }
 241
 242 void OpenSlesInput::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) {
 243   audio_buffer_ = audioBuffer;
 244 }
 245
 246 int OpenSlesInput::InitSampleRate() {
 247   UpdateSampleRate();
 248   audio_buffer_->SetRecordingSampleRate(rec_sampling_rate_);
 249   audio_buffer_->SetRecordingChannels(kNumChannels);
 250   UpdateRecordingDelay();
 251   return 0;
 252 }
 253
 254 int OpenSlesInput::buffer_size_samples() const {
 255   // Since there is no low latency recording, use buffer size corresponding to
 256   // 10ms of data since that's the framesize WebRTC uses. Getting any other
 257   // size would require patching together buffers somewhere before passing them
 258   // to WebRTC.
 259   return rec_sampling_rate_ * 10 / 1000;
 260 }
 261
 262 int OpenSlesInput::buffer_size_bytes() const {
 263   return buffer_size_samples() * kNumChannels * sizeof(int16_t);
 264 }
 265
 266 void OpenSlesInput::UpdateRecordingDelay() {
 267   // TODO(hellner): Add accurate delay estimate.
 268   // On average half the current buffer will have been filled with audio.
 269   int outstanding_samples =
 270       (TotalBuffersUsed() - 0.5) * buffer_size_samples();
 271   recording_delay_ = outstanding_samples / (rec_sampling_rate_ / 1000);
 272 }
 273
 274 void OpenSlesInput::UpdateSampleRate() {
 275   rec_sampling_rate_ = audio_manager_.low_latency_supported() ?
 276       audio_manager_.native_output_sample_rate() : kDefaultSampleRate;
 277 }
 278
 279 void OpenSlesInput::CalculateNumFifoBuffersNeeded() {
 280   // Buffer size is 10ms of data.
 281   num_fifo_buffers_needed_ = kNum10MsToBuffer;
 282 }
 283
 284 void OpenSlesInput::AllocateBuffers() {
 285   // Allocate FIFO to handle passing buffers between processing and OpenSL
 286   // threads.
 287   CalculateNumFifoBuffersNeeded();
 288   assert(num_fifo_buffers_needed_ > 0);
 289   fifo_.reset(new SingleRwFifo(num_fifo_buffers_needed_));
 290
 291   // Allocate the memory area to be used.
 292   rec_buf_.reset(new scoped_ptr<int8_t[]>[TotalBuffersUsed()]);
 293   for (int i = 0; i < TotalBuffersUsed(); ++i) {
 294     rec_buf_[i].reset(new int8_t[buffer_size_bytes()]);
 295   }
 296 }
 297
 298 int OpenSlesInput::TotalBuffersUsed() const {
 299   return num_fifo_buffers_needed_ + kNumOpenSlBuffers;
 300 }
 301
 302 bool OpenSlesInput::EnqueueAllBuffers() {
 303   active_queue_ = 0;
 304   number_overruns_ = 0;
 305   for (int i = 0; i < kNumOpenSlBuffers; ++i) {
 306     memset(rec_buf_[i].get(), 0, buffer_size_bytes());
 307     OPENSL_RETURN_ON_FAILURE(
 308         (*sles_recorder_sbq_itf_)->Enqueue(
 309             sles_recorder_sbq_itf_,
 310             reinterpret_cast<void*>(rec_buf_[i].get()),
 311             buffer_size_bytes()),
 312         false);
 313   }
 314   // In case of underrun the fifo will be at capacity. In case of first enqueue
 315   // no audio can have been returned yet meaning fifo must be empty. Any other
 316   // values are unexpected.
 317   assert(fifo_->size() == fifo_->capacity() ||
 318          fifo_->size() == 0);
 319   // OpenSL recording has been stopped. I.e. only this thread is touching
 320   // |fifo_|.
 321   while (fifo_->size() != 0) {
 322     // Clear the fifo.
 323     fifo_->Pop();
 324   }
 325   return true;
 326 }
 327
 328 bool OpenSlesInput::CreateAudioRecorder() {
 329   if (!event_.Start()) {
 330     assert(false);
 331     return false;
 332   }
 333   SLDataLocator_IODevice micLocator = {
 334     SL_DATALOCATOR_IODEVICE, SL_IODEVICE_AUDIOINPUT,
 335     SL_DEFAULTDEVICEID_AUDIOINPUT, NULL };
 336   SLDataSource audio_source = { &micLocator, NULL };
 337
 338   SLDataLocator_AndroidSimpleBufferQueue simple_buf_queue = {
 339     SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
 340     static_cast<SLuint32>(TotalBuffersUsed())
 341   };
 342   SLDataFormat_PCM configuration =
 343       webrtc_opensl::CreatePcmConfiguration(rec_sampling_rate_);
 344   SLDataSink audio_sink = { &simple_buf_queue, &configuration };
 345
 346   // Interfaces for recording android audio data and Android are needed.
 347   // Note the interfaces still need to be initialized. This only tells OpenSl
 348   // that the interfaces will be needed at some point.
 349   const SLInterfaceID id[kNumInterfaces] = {
 350     SL_IID_ANDROIDSIMPLEBUFFERQUEUE, SL_IID_ANDROIDCONFIGURATION };
 351   const SLboolean req[kNumInterfaces] = {
 352     SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE };
 353   OPENSL_RETURN_ON_FAILURE(
 354       (*sles_engine_itf_)->CreateAudioRecorder(sles_engine_itf_,
 355                                                &sles_recorder_,
 356                                                &audio_source,
 357                                                &audio_sink,
 358                                                kNumInterfaces,
 359                                                id,
 360                                                req),
 361       false);
 362
 363   // Realize the recorder in synchronous mode.
 364   OPENSL_RETURN_ON_FAILURE((*sles_recorder_)->Realize(sles_recorder_,
 365                                                       SL_BOOLEAN_FALSE),
 366                            false);
 367   OPENSL_RETURN_ON_FAILURE(
 368       (*sles_recorder_)->GetInterface(sles_recorder_, SL_IID_RECORD,
 369                                       static_cast<void*>(&sles_recorder_itf_)),
 370       false);
 371   OPENSL_RETURN_ON_FAILURE(
 372       (*sles_recorder_)->GetInterface(
 373           sles_recorder_,
 374           SL_IID_ANDROIDSIMPLEBUFFERQUEUE,
 375           static_cast<void*>(&sles_recorder_sbq_itf_)),
 376       false);
 377   return true;
 378 }
 379
 380 void OpenSlesInput::DestroyAudioRecorder() {
 381   event_.Stop();
 382   if (sles_recorder_sbq_itf_) {
 383     // Release all buffers currently queued up.
 384     OPENSL_RETURN_ON_FAILURE(
 385         (*sles_recorder_sbq_itf_)->Clear(sles_recorder_sbq_itf_),
 386         VOID_RETURN);
 387     sles_recorder_sbq_itf_ = NULL;
 388   }
 389   sles_recorder_itf_ = NULL;
 390
 391   if (sles_recorder_) {
 392     (*sles_recorder_)->Destroy(sles_recorder_);
 393     sles_recorder_ = NULL;
 394   }
 395 }
 396
 397 bool OpenSlesInput::HandleOverrun(int event_id, int event_msg) {
 398   if (!recording_) {
 399     return false;
 400   }
 401   if (event_id == kNoOverrun) {
 402     return false;
 403   }
 404   WEBRTC_TRACE(kTraceWarning, kTraceAudioDevice, id_, "Audio overrun");
 405   assert(event_id == kOverrun);
 406   assert(event_msg > 0);
 407   // Wait for all enqueued buffers be flushed.
 408   if (event_msg != kNumOpenSlBuffers) {
 409     return true;
 410   }
 411   // All buffers passed to OpenSL have been flushed. Restart the audio from
 412   // scratch.
 413   // No need to check sles_recorder_itf_ as recording_ would be false before it
 414   // is set to NULL.
 415   OPENSL_RETURN_ON_FAILURE(
 416       (*sles_recorder_itf_)->SetRecordState(sles_recorder_itf_,
 417                                             SL_RECORDSTATE_STOPPED),
 418       true);
 419   EnqueueAllBuffers();
 420   OPENSL_RETURN_ON_FAILURE(
 421       (*sles_recorder_itf_)->SetRecordState(sles_recorder_itf_,
 422                                             SL_RECORDSTATE_RECORDING),
 423       true);
 424   return true;
 425 }
 426
 427 void OpenSlesInput::RecorderSimpleBufferQueueCallback(
 428     SLAndroidSimpleBufferQueueItf queue_itf,
 429     void* context) {
 430   OpenSlesInput* audio_device = reinterpret_cast<OpenSlesInput*>(context);
 431   audio_device->RecorderSimpleBufferQueueCallbackHandler(queue_itf);
 432 }
 433
 434 void OpenSlesInput::RecorderSimpleBufferQueueCallbackHandler(
 435     SLAndroidSimpleBufferQueueItf queue_itf) {
 436   if (fifo_->size() >= fifo_->capacity() || number_overruns_ > 0) {
 437     ++number_overruns_;
 438     event_.SignalEvent(kOverrun, number_overruns_);
 439     return;
 440   }
 441   int8_t* audio = rec_buf_[active_queue_].get();
 442   // There is at least one spot available in the fifo.
 443   fifo_->Push(audio);
 444   active_queue_ = (active_queue_ + 1) % TotalBuffersUsed();
 445   event_.SignalEvent(kNoOverrun, 0);
 446   // active_queue_ is indexing the next buffer to record to. Since the current
 447   // buffer has been recorded it means that the buffer index
 448   // kNumOpenSlBuffers - 1 past |active_queue_| contains the next free buffer.
 449   // Since |fifo_| wasn't at capacity, at least one buffer is free to be used.
 450   int next_free_buffer =
 451       (active_queue_ + kNumOpenSlBuffers - 1) % TotalBuffersUsed();
 452   OPENSL_RETURN_ON_FAILURE(
 453       (*sles_recorder_sbq_itf_)->Enqueue(
 454           sles_recorder_sbq_itf_,
 455           reinterpret_cast<void*>(rec_buf_[next_free_buffer].get()),
 456           buffer_size_bytes()),
 457       VOID_RETURN);
 458 }
 459
 460 bool OpenSlesInput::StartCbThreads() {
 461   rec_thread_.reset(ThreadWrapper::CreateThread(CbThread,
 462                                                 this,
 463                                                 kRealtimePriority,
 464                                                 "opensl_rec_thread"));
 465   assert(rec_thread_.get());
 466   unsigned int thread_id = 0;
 467   if (!rec_thread_->Start(thread_id)) {
 468     assert(false);
 469     return false;
 470   }
 471   OPENSL_RETURN_ON_FAILURE(
 472       (*sles_recorder_itf_)->SetRecordState(sles_recorder_itf_,
 473                                             SL_RECORDSTATE_RECORDING),
 474       false);
 475   return true;
 476 }
 477
 478 void OpenSlesInput::StopCbThreads() {
 479   {
 480     CriticalSectionScoped lock(crit_sect_.get());
 481     recording_ = false;
 482   }
 483   if (sles_recorder_itf_) {
 484     OPENSL_RETURN_ON_FAILURE(
 485         (*sles_recorder_itf_)->SetRecordState(sles_recorder_itf_,
 486                                               SL_RECORDSTATE_STOPPED),
 487         VOID_RETURN);
 488   }
 489   if (rec_thread_.get() == NULL) {
 490     return;
 491   }
 492   event_.Stop();
 493   if (rec_thread_->Stop()) {
 494     rec_thread_.reset();
 495   } else {
 496     assert(false);
 497   }
 498 }
 499
 500 bool OpenSlesInput::CbThread(void* context) {
 501   return reinterpret_cast<OpenSlesInput*>(context)->CbThreadImpl();
 502 }
 503
 504 bool OpenSlesInput::CbThreadImpl() {
 505   int event_id;
 506   int event_msg;
 507   // event_ must not be waited on while a lock has been taken.
 508   event_.WaitOnEvent(&event_id, &event_msg);
 509
 510   CriticalSectionScoped lock(crit_sect_.get());
 511   if (HandleOverrun(event_id, event_msg)) {
 512     return recording_;
 513   }
 514   // If the fifo_ has audio data process it.
 515   while (fifo_->size() > 0 && recording_) {
 516     int8_t* audio = fifo_->Pop();
 517     audio_buffer_->SetRecordedBuffer(audio, buffer_size_samples());
 518     audio_buffer_->SetVQEData(delay_provider_->PlayoutDelayMs(),
 519                               recording_delay_, 0);
 520     audio_buffer_->DeliverRecordedData();
 521   }
 522   return recording_;
 523 }
 524
 525 }  // namespace webrtc