src/content/renderer/media/speech_recognition_audio_sink.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/speech_recognition_audio_sink.h"
   6
   7 #include "base/logging.h"
   8 #include "base/memory/shared_memory.h"
   9 #include "base/time/time.h"
  10 #include "content/renderer/media/media_stream_audio_source.h"
  11 #include "media/audio/audio_parameters.h"
  12 #include "media/base/audio_fifo.h"
  13
  14 namespace content {
  15
  16 SpeechRecognitionAudioSink::SpeechRecognitionAudioSink(
  17     const blink::WebMediaStreamTrack& track,
  18     const media::AudioParameters& params,
  19     const base::SharedMemoryHandle memory,
  20     scoped_ptr<base::SyncSocket> socket,
  21     const OnStoppedCB& on_stopped_cb)
  22     : track_(track),
  23       shared_memory_(memory, false),
  24       socket_(socket.Pass()),
  25       output_params_(params),
  26       track_stopped_(false),
  27       buffer_index_(0),
  28       on_stopped_cb_(on_stopped_cb) {
  29   DCHECK(socket_.get());
  30   DCHECK(main_render_thread_checker_.CalledOnValidThread());
  31   DCHECK(params.IsValid());
  32   DCHECK(IsSupportedTrack(track));
  33   const size_t kSharedMemorySize = sizeof(media::AudioInputBufferParameters) +
  34                                    media::AudioBus::CalculateMemorySize(params);
  35   CHECK(shared_memory_.Map(kSharedMemorySize));
  36
  37   media::AudioInputBuffer* buffer =
  38       static_cast<media::AudioInputBuffer*>(shared_memory_.memory());
  39
  40   // The peer must manage their own counter and reset it to 0.
  41   DCHECK_EQ(0U, buffer->params.size);
  42   output_bus_ = media::AudioBus::WrapMemory(params, buffer->audio);
  43
  44   // Connect this audio sink to the track
  45   MediaStreamAudioSink::AddToAudioTrack(this, track_);
  46 }
  47
  48 SpeechRecognitionAudioSink::~SpeechRecognitionAudioSink() {
  49   DCHECK(main_render_thread_checker_.CalledOnValidThread());
  50   if (audio_converter_.get())
  51     audio_converter_->RemoveInput(this);
  52
  53   // Notify the track before this sink goes away.
  54   if (!track_stopped_)
  55     MediaStreamAudioSink::RemoveFromAudioTrack(this, track_);
  56 }
  57
  58 // static
  59 bool SpeechRecognitionAudioSink::IsSupportedTrack(
  60     const blink::WebMediaStreamTrack& track) {
  61   if (track.source().type() != blink::WebMediaStreamSource::TypeAudio)
  62     return false;
  63
  64   MediaStreamAudioSource* native_source =
  65       static_cast<MediaStreamAudioSource*>(track.source().extraData());
  66   if (!native_source)
  67     return false;
  68
  69   const StreamDeviceInfo& device_info = native_source->device_info();
  70   // Purposely only support tracks from an audio device. Dissallow WebAudio.
  71   return (device_info.device.type == content::MEDIA_DEVICE_AUDIO_CAPTURE);
  72 }
  73
  74 void SpeechRecognitionAudioSink::OnSetFormat(
  75     const media::AudioParameters& input_params) {
  76   DCHECK(input_params.IsValid());
  77   DCHECK_LE(
  78       input_params.frames_per_buffer() * 1000 / input_params.sample_rate(),
  79       output_params_.frames_per_buffer() * 1000 / output_params_.sample_rate());
  80
  81   // Detach the thread here because it will be a new capture thread
  82   // calling OnSetFormat() and OnData() if the source is restarted.
  83   capture_thread_checker_.DetachFromThread();
  84
  85   input_params_ = input_params;
  86   fifo_buffer_size_ =
  87       std::ceil(output_params_.frames_per_buffer() *
  88                 static_cast<double>(input_params_.sample_rate()) /
  89                     output_params_.sample_rate());
  90   DCHECK_GE(fifo_buffer_size_, input_params_.frames_per_buffer());
  91
  92   // Allows for some delays on the peer.
  93   static const int kNumberOfBuffersInFifo = 2;
  94   int frames_in_fifo = kNumberOfBuffersInFifo * fifo_buffer_size_;
  95   fifo_.reset(new media::AudioFifo(input_params.channels(), frames_in_fifo));
  96   input_bus_ = media::AudioBus::Create(input_params.channels(),
  97                                        input_params.frames_per_buffer());
  98
  99   // Create the audio converter with |disable_fifo| as false so that the
 100   // converter will request input_params.frames_per_buffer() each time.
 101   // This will not increase the complexity as there is only one client to
 102   // the converter.
 103   audio_converter_.reset(
 104       new media::AudioConverter(input_params, output_params_, false));
 105   audio_converter_->AddInput(this);
 106 }
 107
 108 void SpeechRecognitionAudioSink::OnReadyStateChanged(
 109     blink::WebMediaStreamSource::ReadyState state) {
 110   DCHECK(main_render_thread_checker_.CalledOnValidThread());
 111   DCHECK(!track_stopped_);
 112
 113   if (state == blink::WebMediaStreamSource::ReadyStateEnded) {
 114     track_stopped_ = true;
 115
 116     if (!on_stopped_cb_.is_null())
 117       on_stopped_cb_.Run();
 118   }
 119 }
 120
 121 void SpeechRecognitionAudioSink::OnData(const int16* audio_data,
 122                                         int sample_rate,
 123                                         int number_of_channels,
 124                                         int number_of_frames) {
 125   DCHECK(capture_thread_checker_.CalledOnValidThread());
 126   DCHECK_EQ(input_bus_->frames(), number_of_frames);
 127   DCHECK_EQ(input_bus_->channels(), number_of_channels);
 128   if (fifo_->frames() + number_of_frames > fifo_->max_frames()) {
 129     // This would indicate a serious issue with the browser process or the
 130     // SyncSocket and/or SharedMemory. We drop any previous buffers and try to
 131     // recover by resuming where the peer left of.
 132     DLOG(ERROR) << "Audio FIFO overflow";
 133     fifo_->Clear();
 134     buffer_index_ = GetAudioInputBuffer()->params.size;
 135   }
 136   // TODO(xians): A better way to handle the interleaved and deinterleaved
 137   // format switching, see issue/317710.
 138   input_bus_->FromInterleaved(audio_data, number_of_frames,
 139                               sizeof(audio_data[0]));
 140
 141   fifo_->Push(input_bus_.get());
 142   // Wait for FIFO to have at least |fifo_buffer_size_| frames ready.
 143   if (fifo_->frames() < fifo_buffer_size_)
 144     return;
 145
 146   // Make sure the previous output buffer was consumed by the peer before we
 147   // send the next buffer.
 148   // The peer must write to it (incrementing by 1) once the the buffer was
 149   // consumed. This is intentional not to block this audio capturing thread.
 150   if (buffer_index_ != GetAudioInputBuffer()->params.size) {
 151     DVLOG(1) << "Buffer synchronization lag";
 152     return;
 153   }
 154
 155   audio_converter_->Convert(output_bus_.get());
 156
 157   // Notify peer to consume buffer |buffer_index_| on |output_bus_|.
 158   const size_t bytes_sent =
 159       socket_->Send(&buffer_index_, sizeof(buffer_index_));
 160   if (bytes_sent != sizeof(buffer_index_)) {
 161     // The send ocasionally fails if the user changes their input audio device.
 162     DVLOG(1) << "Failed sending buffer index to peer";
 163     // We have discarded this buffer, but could still recover on the next one.
 164     return;
 165   }
 166
 167   // Count the sent buffer. We expect the peer to do the same on their end.
 168   ++buffer_index_;
 169 }
 170
 171 double SpeechRecognitionAudioSink::ProvideInput(media::AudioBus* audio_bus,
 172                                                 base::TimeDelta buffer_delay) {
 173   DCHECK(capture_thread_checker_.CalledOnValidThread());
 174   if (fifo_->frames() >= audio_bus->frames())
 175     fifo_->Consume(audio_bus, 0, audio_bus->frames());
 176   else
 177     audio_bus->Zero();
 178
 179   // Return volume greater than zero to indicate we have more data.
 180   return 1.0;
 181 }
 182
 183 media::AudioInputBuffer*
 184 SpeechRecognitionAudioSink::GetAudioInputBuffer() const {
 185   DCHECK(capture_thread_checker_.CalledOnValidThread());
 186   DCHECK(shared_memory_.memory());
 187   return static_cast<media::AudioInputBuffer*>(shared_memory_.memory());
 188 }
 189
 190 }  // namespace content