content/renderer/pepper/pepper_media_stream_audio_track_host.cc

   1 // Copyright 2014 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/pepper/pepper_media_stream_audio_track_host.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/check_op.h"
  11 #include "base/location.h"
  12 #include "base/numerics/ostream_operators.h"
  13 #include "base/numerics/safe_math.h"
  14 #include "base/task/single_thread_task_runner.h"
  15 #include "base/threading/thread_task_runner_handle.h"
  16 #include "media/base/audio_bus.h"
  17 #include "ppapi/c/pp_errors.h"
  18 #include "ppapi/c/ppb_audio_buffer.h"
  19 #include "ppapi/host/dispatch_host_message.h"
  20 #include "ppapi/host/host_message_context.h"
  21 #include "ppapi/host/ppapi_host.h"
  22 #include "ppapi/proxy/ppapi_messages.h"
  23 #include "ppapi/shared_impl/media_stream_audio_track_shared.h"
  24 #include "ppapi/shared_impl/media_stream_buffer.h"
  25 #include "ppapi/shared_impl/ppb_audio_config_shared.h"
  26
  27 using media::AudioParameters;
  28 using ppapi::host::HostMessageContext;
  29 using ppapi::MediaStreamAudioTrackShared;
  30
  31 namespace {
  32
  33 // Audio buffer durations in milliseconds.
  34 const uint32_t kMinDuration = 10;
  35 const uint32_t kDefaultDuration = 10;
  36
  37 const int32_t kDefaultNumberOfAudioBuffers = 4;
  38 const int32_t kMaxNumberOfAudioBuffers = 1000;  // 10 sec
  39
  40 // Returns true if the |sample_rate| is supported in
  41 // |PP_AudioBuffer_SampleRate|, otherwise false.
  42 PP_AudioBuffer_SampleRate GetPPSampleRate(int sample_rate) {
  43   switch (sample_rate) {
  44     case 8000:
  45       return PP_AUDIOBUFFER_SAMPLERATE_8000;
  46     case 16000:
  47       return PP_AUDIOBUFFER_SAMPLERATE_16000;
  48     case 22050:
  49       return PP_AUDIOBUFFER_SAMPLERATE_22050;
  50     case 32000:
  51       return PP_AUDIOBUFFER_SAMPLERATE_32000;
  52     case 44100:
  53       return PP_AUDIOBUFFER_SAMPLERATE_44100;
  54     case 48000:
  55       return PP_AUDIOBUFFER_SAMPLERATE_48000;
  56     case 96000:
  57       return PP_AUDIOBUFFER_SAMPLERATE_96000;
  58     case 192000:
  59       return PP_AUDIOBUFFER_SAMPLERATE_192000;
  60     default:
  61       return PP_AUDIOBUFFER_SAMPLERATE_UNKNOWN;
  62   }
  63 }
  64
  65 }  // namespace
  66
  67 namespace content {
  68
  69 PepperMediaStreamAudioTrackHost::AudioSink::AudioSink(
  70     PepperMediaStreamAudioTrackHost* host)
  71     : host_(host),
  72       active_buffer_index_(-1),
  73       active_buffers_generation_(0),
  74       active_buffer_frame_offset_(0),
  75       buffers_generation_(0),
  76       output_buffer_size_(0),
  77       main_task_runner_(base::ThreadTaskRunnerHandle::Get()),
  78       number_of_buffers_(kDefaultNumberOfAudioBuffers),
  79       bytes_per_second_(0),
  80       bytes_per_frame_(0),
  81       user_buffer_duration_(kDefaultDuration) {}
  82
  83 PepperMediaStreamAudioTrackHost::AudioSink::~AudioSink() {
  84   DCHECK_EQ(main_task_runner_, base::ThreadTaskRunnerHandle::Get());
  85 }
  86
  87 void PepperMediaStreamAudioTrackHost::AudioSink::EnqueueBuffer(int32_t index) {
  88   DCHECK_EQ(main_task_runner_, base::ThreadTaskRunnerHandle::Get());
  89   DCHECK_GE(index, 0);
  90   DCHECK_LT(index, host_->buffer_manager()->number_of_buffers());
  91   base::AutoLock lock(lock_);
  92   buffers_.push_back(index);
  93 }
  94
  95 int32_t PepperMediaStreamAudioTrackHost::AudioSink::Configure(
  96     int32_t number_of_buffers, int32_t duration,
  97     const ppapi::host::ReplyMessageContext& context) {
  98   DCHECK_EQ(main_task_runner_, base::ThreadTaskRunnerHandle::Get());
  99
 100   if (pending_configure_reply_.is_valid()) {
 101     LOG(ERROR) << "Invalid ending reply context";
 102     return PP_ERROR_INPROGRESS;
 103   }
 104   pending_configure_reply_ = context;
 105
 106   bool changed = false;
 107   if (number_of_buffers != number_of_buffers_)
 108     changed = true;
 109   if (duration != 0 && duration != user_buffer_duration_) {
 110     user_buffer_duration_ = duration;
 111     changed = true;
 112   }
 113   number_of_buffers_ = number_of_buffers;
 114
 115   if (changed) {
 116     // Initialize later in OnSetFormat if bytes_per_second_ is not known yet.
 117     if (bytes_per_second_ > 0 && bytes_per_frame_ > 0)
 118       InitBuffers();
 119   } else {
 120     SendConfigureReply(PP_OK);
 121   }
 122   return PP_OK_COMPLETIONPENDING;
 123 }
 124
 125 void PepperMediaStreamAudioTrackHost::AudioSink::SendConfigureReply(
 126     int32_t result) {
 127   if (pending_configure_reply_.is_valid()) {
 128     pending_configure_reply_.params.set_result(result);
 129     host_->host()->SendReply(
 130         pending_configure_reply_,
 131         PpapiPluginMsg_MediaStreamAudioTrack_ConfigureReply());
 132     pending_configure_reply_ = ppapi::host::ReplyMessageContext();
 133   }
 134 }
 135
 136 void PepperMediaStreamAudioTrackHost::AudioSink::SetFormatOnMainThread(
 137     int bytes_per_second, int bytes_per_frame) {
 138   bytes_per_second_ = bytes_per_second;
 139   bytes_per_frame_ = bytes_per_frame;
 140   InitBuffers();
 141 }
 142
 143 void PepperMediaStreamAudioTrackHost::AudioSink::InitBuffers() {
 144   DCHECK_EQ(main_task_runner_, base::ThreadTaskRunnerHandle::Get());
 145   {
 146     base::AutoLock lock(lock_);
 147     // Clear |buffers_|, so the audio thread will drop all incoming audio data.
 148     buffers_.clear();
 149     buffers_generation_++;
 150   }
 151   int32_t frame_rate = bytes_per_second_ / bytes_per_frame_;
 152   base::CheckedNumeric<int32_t> frames_per_buffer = user_buffer_duration_;
 153   frames_per_buffer *= frame_rate;
 154   frames_per_buffer /= base::Time::kMillisecondsPerSecond;
 155   base::CheckedNumeric<int32_t> buffer_audio_size =
 156       frames_per_buffer * bytes_per_frame_;
 157   // The size is slightly bigger than necessary, because 8 extra bytes are
 158   // added into the struct. Also see |MediaStreamBuffer|. Also, the size of the
 159   // buffer may be larger than requested, since the size of each buffer will be
 160   // 4-byte aligned.
 161   base::CheckedNumeric<int32_t> buffer_size = buffer_audio_size;
 162   buffer_size += sizeof(ppapi::MediaStreamBuffer::Audio);
 163   DCHECK_GT(buffer_size.ValueOrDie(), 0);
 164
 165   // We don't need to hold |lock_| during |host->InitBuffers()| call, because
 166   // we just cleared |buffers_| , so the audio thread will drop all incoming
 167   // audio data, and not use buffers in |host_|.
 168   bool result = host_->InitBuffers(number_of_buffers_,
 169                                    buffer_size.ValueOrDie(),
 170                                    kRead);
 171   if (!result) {
 172     SendConfigureReply(PP_ERROR_NOMEMORY);
 173     return;
 174   }
 175
 176   // Fill the |buffers_|, so the audio thread can continue receiving audio data.
 177   base::AutoLock lock(lock_);
 178   output_buffer_size_ = buffer_audio_size.ValueOrDie();
 179   for (int32_t i = 0; i < number_of_buffers_; ++i) {
 180     int32_t index = host_->buffer_manager()->DequeueBuffer();
 181     DCHECK_GE(index, 0);
 182     buffers_.push_back(index);
 183   }
 184
 185   SendConfigureReply(PP_OK);
 186 }
 187
 188 void PepperMediaStreamAudioTrackHost::AudioSink::
 189     SendEnqueueBufferMessageOnMainThread(int32_t index,
 190                                          int32_t buffers_generation) {
 191   DCHECK_EQ(main_task_runner_, base::ThreadTaskRunnerHandle::Get());
 192   // If |InitBuffers()| is called after this task being posted from the audio
 193   // thread, the buffer should become invalid already. We should ignore it.
 194   // And because only the main thread modifies the |buffers_generation_|,
 195   // so we don't need to lock |lock_| here (main thread).
 196   if (buffers_generation == buffers_generation_)
 197     host_->SendEnqueueBufferMessageToPlugin(index);
 198 }
 199
 200 void PepperMediaStreamAudioTrackHost::AudioSink::OnData(
 201     const media::AudioBus& audio_bus,
 202     base::TimeTicks estimated_capture_time) {
 203   DCHECK(audio_thread_checker_.CalledOnValidThread());
 204   DCHECK(audio_params_.IsValid());
 205   DCHECK_EQ(audio_bus.channels(), audio_params_.channels());
 206   // Here, |audio_params_.frames_per_buffer()| refers to the incomming audio
 207   // buffer. However, this doesn't necessarily equal
 208   // |buffer->number_of_samples|, which is configured by the user when they set
 209   // buffer duration.
 210   DCHECK_EQ(audio_bus.frames(), audio_params_.frames_per_buffer());
 211   DCHECK(!estimated_capture_time.is_null());
 212
 213   if (first_frame_capture_time_.is_null())
 214     first_frame_capture_time_ = estimated_capture_time;
 215
 216   base::AutoLock lock(lock_);
 217   for (int frame_offset = 0; frame_offset < audio_bus.frames(); ) {
 218     if (active_buffers_generation_ != buffers_generation_) {
 219       // Buffers have changed, so drop the active buffer.
 220       active_buffer_index_ = -1;
 221     }
 222     if (active_buffer_index_ == -1 && !buffers_.empty()) {
 223       active_buffers_generation_ = buffers_generation_;
 224       active_buffer_frame_offset_ = 0;
 225       active_buffer_index_ = buffers_.front();
 226       buffers_.pop_front();
 227     }
 228     if (active_buffer_index_ == -1) {
 229       // Eek! We're dropping frames. Bad, bad, bad!
 230       break;
 231     }
 232
 233     // TODO(penghuang): Support re-sampling and channel mixing by using
 234     // media::AudioConverter.
 235     ppapi::MediaStreamBuffer::Audio* buffer =
 236         &(host_->buffer_manager()->GetBufferPointer(active_buffer_index_)
 237           ->audio);
 238     if (active_buffer_frame_offset_ == 0) {
 239       // The active buffer is new, so initialise the header and metadata fields.
 240       buffer->header.size = host_->buffer_manager()->buffer_size();
 241       buffer->header.type = ppapi::MediaStreamBuffer::TYPE_AUDIO;
 242       const base::TimeTicks time_at_offset =
 243           estimated_capture_time +
 244           frame_offset * base::Seconds(1) / audio_params_.sample_rate();
 245       buffer->timestamp =
 246           (time_at_offset - first_frame_capture_time_).InSecondsF();
 247       buffer->sample_rate =
 248           static_cast<PP_AudioBuffer_SampleRate>(audio_params_.sample_rate());
 249       buffer->data_size = output_buffer_size_;
 250       buffer->number_of_channels = audio_params_.channels();
 251       buffer->number_of_samples =
 252           buffer->data_size * audio_params_.channels() / bytes_per_frame_;
 253     }
 254
 255     const int frames_per_buffer =
 256         buffer->number_of_samples / audio_params_.channels();
 257     const int frames_to_copy =
 258         std::min(frames_per_buffer - active_buffer_frame_offset_,
 259                  audio_bus.frames() - frame_offset);
 260     audio_bus.ToInterleavedPartial<media::SignedInt16SampleTypeTraits>(
 261         frame_offset, frames_to_copy,
 262         reinterpret_cast<int16_t*>(buffer->data + active_buffer_frame_offset_ *
 263                                                       bytes_per_frame_));
 264     active_buffer_frame_offset_ += frames_to_copy;
 265     frame_offset += frames_to_copy;
 266
 267     DCHECK_LE(active_buffer_frame_offset_, frames_per_buffer);
 268     if (active_buffer_frame_offset_ == frames_per_buffer) {
 269       main_task_runner_->PostTask(
 270           FROM_HERE,
 271           base::BindOnce(&AudioSink::SendEnqueueBufferMessageOnMainThread,
 272                          weak_factory_.GetWeakPtr(), active_buffer_index_,
 273                          buffers_generation_));
 274       active_buffer_index_ = -1;
 275     }
 276   }
 277 }
 278
 279 void PepperMediaStreamAudioTrackHost::AudioSink::OnSetFormat(
 280     const AudioParameters& params) {
 281   DCHECK(params.IsValid());
 282   // TODO(amistry): How do you handle the case where the user configures a
 283   // duration that's shorter than the received buffer duration? One option is to
 284   // double buffer, where the size of the intermediate ring buffer is at least
 285   // max(user requested duration, received buffer duration). There are other
 286   // ways of dealing with it, but which one is "correct"?
 287   DCHECK_LE(params.GetBufferDuration().InMilliseconds(), kMinDuration);
 288   DCHECK_NE(GetPPSampleRate(params.sample_rate()),
 289             PP_AUDIOBUFFER_SAMPLERATE_UNKNOWN);
 290
 291   // TODO(penghuang): support setting format more than once.
 292   if (audio_params_.IsValid()) {
 293     DCHECK_EQ(params.sample_rate(), audio_params_.sample_rate());
 294     DCHECK_EQ(params.channels(), audio_params_.channels());
 295   } else {
 296     audio_thread_checker_.DetachFromThread();
 297     audio_params_ = params;
 298
 299     static_assert(ppapi::kBitsPerAudioOutputSample == 16,
 300                   "Data must be pcm_s16le.");
 301     int bytes_per_frame = params.GetBytesPerFrame(media::kSampleFormatS16);
 302     int bytes_per_second = params.sample_rate() * bytes_per_frame;
 303     main_task_runner_->PostTask(
 304         FROM_HERE, base::BindOnce(&AudioSink::SetFormatOnMainThread,
 305                                   weak_factory_.GetWeakPtr(), bytes_per_second,
 306                                   bytes_per_frame));
 307   }
 308 }
 309
 310 PepperMediaStreamAudioTrackHost::PepperMediaStreamAudioTrackHost(
 311     RendererPpapiHost* host,
 312     PP_Instance instance,
 313     PP_Resource resource,
 314     const blink::WebMediaStreamTrack& track)
 315     : PepperMediaStreamTrackHostBase(host, instance, resource),
 316       track_(track),
 317       connected_(false),
 318       audio_sink_(this) {
 319   DCHECK(!track_.IsNull());
 320 }
 321
 322 PepperMediaStreamAudioTrackHost::~PepperMediaStreamAudioTrackHost() {
 323   OnClose();
 324 }
 325
 326 int32_t PepperMediaStreamAudioTrackHost::OnResourceMessageReceived(
 327     const IPC::Message& msg,
 328     HostMessageContext* context) {
 329   PPAPI_BEGIN_MESSAGE_MAP(PepperMediaStreamAudioTrackHost, msg)
 330     PPAPI_DISPATCH_HOST_RESOURCE_CALL(
 331         PpapiHostMsg_MediaStreamAudioTrack_Configure, OnHostMsgConfigure)
 332   PPAPI_END_MESSAGE_MAP()
 333   return PepperMediaStreamTrackHostBase::OnResourceMessageReceived(msg,
 334                                                                    context);
 335 }
 336
 337 int32_t PepperMediaStreamAudioTrackHost::OnHostMsgConfigure(
 338     HostMessageContext* context,
 339     const MediaStreamAudioTrackShared::Attributes& attributes) {
 340   if (!MediaStreamAudioTrackShared::VerifyAttributes(attributes)) {
 341     LOG(ERROR) << "Invalid attributes";
 342     return PP_ERROR_BADARGUMENT;
 343   }
 344
 345   int32_t buffers = attributes.buffers
 346                         ? std::min(kMaxNumberOfAudioBuffers, attributes.buffers)
 347                         : kDefaultNumberOfAudioBuffers;
 348   return audio_sink_.Configure(buffers, attributes.duration,
 349                                context->MakeReplyMessageContext());
 350 }
 351
 352 void PepperMediaStreamAudioTrackHost::OnClose() {
 353   if (connected_) {
 354     blink::WebMediaStreamAudioSink::RemoveFromAudioTrack(&audio_sink_, track_);
 355     connected_ = false;
 356   }
 357   audio_sink_.SendConfigureReply(PP_ERROR_ABORTED);
 358 }
 359
 360 void PepperMediaStreamAudioTrackHost::OnNewBufferEnqueued() {
 361   int32_t index = buffer_manager()->DequeueBuffer();
 362   DCHECK_GE(index, 0);
 363   audio_sink_.EnqueueBuffer(index);
 364 }
 365
 366 void PepperMediaStreamAudioTrackHost::DidConnectPendingHostToResource() {
 367   if (!connected_) {
 368     media::AudioParameters format =
 369         blink::WebMediaStreamAudioSink::GetFormatFromAudioTrack(track_);
 370     // Although this should only be called on the audio capture thread, that
 371     // can't happen until the sink is added to the audio track below.
 372     if (format.IsValid())
 373       audio_sink_.OnSetFormat(format);
 374
 375     blink::WebMediaStreamAudioSink::AddToAudioTrack(&audio_sink_, track_);
 376     connected_ = true;
 377   }
 378 }
 379
 380 }  // namespace content