1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/filters/audio_renderer_impl.h"
11 #include "base/bind.h"
12 #include "base/callback.h"
13 #include "base/callback_helpers.h"
14 #include "base/logging.h"
15 #include "base/metrics/histogram.h"
16 #include "base/single_thread_task_runner.h"
17 #include "media/base/audio_buffer.h"
18 #include "media/base/audio_buffer_converter.h"
19 #include "media/base/audio_hardware_config.h"
20 #include "media/base/audio_splicer.h"
21 #include "media/base/bind_to_current_loop.h"
22 #include "media/base/demuxer_stream.h"
23 #include "media/filters/audio_clock.h"
24 #include "media/filters/decrypting_demuxer_stream.h"
30 enum AudioRendererEvent {
33 RENDER_EVENT_MAX = RENDER_ERROR,
36 void HistogramRendererEvent(AudioRendererEvent event) {
37 UMA_HISTOGRAM_ENUMERATION(
38 "Media.AudioRendererEvents", event, RENDER_EVENT_MAX + 1);
43 AudioRendererImpl::AudioRendererImpl(
44 const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
45 media::AudioRendererSink* sink,
46 ScopedVector<AudioDecoder> decoders,
47 const SetDecryptorReadyCB& set_decryptor_ready_cb,
48 AudioHardwareConfig* hardware_config)
49 : task_runner_(task_runner),
51 audio_buffer_stream_(task_runner,
53 set_decryptor_ready_cb),
54 hardware_config_(hardware_config),
55 now_cb_(base::Bind(&base::TimeTicks::Now)),
56 state_(kUninitialized),
59 received_end_of_stream_(false),
60 rendered_end_of_stream_(false),
61 underflow_disabled_(false),
62 preroll_aborted_(false),
64 audio_buffer_stream_.set_splice_observer(base::Bind(
65 &AudioRendererImpl::OnNewSpliceBuffer, weak_factory_.GetWeakPtr()));
66 audio_buffer_stream_.set_config_change_observer(base::Bind(
67 &AudioRendererImpl::OnConfigChange, weak_factory_.GetWeakPtr()));
70 AudioRendererImpl::~AudioRendererImpl() {
71 // Stop() should have been called and |algorithm_| should have been destroyed.
72 DCHECK(state_ == kUninitialized || state_ == kStopped);
73 DCHECK(!algorithm_.get());
76 void AudioRendererImpl::Play(const base::Closure& callback) {
77 DCHECK(task_runner_->BelongsToCurrentThread());
79 base::AutoLock auto_lock(lock_);
80 DCHECK_EQ(state_, kPaused);
81 ChangeState_Locked(kPlaying);
83 earliest_end_time_ = now_cb_.Run();
85 if (algorithm_->playback_rate() != 0)
88 DCHECK(!sink_playing_);
91 void AudioRendererImpl::DoPlay_Locked() {
92 DCHECK(task_runner_->BelongsToCurrentThread());
93 lock_.AssertAcquired();
94 earliest_end_time_ = now_cb_.Run();
96 if ((state_ == kPlaying || state_ == kRebuffering || state_ == kUnderflow) &&
99 base::AutoUnlock auto_unlock(lock_);
103 sink_playing_ = true;
107 void AudioRendererImpl::Pause(const base::Closure& callback) {
108 DCHECK(task_runner_->BelongsToCurrentThread());
110 base::AutoLock auto_lock(lock_);
111 DCHECK(state_ == kPlaying || state_ == kUnderflow ||
112 state_ == kRebuffering) << "state_ == " << state_;
113 ChangeState_Locked(kPaused);
120 void AudioRendererImpl::DoPause_Locked() {
121 DCHECK(task_runner_->BelongsToCurrentThread());
122 lock_.AssertAcquired();
126 base::AutoUnlock auto_unlock(lock_);
129 sink_playing_ = false;
133 void AudioRendererImpl::Flush(const base::Closure& callback) {
134 DCHECK(task_runner_->BelongsToCurrentThread());
136 base::AutoLock auto_lock(lock_);
137 DCHECK_EQ(state_, kPaused);
138 DCHECK(flush_cb_.is_null());
140 flush_cb_ = callback;
143 ChangeState_Locked(kFlushing);
150 void AudioRendererImpl::DoFlush_Locked() {
151 DCHECK(task_runner_->BelongsToCurrentThread());
152 lock_.AssertAcquired();
154 DCHECK(!pending_read_);
155 DCHECK_EQ(state_, kPaused);
157 audio_buffer_stream_.Reset(base::Bind(&AudioRendererImpl::ResetDecoderDone,
158 weak_factory_.GetWeakPtr()));
161 void AudioRendererImpl::ResetDecoderDone() {
162 DCHECK(task_runner_->BelongsToCurrentThread());
164 base::AutoLock auto_lock(lock_);
165 if (state_ == kStopped)
168 DCHECK_EQ(state_, kPaused);
169 DCHECK(!flush_cb_.is_null());
171 audio_clock_.reset(new AudioClock(audio_parameters_.sample_rate()));
172 received_end_of_stream_ = false;
173 rendered_end_of_stream_ = false;
174 preroll_aborted_ = false;
176 earliest_end_time_ = now_cb_.Run();
178 if (buffer_converter_)
179 buffer_converter_->Reset();
180 algorithm_->FlushBuffers();
182 base::ResetAndReturn(&flush_cb_).Run();
185 void AudioRendererImpl::Stop(const base::Closure& callback) {
186 DCHECK(task_runner_->BelongsToCurrentThread());
187 DCHECK(!callback.is_null());
189 // TODO(scherkus): Consider invalidating |weak_factory_| and replacing
190 // task-running guards that check |state_| with DCHECK().
193 base::AutoLock auto_lock(lock_);
195 if (state_ == kStopped) {
196 task_runner_->PostTask(FROM_HERE, callback);
200 ChangeState_Locked(kStopped);
202 underflow_cb_.Reset();
212 audio_buffer_stream_.Stop(callback);
215 void AudioRendererImpl::Preroll(base::TimeDelta time,
216 const PipelineStatusCB& cb) {
217 DCHECK(task_runner_->BelongsToCurrentThread());
219 base::AutoLock auto_lock(lock_);
220 DCHECK(!sink_playing_);
221 DCHECK_EQ(state_, kPaused);
222 DCHECK(!pending_read_) << "Pending read must complete before seeking";
223 DCHECK(preroll_cb_.is_null());
225 ChangeState_Locked(kPrerolling);
227 preroll_timestamp_ = time;
229 AttemptRead_Locked();
232 void AudioRendererImpl::Initialize(DemuxerStream* stream,
233 const PipelineStatusCB& init_cb,
234 const StatisticsCB& statistics_cb,
235 const base::Closure& underflow_cb,
236 const TimeCB& time_cb,
237 const base::Closure& ended_cb,
238 const PipelineStatusCB& error_cb) {
239 DCHECK(task_runner_->BelongsToCurrentThread());
241 DCHECK_EQ(stream->type(), DemuxerStream::AUDIO);
242 DCHECK(!init_cb.is_null());
243 DCHECK(!statistics_cb.is_null());
244 DCHECK(!underflow_cb.is_null());
245 DCHECK(!time_cb.is_null());
246 DCHECK(!ended_cb.is_null());
247 DCHECK(!error_cb.is_null());
248 DCHECK_EQ(kUninitialized, state_);
251 state_ = kInitializing;
254 underflow_cb_ = underflow_cb;
256 ended_cb_ = ended_cb;
257 error_cb_ = error_cb;
259 expecting_config_changes_ = stream->SupportsConfigChanges();
260 if (!expecting_config_changes_) {
261 // The actual buffer size is controlled via the size of the AudioBus
262 // provided to Render(), so just choose something reasonable here for looks.
263 int buffer_size = stream->audio_decoder_config().samples_per_second() / 100;
264 audio_parameters_.Reset(
265 AudioParameters::AUDIO_PCM_LOW_LATENCY,
266 stream->audio_decoder_config().channel_layout(),
267 ChannelLayoutToChannelCount(
268 stream->audio_decoder_config().channel_layout()),
270 stream->audio_decoder_config().samples_per_second(),
271 stream->audio_decoder_config().bits_per_channel(),
273 buffer_converter_.reset();
275 // TODO(rileya): Support hardware config changes
276 const AudioParameters& hw_params = hardware_config_->GetOutputConfig();
277 audio_parameters_.Reset(hw_params.format(),
278 hw_params.channel_layout(),
279 hw_params.channels(),
280 hw_params.input_channels(),
281 hw_params.sample_rate(),
282 hw_params.bits_per_sample(),
283 hardware_config_->GetHighLatencyBufferSize());
286 audio_clock_.reset(new AudioClock(audio_parameters_.sample_rate()));
288 audio_buffer_stream_.Initialize(
292 base::Bind(&AudioRendererImpl::OnAudioBufferStreamInitialized,
293 weak_factory_.GetWeakPtr()));
296 void AudioRendererImpl::OnAudioBufferStreamInitialized(bool success) {
297 DCHECK(task_runner_->BelongsToCurrentThread());
299 base::AutoLock auto_lock(lock_);
301 if (state_ == kStopped) {
302 base::ResetAndReturn(&init_cb_).Run(PIPELINE_ERROR_ABORT);
307 state_ = kUninitialized;
308 base::ResetAndReturn(&init_cb_).Run(DECODER_ERROR_NOT_SUPPORTED);
312 if (!audio_parameters_.IsValid()) {
313 ChangeState_Locked(kUninitialized);
314 base::ResetAndReturn(&init_cb_).Run(PIPELINE_ERROR_INITIALIZATION_FAILED);
318 if (expecting_config_changes_)
319 buffer_converter_.reset(new AudioBufferConverter(audio_parameters_));
320 splicer_.reset(new AudioSplicer(audio_parameters_.sample_rate()));
322 // We're all good! Continue initializing the rest of the audio renderer
323 // based on the decoder format.
324 algorithm_.reset(new AudioRendererAlgorithm());
325 algorithm_->Initialize(0, audio_parameters_);
327 ChangeState_Locked(kPaused);
329 HistogramRendererEvent(INITIALIZED);
332 base::AutoUnlock auto_unlock(lock_);
333 sink_->Initialize(audio_parameters_, this);
336 // Some sinks play on start...
340 DCHECK(!sink_playing_);
342 base::ResetAndReturn(&init_cb_).Run(PIPELINE_OK);
345 void AudioRendererImpl::ResumeAfterUnderflow() {
346 DCHECK(task_runner_->BelongsToCurrentThread());
347 base::AutoLock auto_lock(lock_);
348 if (state_ == kUnderflow) {
349 // The "!preroll_aborted_" is a hack. If preroll is aborted, then we
350 // shouldn't even reach the kUnderflow state to begin with. But for now
351 // we're just making sure that the audio buffer capacity (i.e. the
352 // number of bytes that need to be buffered for preroll to complete)
353 // does not increase due to an aborted preroll.
354 // TODO(vrk): Fix this bug correctly! (crbug.com/151352)
355 if (!preroll_aborted_)
356 algorithm_->IncreaseQueueCapacity();
358 ChangeState_Locked(kRebuffering);
362 void AudioRendererImpl::SetVolume(float volume) {
363 DCHECK(task_runner_->BelongsToCurrentThread());
365 sink_->SetVolume(volume);
368 void AudioRendererImpl::DecodedAudioReady(
369 AudioBufferStream::Status status,
370 const scoped_refptr<AudioBuffer>& buffer) {
371 DVLOG(1) << __FUNCTION__ << "(" << status << ")";
372 DCHECK(task_runner_->BelongsToCurrentThread());
374 base::AutoLock auto_lock(lock_);
375 DCHECK(state_ != kUninitialized);
377 CHECK(pending_read_);
378 pending_read_ = false;
380 if (status == AudioBufferStream::ABORTED ||
381 status == AudioBufferStream::DEMUXER_READ_ABORTED) {
382 HandleAbortedReadOrDecodeError(false);
386 if (status == AudioBufferStream::DECODE_ERROR) {
387 HandleAbortedReadOrDecodeError(true);
391 DCHECK_EQ(status, AudioBufferStream::OK);
392 DCHECK(buffer.get());
394 if (state_ == kFlushing) {
395 ChangeState_Locked(kPaused);
400 if (expecting_config_changes_) {
401 DCHECK(buffer_converter_);
402 buffer_converter_->AddInput(buffer);
403 while (buffer_converter_->HasNextBuffer()) {
404 if (!splicer_->AddInput(buffer_converter_->GetNextBuffer())) {
405 HandleAbortedReadOrDecodeError(true);
410 if (!splicer_->AddInput(buffer)) {
411 HandleAbortedReadOrDecodeError(true);
416 if (!splicer_->HasNextBuffer()) {
417 AttemptRead_Locked();
421 bool need_another_buffer = false;
422 while (splicer_->HasNextBuffer())
423 need_another_buffer = HandleSplicerBuffer(splicer_->GetNextBuffer());
425 if (!need_another_buffer && !CanRead_Locked())
428 AttemptRead_Locked();
431 bool AudioRendererImpl::HandleSplicerBuffer(
432 const scoped_refptr<AudioBuffer>& buffer) {
433 if (buffer->end_of_stream()) {
434 received_end_of_stream_ = true;
436 // Transition to kPlaying if we are currently handling an underflow since
437 // no more data will be arriving.
438 if (state_ == kUnderflow || state_ == kRebuffering)
439 ChangeState_Locked(kPlaying);
441 if (state_ == kPrerolling) {
442 if (IsBeforePrerollTime(buffer))
445 // Trim off any additional time before the preroll timestamp.
446 const base::TimeDelta trim_time =
447 preroll_timestamp_ - buffer->timestamp();
448 if (trim_time > base::TimeDelta()) {
449 buffer->TrimStart(buffer->frame_count() *
450 (static_cast<double>(trim_time.InMicroseconds()) /
451 buffer->duration().InMicroseconds()));
453 // If the entire buffer was trimmed, request a new one.
454 if (!buffer->frame_count())
458 if (state_ != kUninitialized && state_ != kStopped)
459 algorithm_->EnqueueBuffer(buffer);
470 DCHECK(!pending_read_);
474 if (!buffer->end_of_stream() && !algorithm_->IsQueueFull())
476 ChangeState_Locked(kPaused);
477 base::ResetAndReturn(&preroll_cb_).Run(PIPELINE_OK);
485 if (!algorithm_->IsQueueFull())
487 ChangeState_Locked(kPlaying);
496 void AudioRendererImpl::AttemptRead() {
497 base::AutoLock auto_lock(lock_);
498 AttemptRead_Locked();
501 void AudioRendererImpl::AttemptRead_Locked() {
502 DCHECK(task_runner_->BelongsToCurrentThread());
503 lock_.AssertAcquired();
505 if (!CanRead_Locked())
508 pending_read_ = true;
509 audio_buffer_stream_.Read(base::Bind(&AudioRendererImpl::DecodedAudioReady,
510 weak_factory_.GetWeakPtr()));
513 bool AudioRendererImpl::CanRead_Locked() {
514 lock_.AssertAcquired();
531 return !pending_read_ && !received_end_of_stream_ &&
532 !algorithm_->IsQueueFull();
535 void AudioRendererImpl::SetPlaybackRate(float playback_rate) {
536 DVLOG(1) << __FUNCTION__ << "(" << playback_rate << ")";
537 DCHECK(task_runner_->BelongsToCurrentThread());
538 DCHECK_GE(playback_rate, 0);
541 base::AutoLock auto_lock(lock_);
543 // We have two cases here:
544 // Play: current_playback_rate == 0 && playback_rate != 0
545 // Pause: current_playback_rate != 0 && playback_rate == 0
546 float current_playback_rate = algorithm_->playback_rate();
547 if (current_playback_rate == 0 && playback_rate != 0)
549 else if (current_playback_rate != 0 && playback_rate == 0)
552 algorithm_->SetPlaybackRate(playback_rate);
555 bool AudioRendererImpl::IsBeforePrerollTime(
556 const scoped_refptr<AudioBuffer>& buffer) {
557 DCHECK_EQ(state_, kPrerolling);
558 return buffer && !buffer->end_of_stream() &&
559 (buffer->timestamp() + buffer->duration()) < preroll_timestamp_;
562 int AudioRendererImpl::Render(AudioBus* audio_bus,
563 int audio_delay_milliseconds) {
564 const int requested_frames = audio_bus->frames();
565 base::TimeDelta playback_delay = base::TimeDelta::FromMilliseconds(
566 audio_delay_milliseconds);
567 const int delay_frames = static_cast<int>(playback_delay.InSecondsF() *
568 audio_parameters_.sample_rate());
569 int frames_written = 0;
570 base::Closure time_cb;
571 base::Closure underflow_cb;
573 base::AutoLock auto_lock(lock_);
575 // Ensure Stop() hasn't destroyed our |algorithm_| on the pipeline thread.
577 audio_clock_->WroteSilence(requested_frames, delay_frames);
581 float playback_rate = algorithm_->playback_rate();
582 if (playback_rate == 0) {
583 audio_clock_->WroteSilence(requested_frames, delay_frames);
587 // Mute audio by returning 0 when not playing.
588 if (state_ != kPlaying) {
589 audio_clock_->WroteSilence(requested_frames, delay_frames);
593 // We use the following conditions to determine end of playback:
594 // 1) Algorithm can not fill the audio callback buffer
595 // 2) We received an end of stream buffer
596 // 3) We haven't already signalled that we've ended
597 // 4) Our estimated earliest end time has expired
599 // TODO(enal): we should replace (4) with a check that the browser has no
600 // more audio data or at least use a delayed callback.
602 // We use the following conditions to determine underflow:
603 // 1) Algorithm can not fill the audio callback buffer
604 // 2) We have NOT received an end of stream buffer
605 // 3) We are in the kPlaying state
607 // Otherwise the buffer has data we can send to the device.
608 const base::TimeDelta media_timestamp_before_filling =
609 audio_clock_->CurrentMediaTimestamp();
610 if (algorithm_->frames_buffered() > 0) {
611 frames_written = algorithm_->FillBuffer(audio_bus, requested_frames);
612 audio_clock_->WroteAudio(
613 frames_written, delay_frames, playback_rate, algorithm_->GetTime());
615 audio_clock_->WroteSilence(requested_frames - frames_written, delay_frames);
617 if (frames_written == 0) {
618 const base::TimeTicks now = now_cb_.Run();
620 if (received_end_of_stream_ && !rendered_end_of_stream_ &&
621 now >= earliest_end_time_) {
622 rendered_end_of_stream_ = true;
624 } else if (!received_end_of_stream_ && state_ == kPlaying &&
625 !underflow_disabled_) {
626 ChangeState_Locked(kUnderflow);
627 underflow_cb = underflow_cb_;
629 // We can't write any data this cycle. For example, we may have
630 // sent all available data to the audio device while not reaching
631 // |earliest_end_time_|.
635 if (CanRead_Locked()) {
636 task_runner_->PostTask(FROM_HERE,
637 base::Bind(&AudioRendererImpl::AttemptRead,
638 weak_factory_.GetWeakPtr()));
641 // We only want to execute |time_cb_| if time has progressed and we haven't
642 // signaled end of stream yet.
643 if (media_timestamp_before_filling !=
644 audio_clock_->CurrentMediaTimestamp() &&
645 !rendered_end_of_stream_) {
646 time_cb = base::Bind(time_cb_,
647 audio_clock_->CurrentMediaTimestamp(),
648 audio_clock_->last_endpoint_timestamp());
651 if (frames_written > 0) {
652 UpdateEarliestEndTime_Locked(
653 frames_written, playback_delay, now_cb_.Run());
657 if (!time_cb.is_null())
660 if (!underflow_cb.is_null())
663 DCHECK_LE(frames_written, requested_frames);
664 return frames_written;
667 void AudioRendererImpl::UpdateEarliestEndTime_Locked(
668 int frames_filled, const base::TimeDelta& playback_delay,
669 const base::TimeTicks& time_now) {
670 DCHECK_GT(frames_filled, 0);
672 base::TimeDelta predicted_play_time = base::TimeDelta::FromMicroseconds(
673 static_cast<float>(frames_filled) * base::Time::kMicrosecondsPerSecond /
674 audio_parameters_.sample_rate());
676 lock_.AssertAcquired();
677 earliest_end_time_ = std::max(
678 earliest_end_time_, time_now + playback_delay + predicted_play_time);
681 void AudioRendererImpl::OnRenderError() {
682 // UMA data tells us this happens ~0.01% of the time. Trigger an error instead
683 // of trying to gracefully fall back to a fake sink. It's very likely
684 // OnRenderError() should be removed and the audio stack handle errors without
685 // notifying clients. See http://crbug.com/234708 for details.
686 HistogramRendererEvent(RENDER_ERROR);
687 error_cb_.Run(PIPELINE_ERROR_DECODE);
690 void AudioRendererImpl::DisableUnderflowForTesting() {
691 underflow_disabled_ = true;
694 void AudioRendererImpl::HandleAbortedReadOrDecodeError(bool is_decode_error) {
695 lock_.AssertAcquired();
697 PipelineStatus status = is_decode_error ? PIPELINE_ERROR_DECODE : PIPELINE_OK;
704 if (status != PIPELINE_OK)
705 error_cb_.Run(status);
708 ChangeState_Locked(kPaused);
710 if (status == PIPELINE_OK) {
715 error_cb_.Run(status);
716 base::ResetAndReturn(&flush_cb_).Run();
719 // This is a signal for abort if it's not an error.
720 preroll_aborted_ = !is_decode_error;
721 ChangeState_Locked(kPaused);
722 base::ResetAndReturn(&preroll_cb_).Run(status);
728 if (status != PIPELINE_OK)
729 error_cb_.Run(status);
734 void AudioRendererImpl::ChangeState_Locked(State new_state) {
735 DVLOG(1) << __FUNCTION__ << " : " << state_ << " -> " << new_state;
736 lock_.AssertAcquired();
740 void AudioRendererImpl::OnNewSpliceBuffer(base::TimeDelta splice_timestamp) {
741 DCHECK(task_runner_->BelongsToCurrentThread());
742 splicer_->SetSpliceTimestamp(splice_timestamp);
745 void AudioRendererImpl::OnConfigChange() {
746 DCHECK(task_runner_->BelongsToCurrentThread());
747 DCHECK(expecting_config_changes_);
748 buffer_converter_->ResetTimestampState();
749 // Drain flushed buffers from the converter so the AudioSplicer receives all
750 // data ahead of any OnNewSpliceBuffer() calls. Since discontinuities should
751 // only appear after config changes, AddInput() should never fail here.
752 while (buffer_converter_->HasNextBuffer())
753 CHECK(splicer_->AddInput(buffer_converter_->GetNextBuffer()));