1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/filters/audio_renderer_impl.h"
11 #include "base/bind.h"
12 #include "base/callback.h"
13 #include "base/callback_helpers.h"
14 #include "base/logging.h"
15 #include "base/metrics/histogram.h"
16 #include "base/single_thread_task_runner.h"
17 #include "media/base/audio_buffer.h"
18 #include "media/base/audio_buffer_converter.h"
19 #include "media/base/audio_hardware_config.h"
20 #include "media/base/audio_splicer.h"
21 #include "media/base/bind_to_current_loop.h"
22 #include "media/base/demuxer_stream.h"
23 #include "media/filters/audio_clock.h"
24 #include "media/filters/decrypting_demuxer_stream.h"
30 enum AudioRendererEvent {
33 RENDER_EVENT_MAX = RENDER_ERROR,
36 void HistogramRendererEvent(AudioRendererEvent event) {
37 UMA_HISTOGRAM_ENUMERATION(
38 "Media.AudioRendererEvents", event, RENDER_EVENT_MAX + 1);
43 AudioRendererImpl::AudioRendererImpl(
44 const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
45 media::AudioRendererSink* sink,
46 ScopedVector<AudioDecoder> decoders,
47 const SetDecryptorReadyCB& set_decryptor_ready_cb,
48 const AudioHardwareConfig& hardware_config,
49 const scoped_refptr<MediaLog>& media_log)
50 : task_runner_(task_runner),
51 expecting_config_changes_(false),
53 audio_buffer_stream_(new AudioBufferStream(task_runner,
55 set_decryptor_ready_cb,
57 hardware_config_(hardware_config),
59 state_(kUninitialized),
60 buffering_state_(BUFFERING_HAVE_NOTHING),
64 received_end_of_stream_(false),
65 rendered_end_of_stream_(false),
67 audio_buffer_stream_->set_splice_observer(base::Bind(
68 &AudioRendererImpl::OnNewSpliceBuffer, weak_factory_.GetWeakPtr()));
69 audio_buffer_stream_->set_config_change_observer(base::Bind(
70 &AudioRendererImpl::OnConfigChange, weak_factory_.GetWeakPtr()));
73 AudioRendererImpl::~AudioRendererImpl() {
74 DVLOG(1) << __FUNCTION__;
75 DCHECK(task_runner_->BelongsToCurrentThread());
77 // If Render() is in progress, this call will wait for Render() to finish.
78 // After this call, the |sink_| will not call back into |this| anymore.
81 if (!init_cb_.is_null())
82 base::ResetAndReturn(&init_cb_).Run(PIPELINE_ERROR_ABORT);
85 void AudioRendererImpl::StartTicking() {
86 DVLOG(1) << __FUNCTION__;
87 DCHECK(task_runner_->BelongsToCurrentThread());
91 base::AutoLock auto_lock(lock_);
92 // Wait for an eventual call to SetPlaybackRate() to start rendering.
93 if (playback_rate_ == 0) {
94 DCHECK(!sink_playing_);
98 StartRendering_Locked();
101 void AudioRendererImpl::StartRendering_Locked() {
102 DVLOG(1) << __FUNCTION__;
103 DCHECK(task_runner_->BelongsToCurrentThread());
104 DCHECK_EQ(state_, kPlaying);
105 DCHECK(!sink_playing_);
106 DCHECK_NE(playback_rate_, 0);
107 lock_.AssertAcquired();
109 sink_playing_ = true;
111 base::AutoUnlock auto_unlock(lock_);
115 void AudioRendererImpl::StopTicking() {
116 DVLOG(1) << __FUNCTION__;
117 DCHECK(task_runner_->BelongsToCurrentThread());
121 base::AutoLock auto_lock(lock_);
122 // Rendering should have already been stopped with a zero playback rate.
123 if (playback_rate_ == 0) {
124 DCHECK(!sink_playing_);
128 StopRendering_Locked();
131 void AudioRendererImpl::StopRendering_Locked() {
132 DCHECK(task_runner_->BelongsToCurrentThread());
133 DCHECK_EQ(state_, kPlaying);
134 DCHECK(sink_playing_);
135 lock_.AssertAcquired();
137 sink_playing_ = false;
139 base::AutoUnlock auto_unlock(lock_);
143 void AudioRendererImpl::SetMediaTime(base::TimeDelta time) {
144 DVLOG(1) << __FUNCTION__ << "(" << time.InMicroseconds() << ")";
145 DCHECK(task_runner_->BelongsToCurrentThread());
147 base::AutoLock auto_lock(lock_);
149 DCHECK_EQ(state_, kFlushed);
151 start_timestamp_ = time;
152 ended_timestamp_ = kInfiniteDuration();
153 last_render_ticks_ = base::TimeTicks();
154 audio_clock_.reset(new AudioClock(time, audio_parameters_.sample_rate()));
157 base::TimeDelta AudioRendererImpl::CurrentMediaTime() {
158 // In practice the Render() method is called with a high enough frequency
159 // that returning only the front timestamp is good enough and also prevents
160 // returning values that go backwards in time.
161 base::TimeDelta current_media_time;
163 base::AutoLock auto_lock(lock_);
164 current_media_time = audio_clock_->front_timestamp();
167 DVLOG(3) << __FUNCTION__ << ": " << current_media_time.InMilliseconds()
169 return current_media_time;
172 base::TimeDelta AudioRendererImpl::CurrentMediaTimeForSyncingVideo() {
173 DVLOG(2) << __FUNCTION__;
175 base::AutoLock auto_lock(lock_);
176 if (last_render_ticks_.is_null())
177 return audio_clock_->front_timestamp();
179 return audio_clock_->TimestampSinceWriting(base::TimeTicks::Now() -
183 TimeSource* AudioRendererImpl::GetTimeSource() {
187 void AudioRendererImpl::Flush(const base::Closure& callback) {
188 DVLOG(1) << __FUNCTION__;
189 DCHECK(task_runner_->BelongsToCurrentThread());
191 base::AutoLock auto_lock(lock_);
192 DCHECK_EQ(state_, kPlaying);
193 DCHECK(flush_cb_.is_null());
195 flush_cb_ = callback;
196 ChangeState_Locked(kFlushing);
201 ChangeState_Locked(kFlushed);
205 void AudioRendererImpl::DoFlush_Locked() {
206 DCHECK(task_runner_->BelongsToCurrentThread());
207 lock_.AssertAcquired();
209 DCHECK(!pending_read_);
210 DCHECK_EQ(state_, kFlushed);
212 audio_buffer_stream_->Reset(base::Bind(&AudioRendererImpl::ResetDecoderDone,
213 weak_factory_.GetWeakPtr()));
216 void AudioRendererImpl::ResetDecoderDone() {
217 DCHECK(task_runner_->BelongsToCurrentThread());
219 base::AutoLock auto_lock(lock_);
221 DCHECK_EQ(state_, kFlushed);
222 DCHECK(!flush_cb_.is_null());
224 received_end_of_stream_ = false;
225 rendered_end_of_stream_ = false;
227 // Flush() may have been called while underflowed/not fully buffered.
228 if (buffering_state_ != BUFFERING_HAVE_NOTHING)
229 SetBufferingState_Locked(BUFFERING_HAVE_NOTHING);
232 if (buffer_converter_)
233 buffer_converter_->Reset();
234 algorithm_->FlushBuffers();
237 // Changes in buffering state are always posted. Flush callback must only be
238 // run after buffering state has been set back to nothing.
239 task_runner_->PostTask(FROM_HERE, base::ResetAndReturn(&flush_cb_));
242 void AudioRendererImpl::StartPlaying() {
243 DVLOG(1) << __FUNCTION__;
244 DCHECK(task_runner_->BelongsToCurrentThread());
246 base::AutoLock auto_lock(lock_);
247 DCHECK(!sink_playing_);
248 DCHECK_EQ(state_, kFlushed);
249 DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
250 DCHECK(!pending_read_) << "Pending read must complete before seeking";
252 ChangeState_Locked(kPlaying);
253 AttemptRead_Locked();
256 void AudioRendererImpl::Initialize(DemuxerStream* stream,
257 const PipelineStatusCB& init_cb,
258 const StatisticsCB& statistics_cb,
259 const BufferingStateCB& buffering_state_cb,
260 const base::Closure& ended_cb,
261 const PipelineStatusCB& error_cb) {
262 DVLOG(1) << __FUNCTION__;
263 DCHECK(task_runner_->BelongsToCurrentThread());
265 DCHECK_EQ(stream->type(), DemuxerStream::AUDIO);
266 DCHECK(!init_cb.is_null());
267 DCHECK(!statistics_cb.is_null());
268 DCHECK(!buffering_state_cb.is_null());
269 DCHECK(!ended_cb.is_null());
270 DCHECK(!error_cb.is_null());
271 DCHECK_EQ(kUninitialized, state_);
274 state_ = kInitializing;
276 // Always post |init_cb_| because |this| could be destroyed if initialization
278 init_cb_ = BindToCurrentLoop(init_cb);
280 buffering_state_cb_ = buffering_state_cb;
281 ended_cb_ = ended_cb;
282 error_cb_ = error_cb;
284 expecting_config_changes_ = stream->SupportsConfigChanges();
285 if (!expecting_config_changes_) {
286 // The actual buffer size is controlled via the size of the AudioBus
287 // provided to Render(), so just choose something reasonable here for looks.
288 int buffer_size = stream->audio_decoder_config().samples_per_second() / 100;
289 audio_parameters_.Reset(
290 AudioParameters::AUDIO_PCM_LOW_LATENCY,
291 stream->audio_decoder_config().channel_layout(),
292 ChannelLayoutToChannelCount(
293 stream->audio_decoder_config().channel_layout()),
294 stream->audio_decoder_config().samples_per_second(),
295 stream->audio_decoder_config().bits_per_channel(),
297 buffer_converter_.reset();
299 // TODO(rileya): Support hardware config changes
300 const AudioParameters& hw_params = hardware_config_.GetOutputConfig();
301 audio_parameters_.Reset(
303 // Always use the source's channel layout and channel count to avoid
304 // premature downmixing (http://crbug.com/379288), platform specific
305 // issues around channel layouts (http://crbug.com/266674), and
306 // unnecessary upmixing overhead.
307 stream->audio_decoder_config().channel_layout(),
308 ChannelLayoutToChannelCount(
309 stream->audio_decoder_config().channel_layout()),
310 hw_params.sample_rate(),
311 hw_params.bits_per_sample(),
312 hardware_config_.GetHighLatencyBufferSize());
316 new AudioClock(base::TimeDelta(), audio_parameters_.sample_rate()));
318 audio_buffer_stream_->Initialize(
322 base::Bind(&AudioRendererImpl::OnAudioBufferStreamInitialized,
323 weak_factory_.GetWeakPtr()));
326 void AudioRendererImpl::OnAudioBufferStreamInitialized(bool success) {
327 DVLOG(1) << __FUNCTION__ << ": " << success;
328 DCHECK(task_runner_->BelongsToCurrentThread());
330 base::AutoLock auto_lock(lock_);
333 state_ = kUninitialized;
334 base::ResetAndReturn(&init_cb_).Run(DECODER_ERROR_NOT_SUPPORTED);
338 if (!audio_parameters_.IsValid()) {
339 DVLOG(1) << __FUNCTION__ << ": Invalid audio parameters: "
340 << audio_parameters_.AsHumanReadableString();
341 ChangeState_Locked(kUninitialized);
342 base::ResetAndReturn(&init_cb_).Run(PIPELINE_ERROR_INITIALIZATION_FAILED);
346 if (expecting_config_changes_)
347 buffer_converter_.reset(new AudioBufferConverter(audio_parameters_));
348 splicer_.reset(new AudioSplicer(audio_parameters_.sample_rate()));
350 // We're all good! Continue initializing the rest of the audio renderer
351 // based on the decoder format.
352 algorithm_.reset(new AudioRendererAlgorithm());
353 algorithm_->Initialize(audio_parameters_);
355 ChangeState_Locked(kFlushed);
357 HistogramRendererEvent(INITIALIZED);
360 base::AutoUnlock auto_unlock(lock_);
361 sink_->Initialize(audio_parameters_, this);
364 // Some sinks play on start...
368 DCHECK(!sink_playing_);
369 base::ResetAndReturn(&init_cb_).Run(PIPELINE_OK);
372 void AudioRendererImpl::SetVolume(float volume) {
373 DCHECK(task_runner_->BelongsToCurrentThread());
375 sink_->SetVolume(volume);
378 void AudioRendererImpl::DecodedAudioReady(
379 AudioBufferStream::Status status,
380 const scoped_refptr<AudioBuffer>& buffer) {
381 DVLOG(2) << __FUNCTION__ << "(" << status << ")";
382 DCHECK(task_runner_->BelongsToCurrentThread());
384 base::AutoLock auto_lock(lock_);
385 DCHECK(state_ != kUninitialized);
387 CHECK(pending_read_);
388 pending_read_ = false;
390 if (status == AudioBufferStream::ABORTED ||
391 status == AudioBufferStream::DEMUXER_READ_ABORTED) {
392 HandleAbortedReadOrDecodeError(false);
396 if (status == AudioBufferStream::DECODE_ERROR) {
397 HandleAbortedReadOrDecodeError(true);
401 DCHECK_EQ(status, AudioBufferStream::OK);
402 DCHECK(buffer.get());
404 if (state_ == kFlushing) {
405 ChangeState_Locked(kFlushed);
410 if (expecting_config_changes_) {
411 DCHECK(buffer_converter_);
412 buffer_converter_->AddInput(buffer);
413 while (buffer_converter_->HasNextBuffer()) {
414 if (!splicer_->AddInput(buffer_converter_->GetNextBuffer())) {
415 HandleAbortedReadOrDecodeError(true);
420 if (!splicer_->AddInput(buffer)) {
421 HandleAbortedReadOrDecodeError(true);
426 if (!splicer_->HasNextBuffer()) {
427 AttemptRead_Locked();
431 bool need_another_buffer = false;
432 while (splicer_->HasNextBuffer())
433 need_another_buffer = HandleSplicerBuffer_Locked(splicer_->GetNextBuffer());
435 if (!need_another_buffer && !CanRead_Locked())
438 AttemptRead_Locked();
441 bool AudioRendererImpl::HandleSplicerBuffer_Locked(
442 const scoped_refptr<AudioBuffer>& buffer) {
443 lock_.AssertAcquired();
444 if (buffer->end_of_stream()) {
445 received_end_of_stream_ = true;
447 if (state_ == kPlaying) {
448 if (IsBeforeStartTime(buffer))
451 // Trim off any additional time before the start timestamp.
452 const base::TimeDelta trim_time = start_timestamp_ - buffer->timestamp();
453 if (trim_time > base::TimeDelta()) {
454 buffer->TrimStart(buffer->frame_count() *
455 (static_cast<double>(trim_time.InMicroseconds()) /
456 buffer->duration().InMicroseconds()));
458 // If the entire buffer was trimmed, request a new one.
459 if (!buffer->frame_count())
463 if (state_ != kUninitialized)
464 algorithm_->EnqueueBuffer(buffer);
475 DCHECK(!pending_read_);
479 if (buffer->end_of_stream() || algorithm_->IsQueueFull()) {
480 if (buffering_state_ == BUFFERING_HAVE_NOTHING)
481 SetBufferingState_Locked(BUFFERING_HAVE_ENOUGH);
489 void AudioRendererImpl::AttemptRead() {
490 base::AutoLock auto_lock(lock_);
491 AttemptRead_Locked();
494 void AudioRendererImpl::AttemptRead_Locked() {
495 DCHECK(task_runner_->BelongsToCurrentThread());
496 lock_.AssertAcquired();
498 if (!CanRead_Locked())
501 pending_read_ = true;
502 audio_buffer_stream_->Read(base::Bind(&AudioRendererImpl::DecodedAudioReady,
503 weak_factory_.GetWeakPtr()));
506 bool AudioRendererImpl::CanRead_Locked() {
507 lock_.AssertAcquired();
520 return !pending_read_ && !received_end_of_stream_ &&
521 !algorithm_->IsQueueFull();
524 void AudioRendererImpl::SetPlaybackRate(float playback_rate) {
525 DVLOG(1) << __FUNCTION__ << "(" << playback_rate << ")";
526 DCHECK(task_runner_->BelongsToCurrentThread());
527 DCHECK_GE(playback_rate, 0);
530 base::AutoLock auto_lock(lock_);
532 // We have two cases here:
533 // Play: current_playback_rate == 0 && playback_rate != 0
534 // Pause: current_playback_rate != 0 && playback_rate == 0
535 float current_playback_rate = playback_rate_;
536 playback_rate_ = playback_rate;
541 if (current_playback_rate == 0 && playback_rate != 0) {
542 StartRendering_Locked();
546 if (current_playback_rate != 0 && playback_rate == 0) {
547 StopRendering_Locked();
552 bool AudioRendererImpl::IsBeforeStartTime(
553 const scoped_refptr<AudioBuffer>& buffer) {
554 DCHECK_EQ(state_, kPlaying);
555 return buffer.get() && !buffer->end_of_stream() &&
556 (buffer->timestamp() + buffer->duration()) < start_timestamp_;
559 int AudioRendererImpl::Render(AudioBus* audio_bus,
560 int audio_delay_milliseconds) {
561 const int requested_frames = audio_bus->frames();
562 base::TimeDelta playback_delay = base::TimeDelta::FromMilliseconds(
563 audio_delay_milliseconds);
564 const int delay_frames = static_cast<int>(playback_delay.InSecondsF() *
565 audio_parameters_.sample_rate());
566 int frames_written = 0;
568 base::AutoLock auto_lock(lock_);
569 last_render_ticks_ = base::TimeTicks::Now();
571 // Ensure Stop() hasn't destroyed our |algorithm_| on the pipeline thread.
573 audio_clock_->WroteAudio(
574 0, requested_frames, delay_frames, playback_rate_);
578 if (playback_rate_ == 0) {
579 audio_clock_->WroteAudio(
580 0, requested_frames, delay_frames, playback_rate_);
584 // Mute audio by returning 0 when not playing.
585 if (state_ != kPlaying) {
586 audio_clock_->WroteAudio(
587 0, requested_frames, delay_frames, playback_rate_);
591 // We use the following conditions to determine end of playback:
592 // 1) Algorithm can not fill the audio callback buffer
593 // 2) We received an end of stream buffer
594 // 3) We haven't already signalled that we've ended
595 // 4) We've played all known audio data sent to hardware
597 // We use the following conditions to determine underflow:
598 // 1) Algorithm can not fill the audio callback buffer
599 // 2) We have NOT received an end of stream buffer
600 // 3) We are in the kPlaying state
602 // Otherwise the buffer has data we can send to the device.
603 if (algorithm_->frames_buffered() > 0) {
605 algorithm_->FillBuffer(audio_bus, requested_frames, playback_rate_);
608 // Per the TimeSource API the media time should always increase even after
609 // we've rendered all known audio data. Doing so simplifies scenarios where
610 // we have other sources of media data that need to be scheduled after audio
613 // That being said, we don't want to advance time when underflowed as we
614 // know more decoded frames will eventually arrive. If we did, we would
615 // throw things out of sync when said decoded frames arrive.
616 int frames_after_end_of_stream = 0;
617 if (frames_written == 0) {
618 if (received_end_of_stream_) {
619 if (ended_timestamp_ == kInfiniteDuration())
620 ended_timestamp_ = audio_clock_->back_timestamp();
621 frames_after_end_of_stream = requested_frames;
622 } else if (state_ == kPlaying &&
623 buffering_state_ != BUFFERING_HAVE_NOTHING) {
624 algorithm_->IncreaseQueueCapacity();
625 SetBufferingState_Locked(BUFFERING_HAVE_NOTHING);
629 audio_clock_->WroteAudio(frames_written + frames_after_end_of_stream,
634 if (CanRead_Locked()) {
635 task_runner_->PostTask(FROM_HERE,
636 base::Bind(&AudioRendererImpl::AttemptRead,
637 weak_factory_.GetWeakPtr()));
640 if (audio_clock_->front_timestamp() >= ended_timestamp_ &&
641 !rendered_end_of_stream_) {
642 rendered_end_of_stream_ = true;
643 task_runner_->PostTask(FROM_HERE, ended_cb_);
647 DCHECK_LE(frames_written, requested_frames);
648 return frames_written;
651 void AudioRendererImpl::OnRenderError() {
652 // UMA data tells us this happens ~0.01% of the time. Trigger an error instead
653 // of trying to gracefully fall back to a fake sink. It's very likely
654 // OnRenderError() should be removed and the audio stack handle errors without
655 // notifying clients. See http://crbug.com/234708 for details.
656 HistogramRendererEvent(RENDER_ERROR);
657 // Post to |task_runner_| as this is called on the audio callback thread.
658 task_runner_->PostTask(FROM_HERE,
659 base::Bind(error_cb_, PIPELINE_ERROR_DECODE));
662 void AudioRendererImpl::HandleAbortedReadOrDecodeError(bool is_decode_error) {
663 DCHECK(task_runner_->BelongsToCurrentThread());
664 lock_.AssertAcquired();
666 PipelineStatus status = is_decode_error ? PIPELINE_ERROR_DECODE : PIPELINE_OK;
673 ChangeState_Locked(kFlushed);
674 if (status == PIPELINE_OK) {
679 error_cb_.Run(status);
680 base::ResetAndReturn(&flush_cb_).Run();
685 if (status != PIPELINE_OK)
686 error_cb_.Run(status);
691 void AudioRendererImpl::ChangeState_Locked(State new_state) {
692 DVLOG(1) << __FUNCTION__ << " : " << state_ << " -> " << new_state;
693 lock_.AssertAcquired();
697 void AudioRendererImpl::OnNewSpliceBuffer(base::TimeDelta splice_timestamp) {
698 DCHECK(task_runner_->BelongsToCurrentThread());
699 splicer_->SetSpliceTimestamp(splice_timestamp);
702 void AudioRendererImpl::OnConfigChange() {
703 DCHECK(task_runner_->BelongsToCurrentThread());
704 DCHECK(expecting_config_changes_);
705 buffer_converter_->ResetTimestampState();
706 // Drain flushed buffers from the converter so the AudioSplicer receives all
707 // data ahead of any OnNewSpliceBuffer() calls. Since discontinuities should
708 // only appear after config changes, AddInput() should never fail here.
709 while (buffer_converter_->HasNextBuffer())
710 CHECK(splicer_->AddInput(buffer_converter_->GetNextBuffer()));
713 void AudioRendererImpl::SetBufferingState_Locked(
714 BufferingState buffering_state) {
715 DVLOG(1) << __FUNCTION__ << " : " << buffering_state_ << " -> "
717 DCHECK_NE(buffering_state_, buffering_state);
718 lock_.AssertAcquired();
719 buffering_state_ = buffering_state;
721 task_runner_->PostTask(FROM_HERE,
722 base::Bind(buffering_state_cb_, buffering_state_));