media/renderers/audio_renderer_impl.cc

   1 // Copyright 2012 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/renderers/audio_renderer_impl.h"
   6
   7 #include <math.h>
   8 #include <stddef.h>
   9
  10 #include <memory>
  11 #include <utility>
  12
  13 #include "base/command_line.h"
  14 #include "base/functional/bind.h"
  15 #include "base/functional/callback.h"
  16 #include "base/functional/callback_helpers.h"
  17 #include "base/logging.h"
  18 #include "base/metrics/histogram_macros.h"
  19 #include "base/power_monitor/power_monitor.h"
  20 #include "base/ranges/algorithm.h"
  21 #include "base/task/bind_post_task.h"
  22 #include "base/task/sequenced_task_runner.h"
  23 #include "base/time/default_tick_clock.h"
  24 #include "base/time/time.h"
  25 #include "base/trace_event/trace_event.h"
  26 #include "build/build_config.h"
  27 #include "media/audio/null_audio_sink.h"
  28 #include "media/base/audio_buffer.h"
  29 #include "media/base/audio_buffer_converter.h"
  30 #include "media/base/audio_latency.h"
  31 #include "media/base/audio_parameters.h"
  32 #include "media/base/channel_mixing_matrix.h"
  33 #include "media/base/demuxer_stream.h"
  34 #include "media/base/media_client.h"
  35 #include "media/base/media_log.h"
  36 #include "media/base/media_switches.h"
  37 #include "media/base/media_util.h"
  38 #include "media/base/renderer_client.h"
  39 #include "media/base/timestamp_constants.h"
  40 #include "media/filters/audio_clock.h"
  41 #include "media/filters/decrypting_demuxer_stream.h"
  42
  43 namespace media {
  44
  45 AudioRendererImpl::AudioRendererImpl(
  46     const scoped_refptr<base::SequencedTaskRunner>& task_runner,
  47     AudioRendererSink* sink,
  48     const CreateAudioDecodersCB& create_audio_decoders_cb,
  49     MediaLog* media_log,
  50     MediaPlayerLoggingID media_player_id,
  51     SpeechRecognitionClient* speech_recognition_client)
  52     : task_runner_(task_runner),
  53       expecting_config_changes_(false),
  54       sink_(sink),
  55       media_log_(media_log),
  56       player_id_(media_player_id),
  57       client_(nullptr),
  58       tick_clock_(base::DefaultTickClock::GetInstance()),
  59       last_audio_memory_usage_(0),
  60       last_decoded_sample_rate_(0),
  61       last_decoded_channel_layout_(CHANNEL_LAYOUT_NONE),
  62       is_encrypted_(false),
  63       last_decoded_channels_(0),
  64       volume_(1.0f),  // Default unmuted.
  65       playback_rate_(0.0),
  66       state_(kUninitialized),
  67       create_audio_decoders_cb_(create_audio_decoders_cb),
  68       buffering_state_(BUFFERING_HAVE_NOTHING),
  69       rendering_(false),
  70       sink_playing_(false),
  71       pending_read_(false),
  72       received_end_of_stream_(false),
  73       rendered_end_of_stream_(false),
  74       is_suspending_(false),
  75 #if BUILDFLAG(IS_ANDROID)
  76       is_passthrough_(false) {
  77 #else
  78       is_passthrough_(false),
  79       speech_recognition_client_(speech_recognition_client) {
  80 #endif
  81   DCHECK(create_audio_decoders_cb_);
  82   // PowerObserver's must be added and removed from the same thread, but we
  83   // won't remove the observer until we're destructed on |task_runner_| so we
  84   // must post it here if we're on the wrong thread.
  85   if (task_runner_->RunsTasksInCurrentSequence()) {
  86     base::PowerMonitor::AddPowerSuspendObserver(this);
  87   } else {
  88     // Safe to post this without a WeakPtr because this class must be destructed
  89     // on the same thread and construction has not completed yet.
  90     task_runner_->PostTask(
  91         FROM_HERE,
  92         base::BindOnce(
  93             IgnoreResult(&base::PowerMonitor::AddPowerSuspendObserver), this));
  94   }
  95
  96   // Do not add anything below this line since the above actions are only safe
  97   // as the last lines of the constructor.
  98 }
  99
 100 AudioRendererImpl::~AudioRendererImpl() {
 101   DVLOG(1) << __func__;
 102   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 103   base::PowerMonitor::RemovePowerSuspendObserver(this);
 104
 105   // If Render() is in progress, this call will wait for Render() to finish.
 106   // After this call, the |sink_| will not call back into |this| anymore.
 107   sink_->Stop();
 108   if (null_sink_)
 109     null_sink_->Stop();
 110
 111   if (init_cb_)
 112     FinishInitialization(PIPELINE_ERROR_ABORT);
 113 }
 114
 115 void AudioRendererImpl::StartTicking() {
 116   DVLOG(1) << __func__;
 117   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 118
 119   base::AutoLock auto_lock(lock_);
 120
 121   DCHECK(!rendering_);
 122   rendering_ = true;
 123
 124   // Wait for an eventual call to SetPlaybackRate() to start rendering.
 125   if (playback_rate_ == 0) {
 126     DCHECK(!sink_playing_);
 127     return;
 128   }
 129
 130   StartRendering_Locked();
 131 }
 132
 133 void AudioRendererImpl::StartRendering_Locked() {
 134   DVLOG(1) << __func__;
 135   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 136   DCHECK_EQ(state_, kPlaying);
 137   DCHECK(!sink_playing_);
 138   DCHECK_NE(playback_rate_, 0.0);
 139   lock_.AssertAcquired();
 140
 141   sink_playing_ = true;
 142   was_unmuted_ = was_unmuted_ || volume_ != 0;
 143   base::AutoUnlock auto_unlock(lock_);
 144   if (volume_ || !null_sink_)
 145     sink_->Play();
 146   else
 147     null_sink_->Play();
 148 }
 149
 150 void AudioRendererImpl::StopTicking() {
 151   DVLOG(1) << __func__;
 152   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 153
 154   base::AutoLock auto_lock(lock_);
 155
 156   DCHECK(rendering_);
 157   rendering_ = false;
 158
 159   // Rendering should have already been stopped with a zero playback rate.
 160   if (playback_rate_ == 0) {
 161     DCHECK(!sink_playing_);
 162     return;
 163   }
 164
 165   StopRendering_Locked();
 166 }
 167
 168 void AudioRendererImpl::StopRendering_Locked() {
 169   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 170   DCHECK_EQ(state_, kPlaying);
 171   DCHECK(sink_playing_);
 172   lock_.AssertAcquired();
 173
 174   sink_playing_ = false;
 175
 176   base::AutoUnlock auto_unlock(lock_);
 177   if (volume_ || !null_sink_)
 178     sink_->Pause();
 179   else
 180     null_sink_->Pause();
 181
 182   stop_rendering_time_ = last_render_time_;
 183 }
 184
 185 void AudioRendererImpl::SetMediaTime(base::TimeDelta time) {
 186   DVLOG(1) << __func__ << "(" << time << ")";
 187   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 188
 189   base::AutoLock auto_lock(lock_);
 190   DCHECK(!rendering_);
 191   DCHECK_EQ(state_, kFlushed);
 192
 193   start_timestamp_ = time;
 194   ended_timestamp_ = kInfiniteDuration;
 195   last_render_time_ = stop_rendering_time_ = base::TimeTicks();
 196   first_packet_timestamp_ = kNoTimestamp;
 197   audio_clock_ =
 198       std::make_unique<AudioClock>(time, audio_parameters_.sample_rate());
 199 }
 200
 201 base::TimeDelta AudioRendererImpl::CurrentMediaTime() {
 202   base::AutoLock auto_lock(lock_);
 203
 204   // Return the current time based on the known extents of the rendered audio
 205   // data plus an estimate based on the last time those values were calculated.
 206   base::TimeDelta current_media_time = audio_clock_->front_timestamp();
 207   if (!last_render_time_.is_null()) {
 208     current_media_time +=
 209         (tick_clock_->NowTicks() - last_render_time_) * playback_rate_;
 210     if (current_media_time > audio_clock_->back_timestamp())
 211       current_media_time = audio_clock_->back_timestamp();
 212   }
 213
 214   return current_media_time;
 215 }
 216
 217 bool AudioRendererImpl::GetWallClockTimes(
 218     const std::vector<base::TimeDelta>& media_timestamps,
 219     std::vector<base::TimeTicks>* wall_clock_times) {
 220   base::AutoLock auto_lock(lock_);
 221   DCHECK(wall_clock_times->empty());
 222
 223   // When playback is paused (rate is zero), assume a rate of 1.0.
 224   const double playback_rate = playback_rate_ ? playback_rate_ : 1.0;
 225   const bool is_time_moving = sink_playing_ && playback_rate_ &&
 226                               !last_render_time_.is_null() &&
 227                               stop_rendering_time_.is_null() && !is_suspending_;
 228
 229   // Pre-compute the time until playback of the audio buffer extents, since
 230   // these values are frequently used below.
 231   const base::TimeDelta time_until_front =
 232       audio_clock_->TimeUntilPlayback(audio_clock_->front_timestamp());
 233   const base::TimeDelta time_until_back =
 234       audio_clock_->TimeUntilPlayback(audio_clock_->back_timestamp());
 235
 236   if (media_timestamps.empty()) {
 237     // Return the current media time as a wall clock time while accounting for
 238     // frames which may be in the process of play out.
 239     wall_clock_times->push_back(std::min(
 240         std::max(tick_clock_->NowTicks(), last_render_time_ + time_until_front),
 241         last_render_time_ + time_until_back));
 242     return is_time_moving;
 243   }
 244
 245   wall_clock_times->reserve(media_timestamps.size());
 246   for (const auto& media_timestamp : media_timestamps) {
 247     // When time was or is moving and the requested media timestamp is within
 248     // range of played out audio, we can provide an exact conversion.
 249     if (!last_render_time_.is_null() &&
 250         media_timestamp >= audio_clock_->front_timestamp() &&
 251         media_timestamp <= audio_clock_->back_timestamp()) {
 252       wall_clock_times->push_back(
 253           last_render_time_ + audio_clock_->TimeUntilPlayback(media_timestamp));
 254       continue;
 255     }
 256
 257     base::TimeDelta base_timestamp, time_until_playback;
 258     if (media_timestamp < audio_clock_->front_timestamp()) {
 259       base_timestamp = audio_clock_->front_timestamp();
 260       time_until_playback = time_until_front;
 261     } else {
 262       base_timestamp = audio_clock_->back_timestamp();
 263       time_until_playback = time_until_back;
 264     }
 265
 266     // In practice, most calls will be estimates given the relatively small
 267     // window in which clients can get the actual time.
 268     wall_clock_times->push_back(last_render_time_ + time_until_playback +
 269                                 (media_timestamp - base_timestamp) /
 270                                     playback_rate);
 271   }
 272
 273   return is_time_moving;
 274 }
 275
 276 TimeSource* AudioRendererImpl::GetTimeSource() {
 277   return this;
 278 }
 279
 280 void AudioRendererImpl::Flush(base::OnceClosure callback) {
 281   DVLOG(1) << __func__;
 282   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 283   TRACE_EVENT_NESTABLE_ASYNC_BEGIN0("media", "AudioRendererImpl::Flush",
 284                                     TRACE_ID_LOCAL(this));
 285
 286   // Flush |sink_| now.  |sink_| must only be accessed on |task_runner_| and not
 287   // be called under |lock_|.
 288   DCHECK(!sink_playing_);
 289   if (volume_ || !null_sink_)
 290     sink_->Flush();
 291   else
 292     null_sink_->Flush();
 293
 294   base::AutoLock auto_lock(lock_);
 295   DCHECK_EQ(state_, kPlaying);
 296   DCHECK(!flush_cb_);
 297
 298   flush_cb_ = std::move(callback);
 299   ChangeState_Locked(kFlushing);
 300
 301   if (pending_read_)
 302     return;
 303
 304   ChangeState_Locked(kFlushed);
 305   DoFlush_Locked();
 306 }
 307
 308 void AudioRendererImpl::DoFlush_Locked() {
 309   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 310   lock_.AssertAcquired();
 311
 312   DCHECK(!pending_read_);
 313   DCHECK_EQ(state_, kFlushed);
 314
 315   ended_timestamp_ = kInfiniteDuration;
 316   audio_decoder_stream_->Reset(base::BindOnce(
 317       &AudioRendererImpl::ResetDecoderDone, weak_factory_.GetWeakPtr()));
 318 }
 319
 320 void AudioRendererImpl::ResetDecoderDone() {
 321   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 322   {
 323     base::AutoLock auto_lock(lock_);
 324
 325     DCHECK_EQ(state_, kFlushed);
 326     DCHECK(flush_cb_);
 327
 328     received_end_of_stream_ = false;
 329     rendered_end_of_stream_ = false;
 330
 331     // Flush() may have been called while underflowed/not fully buffered.
 332     if (buffering_state_ != BUFFERING_HAVE_NOTHING)
 333       SetBufferingState_Locked(BUFFERING_HAVE_NOTHING);
 334
 335     if (buffer_converter_)
 336       buffer_converter_->Reset();
 337     algorithm_->FlushBuffers();
 338   }
 339   FinishFlush();
 340 }
 341
 342 void AudioRendererImpl::StartPlaying() {
 343   DVLOG(1) << __func__;
 344   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 345
 346   base::AutoLock auto_lock(lock_);
 347   DCHECK(!sink_playing_);
 348   DCHECK_EQ(state_, kFlushed);
 349   DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
 350   DCHECK(!pending_read_) << "Pending read must complete before seeking";
 351
 352   ChangeState_Locked(kPlaying);
 353   AttemptRead_Locked();
 354 }
 355
 356 void AudioRendererImpl::Initialize(DemuxerStream* stream,
 357                                    CdmContext* cdm_context,
 358                                    RendererClient* client,
 359                                    PipelineStatusCallback init_cb) {
 360   DVLOG(1) << __func__;
 361   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 362   DCHECK(client);
 363   DCHECK(stream);
 364   DCHECK_EQ(stream->type(), DemuxerStream::AUDIO);
 365   DCHECK(init_cb);
 366   DCHECK(state_ == kUninitialized || state_ == kFlushed);
 367   DCHECK(sink_);
 368   TRACE_EVENT_NESTABLE_ASYNC_BEGIN0("media", "AudioRendererImpl::Initialize",
 369                                     TRACE_ID_LOCAL(this));
 370
 371   // If we are re-initializing playback (e.g. switching media tracks), stop the
 372   // sink first.
 373   if (state_ == kFlushed) {
 374     num_absurd_delay_warnings_ = 0;
 375     sink_->Stop();
 376     if (null_sink_)
 377       null_sink_->Stop();
 378   }
 379
 380   state_ = kInitializing;
 381   demuxer_stream_ = stream;
 382   client_ = client;
 383
 384   // Always post |init_cb_| because |this| could be destroyed if initialization
 385   // failed.
 386   init_cb_ = base::BindPostTaskToCurrentDefault(std::move(init_cb));
 387
 388   // Retrieve hardware device parameters asynchronously so we don't block the
 389   // media thread on synchronous IPC.
 390   sink_->GetOutputDeviceInfoAsync(
 391       base::BindOnce(&AudioRendererImpl::OnDeviceInfoReceived,
 392                      weak_factory_.GetWeakPtr(), demuxer_stream_, cdm_context));
 393
 394 #if !BUILDFLAG(IS_ANDROID)
 395   if (speech_recognition_client_) {
 396     speech_recognition_client_->SetOnReadyCallback(
 397         base::BindPostTaskToCurrentDefault(
 398             base::BindOnce(&AudioRendererImpl::EnableSpeechRecognition,
 399                            weak_factory_.GetWeakPtr())));
 400   }
 401 #endif
 402 }
 403
 404 void AudioRendererImpl::OnDeviceInfoReceived(
 405     DemuxerStream* stream,
 406     CdmContext* cdm_context,
 407     OutputDeviceInfo output_device_info) {
 408   DVLOG(1) << __func__;
 409   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 410   DCHECK(client_);
 411   DCHECK(stream);
 412   DCHECK_EQ(stream->type(), DemuxerStream::AUDIO);
 413   DCHECK(init_cb_);
 414   DCHECK_EQ(state_, kInitializing);
 415
 416   // Fall-back to a fake audio sink if the audio device can't be setup; this
 417   // allows video playback in cases where there is no audio hardware.
 418   //
 419   // TODO(dalecurtis): We could disable the audio track here too.
 420   UMA_HISTOGRAM_ENUMERATION("Media.AudioRendererImpl.SinkStatus",
 421                             output_device_info.device_status(),
 422                             OUTPUT_DEVICE_STATUS_MAX + 1);
 423   if (output_device_info.device_status() != OUTPUT_DEVICE_STATUS_OK) {
 424     MEDIA_LOG(ERROR, media_log_)
 425         << "Output device error, falling back to null sink. device_status="
 426         << output_device_info.device_status();
 427     sink_ = new NullAudioSink(task_runner_);
 428     output_device_info = sink_->GetOutputDeviceInfo();
 429   } else if (base::FeatureList::IsEnabled(kSuspendMutedAudio)) {
 430     // If playback is muted, we use a fake sink for output until it unmutes.
 431     null_sink_ = new NullAudioSink(task_runner_);
 432   }
 433
 434   current_decoder_config_ = stream->audio_decoder_config();
 435   DCHECK(current_decoder_config_.IsValidConfig());
 436
 437   const AudioParameters& hw_params = output_device_info.output_params();
 438   ChannelLayout hw_channel_layout =
 439       hw_params.IsValid() ? hw_params.channel_layout() : CHANNEL_LAYOUT_NONE;
 440
 441   DVLOG(1) << __func__ << ": " << hw_params.AsHumanReadableString();
 442
 443   AudioCodec codec = stream->audio_decoder_config().codec();
 444   if (auto* mc = GetMediaClient()) {
 445     const auto format = ConvertAudioCodecToBitstreamFormat(codec);
 446     is_passthrough_ = mc->IsSupportedBitstreamAudioCodec(codec) &&
 447                       hw_params.IsFormatSupportedByHardware(format);
 448   } else {
 449     is_passthrough_ = false;
 450   }
 451   expecting_config_changes_ = stream->SupportsConfigChanges();
 452   // AC3/EAC3 windows decoder supports input channel count in the range 1 (mono)
 453   // to 8 (7.1 channel configuration), but output channel config are stereo, 5.1
 454   // and 7.1. There will be channel config changes, so here force
 455   // 'expecting_config_changes_' to true to use 'hw_channel_layout'.
 456   // Refer to
 457   // https://learn.microsoft.com/en-us/windows/win32/medfound/dolby-audio-decoder
 458 #if BUILDFLAG(ENABLE_PLATFORM_AC3_EAC3_AUDIO) && BUILDFLAG(IS_WIN)
 459   if (current_decoder_config_.codec() == AudioCodec::kAC3 ||
 460       current_decoder_config_.codec() == AudioCodec::kEAC3) {
 461     expecting_config_changes_ = true;
 462   }
 463 #endif  // BUILDFLAG(ENABLE_PLATFORM_AC3_EAC3_AUDIO) && BUILDFLAG(IS_WIN)
 464
 465   bool use_stream_params = !expecting_config_changes_ || !hw_params.IsValid() ||
 466                            hw_params.format() == AudioParameters::AUDIO_FAKE ||
 467                            !sink_->IsOptimizedForHardwareParameters();
 468
 469   if (stream->audio_decoder_config().channel_layout() ==
 470           CHANNEL_LAYOUT_DISCRETE &&
 471       sink_->IsOptimizedForHardwareParameters()) {
 472     use_stream_params = false;
 473   }
 474
 475   // Target ~20ms for our buffer size (which is optimal for power efficiency and
 476   // responsiveness to play/pause events), but if the hardware needs something
 477   // even larger (say for Bluetooth devices) prefer that.
 478   //
 479   // Even if |use_stream_params| is true we should choose a value here based on
 480   // hardware parameters since it affects the initial buffer size used by
 481   // AudioRendererAlgorithm. Too small and we will underflow if the hardware
 482   // asks for a buffer larger than the initial algorithm capacity.
 483   const int preferred_buffer_size =
 484       std::max(2 * stream->audio_decoder_config().samples_per_second() / 100,
 485                hw_params.IsValid() ? hw_params.frames_per_buffer() : 0);
 486
 487   SampleFormat target_output_sample_format = kUnknownSampleFormat;
 488   if (is_passthrough_) {
 489     ChannelLayout channel_layout =
 490         stream->audio_decoder_config().channel_layout();
 491     int channels = stream->audio_decoder_config().channels();
 492     int bytes_per_frame = stream->audio_decoder_config().bytes_per_frame();
 493     AudioParameters::Format format = AudioParameters::AUDIO_FAKE;
 494     // For DTS and Dolby formats, set target_output_sample_format to the
 495     // respective bit-stream format so that passthrough decoder will be selected
 496     // by MediaCodecAudioRenderer if this is running on Android.
 497     if (codec == AudioCodec::kAC3) {
 498       format = AudioParameters::AUDIO_BITSTREAM_AC3;
 499       target_output_sample_format = kSampleFormatAc3;
 500     } else if (codec == AudioCodec::kEAC3) {
 501       format = AudioParameters::AUDIO_BITSTREAM_EAC3;
 502       target_output_sample_format = kSampleFormatEac3;
 503     } else if (codec == AudioCodec::kDTS) {
 504       format = AudioParameters::AUDIO_BITSTREAM_DTS;
 505       target_output_sample_format = kSampleFormatDts;
 506       if (hw_params.RequireEncapsulation()) {
 507         bytes_per_frame = 1;
 508         channel_layout = CHANNEL_LAYOUT_MONO;
 509         channels = 1;
 510       }
 511     } else {
 512       NOTREACHED();
 513     }
 514
 515     // If we want the precise PCM frame count here, we have to somehow peek the
 516     // audio bitstream and parse the header ahead of time. Instead, we ensure
 517     // audio bus being large enough to accommodate
 518     // kMaxFramesPerCompressedAudioBuffer frames. The real data size and frame
 519     // count for bitstream formats will be carried in additional fields of
 520     // AudioBus.
 521     const int buffer_size =
 522         AudioParameters::kMaxFramesPerCompressedAudioBuffer * bytes_per_frame;
 523
 524     audio_parameters_.Reset(format, {channel_layout, channels},
 525                             stream->audio_decoder_config().samples_per_second(),
 526                             buffer_size);
 527     buffer_converter_.reset();
 528   } else if (use_stream_params) {
 529     audio_parameters_.Reset(AudioParameters::AUDIO_PCM_LOW_LATENCY,
 530                             {stream->audio_decoder_config().channel_layout(),
 531                              stream->audio_decoder_config().channels()},
 532                             stream->audio_decoder_config().samples_per_second(),
 533                             preferred_buffer_size);
 534     buffer_converter_.reset();
 535   } else {
 536     // To allow for seamless sample rate adaptations (i.e. changes from say
 537     // 16kHz to 48kHz), always resample to the hardware rate.
 538     int sample_rate = hw_params.sample_rate();
 539
 540     // If supported by the OS and the initial sample rate is not too low, let
 541     // the OS level resampler handle resampling for power efficiency.
 542     if (AudioLatency::IsResamplingPassthroughSupported(
 543             AudioLatency::Type::kPlayback) &&
 544         stream->audio_decoder_config().samples_per_second() >= 44100) {
 545       sample_rate = stream->audio_decoder_config().samples_per_second();
 546     }
 547
 548     int stream_channel_count = stream->audio_decoder_config().channels();
 549
 550     bool try_supported_channel_layouts = false;
 551 #if BUILDFLAG(IS_WIN)
 552     try_supported_channel_layouts =
 553         base::CommandLine::ForCurrentProcess()->HasSwitch(
 554             switches::kTrySupportedChannelLayouts);
 555 #endif
 556
 557     // We don't know how to up-mix for DISCRETE layouts (fancy multichannel
 558     // hardware with non-standard speaker arrangement). Instead, pretend the
 559     // hardware layout is stereo and let the OS take care of further up-mixing
 560     // to the discrete layout (http://crbug.com/266674). Additionally, pretend
 561     // hardware is stereo whenever kTrySupportedChannelLayouts is set. This flag
 562     // is for savvy users who want stereo content to output in all surround
 563     // speakers. Using the actual layout (likely 5.1 or higher) will mean our
 564     // mixer will attempt to up-mix stereo source streams to just the left/right
 565     // speaker of the 5.1 setup, nulling out the other channels
 566     // (http://crbug.com/177872).
 567     hw_channel_layout = hw_params.channel_layout() == CHANNEL_LAYOUT_DISCRETE ||
 568                                 try_supported_channel_layouts
 569                             ? CHANNEL_LAYOUT_STEREO
 570                             : hw_params.channel_layout();
 571     int hw_channel_count = ChannelLayoutToChannelCount(hw_channel_layout);
 572
 573     // The layout we pass to |audio_parameters_| will be used for the lifetime
 574     // of this audio renderer, regardless of changes to hardware and/or stream
 575     // properties. Below we choose the max of stream layout vs. hardware layout
 576     // to leave room for changes to the hardware and/or stream (i.e. avoid
 577     // premature down-mixing - http://crbug.com/379288).
 578     // If stream_channels < hw_channels:
 579     //   Taking max means we up-mix to hardware layout. If stream later changes
 580     //   to have more channels, we aren't locked into down-mixing to the
 581     //   initial stream layout.
 582     // If stream_channels > hw_channels:
 583     //   We choose to output stream's layout, meaning mixing is a no-op for the
 584     //   renderer. Browser-side will down-mix to the hardware config. If the
 585     //   hardware later changes to equal stream channels, browser-side will stop
 586     //   down-mixing and use the data from all stream channels.
 587
 588     ChannelLayout stream_channel_layout =
 589         stream->audio_decoder_config().channel_layout();
 590     bool use_stream_channel_layout = hw_channel_count <= stream_channel_count;
 591
 592     ChannelLayoutConfig renderer_channel_layout_config =
 593         use_stream_channel_layout
 594             ? ChannelLayoutConfig(stream_channel_layout, stream_channel_count)
 595             : ChannelLayoutConfig(hw_channel_layout, hw_channel_count);
 596
 597     audio_parameters_.Reset(hw_params.format(), renderer_channel_layout_config,
 598                             sample_rate,
 599                             AudioLatency::GetHighLatencyBufferSize(
 600                                 sample_rate, preferred_buffer_size));
 601   }
 602
 603   audio_parameters_.set_effects(audio_parameters_.effects() |
 604                                 AudioParameters::MULTIZONE);
 605
 606   audio_parameters_.set_latency_tag(AudioLatency::Type::kPlayback);
 607
 608   audio_decoder_stream_ = std::make_unique<AudioDecoderStream>(
 609       std::make_unique<AudioDecoderStream::StreamTraits>(
 610           media_log_, hw_channel_layout, target_output_sample_format),
 611       task_runner_, create_audio_decoders_cb_, media_log_);
 612
 613   audio_decoder_stream_->set_config_change_observer(base::BindRepeating(
 614       &AudioRendererImpl::OnConfigChange, weak_factory_.GetWeakPtr()));
 615
 616   DVLOG(1) << __func__ << ": is_passthrough_=" << is_passthrough_
 617            << " codec=" << codec
 618            << " stream->audio_decoder_config().sample_format="
 619            << stream->audio_decoder_config().sample_format();
 620
 621   if (!client_->IsVideoStreamAvailable()) {
 622     // When video is not available, audio prefetch can be enabled.  See
 623     // crbug/988535.
 624     audio_parameters_.set_effects(audio_parameters_.effects() |
 625                                   AudioParameters::AUDIO_PREFETCH);
 626   }
 627
 628   last_decoded_channel_layout_ =
 629       stream->audio_decoder_config().channel_layout();
 630
 631   is_encrypted_ = stream->audio_decoder_config().is_encrypted();
 632
 633   last_decoded_channels_ = stream->audio_decoder_config().channels();
 634
 635   {
 636     // Set the |audio_clock_| under lock in case this is a reinitialize and some
 637     // external caller to GetWallClockTimes() exists.
 638     base::AutoLock lock(lock_);
 639     audio_clock_ = std::make_unique<AudioClock>(
 640         base::TimeDelta(), audio_parameters_.sample_rate());
 641   }
 642
 643   audio_decoder_stream_->Initialize(
 644       stream,
 645       base::BindOnce(&AudioRendererImpl::OnAudioDecoderStreamInitialized,
 646                      weak_factory_.GetWeakPtr()),
 647       cdm_context,
 648       base::BindRepeating(&AudioRendererImpl::OnStatisticsUpdate,
 649                           weak_factory_.GetWeakPtr()),
 650       base::BindRepeating(&AudioRendererImpl::OnWaiting,
 651                           weak_factory_.GetWeakPtr()));
 652 }
 653
 654 void AudioRendererImpl::OnAudioDecoderStreamInitialized(bool success) {
 655   DVLOG(1) << __func__ << ": " << success;
 656   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 657   base::AutoLock auto_lock(lock_);
 658
 659   if (!success) {
 660     state_ = kUninitialized;
 661     FinishInitialization(DECODER_ERROR_NOT_SUPPORTED);
 662     return;
 663   }
 664
 665   if (!audio_parameters_.IsValid()) {
 666     DVLOG(1) << __func__ << ": Invalid audio parameters: "
 667              << audio_parameters_.AsHumanReadableString();
 668     ChangeState_Locked(kUninitialized);
 669
 670     // TODO(flim): If the channel layout is discrete but channel count is 0, a
 671     // possible cause is that the input stream has > 8 channels but there is no
 672     // Web Audio renderer attached and no channel mixing matrices defined for
 673     // hardware renderers. Adding one for previewing content could be useful.
 674     FinishInitialization(PIPELINE_ERROR_INITIALIZATION_FAILED);
 675     return;
 676   }
 677
 678   if (expecting_config_changes_) {
 679     buffer_converter_ =
 680         std::make_unique<AudioBufferConverter>(audio_parameters_);
 681   }
 682
 683   // We're all good! Continue initializing the rest of the audio renderer
 684   // based on the decoder format.
 685   auto* media_client = GetMediaClient();
 686   auto params =
 687       (media_client ? media_client->GetAudioRendererAlgorithmParameters(
 688                           audio_parameters_)
 689                     : absl::nullopt);
 690   if (params && !client_->IsVideoStreamAvailable()) {
 691     algorithm_ =
 692         std::make_unique<AudioRendererAlgorithm>(media_log_, params.value());
 693   } else {
 694     algorithm_ = std::make_unique<AudioRendererAlgorithm>(media_log_);
 695   }
 696   algorithm_->Initialize(audio_parameters_, is_encrypted_);
 697   if (latency_hint_)
 698     algorithm_->SetLatencyHint(latency_hint_);
 699
 700   algorithm_->SetPreservesPitch(preserves_pitch_);
 701   ConfigureChannelMask();
 702
 703   ChangeState_Locked(kFlushed);
 704
 705   {
 706     base::AutoUnlock auto_unlock(lock_);
 707     sink_->Initialize(audio_parameters_, this);
 708     if (null_sink_) {
 709       null_sink_->Initialize(audio_parameters_, this);
 710       null_sink_->Start();  // Does nothing but reduce state bookkeeping.
 711       real_sink_needs_start_ = true;
 712     } else {
 713       // Even when kSuspendMutedAudio is enabled, we can hit this path if we are
 714       // exclusively using NullAudioSink due to OnDeviceInfoReceived() failure.
 715       sink_->Start();
 716       sink_->Pause();  // Sinks play on start.
 717     }
 718     SetVolume(volume_);
 719   }
 720
 721   DCHECK(!sink_playing_);
 722   FinishInitialization(PIPELINE_OK);
 723 }
 724
 725 void AudioRendererImpl::FinishInitialization(PipelineStatus status) {
 726   DCHECK(init_cb_);
 727   TRACE_EVENT_NESTABLE_ASYNC_END1("media", "AudioRendererImpl::Initialize",
 728                                   TRACE_ID_LOCAL(this), "status",
 729                                   PipelineStatusToString(status));
 730   std::move(init_cb_).Run(status);
 731 }
 732
 733 void AudioRendererImpl::FinishFlush() {
 734   DCHECK(flush_cb_);
 735   TRACE_EVENT_NESTABLE_ASYNC_END0("media", "AudioRendererImpl::Flush",
 736                                   TRACE_ID_LOCAL(this));
 737   // The |flush_cb_| must always post in order to avoid deadlocking, as some of
 738   // the functions which may be bound here are re-entrant into lock-acquiring
 739   // methods of AudioRendererImpl, and FinishFlush may be called while holding
 740   // the lock. See crbug.com/c/1163459 for a detailed explanation of this.
 741   task_runner_->PostTask(FROM_HERE, std::move(flush_cb_));
 742 }
 743
 744 void AudioRendererImpl::OnPlaybackError(PipelineStatus error) {
 745   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 746   client_->OnError(error);
 747 }
 748
 749 void AudioRendererImpl::OnPlaybackEnded() {
 750   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 751   client_->OnEnded();
 752 }
 753
 754 void AudioRendererImpl::OnStatisticsUpdate(const PipelineStatistics& stats) {
 755   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 756   client_->OnStatisticsUpdate(stats);
 757 }
 758
 759 void AudioRendererImpl::OnBufferingStateChange(BufferingState buffering_state) {
 760   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 761
 762   // "Underflow" is only possible when playing. This avoids noise like blaming
 763   // the decoder for an "underflow" that is really just a seek.
 764   BufferingStateChangeReason reason = BUFFERING_CHANGE_REASON_UNKNOWN;
 765   if (state_ == kPlaying && buffering_state == BUFFERING_HAVE_NOTHING) {
 766     reason = audio_decoder_stream_->is_demuxer_read_pending()
 767                  ? DEMUXER_UNDERFLOW
 768                  : DECODER_UNDERFLOW;
 769   }
 770
 771   media_log_->AddEvent<MediaLogEvent::kBufferingStateChanged>(
 772       SerializableBufferingState<SerializableBufferingStateType::kAudio>{
 773           buffering_state, reason});
 774
 775   client_->OnBufferingStateChange(buffering_state, reason);
 776 }
 777
 778 void AudioRendererImpl::OnWaiting(WaitingReason reason) {
 779   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 780   client_->OnWaiting(reason);
 781 }
 782
 783 void AudioRendererImpl::SetVolume(float volume) {
 784   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 785
 786   // Only consider audio as unmuted if the volume is set to a non-zero value
 787   // when the state is kPlaying.
 788   if (state_ == kPlaying) {
 789     was_unmuted_ = was_unmuted_ || volume != 0;
 790   }
 791
 792   if (state_ == kUninitialized || state_ == kInitializing) {
 793     volume_ = volume;
 794     return;
 795   }
 796
 797   sink_->SetVolume(volume);
 798   if (!null_sink_) {
 799     // Either null sink suspension is not enabled or we're already on the null
 800     // sink due to failing to get device parameters.
 801     return;
 802   }
 803
 804   null_sink_->SetVolume(volume);
 805
 806   // Two cases to handle:
 807   //   1. Changing from muted to unmuted state.
 808   //   2. Unmuted startup case.
 809   if ((!volume_ && volume) || (volume && real_sink_needs_start_)) {
 810     // Suspend null audio sink (does nothing if unused).
 811     null_sink_->Pause();
 812
 813     // Complete startup for the real sink if needed.
 814     if (real_sink_needs_start_) {
 815       sink_->Start();
 816       if (!sink_playing_)
 817         sink_->Pause();  // Sinks play on start.
 818       real_sink_needs_start_ = false;
 819     }
 820
 821     // Start sink playback if needed.
 822     if (sink_playing_)
 823       sink_->Play();
 824   } else if (volume_ && !volume) {
 825     // Suspend the real sink (does nothing if unused).
 826     sink_->Pause();
 827
 828     // Start fake sink playback if needed.
 829     if (sink_playing_)
 830       null_sink_->Play();
 831   }
 832
 833   volume_ = volume;
 834 }
 835
 836 void AudioRendererImpl::SetLatencyHint(
 837     absl::optional<base::TimeDelta> latency_hint) {
 838   base::AutoLock auto_lock(lock_);
 839
 840   latency_hint_ = latency_hint;
 841
 842   if (algorithm_) {
 843     algorithm_->SetLatencyHint(latency_hint);
 844
 845     // See if we need further reads to fill up to the new playback threshold.
 846     // This may be needed if rendering isn't active to schedule regular reads.
 847     AttemptRead_Locked();
 848   }
 849 }
 850
 851 void AudioRendererImpl::SetPreservesPitch(bool preserves_pitch) {
 852   base::AutoLock auto_lock(lock_);
 853
 854   preserves_pitch_ = preserves_pitch;
 855
 856   if (algorithm_)
 857     algorithm_->SetPreservesPitch(preserves_pitch);
 858 }
 859
 860 void AudioRendererImpl::SetWasPlayedWithUserActivation(
 861     bool was_played_with_user_activation) {
 862   base::AutoLock auto_lock(lock_);
 863   was_played_with_user_activation_ = was_played_with_user_activation;
 864 }
 865
 866 void AudioRendererImpl::OnSuspend() {
 867   base::AutoLock auto_lock(lock_);
 868   is_suspending_ = true;
 869 }
 870
 871 void AudioRendererImpl::OnResume() {
 872   base::AutoLock auto_lock(lock_);
 873   is_suspending_ = false;
 874 }
 875
 876 void AudioRendererImpl::SetPlayDelayCBForTesting(PlayDelayCBForTesting cb) {
 877   DCHECK_EQ(state_, kUninitialized);
 878   play_delay_cb_for_testing_ = std::move(cb);
 879 }
 880
 881 void AudioRendererImpl::DecodedAudioReady(
 882     AudioDecoderStream::ReadResult result) {
 883   DVLOG(2) << __func__ << "(" << static_cast<int>(result.code()) << ")";
 884   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 885
 886   base::AutoLock auto_lock(lock_);
 887   DCHECK(state_ != kUninitialized);
 888
 889   CHECK(pending_read_);
 890   pending_read_ = false;
 891
 892   if (!result.has_value()) {
 893     auto status = PIPELINE_ERROR_DECODE;
 894     if (result.code() == DecoderStatus::Codes::kAborted)
 895       status = PIPELINE_OK;
 896     else if (result.code() == DecoderStatus::Codes::kDisconnected)
 897       status = PIPELINE_ERROR_DISCONNECTED;
 898
 899     HandleAbortedReadOrDecodeError(status);
 900     return;
 901   }
 902
 903   scoped_refptr<AudioBuffer> buffer = std::move(result).value();
 904   DCHECK(buffer);
 905
 906   if (state_ == kFlushing) {
 907     ChangeState_Locked(kFlushed);
 908     DoFlush_Locked();
 909     return;
 910   }
 911
 912   bool need_another_buffer = true;
 913
 914   // FFmpeg allows "channel pair element" and "single channel element" type
 915   // AAC streams to masquerade as mono and stereo respectively. Allow these
 916   // specific exceptions to avoid playback errors.
 917   bool allow_config_changes = expecting_config_changes_;
 918   if (!expecting_config_changes_ && !buffer->end_of_stream() &&
 919       current_decoder_config_.codec() == AudioCodec::kAAC &&
 920       buffer->sample_rate() == audio_parameters_.sample_rate()) {
 921     const bool is_mono_to_stereo =
 922         buffer->channel_layout() == CHANNEL_LAYOUT_MONO &&
 923         audio_parameters_.channel_layout() == CHANNEL_LAYOUT_STEREO;
 924     const bool is_stereo_to_mono =
 925         buffer->channel_layout() == CHANNEL_LAYOUT_STEREO &&
 926         audio_parameters_.channel_layout() == CHANNEL_LAYOUT_MONO;
 927     if (is_mono_to_stereo || is_stereo_to_mono) {
 928       if (!buffer_converter_) {
 929         buffer_converter_ =
 930             std::make_unique<AudioBufferConverter>(audio_parameters_);
 931       }
 932       allow_config_changes = true;
 933     }
 934   }
 935
 936   if (allow_config_changes) {
 937     if (!buffer->end_of_stream()) {
 938       if (last_decoded_sample_rate_ &&
 939           buffer->sample_rate() != last_decoded_sample_rate_) {
 940         DVLOG(1) << __func__ << " Updating audio sample_rate."
 941                  << " ts:" << buffer->timestamp().InMicroseconds()
 942                  << " old:" << last_decoded_sample_rate_
 943                  << " new:" << buffer->sample_rate();
 944         // Send a bogus config to reset timestamp state.
 945         OnConfigChange(AudioDecoderConfig());
 946       }
 947       last_decoded_sample_rate_ = buffer->sample_rate();
 948
 949       if (last_decoded_channel_layout_ != buffer->channel_layout()) {
 950         if (buffer->channel_layout() == CHANNEL_LAYOUT_DISCRETE) {
 951           MEDIA_LOG(ERROR, media_log_)
 952               << "Unsupported midstream configuration change! Discrete channel"
 953               << " layout not allowed by sink.";
 954           HandleAbortedReadOrDecodeError(PIPELINE_ERROR_DECODE);
 955           return;
 956         } else {
 957           last_decoded_channel_layout_ = buffer->channel_layout();
 958           last_decoded_channels_ = buffer->channel_count();
 959           ConfigureChannelMask();
 960         }
 961       }
 962     }
 963
 964     DCHECK(buffer_converter_);
 965     buffer_converter_->AddInput(std::move(buffer));
 966
 967     while (buffer_converter_->HasNextBuffer()) {
 968       need_another_buffer =
 969           HandleDecodedBuffer_Locked(buffer_converter_->GetNextBuffer());
 970     }
 971   } else {
 972     // TODO(chcunningham, tguilbert): Figure out if we want to support implicit
 973     // config changes during src=. Doing so requires resampling each individual
 974     // stream which is inefficient when there are many tags in a page.
 975     //
 976     // Check if the buffer we received matches the expected configuration.
 977     // Note: We explicitly do not check channel layout here to avoid breaking
 978     // weird behavior with multichannel wav files: http://crbug.com/600538.
 979     if (!buffer->end_of_stream() &&
 980         (buffer->sample_rate() != audio_parameters_.sample_rate() ||
 981          buffer->channel_count() != audio_parameters_.channels())) {
 982       MEDIA_LOG(ERROR, media_log_)
 983           << "Unsupported midstream configuration change!"
 984           << " Sample Rate: " << buffer->sample_rate() << " vs "
 985           << audio_parameters_.sample_rate()
 986           << ", Channels: " << buffer->channel_count() << " vs "
 987           << audio_parameters_.channels();
 988       HandleAbortedReadOrDecodeError(PIPELINE_ERROR_DECODE);
 989       return;
 990     }
 991
 992     need_another_buffer = HandleDecodedBuffer_Locked(std::move(buffer));
 993   }
 994
 995   if (!need_another_buffer && !CanRead_Locked())
 996     return;
 997
 998   AttemptRead_Locked();
 999 }
1000
1001 bool AudioRendererImpl::HandleDecodedBuffer_Locked(
1002     scoped_refptr<AudioBuffer> buffer) {
1003   lock_.AssertAcquired();
1004   bool should_render_end_of_stream = false;
1005   if (buffer->end_of_stream()) {
1006     received_end_of_stream_ = true;
1007     algorithm_->MarkEndOfStream();
1008
1009     // We received no audio to play before EOS, so enter the ended state.
1010     if (first_packet_timestamp_ == kNoTimestamp)
1011       should_render_end_of_stream = true;
1012   } else {
1013     if (buffer->IsBitstreamFormat() && state_ == kPlaying) {
1014       if (IsBeforeStartTime(*buffer))
1015         return true;
1016
1017       // Adjust the start time since we are unable to trim a compressed audio
1018       // buffer.
1019       if (buffer->timestamp() < start_timestamp_ &&
1020           (buffer->timestamp() + buffer->duration()) > start_timestamp_) {
1021         start_timestamp_ = buffer->timestamp();
1022         audio_clock_ = std::make_unique<AudioClock>(
1023             buffer->timestamp(), audio_parameters_.sample_rate());
1024       }
1025     } else if (state_ == kPlaying) {
1026       if (IsBeforeStartTime(*buffer))
1027         return true;
1028
1029       // Trim off any additional time before the start timestamp.
1030       const base::TimeDelta trim_time = start_timestamp_ - buffer->timestamp();
1031       if (trim_time.is_positive()) {
1032         const int frames_to_trim = AudioTimestampHelper::TimeToFrames(
1033             trim_time, buffer->sample_rate());
1034         DVLOG(1) << __func__ << ": Trimming first audio buffer by "
1035                  << frames_to_trim << " frames so it starts at "
1036                  << start_timestamp_;
1037
1038         buffer->TrimStart(frames_to_trim);
1039         buffer->set_timestamp(start_timestamp_);
1040       }
1041       // If the entire buffer was trimmed, request a new one.
1042       if (!buffer->frame_count())
1043         return true;
1044     }
1045
1046     // Store the timestamp of the first packet so we know when to start actual
1047     // audio playback.
1048     if (first_packet_timestamp_ == kNoTimestamp)
1049       first_packet_timestamp_ = buffer->timestamp();
1050
1051 #if !BUILDFLAG(IS_ANDROID)
1052     // Do not transcribe muted streams initiated by autoplay if the stream was
1053     // never unmuted.
1054     if (transcribe_audio_callback_ &&
1055         (was_played_with_user_activation_ || was_unmuted_)) {
1056       transcribe_audio_callback_.Run(buffer);
1057     }
1058 #endif
1059
1060     if (state_ != kUninitialized)
1061       algorithm_->EnqueueBuffer(std::move(buffer));
1062   }
1063
1064   const size_t memory_usage = algorithm_->GetMemoryUsage();
1065   PipelineStatistics stats;
1066   stats.audio_memory_usage = memory_usage - last_audio_memory_usage_;
1067   last_audio_memory_usage_ = memory_usage;
1068   task_runner_->PostTask(FROM_HERE,
1069                          base::BindOnce(&AudioRendererImpl::OnStatisticsUpdate,
1070                                         weak_factory_.GetWeakPtr(), stats));
1071
1072   switch (state_) {
1073     case kUninitialized:
1074     case kInitializing:
1075     case kFlushing:
1076       NOTREACHED_NORETURN();
1077
1078     case kFlushed:
1079       DCHECK(!pending_read_);
1080       return false;
1081
1082     case kPlaying:
1083       if (received_end_of_stream_ || algorithm_->IsQueueAdequateForPlayback()) {
1084         if (buffering_state_ == BUFFERING_HAVE_NOTHING)
1085           SetBufferingState_Locked(BUFFERING_HAVE_ENOUGH);
1086         // This must be done after SetBufferingState_Locked() to ensure the
1087         // proper state transitions for higher levels.
1088         if (should_render_end_of_stream) {
1089           task_runner_->PostTask(
1090               FROM_HERE, base::BindOnce(&AudioRendererImpl::OnPlaybackEnded,
1091                                         weak_factory_.GetWeakPtr()));
1092         }
1093         return false;
1094       }
1095       return true;
1096   }
1097   return false;
1098 }
1099
1100 void AudioRendererImpl::AttemptRead() {
1101   base::AutoLock auto_lock(lock_);
1102   AttemptRead_Locked();
1103 }
1104
1105 void AudioRendererImpl::AttemptRead_Locked() {
1106   DCHECK(task_runner_->RunsTasksInCurrentSequence());
1107   lock_.AssertAcquired();
1108
1109   if (!CanRead_Locked())
1110     return;
1111
1112   pending_read_ = true;
1113
1114   // Don't hold the lock while calling Read(), if the demuxer is busy this will
1115   // block audio rendering for an extended period of time.
1116   // |audio_decoder_stream_| is only accessed on |task_runner_| so this is safe.
1117   base::AutoUnlock auto_unlock(lock_);
1118   audio_decoder_stream_->Read(base::BindOnce(
1119       &AudioRendererImpl::DecodedAudioReady, weak_factory_.GetWeakPtr()));
1120 }
1121
1122 bool AudioRendererImpl::CanRead_Locked() {
1123   lock_.AssertAcquired();
1124
1125   switch (state_) {
1126     case kUninitialized:
1127     case kInitializing:
1128     case kFlushing:
1129     case kFlushed:
1130       return false;
1131
1132     case kPlaying:
1133       break;
1134   }
1135
1136   return !pending_read_ && !received_end_of_stream_ &&
1137          !algorithm_->IsQueueFull();
1138 }
1139
1140 void AudioRendererImpl::SetPlaybackRate(double playback_rate) {
1141   DVLOG(1) << __func__ << "(" << playback_rate << ")";
1142   DCHECK(task_runner_->RunsTasksInCurrentSequence());
1143   DCHECK_GE(playback_rate, 0);
1144   DCHECK(sink_);
1145
1146   base::AutoLock auto_lock(lock_);
1147
1148   if (is_passthrough_ && playback_rate != 0 && playback_rate != 1) {
1149     MEDIA_LOG(INFO, media_log_) << "Playback rate changes are not supported "
1150                                    "when output compressed bitstream."
1151                                 << " Playback Rate: " << playback_rate;
1152     return;
1153   }
1154
1155   // We have two cases here:
1156   // Play: current_playback_rate == 0 && playback_rate != 0
1157   // Pause: current_playback_rate != 0 && playback_rate == 0
1158   double current_playback_rate = playback_rate_;
1159   playback_rate_ = playback_rate;
1160
1161   if (!rendering_)
1162     return;
1163
1164   if (current_playback_rate == 0 && playback_rate != 0) {
1165     StartRendering_Locked();
1166     return;
1167   }
1168
1169   if (current_playback_rate != 0 && playback_rate == 0) {
1170     StopRendering_Locked();
1171     return;
1172   }
1173 }
1174
1175 bool AudioRendererImpl::IsBeforeStartTime(const AudioBuffer& buffer) {
1176   DCHECK_EQ(state_, kPlaying);
1177   return !buffer.end_of_stream() &&
1178          (buffer.timestamp() + buffer.duration()) < start_timestamp_;
1179 }
1180
1181 int AudioRendererImpl::Render(base::TimeDelta delay,
1182                               base::TimeTicks delay_timestamp,
1183                               const AudioGlitchInfo& glitch_info,
1184                               AudioBus* audio_bus) {
1185   TRACE_EVENT1("media", "AudioRendererImpl::Render", "id", player_id_);
1186   int frames_requested = audio_bus->frames();
1187   DVLOG(4) << __func__ << " delay:" << delay << " glitch_info:["
1188            << glitch_info.ToString() << "]"
1189            << " frames_requested:" << frames_requested;
1190
1191   // Since this information is coming from the OS or potentially a fake stream,
1192   // it may end up with spurious values.
1193   if (delay.is_negative()) {
1194     delay = base::TimeDelta();
1195   }
1196
1197   if (delay > base::Seconds(1)) {
1198     LIMITED_MEDIA_LOG(WARNING, media_log_, num_absurd_delay_warnings_, 1)
1199         << "Large rendering delay (" << delay.InSecondsF()
1200         << "s) detected; video may stall or be otherwise out of sync with "
1201            "audio.";
1202   }
1203
1204   int frames_written = 0;
1205   {
1206     base::AutoLock auto_lock(lock_);
1207     last_render_time_ = tick_clock_->NowTicks();
1208
1209     int64_t frames_delayed = AudioTimestampHelper::TimeToFrames(
1210         delay, audio_parameters_.sample_rate());
1211
1212     if (!stop_rendering_time_.is_null()) {
1213       audio_clock_->CompensateForSuspendedWrites(
1214           last_render_time_ - stop_rendering_time_, frames_delayed);
1215       stop_rendering_time_ = base::TimeTicks();
1216     }
1217
1218     // Ensure Stop() hasn't destroyed our |algorithm_| on the pipeline thread.
1219     if (!algorithm_) {
1220       audio_clock_->WroteAudio(0, frames_requested, frames_delayed,
1221                                playback_rate_);
1222       return 0;
1223     }
1224
1225     if (playback_rate_ == 0 || is_suspending_) {
1226       audio_clock_->WroteAudio(0, frames_requested, frames_delayed,
1227                                playback_rate_);
1228       return 0;
1229     }
1230
1231     // Mute audio by returning 0 when not playing.
1232     if (state_ != kPlaying) {
1233       audio_clock_->WroteAudio(0, frames_requested, frames_delayed,
1234                                playback_rate_);
1235       return 0;
1236     }
1237
1238     if (is_passthrough_ && algorithm_->BufferedFrames() > 0) {
1239       // TODO(tsunghung): For compressed bitstream formats, play zeroed buffer
1240       // won't generate delay. It could be discarded immediately. Need another
1241       // way to generate audio delay.
1242       const base::TimeDelta play_delay =
1243           first_packet_timestamp_ - audio_clock_->back_timestamp();
1244       if (play_delay.is_positive()) {
1245         MEDIA_LOG(ERROR, media_log_)
1246             << "Cannot add delay for compressed audio bitstream foramt."
1247             << " Requested delay: " << play_delay;
1248       }
1249
1250       frames_written += algorithm_->FillBuffer(audio_bus, 0, frames_requested,
1251                                                playback_rate_);
1252
1253       // See Initialize(), the |audio_bus| should be bigger than we need in
1254       // bitstream cases. Fix |frames_requested| to avoid incorrent time
1255       // calculation of |audio_clock_| below.
1256       frames_requested = frames_written;
1257     } else if (algorithm_->BufferedFrames() > 0) {
1258       // Delay playback by writing silence if we haven't reached the first
1259       // timestamp yet; this can occur if the video starts before the audio.
1260       CHECK_NE(first_packet_timestamp_, kNoTimestamp);
1261       CHECK_GE(first_packet_timestamp_, base::TimeDelta());
1262       const base::TimeDelta play_delay =
1263           first_packet_timestamp_ - audio_clock_->back_timestamp();
1264       if (play_delay.is_positive()) {
1265         DCHECK_EQ(frames_written, 0);
1266
1267         if (!play_delay_cb_for_testing_.is_null())
1268           play_delay_cb_for_testing_.Run(play_delay);
1269
1270         // Don't multiply |play_delay| out since it can be a huge value on
1271         // poorly encoded media and multiplying by the sample rate could cause
1272         // the value to overflow.
1273         if (play_delay.InSecondsF() > static_cast<double>(frames_requested) /
1274                                           audio_parameters_.sample_rate()) {
1275           frames_written = frames_requested;
1276         } else {
1277           frames_written =
1278               play_delay.InSecondsF() * audio_parameters_.sample_rate();
1279         }
1280
1281         audio_bus->ZeroFramesPartial(0, frames_written);
1282       }
1283
1284       // If there's any space left, actually render the audio; this is where the
1285       // aural magic happens.
1286       if (frames_written < frames_requested) {
1287         frames_written += algorithm_->FillBuffer(
1288             audio_bus, frames_written, frames_requested - frames_written,
1289             playback_rate_);
1290       }
1291     }
1292
1293     // We use the following conditions to determine end of playback:
1294     //   1) Algorithm can not fill the audio callback buffer
1295     //   2) We received an end of stream buffer
1296     //   3) We haven't already signalled that we've ended
1297     //   4) We've played all known audio data sent to hardware
1298     //
1299     // We use the following conditions to determine underflow:
1300     //   1) Algorithm can not fill the audio callback buffer
1301     //   2) We have NOT received an end of stream buffer
1302     //   3) We are in the kPlaying state
1303     //
1304     // Otherwise the buffer has data we can send to the device.
1305     //
1306     // Per the TimeSource API the media time should always increase even after
1307     // we've rendered all known audio data. Doing so simplifies scenarios where
1308     // we have other sources of media data that need to be scheduled after audio
1309     // data has ended.
1310     //
1311     // That being said, we don't want to advance time when underflowed as we
1312     // know more decoded frames will eventually arrive. If we did, we would
1313     // throw things out of sync when said decoded frames arrive.
1314     int frames_after_end_of_stream = 0;
1315     if (frames_written == 0) {
1316       if (received_end_of_stream_) {
1317         if (ended_timestamp_ == kInfiniteDuration)
1318           ended_timestamp_ = audio_clock_->back_timestamp();
1319         frames_after_end_of_stream = frames_requested;
1320       } else if (state_ == kPlaying &&
1321                  buffering_state_ != BUFFERING_HAVE_NOTHING) {
1322         // Don't increase queue capacity if the queue latency is explicitly
1323         // specified.
1324         if (!latency_hint_)
1325           algorithm_->IncreasePlaybackThreshold();
1326
1327         SetBufferingState_Locked(BUFFERING_HAVE_NOTHING);
1328       }
1329     } else if (frames_written < frames_requested && !received_end_of_stream_ &&
1330                state_ == kPlaying &&
1331                buffering_state_ != BUFFERING_HAVE_NOTHING) {
1332       // If we only partially filled the request and should have more data, go
1333       // ahead and increase queue capacity to try and meet the next request.
1334       // Trigger underflow to give us a chance to refill up to the new cap.
1335       // When a latency hint is present, don't override the user's preference
1336       // with a queue increase, but still signal HAVE_NOTHING for them to take
1337       // action if they choose.
1338
1339       if (!latency_hint_)
1340         algorithm_->IncreasePlaybackThreshold();
1341
1342       SetBufferingState_Locked(BUFFERING_HAVE_NOTHING);
1343     }
1344
1345     audio_clock_->WroteAudio(frames_written + frames_after_end_of_stream,
1346                              frames_requested, frames_delayed, playback_rate_);
1347
1348     if (CanRead_Locked()) {
1349       task_runner_->PostTask(FROM_HERE,
1350                              base::BindOnce(&AudioRendererImpl::AttemptRead,
1351                                             weak_factory_.GetWeakPtr()));
1352     }
1353
1354     if (audio_clock_->front_timestamp() >= ended_timestamp_ &&
1355         !rendered_end_of_stream_) {
1356       rendered_end_of_stream_ = true;
1357       task_runner_->PostTask(FROM_HERE,
1358                              base::BindOnce(&AudioRendererImpl::OnPlaybackEnded,
1359                                             weak_factory_.GetWeakPtr()));
1360     }
1361   }
1362
1363   DCHECK_LE(frames_written, frames_requested);
1364   return frames_written;
1365 }
1366
1367 void AudioRendererImpl::OnRenderError() {
1368   MEDIA_LOG(ERROR, media_log_) << "audio render error";
1369
1370   // Post to |task_runner_| as this is called on the audio callback thread.
1371   task_runner_->PostTask(
1372       FROM_HERE,
1373       base::BindOnce(&AudioRendererImpl::OnPlaybackError,
1374                      weak_factory_.GetWeakPtr(), AUDIO_RENDERER_ERROR));
1375 }
1376
1377 void AudioRendererImpl::HandleAbortedReadOrDecodeError(PipelineStatus status) {
1378   DCHECK(task_runner_->RunsTasksInCurrentSequence());
1379   lock_.AssertAcquired();
1380
1381   switch (state_) {
1382     case kUninitialized:
1383     case kInitializing:
1384       NOTREACHED_NORETURN();
1385     case kFlushing:
1386       ChangeState_Locked(kFlushed);
1387       if (status == PIPELINE_OK) {
1388         DoFlush_Locked();
1389         return;
1390       }
1391
1392       MEDIA_LOG(ERROR, media_log_)
1393           << "audio error during flushing, status: " << status;
1394       client_->OnError(status);
1395       FinishFlush();
1396       return;
1397
1398     case kFlushed:
1399     case kPlaying:
1400       if (status != PIPELINE_OK) {
1401         MEDIA_LOG(ERROR, media_log_)
1402             << "audio error during playing, status: " << status;
1403         client_->OnError(status);
1404       }
1405       return;
1406   }
1407 }
1408
1409 void AudioRendererImpl::ChangeState_Locked(State new_state) {
1410   DVLOG(1) << __func__ << " : " << state_ << " -> " << new_state;
1411   lock_.AssertAcquired();
1412   state_ = new_state;
1413 }
1414
1415 void AudioRendererImpl::OnConfigChange(const AudioDecoderConfig& config) {
1416   DCHECK(task_runner_->RunsTasksInCurrentSequence());
1417   DCHECK(expecting_config_changes_);
1418   buffer_converter_->ResetTimestampState();
1419
1420   // An invalid config may be supplied by callers who simply want to reset
1421   // internal state outside of detecting a new config from the demuxer stream.
1422   // RendererClient only cares to know about config changes that differ from
1423   // previous configs.
1424   if (config.IsValidConfig() && !current_decoder_config_.Matches(config)) {
1425     current_decoder_config_ = config;
1426     client_->OnAudioConfigChange(config);
1427   }
1428 }
1429
1430 void AudioRendererImpl::SetBufferingState_Locked(
1431     BufferingState buffering_state) {
1432   DVLOG(1) << __func__ << " : " << buffering_state_ << " -> "
1433            << buffering_state;
1434   DCHECK_NE(buffering_state_, buffering_state);
1435   lock_.AssertAcquired();
1436   buffering_state_ = buffering_state;
1437
1438   task_runner_->PostTask(
1439       FROM_HERE, base::BindOnce(&AudioRendererImpl::OnBufferingStateChange,
1440                                 weak_factory_.GetWeakPtr(), buffering_state_));
1441 }
1442
1443 void AudioRendererImpl::ConfigureChannelMask() {
1444   DCHECK(algorithm_);
1445   DCHECK(audio_parameters_.IsValid());
1446   DCHECK_NE(last_decoded_channel_layout_, CHANNEL_LAYOUT_NONE);
1447   DCHECK_NE(last_decoded_channel_layout_, CHANNEL_LAYOUT_UNSUPPORTED);
1448
1449   // If we're actually downmixing the signal, no mask is necessary, but ensure
1450   // we clear any existing mask if present.
1451   if (last_decoded_channels_ >= audio_parameters_.channels()) {
1452     algorithm_->SetChannelMask(
1453         std::vector<bool>(audio_parameters_.channels(), true));
1454     return;
1455   }
1456
1457   // Determine the matrix used to upmix the channels.
1458   std::vector<std::vector<float>> matrix;
1459   ChannelMixingMatrix(last_decoded_channel_layout_, last_decoded_channels_,
1460                       audio_parameters_.channel_layout(),
1461                       audio_parameters_.channels())
1462       .CreateTransformationMatrix(&matrix);
1463
1464   // All channels with a zero mix are muted and can be ignored.
1465   std::vector<bool> channel_mask(audio_parameters_.channels(), false);
1466   for (size_t ch = 0; ch < matrix.size(); ++ch) {
1467     channel_mask[ch] =
1468         base::ranges::any_of(matrix[ch], [](float mix) { return !!mix; });
1469   }
1470   algorithm_->SetChannelMask(std::move(channel_mask));
1471 }
1472
1473 void AudioRendererImpl::EnableSpeechRecognition() {
1474 #if !BUILDFLAG(IS_ANDROID)
1475   DCHECK(task_runner_->RunsTasksInCurrentSequence());
1476   transcribe_audio_callback_ = base::BindRepeating(
1477       &AudioRendererImpl::TranscribeAudio, weak_factory_.GetWeakPtr());
1478 #endif
1479 }
1480
1481 void AudioRendererImpl::TranscribeAudio(
1482     scoped_refptr<media::AudioBuffer> buffer) {
1483 #if !BUILDFLAG(IS_ANDROID)
1484   DCHECK(task_runner_->RunsTasksInCurrentSequence());
1485   if (speech_recognition_client_)
1486     speech_recognition_client_->AddAudio(std::move(buffer));
1487 #endif
1488 }
1489
1490 }  // namespace media