1 // Copyright 2012 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/filters/audio_renderer_algorithm.h"
10 #include "base/functional/bind.h"
11 #include "base/logging.h"
12 #include "cc/base/math_util.h"
13 #include "media/base/audio_bus.h"
14 #include "media/base/audio_timestamp_helper.h"
15 #include "media/base/limits.h"
16 #include "media/base/media_switches.h"
17 #include "media/filters/wsola_internals.h"
21 // Waveform Similarity Overlap-and-add (WSOLA).
23 // One WSOLA iteration
25 // 1) Extract |target_block_| as input frames at indices
26 // [|target_block_index_|, |target_block_index_| + |ola_window_size_|).
27 // Note that |target_block_| is the "natural" continuation of the output.
29 // 2) Extract |search_block_| as input frames at indices
30 // [|search_block_index_|,
31 // |search_block_index_| + |num_candidate_blocks_| + |ola_window_size_|).
33 // 3) Find a block within the |search_block_| that is most similar
34 // to |target_block_|. Let |optimal_index| be the index of such block and
35 // write it to |optimal_block_|.
38 // |optimal_block_| = |transition_window_| * |target_block_| +
39 // (1 - |transition_window_|) * |optimal_block_|.
41 // 5) Overlap-and-add |optimal_block_| to the |wsola_output_|.
44 // |target_block_| = |optimal_index| + |ola_window_size_| / 2.
45 // |output_index_| = |output_index_| + |ola_window_size_| / 2,
46 // |search_block_center_index| = |output_index_| * |playback_rate|, and
47 // |search_block_index_| = |search_block_center_index| -
48 // |search_block_center_offset_|.
50 // Overlap-and-add window size in milliseconds.
51 constexpr base::TimeDelta kOlaWindowSize = base::Milliseconds(20);
53 // Size of search interval in milliseconds. The search interval is
54 // [-delta delta] around |output_index_| * |playback_rate|. So the search
55 // interval is 2 * delta.
56 constexpr base::TimeDelta kWsolaSearchInterval = base::Milliseconds(30);
58 // The maximum size for the |audio_buffer_|. Arbitrarily determined.
59 constexpr base::TimeDelta kMaxCapacity = base::Seconds(3);
61 // The minimum size for the |audio_buffer_|. Arbitrarily determined.
62 constexpr base::TimeDelta kStartingCapacity = base::Milliseconds(200);
64 // The minimum size for the |audio_buffer_| for encrypted streams.
65 // Set this to be larger than |kStartingCapacity| because the performance of
66 // encrypted playback is always worse than clear playback, due to decryption and
67 // potentially IPC overhead. For the context, see https://crbug.com/403462,
68 // https://crbug.com/718161 and https://crbug.com/879970.
69 constexpr base::TimeDelta kMinStartingCapacityForEncrypted =
70 base::Milliseconds(500);
72 AudioRendererAlgorithm::AudioRendererAlgorithm(MediaLog* media_log)
73 : AudioRendererAlgorithm(
75 {kMaxCapacity, kStartingCapacity,
77 kMinStartingCapacityForEncrypted,
78 kAudioRendererAlgorithmStartingCapacityForEncrypted.Get())}) {}
80 AudioRendererAlgorithm::AudioRendererAlgorithm(
82 AudioRendererAlgorithmParameters params)
83 : media_log_(media_log),
84 audio_renderer_algorithm_params_(std::move(params)),
86 samples_per_second_(0),
87 is_bitstream_format_(false),
90 search_block_center_offset_(0),
91 search_block_index_(0),
92 num_candidate_blocks_(0),
93 target_block_index_(0),
96 num_complete_frames_(0),
100 AudioRendererAlgorithm::~AudioRendererAlgorithm() = default;
102 void AudioRendererAlgorithm::Initialize(const AudioParameters& params,
104 CHECK(params.IsValid());
106 channels_ = params.channels();
107 samples_per_second_ = params.sample_rate();
108 is_bitstream_format_ = params.IsBitstreamFormat();
109 min_playback_threshold_ = params.frames_per_buffer() * 2;
110 initial_capacity_ = capacity_ = playback_threshold_ = std::max(
111 min_playback_threshold_,
112 AudioTimestampHelper::TimeToFrames(
114 ? audio_renderer_algorithm_params_.starting_capacity_for_encrypted
115 : audio_renderer_algorithm_params_.starting_capacity,
116 samples_per_second_));
117 max_capacity_ = std::max(
119 AudioTimestampHelper::TimeToFrames(
120 audio_renderer_algorithm_params_.max_capacity, samples_per_second_));
121 num_candidate_blocks_ = AudioTimestampHelper::TimeToFrames(
122 kWsolaSearchInterval, samples_per_second_);
124 AudioTimestampHelper::TimeToFrames(kOlaWindowSize, samples_per_second_);
126 // Make sure window size is an even number.
127 ola_window_size_ += ola_window_size_ & 1;
128 ola_hop_size_ = ola_window_size_ / 2;
130 // |num_candidate_blocks_| / 2 is the offset of the center of the search
131 // block to the center of the first (left most) candidate block. The offset
132 // of the center of a candidate block to its left most point is
133 // |ola_window_size_| / 2 - 1. Note that |ola_window_size_| is even and in
134 // our convention the center belongs to the left half, so we need to subtract
135 // one frame to get the correct offset.
138 // <------------------------------------------->
140 // |ola_window_size_| / 2 - 1
143 // |num_candidate_blocks_| / 2
146 // X----X----------------X---------------X-----X
147 // <----------> <---------->
148 // Candidate ... Candidate
149 // 1, ... |num_candidate_blocks_|
150 search_block_center_offset_ =
151 num_candidate_blocks_ / 2 + (ola_window_size_ / 2 - 1);
153 // If no mask is provided, assume all channels are valid.
154 if (channel_mask_.empty())
155 SetChannelMask(std::vector<bool>(channels_, true));
158 void AudioRendererAlgorithm::SetChannelMask(std::vector<bool> channel_mask) {
159 DCHECK_EQ(channel_mask.size(), static_cast<size_t>(channels_));
160 channel_mask_ = std::move(channel_mask);
162 CreateSearchWrappers();
165 void AudioRendererAlgorithm::OnResamplerRead(int frame_delay,
166 AudioBus* audio_bus) {
167 const int requested_frames = audio_bus->frames();
168 int read_frames = audio_buffer_.ReadFrames(requested_frames, 0, audio_bus);
170 if (read_frames < requested_frames) {
171 // We should only be filling up |resampler_| with silence if we are playing
172 // out all remaining frames.
173 DCHECK(reached_end_of_stream_);
174 audio_bus->ZeroFramesPartial(read_frames, requested_frames - read_frames);
177 resampler_only_has_silence_ = !read_frames;
180 void AudioRendererAlgorithm::MarkEndOfStream() {
181 reached_end_of_stream_ = true;
184 int AudioRendererAlgorithm::ResampleAndFill(AudioBus* dest,
186 int requested_frames,
187 double playback_rate) {
188 SetFillBufferMode(FillBufferMode::kResampler);
190 resampler_ = std::make_unique<MultiChannelResampler>(
191 channels_, playback_rate, SincResampler::kDefaultRequestSize,
192 base::BindRepeating(&AudioRendererAlgorithm::OnResamplerRead,
193 base::Unretained(this)));
196 if (reached_end_of_stream_ && resampler_only_has_silence_ &&
197 !audio_buffer_.frames()) {
198 // Previous calls to ResampleAndFill() and OnResamplerRead() have used all
199 // of the available buffers from |audio_buffer_|. We have also played out
200 // all remaining frames, and |resampler_| only contains silence.
204 // |resampler_| can request more than |requested_frames|, due to the
205 // requests size not being aligned. To prevent having to fill it with silence,
206 // we find the max number of reads it could request, and make sure we have
207 // enough data to satisfy all of those reads.
208 if (!reached_end_of_stream_ &&
209 audio_buffer_.frames() <
210 resampler_->GetMaxInputFramesRequested(requested_frames)) {
211 // Exit early, forgoing at most a total of |audio_buffer_.frames()| +
212 // |resampler_->BufferedFrames()|.
213 // If we have reached the end of stream, |resampler_| will output silence
214 // after running out of frames, which is ok.
217 resampler_->SetRatio(playback_rate);
219 // Directly use |dest| for the most common case of having 0 offset.
221 resampler_->Resample(requested_frames, dest);
222 return requested_frames;
225 // This is only really used once, at the beginning of a stream, which means
226 // we can use a temporary variable, rather than saving it as a member.
227 // NOTE: We don't wrap |dest|'s channel data in an AudioBus wrapper, because
228 // |dest_offset| isn't aligned always with AudioBus::kChannelAlignment.
229 std::unique_ptr<AudioBus> resampler_output =
230 AudioBus::Create(channels_, requested_frames);
232 resampler_->Resample(requested_frames, resampler_output.get());
233 resampler_output->CopyPartialFramesTo(0, requested_frames, dest_offset, dest);
235 return requested_frames;
238 int AudioRendererAlgorithm::FillBuffer(AudioBus* dest,
240 int requested_frames,
241 double playback_rate) {
242 if (playback_rate == 0)
245 DCHECK_GT(playback_rate, 0);
246 DCHECK_EQ(channels_, dest->channels());
248 // In case of compressed bitstream formats, no post processing is allowed.
249 if (is_bitstream_format_)
250 return audio_buffer_.ReadFrames(requested_frames, dest_offset, dest);
252 int slower_step = ceil(ola_window_size_ * playback_rate);
253 int faster_step = ceil(ola_window_size_ / playback_rate);
255 // Optimize the most common |playback_rate| ~= 1 case to use a single copy
256 // instead of copying frame by frame.
257 if (ola_window_size_ <= faster_step && slower_step >= ola_window_size_) {
258 SetFillBufferMode(FillBufferMode::kPassthrough);
260 const int frames_to_copy =
261 std::min(audio_buffer_.frames(), requested_frames);
262 const int frames_read =
263 audio_buffer_.ReadFrames(frames_to_copy, dest_offset, dest);
264 DCHECK_EQ(frames_read, frames_to_copy);
268 // Use resampling when no pitch adjustments are needed.
269 if (!preserves_pitch_)
270 return ResampleAndFill(dest, dest_offset, requested_frames, playback_rate);
272 SetFillBufferMode(FillBufferMode::kWSOLA);
274 // Allocate structures on first non-1.0 playback rate; these can eat a fair
275 // chunk of memory. ~56kB for stereo 48kHz, up to ~765kB for 7.1 192kHz.
277 ola_window_.reset(new float[ola_window_size_]);
278 internal::GetPeriodicHanningWindow(ola_window_size_, ola_window_.get());
280 transition_window_.reset(new float[ola_window_size_ * 2]);
281 internal::GetPeriodicHanningWindow(2 * ola_window_size_,
282 transition_window_.get());
284 // Initialize for overlap-and-add of the first block.
286 AudioBus::Create(channels_, ola_window_size_ + ola_hop_size_);
287 wsola_output_->Zero();
289 // Auxiliary containers.
290 optimal_block_ = AudioBus::Create(channels_, ola_window_size_);
291 search_block_ = AudioBus::Create(
292 channels_, num_candidate_blocks_ + (ola_window_size_ - 1));
293 target_block_ = AudioBus::Create(channels_, ola_window_size_);
295 // Create potentially smaller wrappers for playback rate adaptation.
296 CreateSearchWrappers();
299 // Silent audio can contain non-zero samples small enough to result in
300 // subnormals internalls. Disabling subnormals can be significantly faster in
302 cc::ScopedSubnormalFloatDisabler disable_subnormals;
304 int rendered_frames = 0;
307 WriteCompletedFramesTo(requested_frames - rendered_frames,
308 dest_offset + rendered_frames, dest);
309 } while (rendered_frames < requested_frames &&
310 RunOneWsolaIteration(playback_rate));
311 return rendered_frames;
314 void AudioRendererAlgorithm::SetFillBufferMode(FillBufferMode mode) {
315 if (last_mode_ == mode)
318 // Clear any state from other fill modes so that we don't produce outdated
320 if (last_mode_ == FillBufferMode::kWSOLA) {
322 search_block_index_ = 0;
323 target_block_index_ = 0;
325 wsola_output_->Zero();
326 num_complete_frames_ = 0;
333 void AudioRendererAlgorithm::FlushBuffers() {
334 // Clear the queue of decoded packets (releasing the buffers).
335 audio_buffer_.Clear();
337 search_block_index_ = 0;
338 target_block_index_ = 0;
340 wsola_output_->Zero();
341 num_complete_frames_ = 0;
344 reached_end_of_stream_ = false;
346 // Reset |capacity_| and |playback_threshold_| so growth triggered by
347 // underflows doesn't penalize seek time. When |latency_hint_| is set we don't
348 // increase the queue for underflow, so avoid resetting it on flush.
349 if (!latency_hint_) {
350 capacity_ = playback_threshold_ = initial_capacity_;
354 void AudioRendererAlgorithm::EnqueueBuffer(
355 scoped_refptr<AudioBuffer> buffer_in) {
356 DCHECK(!buffer_in->end_of_stream());
357 audio_buffer_.Append(std::move(buffer_in));
360 void AudioRendererAlgorithm::SetLatencyHint(
361 absl::optional<base::TimeDelta> latency_hint) {
362 DCHECK_GE(playback_threshold_, min_playback_threshold_);
363 DCHECK_LE(playback_threshold_, capacity_);
364 DCHECK_LE(capacity_, max_capacity_);
366 latency_hint_ = latency_hint;
369 // Restore default values.
370 playback_threshold_ = capacity_ = initial_capacity_;
372 MEDIA_LOG(DEBUG, media_log_)
373 << "Audio latency hint cleared. Default buffer size ("
374 << AudioTimestampHelper::FramesToTime(playback_threshold_,
380 int latency_hint_frames =
381 AudioTimestampHelper::TimeToFrames(*latency_hint_, samples_per_second_);
383 // Set |plabyack_threshold_| using hint, clamped between
384 // [min_playback_threshold_, max_capacity_].
385 std::string clamp_string;
386 if (latency_hint_frames > max_capacity_) {
387 playback_threshold_ = max_capacity_;
388 clamp_string = " (clamped to max)";
389 } else if (latency_hint_frames < min_playback_threshold_) {
390 playback_threshold_ = min_playback_threshold_;
391 clamp_string = " (clamped to min)";
393 playback_threshold_ = latency_hint_frames;
396 // Use |initial_capacity_| if possible. Increase if needed.
397 capacity_ = std::max(playback_threshold_, initial_capacity_);
399 MEDIA_LOG(DEBUG, media_log_)
400 << "Audio latency hint set:" << *latency_hint << ". "
401 << "Effective buffering latency:"
402 << AudioTimestampHelper::FramesToTime(playback_threshold_,
406 DCHECK_GE(playback_threshold_, min_playback_threshold_);
407 DCHECK_LE(playback_threshold_, capacity_);
408 DCHECK_LE(capacity_, max_capacity_);
411 bool AudioRendererAlgorithm::IsQueueAdequateForPlayback() {
412 return audio_buffer_.frames() >= playback_threshold_;
415 bool AudioRendererAlgorithm::IsQueueFull() {
416 return audio_buffer_.frames() >= capacity_;
419 void AudioRendererAlgorithm::IncreasePlaybackThreshold() {
420 DCHECK(!latency_hint_) << "Don't override the user specified latency";
421 DCHECK_EQ(playback_threshold_, capacity_);
422 DCHECK_LE(capacity_, max_capacity_);
424 playback_threshold_ = capacity_ = std::min(2 * capacity_, max_capacity_);
427 int64_t AudioRendererAlgorithm::GetMemoryUsage() const {
428 return BufferedFrames() * channels_ * sizeof(float);
431 int AudioRendererAlgorithm::BufferedFrames() const {
432 return audio_buffer_.frames() +
433 (resampler_ ? static_cast<int>(resampler_->BufferedFrames()) : 0);
436 double AudioRendererAlgorithm::DelayInFrames(double playback_rate) const {
437 int slower_step = std::ceil(ola_window_size_ * playback_rate);
438 int faster_step = std::ceil(ola_window_size_ / playback_rate);
440 // When |playback_rate| ~= 1, we read directly from |audio_buffer_|.
441 if (ola_window_size_ <= faster_step && slower_step >= ola_window_size_)
442 return audio_buffer_.frames();
444 const float buffered_output_frames = BufferedFrames() / playback_rate;
445 const float unconverted_output_frames = buffered_output_frames - output_time_;
446 return unconverted_output_frames + num_complete_frames_;
449 bool AudioRendererAlgorithm::CanPerformWsola() const {
450 const int search_block_size = num_candidate_blocks_ + (ola_window_size_ - 1);
451 const int frames = audio_buffer_.frames();
452 return target_block_index_ + ola_window_size_ <= frames &&
453 search_block_index_ + search_block_size <= frames;
456 bool AudioRendererAlgorithm::RunOneWsolaIteration(double playback_rate) {
457 if (!CanPerformWsola())
463 for (int k = 0; k < channels_; ++k) {
464 if (!channel_mask_[k])
467 const float* const ch_opt_frame = optimal_block_->channel(k);
468 float* ch_output = wsola_output_->channel(k) + num_complete_frames_;
469 for (int n = 0; n < ola_hop_size_; ++n) {
470 ch_output[n] = ch_output[n] * ola_window_[ola_hop_size_ + n] +
471 ch_opt_frame[n] * ola_window_[n];
474 // Copy the second half to the output.
475 memcpy(&ch_output[ola_hop_size_], &ch_opt_frame[ola_hop_size_],
476 sizeof(*ch_opt_frame) * ola_hop_size_);
479 num_complete_frames_ += ola_hop_size_;
480 UpdateOutputTime(playback_rate, ola_hop_size_);
481 RemoveOldInputFrames(playback_rate);
485 void AudioRendererAlgorithm::UpdateOutputTime(double playback_rate,
486 double time_change) {
487 output_time_ += time_change;
488 // Center of the search region, in frames.
489 const int search_block_center_index = static_cast<int>(
490 output_time_ * playback_rate + 0.5);
491 search_block_index_ = search_block_center_index - search_block_center_offset_;
494 void AudioRendererAlgorithm::RemoveOldInputFrames(double playback_rate) {
495 const int earliest_used_index = std::min(target_block_index_,
496 search_block_index_);
497 if (earliest_used_index <= 0)
498 return; // Nothing to remove.
500 // Remove frames from input and adjust indices accordingly.
501 audio_buffer_.SeekFrames(earliest_used_index);
502 target_block_index_ -= earliest_used_index;
504 // Adjust output index.
505 double output_time_change = static_cast<double>(earliest_used_index) /
507 CHECK_GE(output_time_, output_time_change);
508 UpdateOutputTime(playback_rate, -output_time_change);
511 int AudioRendererAlgorithm::WriteCompletedFramesTo(
512 int requested_frames, int dest_offset, AudioBus* dest) {
513 int rendered_frames = std::min(num_complete_frames_, requested_frames);
515 if (rendered_frames == 0)
516 return 0; // There is nothing to read from |wsola_output_|, return.
518 wsola_output_->CopyPartialFramesTo(0, rendered_frames, dest_offset, dest);
520 // Remove the frames which are read.
521 int frames_to_move = wsola_output_->frames() - rendered_frames;
522 for (int k = 0; k < channels_; ++k) {
523 if (!channel_mask_[k])
525 float* ch = wsola_output_->channel(k);
526 memmove(ch, &ch[rendered_frames], sizeof(*ch) * frames_to_move);
528 num_complete_frames_ -= rendered_frames;
529 return rendered_frames;
532 bool AudioRendererAlgorithm::TargetIsWithinSearchRegion() const {
533 const int search_block_size = num_candidate_blocks_ + (ola_window_size_ - 1);
535 return target_block_index_ >= search_block_index_ &&
536 target_block_index_ + ola_window_size_ <=
537 search_block_index_ + search_block_size;
540 void AudioRendererAlgorithm::GetOptimalBlock() {
541 int optimal_index = 0;
543 // An interval around last optimal block which is excluded from the search.
544 // This is to reduce the buzzy sound. The number 160 is rather arbitrary and
545 // derived heuristically.
546 const int kExcludeIntervalLengthFrames = 160;
547 if (TargetIsWithinSearchRegion()) {
548 optimal_index = target_block_index_;
549 PeekAudioWithZeroPrepend(optimal_index, optimal_block_.get());
551 PeekAudioWithZeroPrepend(target_block_index_, target_block_.get());
552 PeekAudioWithZeroPrepend(search_block_index_, search_block_.get());
554 target_block_index_ - ola_hop_size_ - search_block_index_;
555 internal::Interval exclude_interval =
556 std::make_pair(last_optimal - kExcludeIntervalLengthFrames / 2,
557 last_optimal + kExcludeIntervalLengthFrames / 2);
559 // |optimal_index| is in frames and it is relative to the beginning of the
562 internal::OptimalIndex(search_block_wrapper_.get(),
563 target_block_wrapper_.get(), exclude_interval);
565 // Translate |index| w.r.t. the beginning of |audio_buffer_| and extract the
567 optimal_index += search_block_index_;
568 PeekAudioWithZeroPrepend(optimal_index, optimal_block_.get());
570 // Make a transition from target block to the optimal block if different.
571 // Target block has the best continuation to the current output.
572 // Optimal block is the most similar block to the target, however, it might
573 // introduce some discontinuity when over-lap-added. Therefore, we combine
574 // them for a smoother transition. The length of transition window is twice
575 // as that of the optimal-block which makes it like a weighting function
576 // where target-block has higher weight close to zero (weight of 1 at index
577 // 0) and lower weight close the end.
578 for (int k = 0; k < channels_; ++k) {
579 if (!channel_mask_[k])
581 float* ch_opt = optimal_block_->channel(k);
582 const float* const ch_target = target_block_->channel(k);
583 for (int n = 0; n < ola_window_size_; ++n) {
584 ch_opt[n] = ch_opt[n] * transition_window_[n] +
585 ch_target[n] * transition_window_[ola_window_size_ + n];
590 // Next target is one hop ahead of the current optimal.
591 target_block_index_ = optimal_index + ola_hop_size_;
594 void AudioRendererAlgorithm::PeekAudioWithZeroPrepend(
595 int read_offset_frames, AudioBus* dest) {
596 CHECK_LE(read_offset_frames + dest->frames(), audio_buffer_.frames());
598 int write_offset = 0;
599 int num_frames_to_read = dest->frames();
600 if (read_offset_frames < 0) {
601 int num_zero_frames_appended = std::min(-read_offset_frames,
603 read_offset_frames = 0;
604 num_frames_to_read -= num_zero_frames_appended;
605 write_offset = num_zero_frames_appended;
606 dest->ZeroFrames(num_zero_frames_appended);
608 audio_buffer_.PeekFrames(num_frames_to_read, read_offset_frames,
612 void AudioRendererAlgorithm::CreateSearchWrappers() {
613 // WSOLA is quite expensive to run, so if a channel mask exists, use it to
614 // reduce the size of our search space.
615 std::vector<float*> active_target_channels;
616 std::vector<float*> active_search_channels;
617 for (int ch = 0; ch < channels_; ++ch) {
618 if (channel_mask_[ch]) {
619 active_target_channels.push_back(target_block_->channel(ch));
620 active_search_channels.push_back(search_block_->channel(ch));
624 target_block_wrapper_ =
625 AudioBus::WrapVector(target_block_->frames(), active_target_channels);
626 search_block_wrapper_ =
627 AudioBus::WrapVector(search_block_->frames(), active_search_channels);
630 void AudioRendererAlgorithm::SetPreservesPitch(bool preserves_pitch) {
631 preserves_pitch_ = preserves_pitch;