media/filters/decoder_selector.cc

   1 // Copyright 2014 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/filters/decoder_selector.h"
   6
   7 #include <algorithm>
   8 #include <utility>
   9
  10 #include "base/bind.h"
  11 #include "base/callback_helpers.h"
  12 #include "base/feature_list.h"
  13 #include "base/location.h"
  14 #include "base/logging.h"
  15 #include "base/metrics/histogram_functions.h"
  16 #include "base/strings/stringprintf.h"
  17 #include "base/task/sequenced_task_runner.h"
  18 #include "base/trace_event/trace_event.h"
  19 #include "build/build_config.h"
  20 #include "build/chromeos_buildflags.h"
  21 #include "media/base/audio_decoder.h"
  22 #include "media/base/cdm_context.h"
  23 #include "media/base/demuxer_stream.h"
  24 #include "media/base/media_log.h"
  25 #include "media/base/media_switches.h"
  26 #include "media/base/video_decoder.h"
  27 #include "media/filters/decoder_stream_traits.h"
  28 #include "media/filters/decrypting_demuxer_stream.h"
  29
  30 namespace media {
  31
  32 namespace {
  33
  34 const char kSelectDecoderTrace[] = "DecoderSelector::SelectDecoder";
  35
  36 bool SkipDecoderForRTC(const AudioDecoderConfig& /*config*/,
  37                        const AudioDecoder& /*decoder*/) {
  38   return false;
  39 }
  40
  41 bool SkipDecoderForRTC(const VideoDecoderConfig& config,
  42                        const VideoDecoder& decoder) {
  43   // Skip non-platform decoders for rtc based on the feature flag.
  44   return config.is_rtc() && !decoder.IsPlatformDecoder() &&
  45          !base::FeatureList::IsEnabled(kExposeSwDecodersToWebRTC);
  46 }
  47
  48 template <typename ConfigT, typename DecoderT>
  49 DecoderPriority NormalDecoderPriority(const ConfigT& config,
  50                                       const DecoderT& decoder) {
  51   if (SkipDecoderForRTC(config, decoder))
  52     return DecoderPriority::kSkipped;
  53
  54   return DecoderPriority::kNormal;
  55 }
  56
  57 DecoderPriority ResolutionBasedDecoderPriority(const VideoDecoderConfig& config,
  58                                                const VideoDecoder& decoder) {
  59 #if BUILDFLAG(IS_ANDROID)
  60   constexpr auto kSoftwareDecoderHeightCutoff = 360;
  61 #elif BUILDFLAG(IS_CHROMEOS_ASH)
  62   constexpr auto kSoftwareDecoderHeightCutoff = 360;
  63 #else
  64   constexpr auto kSoftwareDecoderHeightCutoff = 720;
  65 #endif
  66
  67   if (SkipDecoderForRTC(config, decoder))
  68     return DecoderPriority::kSkipped;
  69
  70   // We only do a height check to err on the side of prioritizing platform
  71   // decoders.
  72   const auto at_or_above_software_cutoff =
  73       config.visible_rect().height() >= kSoftwareDecoderHeightCutoff;
  74
  75   // Platform decoders are deprioritized below the cutoff, and non-platform
  76   // decoders are deprioritized above it.
  77   return at_or_above_software_cutoff == decoder.IsPlatformDecoder()
  78              ? DecoderPriority::kNormal
  79              : DecoderPriority::kDeprioritized;
  80 }
  81
  82 DecoderPriority PreferNonPlatformDecoders(const VideoDecoderConfig& config,
  83                                           const VideoDecoder& decoder) {
  84   // Prefer software decoders over hardware decoders.  This is useful to force
  85   // software fallback for WebRTC, but still use hardware if there's no software
  86   // implementation to choose.
  87   return decoder.IsPlatformDecoder() ? DecoderPriority::kDeprioritized
  88                                      : DecoderPriority::kNormal;
  89 }
  90
  91 DecoderPriority UnifiedDecoderPriority(const VideoDecoderConfig& config,
  92                                        const VideoDecoder& decoder) {
  93   if (config.is_rtc() ||
  94       base::FeatureList::IsEnabled(kResolutionBasedDecoderPriority)) {
  95     return ResolutionBasedDecoderPriority(config, decoder);
  96   } else {
  97     return NormalDecoderPriority(config, decoder);
  98   }
  99 }
 100
 101 template <typename ConfigT, typename DecoderT>
 102 DecoderPriority SkipNonPlatformDecoders(const ConfigT& config,
 103                                         const DecoderT& decoder) {
 104   if (SkipDecoderForRTC(config, decoder))
 105     return DecoderPriority::kSkipped;
 106
 107   return decoder.IsPlatformDecoder() ? DecoderPriority::kNormal
 108                                      : DecoderPriority::kSkipped;
 109 }
 110
 111 void SetDefaultDecoderPriorityCB(
 112     VideoDecoderSelector::DecoderPriorityCB* out,
 113     const DecoderStreamTraits<DemuxerStream::VIDEO>* traits) {
 114   if (base::FeatureList::IsEnabled(kForceHardwareVideoDecoders)) {
 115     *out = base::BindRepeating(
 116         SkipNonPlatformDecoders<VideoDecoderConfig, VideoDecoder>);
 117   } else if (traits->GetPreferNonPlatformDecoders()) {
 118     *out = base::BindRepeating(PreferNonPlatformDecoders);
 119   } else {
 120     *out = base::BindRepeating(UnifiedDecoderPriority);
 121   }
 122 }
 123
 124 void SetDefaultDecoderPriorityCB(
 125     AudioDecoderSelector::DecoderPriorityCB* out,
 126     const DecoderStreamTraits<DemuxerStream::AUDIO>*) {
 127   if (base::FeatureList::IsEnabled(kForceHardwareAudioDecoders)) {
 128     *out = base::BindRepeating(
 129         SkipNonPlatformDecoders<AudioDecoderConfig, AudioDecoder>);
 130   } else {
 131     // Platform audio decoders are not currently prioritized or deprioritized
 132     *out = base::BindRepeating(
 133         NormalDecoderPriority<AudioDecoderConfig, AudioDecoder>);
 134   }
 135 }
 136
 137 }  // namespace
 138
 139 template <DemuxerStream::Type StreamType>
 140 DecoderSelector<StreamType>::DecoderSelector(
 141     scoped_refptr<base::SequencedTaskRunner> task_runner,
 142     CreateDecodersCB create_decoders_cb,
 143     MediaLog* media_log)
 144     : task_runner_(std::move(task_runner)),
 145       create_decoders_cb_(std::move(create_decoders_cb)),
 146       media_log_(media_log) {
 147   DETACH_FROM_SEQUENCE(sequence_checker_);
 148 }
 149
 150 template <DemuxerStream::Type StreamType>
 151 DecoderSelector<StreamType>::~DecoderSelector() {
 152   DVLOG(2) << __func__;
 153   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 154   if (select_decoder_cb_)
 155     ReturnSelectionError(DecoderStatus::Codes::kFailed);
 156 }
 157
 158 template <DemuxerStream::Type StreamType>
 159 void DecoderSelector<StreamType>::Initialize(StreamTraits* traits,
 160                                              DemuxerStream* stream,
 161                                              CdmContext* cdm_context,
 162                                              WaitingCB waiting_cb) {
 163   DVLOG(2) << __func__;
 164   DCHECK(traits);
 165   DCHECK(stream);
 166
 167   traits_ = traits;
 168   stream_ = stream;
 169   cdm_context_ = cdm_context;
 170   waiting_cb_ = std::move(waiting_cb);
 171   // Only set this here if nobody has overridden it for tests.
 172   if (!decoder_priority_cb_)
 173     SetDefaultDecoderPriorityCB(&decoder_priority_cb_, traits_);
 174 }
 175
 176 template <DemuxerStream::Type StreamType>
 177 void DecoderSelector<StreamType>::SelectDecoderInternal(
 178     SelectDecoderCB select_decoder_cb,
 179     typename Decoder::OutputCB output_cb,
 180     bool needs_new_decoders) {
 181   DVLOG(2) << __func__;
 182   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 183   DCHECK(select_decoder_cb);
 184   DCHECK(!select_decoder_cb_);
 185   select_decoder_cb_ = std::move(select_decoder_cb);
 186   output_cb_ = std::move(output_cb);
 187   config_ = traits_->GetDecoderConfig(stream_);
 188
 189   TRACE_EVENT_ASYNC_BEGIN2("media", kSelectDecoderTrace, this, "type",
 190                            DemuxerStream::GetTypeName(StreamType), "config",
 191                            config_.AsHumanReadableString());
 192
 193   if (!config_.IsValidConfig()) {
 194     DLOG(ERROR) << "Invalid stream config";
 195     ReturnSelectionError(DecoderStatus::Codes::kUnsupportedConfig);
 196     return;
 197   }
 198
 199   if (needs_new_decoders) {
 200     decoder_selection_start_ = base::TimeTicks::Now();
 201     decode_failure_reinit_cause_ = absl::nullopt;
 202     CreateDecoders();
 203   }
 204
 205   GetAndInitializeNextDecoder();
 206 }
 207
 208 template <DemuxerStream::Type StreamType>
 209 void DecoderSelector<StreamType>::BeginDecoderSelection(
 210     SelectDecoderCB select_decoder_cb,
 211     typename Decoder::OutputCB output_cb) {
 212   SelectDecoderInternal(std::move(select_decoder_cb), std::move(output_cb),
 213                         /*needs_new_decoders = */ true);
 214 }
 215
 216 template <DemuxerStream::Type StreamType>
 217 void DecoderSelector<StreamType>::ResumeDecoderSelection(
 218     SelectDecoderCB select_decoder_cb,
 219     typename Decoder::OutputCB output_cb,
 220     DecoderStatus&& reinit_cause) {
 221   DVLOG(2) << __func__;
 222   if (!decode_failure_reinit_cause_.has_value())
 223     decode_failure_reinit_cause_ = std::move(reinit_cause);
 224   SelectDecoderInternal(std::move(select_decoder_cb), std::move(output_cb),
 225                         /*needs_new_decoders = */ false);
 226 }
 227
 228 template <DemuxerStream::Type StreamType>
 229 void DecoderSelector<StreamType>::FinalizeDecoderSelection() {
 230   DVLOG(2) << __func__;
 231   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 232   DCHECK(!select_decoder_cb_);
 233
 234   const std::string decoder_type = is_platform_decoder_ ? "HW" : "SW";
 235   const std::string stream_type =
 236       StreamType == DemuxerStream::AUDIO ? "Audio" : "Video";
 237
 238   if (is_selecting_for_config_change_) {
 239     is_selecting_for_config_change_ = false;
 240     base::UmaHistogramTimes("Media.ConfigChangeDecoderSelectionTime." +
 241                                 stream_type + "." + decoder_type,
 242                             base::TimeTicks::Now() - decoder_selection_start_);
 243   } else {
 244     // Initial selection
 245     base::UmaHistogramTimes(
 246         "Media.InitialDecoderSelectionTime." + stream_type + "." + decoder_type,
 247         base::TimeTicks::Now() - decoder_selection_start_);
 248   }
 249
 250   if (is_codec_changing_) {
 251     is_codec_changing_ = false;
 252     base::UmaHistogramTimes(
 253         "Media.MSE.CodecChangeTime." + stream_type + "." + decoder_type,
 254         base::TimeTicks::Now() - codec_change_start_);
 255   }
 256
 257   // Discard any remaining decoder instances, they won't be used.
 258   decoders_.clear();
 259 }
 260
 261 template <DemuxerStream::Type StreamType>
 262 void DecoderSelector<StreamType>::NotifyConfigChanged() {
 263   DVLOG(2) << __func__;
 264   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 265
 266   is_selecting_for_config_change_ = true;
 267
 268   DecoderConfig config = traits_->GetDecoderConfig(stream_);
 269   if (config.codec() != config_.codec()) {
 270     is_codec_changing_ = true;
 271     codec_change_start_ = base::TimeTicks::Now();
 272   }
 273 }
 274
 275 template <DemuxerStream::Type StreamType>
 276 void DecoderSelector<StreamType>::PrependDecoder(
 277     std::unique_ptr<Decoder> decoder) {
 278   DVLOG(2) << __func__;
 279   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 280
 281   // Decoders inserted directly should be given priority over those returned by
 282   // |create_decoders_cb_|.
 283   decoders_.insert(decoders_.begin(), std::move(decoder));
 284   FilterAndSortAvailableDecoders();
 285 }
 286
 287 template <DemuxerStream::Type StreamType>
 288 void DecoderSelector<StreamType>::OverrideDecoderPriorityCBForTesting(
 289     DecoderPriorityCB decoder_priority_cb) {
 290   decoder_priority_cb_ = std::move(decoder_priority_cb);
 291 }
 292
 293 template <DemuxerStream::Type StreamType>
 294 void DecoderSelector<StreamType>::CreateDecoders() {
 295   // Post-insert decoders returned by `create_decoders_cb_`, so that
 296   // any decoders added via `PrependDecoder()` are not overwritten and retain
 297   // priority (even if they are ultimately de-ranked by
 298   // `FilterAndSortAvailableDecoders()`)
 299   auto new_decoders = create_decoders_cb_.Run();
 300   std::move(new_decoders.begin(), new_decoders.end(),
 301             std::inserter(decoders_, decoders_.end()));
 302   FilterAndSortAvailableDecoders();
 303 }
 304
 305 template <DemuxerStream::Type StreamType>
 306 void DecoderSelector<StreamType>::GetAndInitializeNextDecoder() {
 307   DVLOG(2) << __func__;
 308   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 309   DCHECK(!decoder_);
 310
 311   if (decoders_.empty()) {
 312     // Decoder selection failed. If the stream is encrypted, try again using
 313     // DecryptingDemuxerStream.
 314     if (config_.is_encrypted() && cdm_context_) {
 315       InitializeDecryptingDemuxerStream();
 316       return;
 317     }
 318
 319     if (decode_failure_reinit_cause_.has_value()) {
 320       ReturnSelectionError(std::move(*decode_failure_reinit_cause_));
 321     } else {
 322       ReturnSelectionError(DecoderStatus::Codes::kUnsupportedConfig);
 323     }
 324     return;
 325   }
 326
 327   // Initialize the first decoder on the list.
 328   decoder_ = std::move(decoders_.front());
 329   decoders_.erase(decoders_.begin());
 330   is_platform_decoder_ = decoder_->IsPlatformDecoder();
 331   TRACE_EVENT_ASYNC_STEP_INTO0("media", kSelectDecoderTrace, this,
 332                                GetDecoderName(decoder_->GetDecoderType()));
 333
 334   DVLOG(2) << __func__ << ": initializing " << decoder_->GetDecoderType();
 335   const bool is_live = stream_->liveness() == StreamLiveness::kLive;
 336   traits_->InitializeDecoder(
 337       decoder_.get(), config_, is_live, cdm_context_,
 338       base::BindOnce(&DecoderSelector<StreamType>::OnDecoderInitializeDone,
 339                      weak_this_factory_.GetWeakPtr()),
 340       output_cb_, waiting_cb_);
 341 }
 342
 343 template <DemuxerStream::Type StreamType>
 344 void DecoderSelector<StreamType>::OnDecoderInitializeDone(
 345     DecoderStatus status) {
 346   DCHECK(decoder_);
 347   DVLOG(2) << __func__ << ": " << decoder_->GetDecoderType()
 348            << " success=" << static_cast<int>(status.code());
 349   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 350
 351   if (!status.is_ok()) {
 352     // Note: Don't track this decode status, as it is the result of
 353     // initialization failure.
 354     MEDIA_LOG(INFO, media_log_)
 355         << "Failed to initialize " << decoder_->GetDecoderType();
 356
 357     // Try the next decoder on the list.
 358     decoder_ = nullptr;
 359     GetAndInitializeNextDecoder();
 360     return;
 361   }
 362
 363   RunSelectDecoderCB(std::move(decoder_));
 364 }
 365
 366 template <DemuxerStream::Type StreamType>
 367 void DecoderSelector<StreamType>::ReturnSelectionError(DecoderStatus error) {
 368   DVLOG(1) << __func__ << ": No decoder selected";
 369   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 370   DCHECK(!error.is_ok());
 371
 372   decrypting_demuxer_stream_.reset();
 373   decoders_.clear();
 374   RunSelectDecoderCB(std::move(error));
 375 }
 376
 377 template <DemuxerStream::Type StreamType>
 378 void DecoderSelector<StreamType>::InitializeDecryptingDemuxerStream() {
 379   DCHECK(decoders_.empty());
 380   DCHECK(config_.is_encrypted());
 381   DCHECK(cdm_context_);
 382   TRACE_EVENT_ASYNC_STEP_INTO0("media", kSelectDecoderTrace, this,
 383                                "DecryptingDemuxerStream");
 384
 385   decrypting_demuxer_stream_ = std::make_unique<DecryptingDemuxerStream>(
 386       task_runner_, media_log_, waiting_cb_);
 387
 388   decrypting_demuxer_stream_->Initialize(
 389       stream_, cdm_context_,
 390       base::BindOnce(
 391           &DecoderSelector<StreamType>::OnDecryptingDemuxerStreamInitializeDone,
 392           weak_this_factory_.GetWeakPtr()));
 393 }
 394
 395 template <DemuxerStream::Type StreamType>
 396 void DecoderSelector<StreamType>::OnDecryptingDemuxerStreamInitializeDone(
 397     PipelineStatus status) {
 398   DVLOG(2) << __func__ << ": status=" << status;
 399   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 400
 401   if (status != PIPELINE_OK) {
 402     // Since we already tried every potential decoder without DDS, give up.
 403     ReturnSelectionError(
 404         {DecoderStatus::Codes::kUnsupportedEncryptionMode, std::move(status)});
 405     return;
 406   }
 407
 408   // Once DDS is enabled, there is no going back.
 409   // TODO(sandersd): Support transitions from encrypted to unencrypted.
 410   stream_ = decrypting_demuxer_stream_.get();
 411   cdm_context_ = nullptr;
 412
 413   // We'll use the decrypted config from now on.
 414   config_ = traits_->GetDecoderConfig(stream_);
 415   DCHECK(!config_.is_encrypted());
 416
 417   // Try decoder selection again now that DDS is being used.
 418   CreateDecoders();
 419   GetAndInitializeNextDecoder();
 420 }
 421
 422 template <DemuxerStream::Type StreamType>
 423 void DecoderSelector<StreamType>::RunSelectDecoderCB(
 424     DecoderOrError decoder_or_error) {
 425   DCHECK(select_decoder_cb_);
 426   TRACE_EVENT_ASYNC_END2(
 427       "media", kSelectDecoderTrace, this, "type",
 428       DemuxerStream::GetTypeName(StreamType), "decoder",
 429       base::StringPrintf(
 430           "%s (%s)",
 431           decoder_or_error.has_value()
 432               ? GetDecoderName(decoder_or_error->GetDecoderType()).c_str()
 433               : "null",
 434           decrypting_demuxer_stream_ ? "encrypted" : "unencrypted"));
 435
 436   task_runner_->PostTask(
 437       FROM_HERE,
 438       base::BindOnce(std::move(select_decoder_cb_), std::move(decoder_or_error),
 439                      std::move(decrypting_demuxer_stream_)));
 440 }
 441
 442 template <DemuxerStream::Type StreamType>
 443 void DecoderSelector<StreamType>::FilterAndSortAvailableDecoders() {
 444   std::vector<std::unique_ptr<Decoder>> decoders = std::move(decoders_);
 445   std::vector<std::unique_ptr<Decoder>> deprioritized_decoders;
 446   DCHECK(decoder_priority_cb_);
 447
 448   for (auto& decoder : decoders) {
 449     // Skip the decoder if this decoder doesn't support encryption for a
 450     // decrypting config
 451     if (config_.is_encrypted() && !decoder->SupportsDecryption())
 452       continue;
 453
 454     // Run the predicate on this decoder.
 455     switch (decoder_priority_cb_.Run(config_, *decoder)) {
 456       case DecoderPriority::kSkipped:
 457         continue;
 458       case DecoderPriority::kNormal:
 459         decoders_.push_back(std::move(decoder));
 460         break;
 461       case DecoderPriority::kDeprioritized:
 462         deprioritized_decoders.push_back(std::move(decoder));
 463         break;
 464     }
 465   }
 466
 467   // Post-insert deprioritized decoders
 468   std::move(deprioritized_decoders.begin(), deprioritized_decoders.end(),
 469             std::inserter(decoders_, decoders_.end()));
 470 }
 471
 472 // These forward declarations tell the compiler that we will use
 473 // DecoderSelector with these arguments, allowing us to keep these definitions
 474 // in our .cc without causing linker errors. This also means if anyone tries to
 475 // instantiate a DecoderSelector with anything but these two specializations
 476 // they'll most likely get linker errors.
 477 template class DecoderSelector<DemuxerStream::AUDIO>;
 478 template class DecoderSelector<DemuxerStream::VIDEO>;
 479
 480 }  // namespace media