media/mojo/services/mojo_video_decoder_service.cc

   1 // Copyright 2016 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/mojo/services/mojo_video_decoder_service.h"
   6
   7 #include <memory>
   8 #include <utility>
   9
  10 #include "base/functional/bind.h"
  11 #include "base/functional/callback_helpers.h"
  12 #include "base/logging.h"
  13 #include "base/metrics/histogram_functions.h"
  14 #include "base/strings/stringprintf.h"
  15 #include "base/task/single_thread_task_runner.h"
  16 #include "base/trace_event/trace_event.h"
  17 #include "base/types/optional_util.h"
  18 #include "media/base/decoder_buffer.h"
  19 #include "media/base/media_util.h"
  20 #include "media/base/simple_sync_token_client.h"
  21 #include "media/base/video_decoder.h"
  22 #include "media/base/video_decoder_config.h"
  23 #include "media/base/video_frame.h"
  24 #include "media/mojo/common/media_type_converters.h"
  25 #include "media/mojo/common/mojo_decoder_buffer_converter.h"
  26 #include "media/mojo/services/mojo_cdm_service_context.h"
  27 #include "media/mojo/services/mojo_media_client.h"
  28 #include "media/mojo/services/mojo_media_log.h"
  29 #include "mojo/public/c/system/types.h"
  30 #include "mojo/public/cpp/bindings/message.h"
  31 #include "mojo/public/cpp/system/buffer.h"
  32 #include "mojo/public/cpp/system/handle.h"
  33 #include "third_party/abseil-cpp/absl/types/optional.h"
  34
  35 namespace media {
  36
  37 namespace {
  38
  39 // Number of active (Decode() was called at least once)
  40 // MojoVideoDecoderService instances that are alive.
  41 //
  42 // Since MojoVideoDecoderService is constructed only by the MediaFactory,
  43 // this will only ever be accessed from a single thread.
  44 static int32_t g_num_active_mvd_instances = 0;
  45
  46 const char kInitializeTraceName[] = "MojoVideoDecoderService::Initialize";
  47 const char kDecodeTraceName[] = "MojoVideoDecoderService::Decode";
  48 const char kResetTraceName[] = "MojoVideoDecoderService::Reset";
  49
  50 base::debug::CrashKeyString* GetNumVideoDecodersCrashKeyString() {
  51   static base::debug::CrashKeyString* codec_count_crash_key =
  52       base::debug::AllocateCrashKeyString("num-video-decoders",
  53                                           base::debug::CrashKeySize::Size32);
  54   return codec_count_crash_key;
  55 }
  56
  57 }  // namespace
  58
  59 class VideoFrameHandleReleaserImpl final
  60     : public mojom::VideoFrameHandleReleaser {
  61  public:
  62   VideoFrameHandleReleaserImpl() { DVLOG(3) << __func__; }
  63
  64   VideoFrameHandleReleaserImpl(const VideoFrameHandleReleaserImpl&) = delete;
  65   VideoFrameHandleReleaserImpl& operator=(const VideoFrameHandleReleaserImpl&) =
  66       delete;
  67
  68   ~VideoFrameHandleReleaserImpl() final { DVLOG(3) << __func__; }
  69
  70   // Register a VideoFrame to recieve release callbacks. A reference to |frame|
  71   // will be held until the remote client calls ReleaseVideoFrame() or is
  72   // disconnected.
  73   //
  74   // Returns an UnguessableToken which the client must use to release the
  75   // VideoFrame.
  76   base::UnguessableToken RegisterVideoFrame(scoped_refptr<VideoFrame> frame) {
  77     base::UnguessableToken token = base::UnguessableToken::Create();
  78     DVLOG(3) << __func__ << " => " << token.ToString();
  79     video_frames_[token] = std::move(frame);
  80     return token;
  81   }
  82
  83   // mojom::MojoVideoFrameHandleReleaser implementation
  84   void ReleaseVideoFrame(
  85       const base::UnguessableToken& release_token,
  86       const absl::optional<gpu::SyncToken>& release_sync_token) final {
  87     DVLOG(3) << __func__ << "(" << release_token.ToString() << ")";
  88     TRACE_EVENT2("media", "VideoFrameHandleReleaserImpl::ReleaseVideoFrame",
  89                  "release_token", release_token.ToString(),
  90                  "release_sync_token",
  91                  release_sync_token
  92                      ? (release_sync_token->ToDebugString() + ", has_data: " +
  93                         (release_sync_token->HasData() ? "true" : "false"))
  94                      : "null");
  95     auto it = video_frames_.find(release_token);
  96     if (it == video_frames_.end()) {
  97       mojo::ReportBadMessage("Unknown |release_token|.");
  98       return;
  99     }
 100     if (it->second->HasReleaseMailboxCB()) {
 101       if (!release_sync_token) {
 102         mojo::ReportBadMessage(
 103             "A SyncToken is required to release frames that have a callback "
 104             "for releasing mailboxes.");
 105         return;
 106       }
 107       // An empty *|release_sync_token| can be taken as a signal that the
 108       // about-to-be-released VideoFrame was never used by the client.
 109       // Therefore, we should let that frame retain whatever SyncToken it has.
 110       if (release_sync_token->HasData()) {
 111         SimpleSyncTokenClient client(*release_sync_token);
 112         it->second->UpdateReleaseSyncToken(&client);
 113       }
 114     }
 115     video_frames_.erase(it);
 116   }
 117
 118  private:
 119   // TODO(sandersd): Also track age, so that an overall limit can be enforced.
 120   base::flat_map<base::UnguessableToken, scoped_refptr<VideoFrame>>
 121       video_frames_;
 122 };
 123
 124 MojoVideoDecoderService::MojoVideoDecoderService(
 125     MojoMediaClient* mojo_media_client,
 126     MojoCdmServiceContext* mojo_cdm_service_context,
 127     mojo::PendingRemote<stable::mojom::StableVideoDecoder>
 128         oop_video_decoder_pending_remote)
 129     : mojo_media_client_(mojo_media_client),
 130       mojo_cdm_service_context_(mojo_cdm_service_context),
 131       oop_video_decoder_pending_remote_(
 132           std::move(oop_video_decoder_pending_remote)) {
 133   DVLOG(1) << __func__;
 134   DCHECK(mojo_media_client_);
 135   DCHECK(mojo_cdm_service_context_);
 136   weak_this_ = weak_factory_.GetWeakPtr();
 137 }
 138
 139 MojoVideoDecoderService::~MojoVideoDecoderService() {
 140   DVLOG(1) << __func__;
 141
 142   if (init_cb_) {
 143     OnDecoderInitialized(DecoderStatus::Codes::kInterrupted);
 144   }
 145
 146   if (reset_cb_)
 147     OnDecoderReset();
 148
 149   if (is_active_instance_) {
 150     g_num_active_mvd_instances--;
 151     base::debug::SetCrashKeyString(
 152         GetNumVideoDecodersCrashKeyString(),
 153         base::NumberToString(g_num_active_mvd_instances));
 154   }
 155
 156   // Destruct the VideoDecoder here so its destruction duration is included by
 157   // the histogram timer below.
 158   weak_factory_.InvalidateWeakPtrs();
 159   decoder_.reset();
 160
 161   mojo_media_client_ = nullptr;
 162   mojo_cdm_service_context_ = nullptr;
 163 }
 164
 165 void MojoVideoDecoderService::GetSupportedConfigs(
 166     GetSupportedConfigsCallback callback) {
 167   DVLOG(3) << __func__;
 168   TRACE_EVENT0("media", "MojoVideoDecoderService::GetSupportedConfigs");
 169
 170   std::move(callback).Run(mojo_media_client_->GetSupportedVideoDecoderConfigs(),
 171                           mojo_media_client_->GetDecoderImplementationType());
 172 }
 173
 174 void MojoVideoDecoderService::Construct(
 175     mojo::PendingAssociatedRemote<mojom::VideoDecoderClient> client,
 176     mojo::PendingRemote<mojom::MediaLog> media_log,
 177     mojo::PendingReceiver<mojom::VideoFrameHandleReleaser>
 178         video_frame_handle_releaser_receiver,
 179     mojo::ScopedDataPipeConsumerHandle decoder_buffer_pipe,
 180     mojom::CommandBufferIdPtr command_buffer_id,
 181     const gfx::ColorSpace& target_color_space) {
 182   DVLOG(1) << __func__;
 183   TRACE_EVENT0("media", "MojoVideoDecoderService::Construct");
 184
 185   if (media_log_) {
 186     mojo::ReportBadMessage("Construct() already called");
 187     return;
 188   }
 189
 190   client_.Bind(std::move(client));
 191
 192   scoped_refptr<base::SequencedTaskRunner> task_runner =
 193       base::SequencedTaskRunner::GetCurrentDefault();
 194
 195   media_log_ =
 196       std::make_unique<MojoMediaLog>(std::move(media_log), task_runner);
 197
 198   video_frame_handle_releaser_ = mojo::MakeSelfOwnedReceiver(
 199       std::make_unique<VideoFrameHandleReleaserImpl>(),
 200       std::move(video_frame_handle_releaser_receiver));
 201
 202   mojo_decoder_buffer_reader_ =
 203       std::make_unique<MojoDecoderBufferReader>(std::move(decoder_buffer_pipe));
 204
 205   decoder_ = mojo_media_client_->CreateVideoDecoder(
 206       task_runner, media_log_.get(), std::move(command_buffer_id),
 207       base::BindRepeating(
 208           &MojoVideoDecoderService::OnDecoderRequestedOverlayInfo, weak_this_),
 209       target_color_space, std::move(oop_video_decoder_pending_remote_));
 210 }
 211
 212 void MojoVideoDecoderService::Initialize(
 213     const VideoDecoderConfig& config,
 214     bool low_delay,
 215     const absl::optional<base::UnguessableToken>& cdm_id,
 216     InitializeCallback callback) {
 217   DVLOG(1) << __func__ << " config = " << config.AsHumanReadableString()
 218            << ", cdm_id = "
 219            << CdmContext::CdmIdToString(base::OptionalToPtr(cdm_id));
 220   DCHECK(!init_cb_);
 221   DCHECK(callback);
 222
 223   TRACE_EVENT_ASYNC_BEGIN2(
 224       "media", kInitializeTraceName, this, "config",
 225       config.AsHumanReadableString(), "cdm_id",
 226       CdmContext::CdmIdToString(base::OptionalToPtr(cdm_id)));
 227
 228   init_cb_ = std::move(callback);
 229
 230   // Prevent creation of too many hardware decoding instances since it may lead
 231   // to system instability. Note: This will break decoding entirely for codecs
 232   // which don't have software fallback, so we use a conservative limit. Most
 233   // platforms will self-limit and never reach this limit.
 234   if (!config.is_encrypted() && g_num_active_mvd_instances >= 128) {
 235     OnDecoderInitialized(DecoderStatus::Codes::kTooManyDecoders);
 236     return;
 237   }
 238
 239   if (!decoder_) {
 240     OnDecoderInitialized(DecoderStatus::Codes::kFailedToCreateDecoder);
 241     return;
 242   }
 243
 244   // |cdm_context_ref_| must be kept as long as |cdm_context| is used by the
 245   // |decoder_|. We do NOT support resetting |cdm_context_ref_| because in
 246   // general we don't support resetting CDM in the media pipeline.
 247   if (cdm_id) {
 248     if (!cdm_id_) {
 249       DCHECK(!cdm_context_ref_);
 250       cdm_id_ = cdm_id;
 251       cdm_context_ref_ =
 252           mojo_cdm_service_context_->GetCdmContextRef(cdm_id.value());
 253     } else if (cdm_id != cdm_id_) {
 254       // TODO(xhwang): Replace with mojo::ReportBadMessage().
 255       NOTREACHED() << "The caller should not switch CDM";
 256       OnDecoderInitialized(DecoderStatus::Codes::kUnsupportedEncryptionMode);
 257       return;
 258     }
 259   }
 260
 261   // Get CdmContext, which could be null.
 262   CdmContext* cdm_context =
 263       cdm_context_ref_ ? cdm_context_ref_->GetCdmContext() : nullptr;
 264
 265   if (config.is_encrypted() && !cdm_context) {
 266     DVLOG(1) << "CdmContext for "
 267              << CdmContext::CdmIdToString(base::OptionalToPtr(cdm_id))
 268              << " not found for encrypted video";
 269     OnDecoderInitialized(DecoderStatus::Codes::kUnsupportedEncryptionMode);
 270     return;
 271   }
 272
 273   auto gfx_cs = config.color_space_info().ToGfxColorSpace();
 274   codec_string_ = base::StringPrintf(
 275       "name=%s:codec=%s:profile=%d:size=%s:cs=[%d,%d,%d,%d]:hdrm=%d",
 276       GetDecoderName(decoder_->GetDecoderType()).c_str(),
 277       GetCodecName(config.codec()).c_str(), config.profile(),
 278       config.coded_size().ToString().c_str(),
 279       static_cast<int>(gfx_cs.GetPrimaryID()),
 280       static_cast<int>(gfx_cs.GetTransferID()),
 281       static_cast<int>(gfx_cs.GetMatrixID()),
 282       static_cast<int>(gfx_cs.GetRangeID()), config.hdr_metadata().has_value());
 283
 284   using Self = MojoVideoDecoderService;
 285   decoder_->Initialize(
 286       config, low_delay, cdm_context,
 287       base::BindOnce(&Self::OnDecoderInitialized, weak_this_),
 288       base::BindRepeating(&Self::OnDecoderOutput, weak_this_),
 289       base::BindRepeating(&Self::OnDecoderWaiting, weak_this_));
 290 }
 291
 292 void MojoVideoDecoderService::Decode(mojom::DecoderBufferPtr buffer,
 293                                      DecodeCallback callback) {
 294   DVLOG(3) << __func__ << " pts=" << buffer->timestamp.InMilliseconds();
 295   DCHECK(callback);
 296
 297   std::unique_ptr<ScopedDecodeTrace> trace_event;
 298   if (MediaTraceIsEnabled()) {
 299     // Because multiple Decode() calls may be in flight, each call needs a
 300     // unique trace event class to identify it. This scoped event is bound
 301     // into the OnDecodeDone callback to ensure the trace is always closed.
 302     trace_event = std::make_unique<ScopedDecodeTrace>(
 303         kDecodeTraceName, buffer->is_key_frame, buffer->timestamp);
 304   }
 305
 306   if (!decoder_) {
 307     OnDecoderDecoded(std::move(callback), std::move(trace_event),
 308                      DecoderStatus::Codes::kNotInitialized);
 309     return;
 310   }
 311
 312   if (!is_active_instance_) {
 313     is_active_instance_ = true;
 314     g_num_active_mvd_instances++;
 315     base::UmaHistogramExactLinear("Media.MojoVideoDecoder.ActiveInstances",
 316                                   g_num_active_mvd_instances, 64);
 317     base::debug::SetCrashKeyString(
 318         GetNumVideoDecodersCrashKeyString(),
 319         base::NumberToString(g_num_active_mvd_instances));
 320
 321     // This will be overwritten as subsequent decoders are created.
 322     static auto* last_codec_crash_key = base::debug::AllocateCrashKeyString(
 323         "last-video-decoder", base::debug::CrashKeySize::Size256);
 324     base::debug::SetCrashKeyString(last_codec_crash_key, codec_string_);
 325   }
 326
 327   mojo_decoder_buffer_reader_->ReadDecoderBuffer(
 328       std::move(buffer),
 329       base::BindOnce(&MojoVideoDecoderService::OnReaderRead, weak_this_,
 330                      std::move(callback), std::move(trace_event)));
 331 }
 332
 333 void MojoVideoDecoderService::Reset(ResetCallback callback) {
 334   DVLOG(2) << __func__;
 335   TRACE_EVENT_ASYNC_BEGIN0("media", kResetTraceName, this);
 336   DCHECK(callback);
 337   DCHECK(!reset_cb_);
 338
 339   reset_cb_ = std::move(callback);
 340
 341   if (!decoder_) {
 342     OnDecoderReset();
 343     return;
 344   }
 345
 346   // Flush the reader so that pending decodes will be dispatched first.
 347   mojo_decoder_buffer_reader_->Flush(
 348       base::BindOnce(&MojoVideoDecoderService::OnReaderFlushed, weak_this_));
 349 }
 350
 351 void MojoVideoDecoderService::OnDecoderInitialized(DecoderStatus status) {
 352   DVLOG(1) << __func__;
 353   DCHECK(!status.is_ok() || decoder_);
 354   DCHECK(init_cb_);
 355   TRACE_EVENT_ASYNC_END1("media", kInitializeTraceName, this, "success",
 356                          status.code());
 357
 358   if (!status.is_ok()) {
 359     std::move(init_cb_).Run(
 360         status, false, 1,
 361         decoder_ ? decoder_->GetDecoderType() : VideoDecoderType::kUnknown);
 362     return;
 363   }
 364   std::move(init_cb_).Run(status, decoder_->NeedsBitstreamConversion(),
 365                           decoder_->GetMaxDecodeRequests(),
 366                           decoder_->GetDecoderType());
 367 }
 368
 369 void MojoVideoDecoderService::OnReaderRead(
 370     DecodeCallback callback,
 371     std::unique_ptr<ScopedDecodeTrace> trace_event,
 372     scoped_refptr<DecoderBuffer> buffer) {
 373   DVLOG(3) << __func__;
 374   if (trace_event) {
 375     TRACE_EVENT_ASYNC_STEP_PAST1(
 376         "media", kDecodeTraceName, trace_event.get(), "ReadDecoderBuffer",
 377         "decoder_buffer", buffer ? buffer->AsHumanReadableString() : "null");
 378   }
 379
 380   if (!buffer) {
 381     OnDecoderDecoded(std::move(callback), std::move(trace_event),
 382                      DecoderStatus::Codes::kFailedToGetDecoderBuffer);
 383     return;
 384   }
 385
 386   decoder_->Decode(
 387       buffer,
 388       base::BindOnce(&MojoVideoDecoderService::OnDecoderDecoded, weak_this_,
 389                      std::move(callback), std::move(trace_event)));
 390 }
 391
 392 void MojoVideoDecoderService::OnReaderFlushed() {
 393   decoder_->Reset(
 394       base::BindOnce(&MojoVideoDecoderService::OnDecoderReset, weak_this_));
 395 }
 396
 397 void MojoVideoDecoderService::OnDecoderDecoded(
 398     DecodeCallback callback,
 399     std::unique_ptr<ScopedDecodeTrace> trace_event,
 400     media::DecoderStatus status) {
 401   DVLOG(3) << __func__;
 402   if (trace_event) {
 403     TRACE_EVENT_ASYNC_STEP_PAST0("media", kDecodeTraceName, trace_event.get(),
 404                                  "Decode");
 405     trace_event->EndTrace(status);
 406   }
 407
 408   std::move(callback).Run(std::move(status));
 409 }
 410
 411 void MojoVideoDecoderService::OnDecoderReset() {
 412   DVLOG(2) << __func__;
 413   DCHECK(reset_cb_);
 414   TRACE_EVENT_ASYNC_END0("media", kResetTraceName, this);
 415   std::move(reset_cb_).Run();
 416 }
 417
 418 void MojoVideoDecoderService::OnDecoderOutput(scoped_refptr<VideoFrame> frame) {
 419   DVLOG(3) << __func__ << " pts=" << frame->timestamp().InMilliseconds();
 420   DCHECK(client_);
 421   DCHECK(decoder_);
 422   TRACE_EVENT1("media", "MojoVideoDecoderService::OnDecoderOutput",
 423                "video_frame", frame->AsHumanReadableString());
 424
 425   // All MojoVideoDecoder-based decoders are hardware decoders. If you're the
 426   // first to implement an out-of-process decoder that is not power efficent,
 427   // you can remove this DCHECK.
 428   DCHECK(frame->metadata().power_efficient);
 429
 430   absl::optional<base::UnguessableToken> release_token;
 431   if ((decoder_->FramesHoldExternalResources() ||
 432        frame->HasReleaseMailboxCB()) &&
 433       video_frame_handle_releaser_) {
 434     // |video_frame_handle_releaser_| is explicitly constructed with a
 435     // VideoFrameHandleReleaserImpl in Construct().
 436     VideoFrameHandleReleaserImpl* releaser =
 437         static_cast<VideoFrameHandleReleaserImpl*>(
 438             video_frame_handle_releaser_->impl());
 439     release_token = releaser->RegisterVideoFrame(frame);
 440   }
 441
 442   client_->OnVideoFrameDecoded(std::move(frame),
 443                                decoder_->CanReadWithoutStalling(),
 444                                std::move(release_token));
 445 }
 446
 447 void MojoVideoDecoderService::OnDecoderWaiting(WaitingReason reason) {
 448   DVLOG(3) << __func__;
 449   DCHECK(client_);
 450   TRACE_EVENT1("media", "MojoVideoDecoderService::OnDecoderWaiting", "reason",
 451                static_cast<int>(reason));
 452   client_->OnWaiting(reason);
 453 }
 454
 455 void MojoVideoDecoderService::OnOverlayInfoChanged(
 456     const OverlayInfo& overlay_info) {
 457   DVLOG(2) << __func__;
 458   DCHECK(client_);
 459   DCHECK(decoder_);
 460   DCHECK(provide_overlay_info_cb_);
 461   TRACE_EVENT0("media", "MojoVideoDecoderService::OnOverlayInfoChanged");
 462   provide_overlay_info_cb_.Run(overlay_info);
 463 }
 464
 465 void MojoVideoDecoderService::OnDecoderRequestedOverlayInfo(
 466     bool restart_for_transitions,
 467     ProvideOverlayInfoCB provide_overlay_info_cb) {
 468   DVLOG(2) << __func__;
 469   DCHECK(client_);
 470   DCHECK(decoder_);
 471   DCHECK(!provide_overlay_info_cb_);
 472   TRACE_EVENT0("media",
 473                "MojoVideoDecoderService::OnDecoderRequestedOverlayInfo");
 474
 475   provide_overlay_info_cb_ = std::move(provide_overlay_info_cb);
 476   client_->RequestOverlayInfo(restart_for_transitions);
 477 }
 478
 479 }  // namespace media