1 // Copyright 2016 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/mojo/services/mojo_video_decoder_service.h"
10 #include "base/functional/bind.h"
11 #include "base/functional/callback_helpers.h"
12 #include "base/logging.h"
13 #include "base/metrics/histogram_functions.h"
14 #include "base/strings/stringprintf.h"
15 #include "base/task/single_thread_task_runner.h"
16 #include "base/trace_event/trace_event.h"
17 #include "base/types/optional_util.h"
18 #include "media/base/decoder_buffer.h"
19 #include "media/base/media_util.h"
20 #include "media/base/simple_sync_token_client.h"
21 #include "media/base/video_decoder.h"
22 #include "media/base/video_decoder_config.h"
23 #include "media/base/video_frame.h"
24 #include "media/mojo/common/media_type_converters.h"
25 #include "media/mojo/common/mojo_decoder_buffer_converter.h"
26 #include "media/mojo/services/mojo_cdm_service_context.h"
27 #include "media/mojo/services/mojo_media_client.h"
28 #include "media/mojo/services/mojo_media_log.h"
29 #include "mojo/public/c/system/types.h"
30 #include "mojo/public/cpp/bindings/message.h"
31 #include "mojo/public/cpp/system/buffer.h"
32 #include "mojo/public/cpp/system/handle.h"
33 #include "third_party/abseil-cpp/absl/types/optional.h"
39 // Number of active (Decode() was called at least once)
40 // MojoVideoDecoderService instances that are alive.
42 // Since MojoVideoDecoderService is constructed only by the MediaFactory,
43 // this will only ever be accessed from a single thread.
44 static int32_t g_num_active_mvd_instances = 0;
46 const char kInitializeTraceName[] = "MojoVideoDecoderService::Initialize";
47 const char kDecodeTraceName[] = "MojoVideoDecoderService::Decode";
48 const char kResetTraceName[] = "MojoVideoDecoderService::Reset";
50 base::debug::CrashKeyString* GetNumVideoDecodersCrashKeyString() {
51 static base::debug::CrashKeyString* codec_count_crash_key =
52 base::debug::AllocateCrashKeyString("num-video-decoders",
53 base::debug::CrashKeySize::Size32);
54 return codec_count_crash_key;
59 class VideoFrameHandleReleaserImpl final
60 : public mojom::VideoFrameHandleReleaser {
62 VideoFrameHandleReleaserImpl() { DVLOG(3) << __func__; }
64 VideoFrameHandleReleaserImpl(const VideoFrameHandleReleaserImpl&) = delete;
65 VideoFrameHandleReleaserImpl& operator=(const VideoFrameHandleReleaserImpl&) =
68 ~VideoFrameHandleReleaserImpl() final { DVLOG(3) << __func__; }
70 // Register a VideoFrame to recieve release callbacks. A reference to |frame|
71 // will be held until the remote client calls ReleaseVideoFrame() or is
74 // Returns an UnguessableToken which the client must use to release the
76 base::UnguessableToken RegisterVideoFrame(scoped_refptr<VideoFrame> frame) {
77 base::UnguessableToken token = base::UnguessableToken::Create();
78 DVLOG(3) << __func__ << " => " << token.ToString();
79 video_frames_[token] = std::move(frame);
83 // mojom::MojoVideoFrameHandleReleaser implementation
84 void ReleaseVideoFrame(
85 const base::UnguessableToken& release_token,
86 const absl::optional<gpu::SyncToken>& release_sync_token) final {
87 DVLOG(3) << __func__ << "(" << release_token.ToString() << ")";
88 TRACE_EVENT2("media", "VideoFrameHandleReleaserImpl::ReleaseVideoFrame",
89 "release_token", release_token.ToString(),
92 ? (release_sync_token->ToDebugString() + ", has_data: " +
93 (release_sync_token->HasData() ? "true" : "false"))
95 auto it = video_frames_.find(release_token);
96 if (it == video_frames_.end()) {
97 mojo::ReportBadMessage("Unknown |release_token|.");
100 if (it->second->HasReleaseMailboxCB()) {
101 if (!release_sync_token) {
102 mojo::ReportBadMessage(
103 "A SyncToken is required to release frames that have a callback "
104 "for releasing mailboxes.");
107 // An empty *|release_sync_token| can be taken as a signal that the
108 // about-to-be-released VideoFrame was never used by the client.
109 // Therefore, we should let that frame retain whatever SyncToken it has.
110 if (release_sync_token->HasData()) {
111 SimpleSyncTokenClient client(*release_sync_token);
112 it->second->UpdateReleaseSyncToken(&client);
115 video_frames_.erase(it);
119 // TODO(sandersd): Also track age, so that an overall limit can be enforced.
120 base::flat_map<base::UnguessableToken, scoped_refptr<VideoFrame>>
124 MojoVideoDecoderService::MojoVideoDecoderService(
125 MojoMediaClient* mojo_media_client,
126 MojoCdmServiceContext* mojo_cdm_service_context,
127 mojo::PendingRemote<stable::mojom::StableVideoDecoder>
128 oop_video_decoder_pending_remote)
129 : mojo_media_client_(mojo_media_client),
130 mojo_cdm_service_context_(mojo_cdm_service_context),
131 oop_video_decoder_pending_remote_(
132 std::move(oop_video_decoder_pending_remote)) {
133 DVLOG(1) << __func__;
134 DCHECK(mojo_media_client_);
135 DCHECK(mojo_cdm_service_context_);
136 weak_this_ = weak_factory_.GetWeakPtr();
139 MojoVideoDecoderService::~MojoVideoDecoderService() {
140 DVLOG(1) << __func__;
143 OnDecoderInitialized(DecoderStatus::Codes::kInterrupted);
149 if (is_active_instance_) {
150 g_num_active_mvd_instances--;
151 base::debug::SetCrashKeyString(
152 GetNumVideoDecodersCrashKeyString(),
153 base::NumberToString(g_num_active_mvd_instances));
156 // Destruct the VideoDecoder here so its destruction duration is included by
157 // the histogram timer below.
158 weak_factory_.InvalidateWeakPtrs();
161 mojo_media_client_ = nullptr;
162 mojo_cdm_service_context_ = nullptr;
165 void MojoVideoDecoderService::GetSupportedConfigs(
166 GetSupportedConfigsCallback callback) {
167 DVLOG(3) << __func__;
168 TRACE_EVENT0("media", "MojoVideoDecoderService::GetSupportedConfigs");
170 std::move(callback).Run(mojo_media_client_->GetSupportedVideoDecoderConfigs(),
171 mojo_media_client_->GetDecoderImplementationType());
174 void MojoVideoDecoderService::Construct(
175 mojo::PendingAssociatedRemote<mojom::VideoDecoderClient> client,
176 mojo::PendingRemote<mojom::MediaLog> media_log,
177 mojo::PendingReceiver<mojom::VideoFrameHandleReleaser>
178 video_frame_handle_releaser_receiver,
179 mojo::ScopedDataPipeConsumerHandle decoder_buffer_pipe,
180 mojom::CommandBufferIdPtr command_buffer_id,
181 const gfx::ColorSpace& target_color_space) {
182 DVLOG(1) << __func__;
183 TRACE_EVENT0("media", "MojoVideoDecoderService::Construct");
186 mojo::ReportBadMessage("Construct() already called");
190 client_.Bind(std::move(client));
192 scoped_refptr<base::SequencedTaskRunner> task_runner =
193 base::SequencedTaskRunner::GetCurrentDefault();
196 std::make_unique<MojoMediaLog>(std::move(media_log), task_runner);
198 video_frame_handle_releaser_ = mojo::MakeSelfOwnedReceiver(
199 std::make_unique<VideoFrameHandleReleaserImpl>(),
200 std::move(video_frame_handle_releaser_receiver));
202 mojo_decoder_buffer_reader_ =
203 std::make_unique<MojoDecoderBufferReader>(std::move(decoder_buffer_pipe));
205 decoder_ = mojo_media_client_->CreateVideoDecoder(
206 task_runner, media_log_.get(), std::move(command_buffer_id),
208 &MojoVideoDecoderService::OnDecoderRequestedOverlayInfo, weak_this_),
209 target_color_space, std::move(oop_video_decoder_pending_remote_));
212 void MojoVideoDecoderService::Initialize(
213 const VideoDecoderConfig& config,
215 const absl::optional<base::UnguessableToken>& cdm_id,
216 InitializeCallback callback) {
217 DVLOG(1) << __func__ << " config = " << config.AsHumanReadableString()
219 << CdmContext::CdmIdToString(base::OptionalToPtr(cdm_id));
223 TRACE_EVENT_ASYNC_BEGIN2(
224 "media", kInitializeTraceName, this, "config",
225 config.AsHumanReadableString(), "cdm_id",
226 CdmContext::CdmIdToString(base::OptionalToPtr(cdm_id)));
228 init_cb_ = std::move(callback);
230 // Prevent creation of too many hardware decoding instances since it may lead
231 // to system instability. Note: This will break decoding entirely for codecs
232 // which don't have software fallback, so we use a conservative limit. Most
233 // platforms will self-limit and never reach this limit.
234 if (!config.is_encrypted() && g_num_active_mvd_instances >= 128) {
235 OnDecoderInitialized(DecoderStatus::Codes::kTooManyDecoders);
240 OnDecoderInitialized(DecoderStatus::Codes::kFailedToCreateDecoder);
244 // |cdm_context_ref_| must be kept as long as |cdm_context| is used by the
245 // |decoder_|. We do NOT support resetting |cdm_context_ref_| because in
246 // general we don't support resetting CDM in the media pipeline.
249 DCHECK(!cdm_context_ref_);
252 mojo_cdm_service_context_->GetCdmContextRef(cdm_id.value());
253 } else if (cdm_id != cdm_id_) {
254 // TODO(xhwang): Replace with mojo::ReportBadMessage().
255 NOTREACHED() << "The caller should not switch CDM";
256 OnDecoderInitialized(DecoderStatus::Codes::kUnsupportedEncryptionMode);
261 // Get CdmContext, which could be null.
262 CdmContext* cdm_context =
263 cdm_context_ref_ ? cdm_context_ref_->GetCdmContext() : nullptr;
265 if (config.is_encrypted() && !cdm_context) {
266 DVLOG(1) << "CdmContext for "
267 << CdmContext::CdmIdToString(base::OptionalToPtr(cdm_id))
268 << " not found for encrypted video";
269 OnDecoderInitialized(DecoderStatus::Codes::kUnsupportedEncryptionMode);
273 auto gfx_cs = config.color_space_info().ToGfxColorSpace();
274 codec_string_ = base::StringPrintf(
275 "name=%s:codec=%s:profile=%d:size=%s:cs=[%d,%d,%d,%d]:hdrm=%d",
276 GetDecoderName(decoder_->GetDecoderType()).c_str(),
277 GetCodecName(config.codec()).c_str(), config.profile(),
278 config.coded_size().ToString().c_str(),
279 static_cast<int>(gfx_cs.GetPrimaryID()),
280 static_cast<int>(gfx_cs.GetTransferID()),
281 static_cast<int>(gfx_cs.GetMatrixID()),
282 static_cast<int>(gfx_cs.GetRangeID()), config.hdr_metadata().has_value());
284 using Self = MojoVideoDecoderService;
285 decoder_->Initialize(
286 config, low_delay, cdm_context,
287 base::BindOnce(&Self::OnDecoderInitialized, weak_this_),
288 base::BindRepeating(&Self::OnDecoderOutput, weak_this_),
289 base::BindRepeating(&Self::OnDecoderWaiting, weak_this_));
292 void MojoVideoDecoderService::Decode(mojom::DecoderBufferPtr buffer,
293 DecodeCallback callback) {
294 DVLOG(3) << __func__ << " pts=" << buffer->timestamp.InMilliseconds();
297 std::unique_ptr<ScopedDecodeTrace> trace_event;
298 if (MediaTraceIsEnabled()) {
299 // Because multiple Decode() calls may be in flight, each call needs a
300 // unique trace event class to identify it. This scoped event is bound
301 // into the OnDecodeDone callback to ensure the trace is always closed.
302 trace_event = std::make_unique<ScopedDecodeTrace>(
303 kDecodeTraceName, buffer->is_key_frame, buffer->timestamp);
307 OnDecoderDecoded(std::move(callback), std::move(trace_event),
308 DecoderStatus::Codes::kNotInitialized);
312 if (!is_active_instance_) {
313 is_active_instance_ = true;
314 g_num_active_mvd_instances++;
315 base::UmaHistogramExactLinear("Media.MojoVideoDecoder.ActiveInstances",
316 g_num_active_mvd_instances, 64);
317 base::debug::SetCrashKeyString(
318 GetNumVideoDecodersCrashKeyString(),
319 base::NumberToString(g_num_active_mvd_instances));
321 // This will be overwritten as subsequent decoders are created.
322 static auto* last_codec_crash_key = base::debug::AllocateCrashKeyString(
323 "last-video-decoder", base::debug::CrashKeySize::Size256);
324 base::debug::SetCrashKeyString(last_codec_crash_key, codec_string_);
327 mojo_decoder_buffer_reader_->ReadDecoderBuffer(
329 base::BindOnce(&MojoVideoDecoderService::OnReaderRead, weak_this_,
330 std::move(callback), std::move(trace_event)));
333 void MojoVideoDecoderService::Reset(ResetCallback callback) {
334 DVLOG(2) << __func__;
335 TRACE_EVENT_ASYNC_BEGIN0("media", kResetTraceName, this);
339 reset_cb_ = std::move(callback);
346 // Flush the reader so that pending decodes will be dispatched first.
347 mojo_decoder_buffer_reader_->Flush(
348 base::BindOnce(&MojoVideoDecoderService::OnReaderFlushed, weak_this_));
351 void MojoVideoDecoderService::OnDecoderInitialized(DecoderStatus status) {
352 DVLOG(1) << __func__;
353 DCHECK(!status.is_ok() || decoder_);
355 TRACE_EVENT_ASYNC_END1("media", kInitializeTraceName, this, "success",
358 if (!status.is_ok()) {
359 std::move(init_cb_).Run(
361 decoder_ ? decoder_->GetDecoderType() : VideoDecoderType::kUnknown);
364 std::move(init_cb_).Run(status, decoder_->NeedsBitstreamConversion(),
365 decoder_->GetMaxDecodeRequests(),
366 decoder_->GetDecoderType());
369 void MojoVideoDecoderService::OnReaderRead(
370 DecodeCallback callback,
371 std::unique_ptr<ScopedDecodeTrace> trace_event,
372 scoped_refptr<DecoderBuffer> buffer) {
373 DVLOG(3) << __func__;
375 TRACE_EVENT_ASYNC_STEP_PAST1(
376 "media", kDecodeTraceName, trace_event.get(), "ReadDecoderBuffer",
377 "decoder_buffer", buffer ? buffer->AsHumanReadableString() : "null");
381 OnDecoderDecoded(std::move(callback), std::move(trace_event),
382 DecoderStatus::Codes::kFailedToGetDecoderBuffer);
388 base::BindOnce(&MojoVideoDecoderService::OnDecoderDecoded, weak_this_,
389 std::move(callback), std::move(trace_event)));
392 void MojoVideoDecoderService::OnReaderFlushed() {
394 base::BindOnce(&MojoVideoDecoderService::OnDecoderReset, weak_this_));
397 void MojoVideoDecoderService::OnDecoderDecoded(
398 DecodeCallback callback,
399 std::unique_ptr<ScopedDecodeTrace> trace_event,
400 media::DecoderStatus status) {
401 DVLOG(3) << __func__;
403 TRACE_EVENT_ASYNC_STEP_PAST0("media", kDecodeTraceName, trace_event.get(),
405 trace_event->EndTrace(status);
408 std::move(callback).Run(std::move(status));
411 void MojoVideoDecoderService::OnDecoderReset() {
412 DVLOG(2) << __func__;
414 TRACE_EVENT_ASYNC_END0("media", kResetTraceName, this);
415 std::move(reset_cb_).Run();
418 void MojoVideoDecoderService::OnDecoderOutput(scoped_refptr<VideoFrame> frame) {
419 DVLOG(3) << __func__ << " pts=" << frame->timestamp().InMilliseconds();
422 TRACE_EVENT1("media", "MojoVideoDecoderService::OnDecoderOutput",
423 "video_frame", frame->AsHumanReadableString());
425 // All MojoVideoDecoder-based decoders are hardware decoders. If you're the
426 // first to implement an out-of-process decoder that is not power efficent,
427 // you can remove this DCHECK.
428 DCHECK(frame->metadata().power_efficient);
430 absl::optional<base::UnguessableToken> release_token;
431 if ((decoder_->FramesHoldExternalResources() ||
432 frame->HasReleaseMailboxCB()) &&
433 video_frame_handle_releaser_) {
434 // |video_frame_handle_releaser_| is explicitly constructed with a
435 // VideoFrameHandleReleaserImpl in Construct().
436 VideoFrameHandleReleaserImpl* releaser =
437 static_cast<VideoFrameHandleReleaserImpl*>(
438 video_frame_handle_releaser_->impl());
439 release_token = releaser->RegisterVideoFrame(frame);
442 client_->OnVideoFrameDecoded(std::move(frame),
443 decoder_->CanReadWithoutStalling(),
444 std::move(release_token));
447 void MojoVideoDecoderService::OnDecoderWaiting(WaitingReason reason) {
448 DVLOG(3) << __func__;
450 TRACE_EVENT1("media", "MojoVideoDecoderService::OnDecoderWaiting", "reason",
451 static_cast<int>(reason));
452 client_->OnWaiting(reason);
455 void MojoVideoDecoderService::OnOverlayInfoChanged(
456 const OverlayInfo& overlay_info) {
457 DVLOG(2) << __func__;
460 DCHECK(provide_overlay_info_cb_);
461 TRACE_EVENT0("media", "MojoVideoDecoderService::OnOverlayInfoChanged");
462 provide_overlay_info_cb_.Run(overlay_info);
465 void MojoVideoDecoderService::OnDecoderRequestedOverlayInfo(
466 bool restart_for_transitions,
467 ProvideOverlayInfoCB provide_overlay_info_cb) {
468 DVLOG(2) << __func__;
471 DCHECK(!provide_overlay_info_cb_);
472 TRACE_EVENT0("media",
473 "MojoVideoDecoderService::OnDecoderRequestedOverlayInfo");
475 provide_overlay_info_cb_ = std::move(provide_overlay_info_cb);
476 client_->RequestOverlayInfo(restart_for_transitions);