1 // Copyright 2019 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/filters/dav1d_video_decoder.h"
11 #include "base/bits.h"
12 #include "base/functional/bind.h"
13 #include "base/functional/callback.h"
14 #include "base/functional/callback_helpers.h"
15 #include "base/logging.h"
16 #include "base/strings/stringprintf.h"
17 #include "base/task/bind_post_task.h"
18 #include "base/task/sequenced_task_runner.h"
19 #include "media/base/decoder_buffer.h"
20 #include "media/base/limits.h"
21 #include "media/base/media_log.h"
22 #include "media/base/video_aspect_ratio.h"
23 #include "media/base/video_util.h"
26 #include "third_party/dav1d/libdav1d/include/dav1d/dav1d.h"
31 static void GetDecoderThreadCounts(const int coded_height,
34 // Tile thread counts based on currently available content. Recommended by
35 // YouTube, while frame thread values fit within
36 // limits::kMaxVideoDecodeThreads.
37 if (coded_height >= 700) {
39 4; // Current 720p content is encoded in 5 tiles and 1080p content with
40 // 8 tiles, but we'll exceed limits::kMaxVideoDecodeThreads with 5+
41 // tile threads with 3 frame threads (5 * 3 + 3 = 18 threads vs 16
44 // Since 720p playback isn't smooth without 3 frame threads, we've
45 // chosen a slightly lower tile thread count.
47 } else if (coded_height >= 300) {
56 static VideoPixelFormat Dav1dImgFmtToVideoPixelFormat(
57 const Dav1dPictureParameters* pic) {
58 switch (pic->layout) {
59 // Single plane monochrome images will be converted to standard 3 plane ones
60 // since Chromium doesn't support single Y plane images.
61 case DAV1D_PIXEL_LAYOUT_I400:
62 case DAV1D_PIXEL_LAYOUT_I420:
65 return PIXEL_FORMAT_I420;
67 return PIXEL_FORMAT_YUV420P10;
69 return PIXEL_FORMAT_YUV420P12;
71 DLOG(ERROR) << "Unsupported bit depth: " << pic->bpc;
72 return PIXEL_FORMAT_UNKNOWN;
74 case DAV1D_PIXEL_LAYOUT_I422:
77 return PIXEL_FORMAT_I422;
79 return PIXEL_FORMAT_YUV422P10;
81 return PIXEL_FORMAT_YUV422P12;
83 DLOG(ERROR) << "Unsupported bit depth: " << pic->bpc;
84 return PIXEL_FORMAT_UNKNOWN;
86 case DAV1D_PIXEL_LAYOUT_I444:
89 return PIXEL_FORMAT_I444;
91 return PIXEL_FORMAT_YUV444P10;
93 return PIXEL_FORMAT_YUV444P12;
95 DLOG(ERROR) << "Unsupported bit depth: " << pic->bpc;
96 return PIXEL_FORMAT_UNKNOWN;
101 static void ReleaseDecoderBuffer(const uint8_t* buffer, void* opaque) {
103 static_cast<DecoderBuffer*>(opaque)->Release();
106 static void LogDav1dMessage(void* cookie, const char* format, va_list ap) {
107 auto log = base::StringPrintV(format, ap);
111 if (log.back() == '\n')
117 // std::unique_ptr release helpers. We need to release both the containing
118 // structs as well as refs held within the structures.
119 struct ScopedDav1dDataFree {
120 void operator()(void* x) const {
121 auto* data = static_cast<Dav1dData*>(x);
122 dav1d_data_unref(data);
127 struct ScopedDav1dPictureFree {
128 void operator()(void* x) const {
129 auto* pic = static_cast<Dav1dPicture*>(x);
130 dav1d_picture_unref(pic);
136 SupportedVideoDecoderConfigs Dav1dVideoDecoder::SupportedConfigs() {
137 return {{/*profile_min=*/AV1PROFILE_PROFILE_MAIN,
138 /*profile_max=*/AV1PROFILE_PROFILE_HIGH,
139 /*coded_size_min=*/kDefaultSwDecodeSizeMin,
140 /*coded_size_max=*/kDefaultSwDecodeSizeMax,
141 /*allow_encrypted=*/false,
142 /*require_encrypted=*/false}};
145 Dav1dVideoDecoder::Dav1dVideoDecoder(std::unique_ptr<MediaLog> media_log,
146 OffloadState offload_state)
147 : media_log_(std::move(media_log)),
148 bind_callbacks_(offload_state == OffloadState::kNormal) {
149 DETACH_FROM_SEQUENCE(sequence_checker_);
152 Dav1dVideoDecoder::~Dav1dVideoDecoder() {
153 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
157 VideoDecoderType Dav1dVideoDecoder::GetDecoderType() const {
158 return VideoDecoderType::kDav1d;
161 void Dav1dVideoDecoder::Initialize(const VideoDecoderConfig& config,
163 CdmContext* /* cdm_context */,
165 const OutputCB& output_cb,
166 const WaitingCB& /* waiting_cb */) {
167 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
168 DCHECK(config.IsValidConfig());
170 InitCB bound_init_cb =
171 bind_callbacks_ ? base::BindPostTaskToCurrentDefault(std::move(init_cb))
172 : std::move(init_cb);
173 if (config.is_encrypted()) {
174 std::move(bound_init_cb)
175 .Run(DecoderStatus::Codes::kUnsupportedEncryptionMode);
179 if (config.codec() != VideoCodec::kAV1) {
180 std::move(bound_init_cb)
181 .Run(DecoderStatus(DecoderStatus::Codes::kUnsupportedCodec)
182 .WithData("codec", config.codec()));
186 // Clear any previously initialized decoder.
190 dav1d_default_settings(&s);
192 // Compute the ideal thread count values. We'll then clamp these based on the
193 // maximum number of recommended threads (using number of processors, etc).
194 int tile_threads, frame_threads;
195 GetDecoderThreadCounts(config.coded_size().height(), &tile_threads,
198 // While dav1d has switched to a thread pool, preserve the same thread counts
199 // we used when tile and frame threads were configured distinctly. It may be
200 // possible to lower this after some performance analysis of the new system.
201 s.n_threads = VideoDecoder::GetRecommendedThreadCount(frame_threads *
204 // We only want 1 frame thread in low delay mode, since otherwise we'll
205 // require at least two buffers before the first frame can be output.
206 if (low_delay || config.is_rtc())
207 s.max_frame_delay = 1;
209 // Route dav1d internal logs through Chrome's DLOG system.
210 s.logger = {nullptr, &LogDav1dMessage};
212 // Set a maximum frame size limit to avoid OOM'ing fuzzers.
213 s.frame_size_limit = limits::kMaxCanvas;
215 // TODO(tmathmeyer) write the dav1d error into the data for the media error.
216 if (dav1d_open(&dav1d_decoder_, &s) < 0) {
217 std::move(bound_init_cb).Run(DecoderStatus::Codes::kFailedToCreateDecoder);
222 state_ = DecoderState::kNormal;
223 output_cb_ = output_cb;
224 std::move(bound_init_cb).Run(DecoderStatus::Codes::kOk);
227 void Dav1dVideoDecoder::Decode(scoped_refptr<DecoderBuffer> buffer,
228 DecodeCB decode_cb) {
229 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
232 DCHECK_NE(state_, DecoderState::kUninitialized)
233 << "Called Decode() before successful Initialize()";
235 DecodeCB bound_decode_cb =
236 bind_callbacks_ ? base::BindPostTaskToCurrentDefault(std::move(decode_cb))
237 : std::move(decode_cb);
239 if (state_ == DecoderState::kError) {
240 std::move(bound_decode_cb).Run(DecoderStatus::Codes::kFailed);
244 if (!DecodeBuffer(std::move(buffer))) {
245 state_ = DecoderState::kError;
246 std::move(bound_decode_cb).Run(DecoderStatus::Codes::kFailed);
250 // VideoDecoderShim expects |decode_cb| call after |output_cb_|.
251 std::move(bound_decode_cb).Run(DecoderStatus::Codes::kOk);
254 void Dav1dVideoDecoder::Reset(base::OnceClosure reset_cb) {
255 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
256 state_ = DecoderState::kNormal;
257 dav1d_flush(dav1d_decoder_);
260 base::SequencedTaskRunner::GetCurrentDefault()->PostTask(
261 FROM_HERE, std::move(reset_cb));
263 std::move(reset_cb).Run();
266 void Dav1dVideoDecoder::Detach() {
267 // Even though we offload all resolutions of AV1, this may be called in a
268 // transition from clear to encrypted content. Which will subsequently fail
269 // Initialize() since encrypted content isn't supported by this decoder.
271 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
272 DCHECK(!bind_callbacks_);
275 DETACH_FROM_SEQUENCE(sequence_checker_);
278 void Dav1dVideoDecoder::CloseDecoder() {
279 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
282 dav1d_close(&dav1d_decoder_);
283 DCHECK(!dav1d_decoder_);
286 bool Dav1dVideoDecoder::DecodeBuffer(scoped_refptr<DecoderBuffer> buffer) {
287 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
289 using ScopedPtrDav1dData = std::unique_ptr<Dav1dData, ScopedDav1dDataFree>;
290 ScopedPtrDav1dData input_buffer;
292 if (!buffer->end_of_stream()) {
293 input_buffer.reset(new Dav1dData{0});
294 if (dav1d_data_wrap(input_buffer.get(), buffer->data(), buffer->data_size(),
295 &ReleaseDecoderBuffer, buffer.get()) < 0) {
298 input_buffer->m.timestamp = buffer->timestamp().InMicroseconds();
302 // Used to DCHECK that dav1d_send_data() actually takes the packet. If we exit
303 // this function without sending |input_buffer| that packet will be lost. We
304 // have no packet to send at end of stream.
305 bool send_data_completed = buffer->end_of_stream();
307 while (!input_buffer || input_buffer->sz) {
309 const int res = dav1d_send_data(dav1d_decoder_, input_buffer.get());
310 if (res < 0 && res != -EAGAIN) {
311 MEDIA_LOG(ERROR, media_log_) << "dav1d_send_data() failed on "
312 << buffer->AsHumanReadableString();
317 send_data_completed = true;
319 // Even if dav1d_send_data() returned EAGAIN, try dav1d_get_picture().
322 using ScopedPtrDav1dPicture =
323 std::unique_ptr<Dav1dPicture, ScopedDav1dPictureFree>;
324 ScopedPtrDav1dPicture p(new Dav1dPicture{0});
326 const int res = dav1d_get_picture(dav1d_decoder_, p.get());
328 if (res != -EAGAIN) {
329 MEDIA_LOG(ERROR, media_log_) << "dav1d_get_picture() failed on "
330 << buffer->AsHumanReadableString();
334 // We've reached end of stream and no frames remain to drain.
336 DCHECK(send_data_completed);
343 auto frame = BindImageToVideoFrame(p.get());
345 MEDIA_LOG(DEBUG, media_log_)
346 << "Failed to produce video frame from Dav1dPicture.";
350 // AV1 color space defines match ISO 23001-8:2016 via ISO/IEC 23091-4/ITU-T
351 // H.273. https://aomediacodec.github.io/av1-spec/#color-config-semantics
352 VideoColorSpace color_space(
353 p->seq_hdr->pri, p->seq_hdr->trc, p->seq_hdr->mtrx,
354 p->seq_hdr->color_range ? gfx::ColorSpace::RangeID::FULL
355 : gfx::ColorSpace::RangeID::LIMITED);
357 // If the frame doesn't specify a color space, use the container's.
358 auto gfx_cs = color_space.ToGfxColorSpace();
359 if (!gfx_cs.IsValid()) {
360 gfx_cs = config_.color_space_info().ToGfxColorSpace();
363 frame->set_color_space(gfx_cs);
364 frame->metadata().power_efficient = false;
365 frame->set_hdr_metadata(config_.hdr_metadata());
367 // When we use bind mode, our image data is dependent on the Dav1dPicture,
368 // so we must ensure it stays alive along enough.
369 frame->AddDestructionObserver(base::DoNothingWithBoundArgs(std::move(p)));
370 output_cb_.Run(std::move(frame));
373 DCHECK(send_data_completed);
377 scoped_refptr<VideoFrame> Dav1dVideoDecoder::BindImageToVideoFrame(
378 const Dav1dPicture* pic) {
379 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
380 const gfx::Size visible_size(pic->p.w, pic->p.h);
382 VideoPixelFormat pixel_format = Dav1dImgFmtToVideoPixelFormat(&pic->p);
383 if (pixel_format == PIXEL_FORMAT_UNKNOWN)
386 auto uv_plane_stride = pic->stride[1];
387 auto* u_plane = static_cast<uint8_t*>(pic->data[1]);
388 auto* v_plane = static_cast<uint8_t*>(pic->data[2]);
390 const bool needs_fake_uv_planes = pic->p.layout == DAV1D_PIXEL_LAYOUT_I400;
391 if (needs_fake_uv_planes) {
392 // UV planes are half the size of the Y plane.
393 uv_plane_stride = base::bits::AlignUp(pic->stride[0] / 2, ptrdiff_t{2});
394 const auto uv_plane_height = (pic->p.h + 1) / 2;
395 const size_t size_needed = uv_plane_stride * uv_plane_height;
397 if (!fake_uv_data_ || fake_uv_data_->size() != size_needed) {
398 if (pic->p.bpc == 8) {
399 // Avoid having base::RefCountedBytes zero initialize the memory just to
400 // fill it with a different value.
401 constexpr uint8_t kBlankUV = 256 / 2;
402 std::vector<unsigned char> empty_data(size_needed, kBlankUV);
404 // When we resize, existing frames will keep their refs on the old data.
405 fake_uv_data_ = base::RefCountedBytes::TakeVector(&empty_data);
407 DCHECK(pic->p.bpc == 10 || pic->p.bpc == 12);
408 const uint16_t kBlankUV = (1 << pic->p.bpc) / 2;
410 base::MakeRefCounted<base::RefCountedBytes>(size_needed);
412 uint16_t* data = fake_uv_data_->front_as<uint16_t>();
413 std::fill(data, data + size_needed / 2, kBlankUV);
417 u_plane = v_plane = fake_uv_data_->front_as<uint8_t>();
420 auto frame = VideoFrame::WrapExternalYuvData(
421 pixel_format, visible_size, gfx::Rect(visible_size),
422 config_.aspect_ratio().GetNaturalSize(gfx::Rect(visible_size)),
423 pic->stride[0], uv_plane_stride, uv_plane_stride,
424 static_cast<uint8_t*>(pic->data[0]), u_plane, v_plane,
425 base::Microseconds(pic->m.timestamp));
429 // Each frame needs a ref on the fake UV data to keep it alive until done.
430 if (needs_fake_uv_planes) {
431 frame->AddDestructionObserver(base::BindOnce(
432 [](scoped_refptr<base::RefCountedBytes>) {}, fake_uv_data_));