media/filters/dav1d_video_decoder.cc

   1 // Copyright 2019 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/filters/dav1d_video_decoder.h"
   6
   7 #include <memory>
   8 #include <string>
   9 #include <utility>
  10
  11 #include "base/bits.h"
  12 #include "base/functional/bind.h"
  13 #include "base/functional/callback.h"
  14 #include "base/functional/callback_helpers.h"
  15 #include "base/logging.h"
  16 #include "base/strings/stringprintf.h"
  17 #include "base/task/bind_post_task.h"
  18 #include "base/task/sequenced_task_runner.h"
  19 #include "media/base/decoder_buffer.h"
  20 #include "media/base/limits.h"
  21 #include "media/base/media_log.h"
  22 #include "media/base/video_aspect_ratio.h"
  23 #include "media/base/video_util.h"
  24
  25 extern "C" {
  26 #include "third_party/dav1d/libdav1d/include/dav1d/dav1d.h"
  27 }
  28
  29 namespace media {
  30
  31 static void GetDecoderThreadCounts(const int coded_height,
  32                                    int* tile_threads,
  33                                    int* frame_threads) {
  34   // Tile thread counts based on currently available content. Recommended by
  35   // YouTube, while frame thread values fit within
  36   // limits::kMaxVideoDecodeThreads.
  37   if (coded_height >= 700) {
  38     *tile_threads =
  39         4;  // Current 720p content is encoded in 5 tiles and 1080p content with
  40             // 8 tiles, but we'll exceed limits::kMaxVideoDecodeThreads with 5+
  41             // tile threads with 3 frame threads (5 * 3 + 3 = 18 threads vs 16
  42             // max).
  43             //
  44             // Since 720p playback isn't smooth without 3 frame threads, we've
  45             // chosen a slightly lower tile thread count.
  46     *frame_threads = 3;
  47   } else if (coded_height >= 300) {
  48     *tile_threads = 3;
  49     *frame_threads = 2;
  50   } else {
  51     *tile_threads = 2;
  52     *frame_threads = 2;
  53   }
  54 }
  55
  56 static VideoPixelFormat Dav1dImgFmtToVideoPixelFormat(
  57     const Dav1dPictureParameters* pic) {
  58   switch (pic->layout) {
  59     // Single plane monochrome images will be converted to standard 3 plane ones
  60     // since Chromium doesn't support single Y plane images.
  61     case DAV1D_PIXEL_LAYOUT_I400:
  62     case DAV1D_PIXEL_LAYOUT_I420:
  63       switch (pic->bpc) {
  64         case 8:
  65           return PIXEL_FORMAT_I420;
  66         case 10:
  67           return PIXEL_FORMAT_YUV420P10;
  68         case 12:
  69           return PIXEL_FORMAT_YUV420P12;
  70         default:
  71           DLOG(ERROR) << "Unsupported bit depth: " << pic->bpc;
  72           return PIXEL_FORMAT_UNKNOWN;
  73       }
  74     case DAV1D_PIXEL_LAYOUT_I422:
  75       switch (pic->bpc) {
  76         case 8:
  77           return PIXEL_FORMAT_I422;
  78         case 10:
  79           return PIXEL_FORMAT_YUV422P10;
  80         case 12:
  81           return PIXEL_FORMAT_YUV422P12;
  82         default:
  83           DLOG(ERROR) << "Unsupported bit depth: " << pic->bpc;
  84           return PIXEL_FORMAT_UNKNOWN;
  85       }
  86     case DAV1D_PIXEL_LAYOUT_I444:
  87       switch (pic->bpc) {
  88         case 8:
  89           return PIXEL_FORMAT_I444;
  90         case 10:
  91           return PIXEL_FORMAT_YUV444P10;
  92         case 12:
  93           return PIXEL_FORMAT_YUV444P12;
  94         default:
  95           DLOG(ERROR) << "Unsupported bit depth: " << pic->bpc;
  96           return PIXEL_FORMAT_UNKNOWN;
  97       }
  98   }
  99 }
 100
 101 static void ReleaseDecoderBuffer(const uint8_t* buffer, void* opaque) {
 102   if (opaque)
 103     static_cast<DecoderBuffer*>(opaque)->Release();
 104 }
 105
 106 static void LogDav1dMessage(void* cookie, const char* format, va_list ap) {
 107   auto log = base::StringPrintV(format, ap);
 108   if (log.empty())
 109     return;
 110
 111   if (log.back() == '\n')
 112     log.pop_back();
 113
 114   DLOG(ERROR) << log;
 115 }
 116
 117 // std::unique_ptr release helpers. We need to release both the containing
 118 // structs as well as refs held within the structures.
 119 struct ScopedDav1dDataFree {
 120   void operator()(void* x) const {
 121     auto* data = static_cast<Dav1dData*>(x);
 122     dav1d_data_unref(data);
 123     delete data;
 124   }
 125 };
 126
 127 struct ScopedDav1dPictureFree {
 128   void operator()(void* x) const {
 129     auto* pic = static_cast<Dav1dPicture*>(x);
 130     dav1d_picture_unref(pic);
 131     delete pic;
 132   }
 133 };
 134
 135 // static
 136 SupportedVideoDecoderConfigs Dav1dVideoDecoder::SupportedConfigs() {
 137   return {{/*profile_min=*/AV1PROFILE_PROFILE_MAIN,
 138            /*profile_max=*/AV1PROFILE_PROFILE_HIGH,
 139            /*coded_size_min=*/kDefaultSwDecodeSizeMin,
 140            /*coded_size_max=*/kDefaultSwDecodeSizeMax,
 141            /*allow_encrypted=*/false,
 142            /*require_encrypted=*/false}};
 143 }
 144
 145 Dav1dVideoDecoder::Dav1dVideoDecoder(std::unique_ptr<MediaLog> media_log,
 146                                      OffloadState offload_state)
 147     : media_log_(std::move(media_log)),
 148       bind_callbacks_(offload_state == OffloadState::kNormal) {
 149   DETACH_FROM_SEQUENCE(sequence_checker_);
 150 }
 151
 152 Dav1dVideoDecoder::~Dav1dVideoDecoder() {
 153   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 154   CloseDecoder();
 155 }
 156
 157 VideoDecoderType Dav1dVideoDecoder::GetDecoderType() const {
 158   return VideoDecoderType::kDav1d;
 159 }
 160
 161 void Dav1dVideoDecoder::Initialize(const VideoDecoderConfig& config,
 162                                    bool low_delay,
 163                                    CdmContext* /* cdm_context */,
 164                                    InitCB init_cb,
 165                                    const OutputCB& output_cb,
 166                                    const WaitingCB& /* waiting_cb */) {
 167   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 168   DCHECK(config.IsValidConfig());
 169
 170   InitCB bound_init_cb =
 171       bind_callbacks_ ? base::BindPostTaskToCurrentDefault(std::move(init_cb))
 172                       : std::move(init_cb);
 173   if (config.is_encrypted()) {
 174     std::move(bound_init_cb)
 175         .Run(DecoderStatus::Codes::kUnsupportedEncryptionMode);
 176     return;
 177   }
 178
 179   if (config.codec() != VideoCodec::kAV1) {
 180     std::move(bound_init_cb)
 181         .Run(DecoderStatus(DecoderStatus::Codes::kUnsupportedCodec)
 182                  .WithData("codec", config.codec()));
 183     return;
 184   }
 185
 186   // Clear any previously initialized decoder.
 187   CloseDecoder();
 188
 189   Dav1dSettings s;
 190   dav1d_default_settings(&s);
 191
 192   // Compute the ideal thread count values. We'll then clamp these based on the
 193   // maximum number of recommended threads (using number of processors, etc).
 194   int tile_threads, frame_threads;
 195   GetDecoderThreadCounts(config.coded_size().height(), &tile_threads,
 196                          &frame_threads);
 197
 198   // While dav1d has switched to a thread pool, preserve the same thread counts
 199   // we used when tile and frame threads were configured distinctly. It may be
 200   // possible to lower this after some performance analysis of the new system.
 201   s.n_threads = VideoDecoder::GetRecommendedThreadCount(frame_threads *
 202                                                         (tile_threads + 1));
 203
 204   // We only want 1 frame thread in low delay mode, since otherwise we'll
 205   // require at least two buffers before the first frame can be output.
 206   if (low_delay || config.is_rtc())
 207     s.max_frame_delay = 1;
 208
 209   // Route dav1d internal logs through Chrome's DLOG system.
 210   s.logger = {nullptr, &LogDav1dMessage};
 211
 212   // Set a maximum frame size limit to avoid OOM'ing fuzzers.
 213   s.frame_size_limit = limits::kMaxCanvas;
 214
 215   // TODO(tmathmeyer) write the dav1d error into the data for the media error.
 216   if (dav1d_open(&dav1d_decoder_, &s) < 0) {
 217     std::move(bound_init_cb).Run(DecoderStatus::Codes::kFailedToCreateDecoder);
 218     return;
 219   }
 220
 221   config_ = config;
 222   state_ = DecoderState::kNormal;
 223   output_cb_ = output_cb;
 224   std::move(bound_init_cb).Run(DecoderStatus::Codes::kOk);
 225 }
 226
 227 void Dav1dVideoDecoder::Decode(scoped_refptr<DecoderBuffer> buffer,
 228                                DecodeCB decode_cb) {
 229   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 230   DCHECK(buffer);
 231   DCHECK(decode_cb);
 232   DCHECK_NE(state_, DecoderState::kUninitialized)
 233       << "Called Decode() before successful Initialize()";
 234
 235   DecodeCB bound_decode_cb =
 236       bind_callbacks_ ? base::BindPostTaskToCurrentDefault(std::move(decode_cb))
 237                       : std::move(decode_cb);
 238
 239   if (state_ == DecoderState::kError) {
 240     std::move(bound_decode_cb).Run(DecoderStatus::Codes::kFailed);
 241     return;
 242   }
 243
 244   if (!DecodeBuffer(std::move(buffer))) {
 245     state_ = DecoderState::kError;
 246     std::move(bound_decode_cb).Run(DecoderStatus::Codes::kFailed);
 247     return;
 248   }
 249
 250   // VideoDecoderShim expects |decode_cb| call after |output_cb_|.
 251   std::move(bound_decode_cb).Run(DecoderStatus::Codes::kOk);
 252 }
 253
 254 void Dav1dVideoDecoder::Reset(base::OnceClosure reset_cb) {
 255   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 256   state_ = DecoderState::kNormal;
 257   dav1d_flush(dav1d_decoder_);
 258
 259   if (bind_callbacks_)
 260     base::SequencedTaskRunner::GetCurrentDefault()->PostTask(
 261         FROM_HERE, std::move(reset_cb));
 262   else
 263     std::move(reset_cb).Run();
 264 }
 265
 266 void Dav1dVideoDecoder::Detach() {
 267   // Even though we offload all resolutions of AV1, this may be called in a
 268   // transition from clear to encrypted content. Which will subsequently fail
 269   // Initialize() since encrypted content isn't supported by this decoder.
 270
 271   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 272   DCHECK(!bind_callbacks_);
 273
 274   CloseDecoder();
 275   DETACH_FROM_SEQUENCE(sequence_checker_);
 276 }
 277
 278 void Dav1dVideoDecoder::CloseDecoder() {
 279   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 280   if (!dav1d_decoder_)
 281     return;
 282   dav1d_close(&dav1d_decoder_);
 283   DCHECK(!dav1d_decoder_);
 284 }
 285
 286 bool Dav1dVideoDecoder::DecodeBuffer(scoped_refptr<DecoderBuffer> buffer) {
 287   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 288
 289   using ScopedPtrDav1dData = std::unique_ptr<Dav1dData, ScopedDav1dDataFree>;
 290   ScopedPtrDav1dData input_buffer;
 291
 292   if (!buffer->end_of_stream()) {
 293     input_buffer.reset(new Dav1dData{0});
 294     if (dav1d_data_wrap(input_buffer.get(), buffer->data(), buffer->data_size(),
 295                         &ReleaseDecoderBuffer, buffer.get()) < 0) {
 296       return false;
 297     }
 298     input_buffer->m.timestamp = buffer->timestamp().InMicroseconds();
 299     buffer->AddRef();
 300   }
 301
 302   // Used to DCHECK that dav1d_send_data() actually takes the packet. If we exit
 303   // this function without sending |input_buffer| that packet will be lost. We
 304   // have no packet to send at end of stream.
 305   bool send_data_completed = buffer->end_of_stream();
 306
 307   while (!input_buffer || input_buffer->sz) {
 308     if (input_buffer) {
 309       const int res = dav1d_send_data(dav1d_decoder_, input_buffer.get());
 310       if (res < 0 && res != -EAGAIN) {
 311         MEDIA_LOG(ERROR, media_log_) << "dav1d_send_data() failed on "
 312                                      << buffer->AsHumanReadableString();
 313         return false;
 314       }
 315
 316       if (res != -EAGAIN)
 317         send_data_completed = true;
 318
 319       // Even if dav1d_send_data() returned EAGAIN, try dav1d_get_picture().
 320     }
 321
 322     using ScopedPtrDav1dPicture =
 323         std::unique_ptr<Dav1dPicture, ScopedDav1dPictureFree>;
 324     ScopedPtrDav1dPicture p(new Dav1dPicture{0});
 325
 326     const int res = dav1d_get_picture(dav1d_decoder_, p.get());
 327     if (res < 0) {
 328       if (res != -EAGAIN) {
 329         MEDIA_LOG(ERROR, media_log_) << "dav1d_get_picture() failed on "
 330                                      << buffer->AsHumanReadableString();
 331         return false;
 332       }
 333
 334       // We've reached end of stream and no frames remain to drain.
 335       if (!input_buffer) {
 336         DCHECK(send_data_completed);
 337         return true;
 338       }
 339
 340       continue;
 341     }
 342
 343     auto frame = BindImageToVideoFrame(p.get());
 344     if (!frame) {
 345       MEDIA_LOG(DEBUG, media_log_)
 346           << "Failed to produce video frame from Dav1dPicture.";
 347       return false;
 348     }
 349
 350     // AV1 color space defines match ISO 23001-8:2016 via ISO/IEC 23091-4/ITU-T
 351     // H.273. https://aomediacodec.github.io/av1-spec/#color-config-semantics
 352     VideoColorSpace color_space(
 353         p->seq_hdr->pri, p->seq_hdr->trc, p->seq_hdr->mtrx,
 354         p->seq_hdr->color_range ? gfx::ColorSpace::RangeID::FULL
 355                                 : gfx::ColorSpace::RangeID::LIMITED);
 356
 357     // If the frame doesn't specify a color space, use the container's.
 358     auto gfx_cs = color_space.ToGfxColorSpace();
 359     if (!gfx_cs.IsValid()) {
 360       gfx_cs = config_.color_space_info().ToGfxColorSpace();
 361     }
 362
 363     frame->set_color_space(gfx_cs);
 364     frame->metadata().power_efficient = false;
 365     frame->set_hdr_metadata(config_.hdr_metadata());
 366
 367     // When we use bind mode, our image data is dependent on the Dav1dPicture,
 368     // so we must ensure it stays alive along enough.
 369     frame->AddDestructionObserver(base::DoNothingWithBoundArgs(std::move(p)));
 370     output_cb_.Run(std::move(frame));
 371   }
 372
 373   DCHECK(send_data_completed);
 374   return true;
 375 }
 376
 377 scoped_refptr<VideoFrame> Dav1dVideoDecoder::BindImageToVideoFrame(
 378     const Dav1dPicture* pic) {
 379   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 380   const gfx::Size visible_size(pic->p.w, pic->p.h);
 381
 382   VideoPixelFormat pixel_format = Dav1dImgFmtToVideoPixelFormat(&pic->p);
 383   if (pixel_format == PIXEL_FORMAT_UNKNOWN)
 384     return nullptr;
 385
 386   auto uv_plane_stride = pic->stride[1];
 387   auto* u_plane = static_cast<uint8_t*>(pic->data[1]);
 388   auto* v_plane = static_cast<uint8_t*>(pic->data[2]);
 389
 390   const bool needs_fake_uv_planes = pic->p.layout == DAV1D_PIXEL_LAYOUT_I400;
 391   if (needs_fake_uv_planes) {
 392     // UV planes are half the size of the Y plane.
 393     uv_plane_stride = base::bits::AlignUp(pic->stride[0] / 2, ptrdiff_t{2});
 394     const auto uv_plane_height = (pic->p.h + 1) / 2;
 395     const size_t size_needed = uv_plane_stride * uv_plane_height;
 396
 397     if (!fake_uv_data_ || fake_uv_data_->size() != size_needed) {
 398       if (pic->p.bpc == 8) {
 399         // Avoid having base::RefCountedBytes zero initialize the memory just to
 400         // fill it with a different value.
 401         constexpr uint8_t kBlankUV = 256 / 2;
 402         std::vector<unsigned char> empty_data(size_needed, kBlankUV);
 403
 404         // When we resize, existing frames will keep their refs on the old data.
 405         fake_uv_data_ = base::RefCountedBytes::TakeVector(&empty_data);
 406       } else {
 407         DCHECK(pic->p.bpc == 10 || pic->p.bpc == 12);
 408         const uint16_t kBlankUV = (1 << pic->p.bpc) / 2;
 409         fake_uv_data_ =
 410             base::MakeRefCounted<base::RefCountedBytes>(size_needed);
 411
 412         uint16_t* data = fake_uv_data_->front_as<uint16_t>();
 413         std::fill(data, data + size_needed / 2, kBlankUV);
 414       }
 415     }
 416
 417     u_plane = v_plane = fake_uv_data_->front_as<uint8_t>();
 418   }
 419
 420   auto frame = VideoFrame::WrapExternalYuvData(
 421       pixel_format, visible_size, gfx::Rect(visible_size),
 422       config_.aspect_ratio().GetNaturalSize(gfx::Rect(visible_size)),
 423       pic->stride[0], uv_plane_stride, uv_plane_stride,
 424       static_cast<uint8_t*>(pic->data[0]), u_plane, v_plane,
 425       base::Microseconds(pic->m.timestamp));
 426   if (!frame)
 427     return nullptr;
 428
 429   // Each frame needs a ref on the fake UV data to keep it alive until done.
 430   if (needs_fake_uv_planes) {
 431     frame->AddDestructionObserver(base::BindOnce(
 432         [](scoped_refptr<base::RefCountedBytes>) {}, fake_uv_data_));
 433   }
 434
 435   return frame;
 436 }
 437
 438 }  // namespace media