media/filters/dav1d_video_decoder.cc

   1 // Copyright 2019 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/filters/dav1d_video_decoder.h"
   6
   7 #include <memory>
   8 #include <string>
   9 #include <utility>
  10
  11 #include "base/bind.h"
  12 #include "base/bits.h"
  13 #include "base/callback.h"
  14 #include "base/logging.h"
  15 #include "base/strings/stringprintf.h"
  16 #include "base/threading/sequenced_task_runner_handle.h"
  17 #include "media/base/bind_to_current_loop.h"
  18 #include "media/base/decoder_buffer.h"
  19 #include "media/base/limits.h"
  20 #include "media/base/media_log.h"
  21 #include "media/base/video_aspect_ratio.h"
  22 #include "media/base/video_util.h"
  23
  24 extern "C" {
  25 #include "third_party/dav1d/libdav1d/include/dav1d/dav1d.h"
  26 }
  27
  28 namespace media {
  29
  30 static void GetDecoderThreadCounts(const int coded_height,
  31                                    int* tile_threads,
  32                                    int* frame_threads) {
  33   // Tile thread counts based on currently available content. Recommended by
  34   // YouTube, while frame thread values fit within
  35   // limits::kMaxVideoDecodeThreads.
  36   if (coded_height >= 700) {
  37     *tile_threads =
  38         4;  // Current 720p content is encoded in 5 tiles and 1080p content with
  39             // 8 tiles, but we'll exceed limits::kMaxVideoDecodeThreads with 5+
  40             // tile threads with 3 frame threads (5 * 3 + 3 = 18 threads vs 16
  41             // max).
  42             //
  43             // Since 720p playback isn't smooth without 3 frame threads, we've
  44             // chosen a slightly lower tile thread count.
  45     *frame_threads = 3;
  46   } else if (coded_height >= 300) {
  47     *tile_threads = 3;
  48     *frame_threads = 2;
  49   } else {
  50     *tile_threads = 2;
  51     *frame_threads = 2;
  52   }
  53 }
  54
  55 static VideoPixelFormat Dav1dImgFmtToVideoPixelFormat(
  56     const Dav1dPictureParameters* pic) {
  57   switch (pic->layout) {
  58     // Single plane monochrome images will be converted to standard 3 plane ones
  59     // since Chromium doesn't support single Y plane images.
  60     case DAV1D_PIXEL_LAYOUT_I400:
  61     case DAV1D_PIXEL_LAYOUT_I420:
  62       switch (pic->bpc) {
  63         case 8:
  64           return PIXEL_FORMAT_I420;
  65         case 10:
  66           return PIXEL_FORMAT_YUV420P10;
  67         case 12:
  68           return PIXEL_FORMAT_YUV420P12;
  69         default:
  70           DLOG(ERROR) << "Unsupported bit depth: " << pic->bpc;
  71           return PIXEL_FORMAT_UNKNOWN;
  72       }
  73     case DAV1D_PIXEL_LAYOUT_I422:
  74       switch (pic->bpc) {
  75         case 8:
  76           return PIXEL_FORMAT_I422;
  77         case 10:
  78           return PIXEL_FORMAT_YUV422P10;
  79         case 12:
  80           return PIXEL_FORMAT_YUV422P12;
  81         default:
  82           DLOG(ERROR) << "Unsupported bit depth: " << pic->bpc;
  83           return PIXEL_FORMAT_UNKNOWN;
  84       }
  85     case DAV1D_PIXEL_LAYOUT_I444:
  86       switch (pic->bpc) {
  87         case 8:
  88           return PIXEL_FORMAT_I444;
  89         case 10:
  90           return PIXEL_FORMAT_YUV444P10;
  91         case 12:
  92           return PIXEL_FORMAT_YUV444P12;
  93         default:
  94           DLOG(ERROR) << "Unsupported bit depth: " << pic->bpc;
  95           return PIXEL_FORMAT_UNKNOWN;
  96       }
  97   }
  98 }
  99
 100 static void ReleaseDecoderBuffer(const uint8_t* buffer, void* opaque) {
 101   if (opaque)
 102     static_cast<DecoderBuffer*>(opaque)->Release();
 103 }
 104
 105 static void LogDav1dMessage(void* cookie, const char* format, va_list ap) {
 106   auto log = base::StringPrintV(format, ap);
 107   if (log.empty())
 108     return;
 109
 110   if (log.back() == '\n')
 111     log.pop_back();
 112
 113   DLOG(ERROR) << log;
 114 }
 115
 116 // std::unique_ptr release helpers. We need to release both the containing
 117 // structs as well as refs held within the structures.
 118 struct ScopedDav1dDataFree {
 119   void operator()(void* x) const {
 120     auto* data = static_cast<Dav1dData*>(x);
 121     dav1d_data_unref(data);
 122     delete data;
 123   }
 124 };
 125
 126 struct ScopedDav1dPictureFree {
 127   void operator()(void* x) const {
 128     auto* pic = static_cast<Dav1dPicture*>(x);
 129     dav1d_picture_unref(pic);
 130     delete pic;
 131   }
 132 };
 133
 134 // static
 135 SupportedVideoDecoderConfigs Dav1dVideoDecoder::SupportedConfigs() {
 136   return {{/*profile_min=*/AV1PROFILE_PROFILE_MAIN,
 137            /*profile_max=*/AV1PROFILE_PROFILE_HIGH,
 138            /*coded_size_min=*/kDefaultSwDecodeSizeMin,
 139            /*coded_size_max=*/kDefaultSwDecodeSizeMax,
 140            /*allow_encrypted=*/false,
 141            /*require_encrypted=*/false}};
 142 }
 143
 144 Dav1dVideoDecoder::Dav1dVideoDecoder(MediaLog* media_log,
 145                                      OffloadState offload_state)
 146     : media_log_(media_log),
 147       bind_callbacks_(offload_state == OffloadState::kNormal) {
 148   DETACH_FROM_SEQUENCE(sequence_checker_);
 149 }
 150
 151 Dav1dVideoDecoder::~Dav1dVideoDecoder() {
 152   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 153   CloseDecoder();
 154 }
 155
 156 VideoDecoderType Dav1dVideoDecoder::GetDecoderType() const {
 157   return VideoDecoderType::kDav1d;
 158 }
 159
 160 void Dav1dVideoDecoder::Initialize(const VideoDecoderConfig& config,
 161                                    bool low_delay,
 162                                    CdmContext* /* cdm_context */,
 163                                    InitCB init_cb,
 164                                    const OutputCB& output_cb,
 165                                    const WaitingCB& /* waiting_cb */) {
 166   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 167   DCHECK(config.IsValidConfig());
 168
 169   InitCB bound_init_cb = bind_callbacks_ ? BindToCurrentLoop(std::move(init_cb))
 170                                          : std::move(init_cb);
 171   if (config.is_encrypted()) {
 172     std::move(bound_init_cb)
 173         .Run(DecoderStatus::Codes::kUnsupportedEncryptionMode);
 174     return;
 175   }
 176
 177   if (config.codec() != VideoCodec::kAV1) {
 178     std::move(bound_init_cb)
 179         .Run(DecoderStatus(DecoderStatus::Codes::kUnsupportedCodec)
 180                  .WithData("codec", config.codec()));
 181     return;
 182   }
 183
 184   // Clear any previously initialized decoder.
 185   CloseDecoder();
 186
 187   Dav1dSettings s;
 188   dav1d_default_settings(&s);
 189
 190   // Compute the ideal thread count values. We'll then clamp these based on the
 191   // maximum number of recommended threads (using number of processors, etc).
 192   int tile_threads, frame_threads;
 193   GetDecoderThreadCounts(config.coded_size().height(), &tile_threads,
 194                          &frame_threads);
 195
 196   // While dav1d has switched to a thread pool, preserve the same thread counts
 197   // we used when tile and frame threads were configured distinctly. It may be
 198   // possible to lower this after some performance analysis of the new system.
 199   s.n_threads = VideoDecoder::GetRecommendedThreadCount(frame_threads *
 200                                                         (tile_threads + 1));
 201
 202   // We only want 1 frame thread in low delay mode, since otherwise we'll
 203   // require at least two buffers before the first frame can be output.
 204   if (low_delay || config.is_rtc())
 205     s.max_frame_delay = 1;
 206
 207   // Route dav1d internal logs through Chrome's DLOG system.
 208   s.logger = {nullptr, &LogDav1dMessage};
 209
 210   // Set a maximum frame size limit to avoid OOM'ing fuzzers.
 211   s.frame_size_limit = limits::kMaxCanvas;
 212
 213   // TODO(tmathmeyer) write the dav1d error into the data for the media error.
 214   if (dav1d_open(&dav1d_decoder_, &s) < 0) {
 215     std::move(bound_init_cb).Run(DecoderStatus::Codes::kFailedToCreateDecoder);
 216     return;
 217   }
 218
 219   config_ = config;
 220   state_ = DecoderState::kNormal;
 221   output_cb_ = output_cb;
 222   std::move(bound_init_cb).Run(DecoderStatus::Codes::kOk);
 223 }
 224
 225 void Dav1dVideoDecoder::Decode(scoped_refptr<DecoderBuffer> buffer,
 226                                DecodeCB decode_cb) {
 227   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 228   DCHECK(buffer);
 229   DCHECK(decode_cb);
 230   DCHECK_NE(state_, DecoderState::kUninitialized)
 231       << "Called Decode() before successful Initialize()";
 232
 233   DecodeCB bound_decode_cb = bind_callbacks_
 234                                  ? BindToCurrentLoop(std::move(decode_cb))
 235                                  : std::move(decode_cb);
 236
 237   if (state_ == DecoderState::kError) {
 238     std::move(bound_decode_cb).Run(DecoderStatus::Codes::kFailed);
 239     return;
 240   }
 241
 242   if (!DecodeBuffer(std::move(buffer))) {
 243     state_ = DecoderState::kError;
 244     std::move(bound_decode_cb).Run(DecoderStatus::Codes::kFailed);
 245     return;
 246   }
 247
 248   // VideoDecoderShim expects |decode_cb| call after |output_cb_|.
 249   std::move(bound_decode_cb).Run(DecoderStatus::Codes::kOk);
 250 }
 251
 252 void Dav1dVideoDecoder::Reset(base::OnceClosure reset_cb) {
 253   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 254   state_ = DecoderState::kNormal;
 255   dav1d_flush(dav1d_decoder_);
 256
 257   if (bind_callbacks_)
 258     base::SequencedTaskRunnerHandle::Get()->PostTask(FROM_HERE,
 259                                                      std::move(reset_cb));
 260   else
 261     std::move(reset_cb).Run();
 262 }
 263
 264 void Dav1dVideoDecoder::Detach() {
 265   // Even though we offload all resolutions of AV1, this may be called in a
 266   // transition from clear to encrypted content. Which will subsequently fail
 267   // Initialize() since encrypted content isn't supported by this decoder.
 268
 269   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 270   DCHECK(!bind_callbacks_);
 271
 272   CloseDecoder();
 273   DETACH_FROM_SEQUENCE(sequence_checker_);
 274 }
 275
 276 void Dav1dVideoDecoder::CloseDecoder() {
 277   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 278   if (!dav1d_decoder_)
 279     return;
 280   dav1d_close(&dav1d_decoder_);
 281   DCHECK(!dav1d_decoder_);
 282 }
 283
 284 bool Dav1dVideoDecoder::DecodeBuffer(scoped_refptr<DecoderBuffer> buffer) {
 285   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 286
 287   using ScopedPtrDav1dData = std::unique_ptr<Dav1dData, ScopedDav1dDataFree>;
 288   ScopedPtrDav1dData input_buffer;
 289
 290   if (!buffer->end_of_stream()) {
 291     input_buffer.reset(new Dav1dData{0});
 292     if (dav1d_data_wrap(input_buffer.get(), buffer->data(), buffer->data_size(),
 293                         &ReleaseDecoderBuffer, buffer.get()) < 0) {
 294       return false;
 295     }
 296     input_buffer->m.timestamp = buffer->timestamp().InMicroseconds();
 297     buffer->AddRef();
 298   }
 299
 300   // Used to DCHECK that dav1d_send_data() actually takes the packet. If we exit
 301   // this function without sending |input_buffer| that packet will be lost. We
 302   // have no packet to send at end of stream.
 303   bool send_data_completed = buffer->end_of_stream();
 304
 305   while (!input_buffer || input_buffer->sz) {
 306     if (input_buffer) {
 307       const int res = dav1d_send_data(dav1d_decoder_, input_buffer.get());
 308       if (res < 0 && res != -EAGAIN) {
 309         MEDIA_LOG(ERROR, media_log_) << "dav1d_send_data() failed on "
 310                                      << buffer->AsHumanReadableString();
 311         return false;
 312       }
 313
 314       if (res != -EAGAIN)
 315         send_data_completed = true;
 316
 317       // Even if dav1d_send_data() returned EAGAIN, try dav1d_get_picture().
 318     }
 319
 320     using ScopedPtrDav1dPicture =
 321         std::unique_ptr<Dav1dPicture, ScopedDav1dPictureFree>;
 322     ScopedPtrDav1dPicture p(new Dav1dPicture{0});
 323
 324     const int res = dav1d_get_picture(dav1d_decoder_, p.get());
 325     if (res < 0) {
 326       if (res != -EAGAIN) {
 327         MEDIA_LOG(ERROR, media_log_) << "dav1d_get_picture() failed on "
 328                                      << buffer->AsHumanReadableString();
 329         return false;
 330       }
 331
 332       // We've reached end of stream and no frames remain to drain.
 333       if (!input_buffer) {
 334         DCHECK(send_data_completed);
 335         return true;
 336       }
 337
 338       continue;
 339     }
 340
 341     auto frame = BindImageToVideoFrame(p.get());
 342     if (!frame) {
 343       MEDIA_LOG(DEBUG, media_log_)
 344           << "Failed to produce video frame from Dav1dPicture.";
 345       return false;
 346     }
 347
 348     // AV1 color space defines match ISO 23001-8:2016 via ISO/IEC 23091-4/ITU-T
 349     // H.273. https://aomediacodec.github.io/av1-spec/#color-config-semantics
 350     VideoColorSpace color_space(
 351         p->seq_hdr->pri, p->seq_hdr->trc, p->seq_hdr->mtrx,
 352         p->seq_hdr->color_range ? gfx::ColorSpace::RangeID::FULL
 353                                 : gfx::ColorSpace::RangeID::LIMITED);
 354
 355     // If the frame doesn't specify a color space, use the container's.
 356     if (!color_space.IsSpecified())
 357       color_space = config_.color_space_info();
 358
 359     frame->set_color_space(color_space.ToGfxColorSpace());
 360     frame->metadata().power_efficient = false;
 361     frame->set_hdr_metadata(config_.hdr_metadata());
 362
 363     // When we use bind mode, our image data is dependent on the Dav1dPicture,
 364     // so we must ensure it stays alive along enough.
 365     frame->AddDestructionObserver(
 366         base::BindOnce([](ScopedPtrDav1dPicture) {}, std::move(p)));
 367     output_cb_.Run(std::move(frame));
 368   }
 369
 370   DCHECK(send_data_completed);
 371   return true;
 372 }
 373
 374 scoped_refptr<VideoFrame> Dav1dVideoDecoder::BindImageToVideoFrame(
 375     const Dav1dPicture* pic) {
 376   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 377   const gfx::Size visible_size(pic->p.w, pic->p.h);
 378
 379   VideoPixelFormat pixel_format = Dav1dImgFmtToVideoPixelFormat(&pic->p);
 380   if (pixel_format == PIXEL_FORMAT_UNKNOWN)
 381     return nullptr;
 382
 383   auto uv_plane_stride = pic->stride[1];
 384   auto* u_plane = static_cast<uint8_t*>(pic->data[1]);
 385   auto* v_plane = static_cast<uint8_t*>(pic->data[2]);
 386
 387   const bool needs_fake_uv_planes = pic->p.layout == DAV1D_PIXEL_LAYOUT_I400;
 388   if (needs_fake_uv_planes) {
 389     // UV planes are half the size of the Y plane.
 390     uv_plane_stride = base::bits::AlignUp(pic->stride[0] / 2, ptrdiff_t{2});
 391     const auto uv_plane_height = (pic->p.h + 1) / 2;
 392     const size_t size_needed = uv_plane_stride * uv_plane_height;
 393
 394     if (!fake_uv_data_ || fake_uv_data_->size() != size_needed) {
 395       if (pic->p.bpc == 8) {
 396         // Avoid having base::RefCountedBytes zero initialize the memory just to
 397         // fill it with a different value.
 398         constexpr uint8_t kBlankUV = 256 / 2;
 399         std::vector<unsigned char> empty_data(size_needed, kBlankUV);
 400
 401         // When we resize, existing frames will keep their refs on the old data.
 402         fake_uv_data_ = base::RefCountedBytes::TakeVector(&empty_data);
 403       } else {
 404         DCHECK(pic->p.bpc == 10 || pic->p.bpc == 12);
 405         const uint16_t kBlankUV = (1 << pic->p.bpc) / 2;
 406         fake_uv_data_ =
 407             base::MakeRefCounted<base::RefCountedBytes>(size_needed);
 408
 409         uint16_t* data = fake_uv_data_->front_as<uint16_t>();
 410         std::fill(data, data + size_needed / 2, kBlankUV);
 411       }
 412     }
 413
 414     u_plane = v_plane = fake_uv_data_->front_as<uint8_t>();
 415   }
 416
 417   auto frame = VideoFrame::WrapExternalYuvData(
 418       pixel_format, visible_size, gfx::Rect(visible_size),
 419       config_.aspect_ratio().GetNaturalSize(gfx::Rect(visible_size)),
 420       pic->stride[0], uv_plane_stride, uv_plane_stride,
 421       static_cast<uint8_t*>(pic->data[0]), u_plane, v_plane,
 422       base::Microseconds(pic->m.timestamp));
 423   if (!frame)
 424     return nullptr;
 425
 426   // Each frame needs a ref on the fake UV data to keep it alive until done.
 427   if (needs_fake_uv_planes) {
 428     frame->AddDestructionObserver(base::BindOnce(
 429         [](scoped_refptr<base::RefCountedBytes>) {}, fake_uv_data_));
 430   }
 431
 432   return frame;
 433 }
 434
 435 }  // namespace media