[TTVD] Trigger lazy frame when requests exceeds max DPB count

Some contents might have gaps in timestamps which is
problematic for lazy frame mechanism. We're not able to
correctly determine whether we have sufficient data in
decoder to being able to return next video frame because
of that.

The main problem is that several codec might have
bidirectional frames, so need to be sure it might be
really decoded by decoder (all reference frames are already
in the decoder). That's mainly for H264 and HEVC,so other
codecs should not be affected by lazy frame issue now.
This change introduces additional check as last resort
how many frames are in the decoder without lazy frame
generated. If it exceeeds maximal value of DPB (Decoded
Picture Buffering) count, then we can be sure, that at
least single frame with the lowest timestamp can be decoded
(and returned as lazy frame).

Bug: https://jira-eu.sec.samsung.net/browse/VDGAME-565
Change-Id: Ia04dfee98f3b206745e52103cd7b60f80cc86477
Signed-off-by: Jakub Gajownik <j.gajownik2@samsung.com>

diff --git a/media/filters/tizen/lazy_frame_ranges.cc b/media/filters/tizen/lazy_frame_ranges.cc
index 48e3a893edb36827049d3d49b4307bdd054fc54a..51d53adca62d445527d23a72dc81fbc783c4f170 100644 (file)
--- a/media/filters/tizen/lazy_frame_ranges.cc
+++ b/media/filters/tizen/lazy_frame_ranges.cc
@@ -56,6 +56,13 @@ void LazyFrameRanges::Clear() {
   ranges_.clear();
 }
 
+void LazyFrameRanges::FillGapUpTo(base::TimeDelta timestamp) {
+  if (ranges_.size() == 0) {
+    return;
+  }
+  ranges_.Add(ranges_.start(0), timestamp);
+}
+
 bool LazyFrameRanges::IsInRange(base::TimeDelta timestamp) const {
   if (ranges_.size() == 0) {
     return false;

diff --git a/media/filters/tizen/lazy_frame_ranges.h b/media/filters/tizen/lazy_frame_ranges.h
index bc5b9eeb00998bbd387e920200e8172ab3d6a125..cd1e3a6724bf1ca9425a91f40121c7d91b769b32 100644 (file)
--- a/media/filters/tizen/lazy_frame_ranges.h
+++ b/media/filters/tizen/lazy_frame_ranges.h
@@ -18,6 +18,10 @@ class LazyFrameRanges {
   void Insert(const DecoderBuffer& buffer);
   void Clear();
 
+  // Inform that |timestamp| should be in the range. It's intended to fill
+  // gaps in content using this method only.
+  void FillGapUpTo(base::TimeDelta timestamp);
+
   bool IsInRange(base::TimeDelta timestamp) const;
 
  private:

diff --git a/media/filters/tizen/lazy_frame_ranges_test.cc b/media/filters/tizen/lazy_frame_ranges_test.cc
index 7f54559fa70824ab97947774dcbc3521a3c58fbd..9642aa9ca61a181eb228b1461b11e7f27c911ac1 100644 (file)
--- a/media/filters/tizen/lazy_frame_ranges_test.cc
+++ b/media/filters/tizen/lazy_frame_ranges_test.cc
@@ -165,4 +165,15 @@ TEST_F(LazyFrameRangesTest, KeyframeFinishesMultipleBidirectional) {
   EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(60)));
 }
 
+TEST_F(LazyFrameRangesTest, FillGap) {
+  InsertKeyframe(base::Milliseconds(0), base::Milliseconds(0));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(0)));
+
+  InsertDelta(base::Milliseconds(15), base::Milliseconds(10));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(15)));
+
+  ranges.FillGapUpTo(base::Milliseconds(15));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(15)));
+}
+
 }  // namespace media

diff --git a/media/filters/tizen/ttvd_video_decoder_impl.cc b/media/filters/tizen/ttvd_video_decoder_impl.cc
index 3c7e38148c71661ef0763365debd4e15e0bee41d..f5a320d679de1d67536878efa0c6435b1039de4d 100644 (file)
--- a/media/filters/tizen/ttvd_video_decoder_impl.cc
+++ b/media/filters/tizen/ttvd_video_decoder_impl.cc
@@ -65,6 +65,25 @@ constexpr static size_t kMaxUsedDecodedFrames = 20;
 // Minimum duration of syncing state. It serves the role to ensure
 // that overlay and texture output is synchronized on display.
 constexpr static auto kMinimumSyncingDuration = base::Milliseconds(300);
+
+size_t MaxForwardReferenceFrameDistance(MediaVideoCodec codec) {
+  switch (codec) {
+    case MediaVideoCodec::kCodecUnknown:
+    case MediaVideoCodec::kCodecVP8:
+    case MediaVideoCodec::kCodecVP9:
+    case MediaVideoCodec::kCodecAV1:
+    case MediaVideoCodec::kCodecMJPEG:
+      // Those codecs don't have bidirectional frames.
+      return 0;
+    // Both H264 and HEVC have bidirectional decoded picture buffering (DPB)
+    // limited to 16. It means that at most that number of frames might be
+    // hold as an reference for frame decoding.
+    case MediaVideoCodec::kCodecH264:
+    case MediaVideoCodec::kCodecHEVC:
+      return 16;
+  }
+  return 0;
+}
 }  // namespace
 
 void DecodedCollectionOnGpu::NotifyModeChange(gfx::VideoOutputMode mode) {
@@ -403,7 +422,12 @@ bool TTvdVideoDecoderImpl::ShouldTriggerLazyFrame(
   // error of 1us (internal unit of base::TimeDelta).
   // Low delay mode does not have any bi-directional frames, so decoding order
   // is also rendering order.
-  if (!low_delay_ && !ranges_.IsInRange(decoding_request.timestamp)) {
+  if (CanHaveBidirectionalFrames() &&
+      !ranges_.IsInRange(decoding_request.timestamp)) {
+    return false;
+  }
+
+  if (decoding_request.end_of_stream) {
     return false;
   }
 
@@ -417,18 +441,40 @@ bool TTvdVideoDecoderImpl::ShouldTriggerLazyFrame(
   return true;
 }
 
+bool TTvdVideoDecoderImpl::CanHaveBidirectionalFrames() const {
+  if (low_delay_) {
+    return false;
+  }
+  return MaxForwardReferenceFrameDistance(codec_) != 0;
+}
+
 void TTvdVideoDecoderImpl::TriggerLazyFrames() {
   // We want to return frames in proper order (of non-descending timestamp)
   // and they are stored according to decode timestamp. At first, gather all
   // frames that can be returned as lazy frames.
   std::vector<base::TimeDelta> frames_timestamp;
 
-  for (auto it = decoding_requests_.begin(); it != decoding_requests_.end();
-       ++it) {
-    if (!ShouldTriggerLazyFrame(*it)) {
-      continue;
+  for (const auto& decoding_request : decoding_requests_) {
+    if (ShouldTriggerLazyFrame(decoding_request)) {
+      frames_timestamp.push_back(decoding_request.timestamp);
     }
+  }
 
+  if (frames_timestamp.empty() && CanHaveBidirectionalFrames() &&
+      decoding_requests_.size() >= MaxForwardReferenceFrameDistance(codec_) &&
+      std::none_of(decoding_requests_.begin(), decoding_requests_.end(),
+                   [](const DecodingRequest& decoding_request) {
+                     return decoding_request.lazy_frame_returned;
+                   })) {
+    auto it = std::min_element(
+        decoding_requests_.begin(), decoding_requests_.end(),
+        [](const DecodingRequest& a, const DecodingRequest& b) {
+          return a.timestamp < b.timestamp;
+        });
+    TIZEN_MEDIA_LOG_ASSERT(it != decoding_requests_.end());
+    TIZEN_MEDIA_LOG(VERBOSE)
+        << "We should trigger lazy frame for: " << it->timestamp;
+    ranges_.FillGapUpTo(it->timestamp);
     frames_timestamp.push_back(it->timestamp);
   }
 
@@ -447,21 +493,6 @@ void TTvdVideoDecoderImpl::TriggerLazyFrames() {
   // sort and return.
   std::vector<scoped_refptr<VideoFrame>> video_frames;
   for (auto it = decoding_requests_.begin(); it != decoding_requests_.end();) {
-    if (!ShouldTriggerLazyFrame(*it)) {
-      ++it;
-      continue;
-    }
-
-    if (it->end_of_stream) {
-      ++it;
-      continue;
-    }
-
-    if (ttvd_decoded_frame_pool_.empty()) {
-      needs_new_ttvd_decoded_frame = true;
-      break;
-    }
-
     const base::TimeDelta timestamp = it->timestamp;
 
     if (std::find(frames_timestamp.begin(), frames_timestamp.end(),
@@ -582,9 +613,8 @@ void TTvdVideoDecoderImpl::HandleDecodingRequest(
   // buffers are rejected in |Decode| method.
   TIZEN_MEDIA_LOG_ASSERT(buffer->timestamp() != kNoTimestamp);
 
-  // Calculating ranges makes no sense for RTC, since it cannot contain
-  // bidirectional frames.
-  if (!config_.is_rtc()) {
+  // Calculating ranges makes no sense when there won't be bidirectional frames.
+  if (CanHaveBidirectionalFrames()) {
     ranges_.Insert(*buffer);
   }
 
@@ -992,8 +1022,8 @@ bool TTvdVideoDecoderImpl::AllocateDecoder() {
   callbacks.switch_cb = base::BindPostTaskToCurrentDefault(base::BindRepeating(
       &TTvdVideoDecoderImpl::SwitchDecoder, weak_factory_.GetWeakPtr()));
 
-  MediaVideoCodec codec = VideoCodecToMediaVideoCodec(config_.codec());
-  if (codec == MediaVideoCodec::kCodecUnknown) {
+  codec_ = VideoCodecToMediaVideoCodec(config_.codec());
+  if (codec_ == MediaVideoCodec::kCodecUnknown) {
     TIZEN_MEDIA_LOG(ERROR) << "Not supported video codec: " << config_.codec();
   }
 
@@ -1001,7 +1031,7 @@ bool TTvdVideoDecoderImpl::AllocateDecoder() {
   // return that valid resolutions (e.g 3840x2176) are not supported on
   // several boards.
   auto decoder = DecoderPromotion::GetInstance()->SelectDecoder(
-      codec, config_.level(), config_.visible_rect().size(),
+      codec_, config_.level(), config_.visible_rect().size(),
       config_.chroma_sampling, config_.bit_depth, GetLatencyMode(),
       std::move(callbacks), &workarounds_);
   if (!decoder) {

diff --git a/media/filters/tizen/ttvd_video_decoder_impl.h b/media/filters/tizen/ttvd_video_decoder_impl.h
index d581b130b6bee550dae91be77768d7442b2c484d..2a9aac6b96be5ac383a9a7ead8286695ae808b73 100644 (file)
--- a/media/filters/tizen/ttvd_video_decoder_impl.h
+++ b/media/filters/tizen/ttvd_video_decoder_impl.h
@@ -269,6 +269,8 @@ class MEDIA_EXPORT TTvdVideoDecoderImpl {
   // It might be also changed be calls to |Initialize|.
   ExtendedVideoDecoderConfig config_;
 
+  MediaVideoCodec codec_;
+
   // Determines whether decoder should return decoded frame without
   // further |Decode| calls. Note that B-frames are not allowed if
   // it is enabled.
@@ -357,6 +359,10 @@ class MEDIA_EXPORT TTvdVideoDecoderImpl {
 
   bool ShouldTriggerLazyFrame(const DecodingRequest& decoding_request) const;
 
+  // Returns information whether current configuration can have
+  // bidirectional frames.
+  bool CanHaveBidirectionalFrames() const;
+
   // Holds pending video frame decodings, so the onces that were
   // already accepted by Decode method on decoder thread,
   // but no output (media::VideoFrame) was triggered for them.
@@ -410,6 +416,8 @@ class MEDIA_EXPORT TTvdVideoDecoderImpl {
   std::pair<base::TimeDelta, ProcessingAction> last_processing_frame_;
 
   // Ranges of video corresponding data that were accepted by decoder.
+  // Note that it should be used only when bidirectional frames are in stream,
+  // |CanHaveBidirectionalFrames| should be used to check that.
   LazyFrameRanges ranges_;
 
   // Rendering callback gathered from overlay system. It should be