[TTVD] Use ranges instead of comparing dts and pts

Previously used mechanism with comparing PTS and DTS is
sometimes giving wrong results, e.g for frames with
following order:
0, 4, 3, 2, 1
It doesn't check for timestamp continuity. Because of that
some frames might be wrongly returned as lazy frames, even
it we still don't have enough data for them to be decoded
(decoder returns frames according to ordered PTS).

With this change, we're going back to improved range
mechanism. With previous approach there were problems with
duration not provided by some clients (demuxer). Now it's
handled by calculating it using DTS parameter which should
solve all the past issues.

Bug: https://jira-eu.sec.samsung.net/browse/VDGAME-507
Change-Id: I9243a8af707eb65fa9b18bbbdad8752bbc439662
Signed-off-by: Jakub Gajownik <j.gajownik2@samsung.com>

diff --git a/media/filters/BUILD.gn b/media/filters/BUILD.gn
index 25beedabcefb6aa6f61af9b04d80b4a9050bc86f..4002d8ed8ea4eea3af2d92916c936607ffd537b6 100644 (file)
--- a/media/filters/BUILD.gn
+++ b/media/filters/BUILD.gn
@@ -300,6 +300,8 @@ source_set("filters") {
       "tizen/dummy_drm.cc",
       "tizen/dummy_drm.h",
       "tizen/extended_video_decoder_config.h",
+      "tizen/lazy_frame_ranges.cc",
+      "tizen/lazy_frame_ranges.h",
       "tizen/media_video_codec.cc",
       "tizen/media_video_codec.h",
       "tizen/nv12_data.h",

diff --git a/media/filters/tizen/lazy_frame_ranges.cc b/media/filters/tizen/lazy_frame_ranges.cc
new file mode 100644 (file)
index 0000000..48e3a89
--- /dev/null
+++ b/media/filters/tizen/lazy_frame_ranges.cc
@@ -0,0 +1,67 @@
+// Copyright 2024 Samsung Electronics Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/filters/tizen/lazy_frame_ranges.h"
+
+#include "base/logging.h"
+
+namespace media {
+
+void LazyFrameRanges::Insert(const DecoderBuffer& buffer) {
+  // Check for EOS packet, which has |kNoTimestamp| dts.
+  if (buffer.end_of_stream()) {
+    if (ranges_.size() != 0) {
+      ranges_.clear();
+      ranges_.Add(base::TimeDelta::Min(), base::TimeDelta::Max());
+    }
+  } else if (buffer.dts() != kNoTimestamp) {
+    if (last_dts_ != kNoTimestamp) {
+      auto duration = buffer.dts() - last_dts_;
+      ranges_.Add(last_pts_, last_pts_ + duration);
+
+      // Fill small gaps in ranges due to TimeDelta inaccuracy, eg. 1.86853
+      // and 1.86854 Start from the end to omit the problem with changing Ranges
+      // size.
+      for (int i = ranges_.size() - 2; i >= 0; --i) {
+        if (ranges_.end(i) + base::Milliseconds(1) >= ranges_.start(i + 1)) {
+          ranges_.Add(ranges_.end(i), ranges_.start(i + 1));
+        }
+      }
+    }
+
+    if (buffer.is_key_frame()) {
+      if (ranges_.size() == 0) {
+        // Since we don't have information about duration and range won't accept
+        // zero duration range, we need to use possibly small value to immitate
+        // range.
+        ranges_.Add(buffer.timestamp(),
+                    buffer.timestamp() + base::Milliseconds(1));
+      } else {
+        ranges_.Add(ranges_.start(0), buffer.timestamp());
+      }
+    }
+    last_pts_ = buffer.timestamp();
+    last_dts_ = buffer.dts();
+  } else {
+    LOG(ERROR) << "Non-eos buffer without DTS";
+    last_pts_ = kNoTimestamp;
+    last_dts_ = kNoTimestamp;
+  }
+}
+
+void LazyFrameRanges::Clear() {
+  last_pts_ = kNoTimestamp;
+  last_dts_ = kNoTimestamp;
+  ranges_.clear();
+}
+
+bool LazyFrameRanges::IsInRange(base::TimeDelta timestamp) const {
+  if (ranges_.size() == 0) {
+    return false;
+  }
+
+  return timestamp <= ranges_.end(0);
+}
+
+}  // namespace media

diff --git a/media/filters/tizen/lazy_frame_ranges.h b/media/filters/tizen/lazy_frame_ranges.h
new file mode 100644 (file)
index 0000000..bc5b9ee
--- /dev/null
+++ b/media/filters/tizen/lazy_frame_ranges.h
@@ -0,0 +1,31 @@
+// Copyright 2024 Samsung Electronics Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_FILTERS_TIZEN_LAZY_FRAME_RANGES_H_
+#define MEDIA_FILTERS_TIZEN_LAZY_FRAME_RANGES_H_
+
+#include "base/time/time.h"
+#include "media/base/decoder_buffer.h"
+#include "media/base/ranges.h"
+#include "media/base/timestamp_constants.h"
+
+namespace media {
+
+class LazyFrameRanges {
+ public:
+  // Updates |ranges_| with timestamp and duration from |buffer|.
+  void Insert(const DecoderBuffer& buffer);
+  void Clear();
+
+  bool IsInRange(base::TimeDelta timestamp) const;
+
+ private:
+  base::TimeDelta last_pts_ = kNoTimestamp;
+  base::TimeDelta last_dts_ = kNoTimestamp;
+  Ranges<base::TimeDelta> ranges_;
+};
+
+}  // namespace media
+
+#endif  // MEDIA_FILTERS_TIZEN_LAZY_FRAME_RANGES_H_

diff --git a/media/filters/tizen/lazy_frame_ranges_test.cc b/media/filters/tizen/lazy_frame_ranges_test.cc
new file mode 100644 (file)
index 0000000..b31afed
--- /dev/null
+++ b/media/filters/tizen/lazy_frame_ranges_test.cc
@@ -0,0 +1,161 @@
+// Copyright 2024 Samsung Electronics Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/filters/tizen/lazy_frame_ranges.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace media {
+
+class LazyFrameRangesTest : public testing::Test {
+ public:
+  LazyFrameRangesTest() = default;
+  ~LazyFrameRangesTest() = default;
+
+  LazyFrameRangesTest(const LazyFrameRangesTest&) = delete;
+  LazyFrameRangesTest& operator=(const LazyFrameRangesTest&) = delete;
+
+  void InsertKeyframe(base::TimeDelta pts, base::TimeDelta dts) {
+    auto buffer = base::WrapRefCounted(new DecoderBuffer(0));
+    buffer->set_timestamp(pts);
+    buffer->set_dts(dts);
+    buffer->set_is_key_frame(true);
+    ranges.Insert(*buffer);
+  }
+
+  void InsertDelta(base::TimeDelta pts, base::TimeDelta dts) {
+    auto buffer = base::WrapRefCounted(new DecoderBuffer(0));
+    buffer->set_timestamp(pts);
+    buffer->set_dts(dts);
+    ranges.Insert(*buffer);
+  }
+
+  void InsertEOS() {
+    auto buffer = DecoderBuffer::CreateEOSBuffer();
+    ranges.Insert(*buffer);
+  }
+
+  void TearDown() override { ranges.Clear(); }
+
+  LazyFrameRanges ranges;
+};
+
+TEST_F(LazyFrameRangesTest, SimpleKeyframe) {
+  InsertKeyframe(base::Milliseconds(0), base::Milliseconds(0));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(0)));
+}
+
+TEST_F(LazyFrameRangesTest, SingleDeltaAfterKeyframe) {
+  InsertKeyframe(base::Milliseconds(0), base::Milliseconds(0));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(0)));
+  InsertDelta(base::Milliseconds(10), base::Milliseconds(10));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(10)));
+}
+
+TEST_F(LazyFrameRangesTest, DoubleDeltaAfterKeyframe) {
+  InsertKeyframe(base::Milliseconds(0), base::Milliseconds(0));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(0)));
+  InsertDelta(base::Milliseconds(10), base::Milliseconds(10));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(10)));
+  InsertDelta(base::Milliseconds(20), base::Milliseconds(20));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(20)));
+}
+
+TEST_F(LazyFrameRangesTest, BidirectionalAfterKeyframeMissing) {
+  InsertKeyframe(base::Milliseconds(0), base::Milliseconds(0));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(0)));
+  InsertDelta(base::Milliseconds(20), base::Milliseconds(10));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(20)));
+}
+
+TEST_F(LazyFrameRangesTest, BidirectionalAfterKeyframe) {
+  InsertKeyframe(base::Milliseconds(0), base::Milliseconds(0));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(0)));
+
+  InsertDelta(base::Milliseconds(20), base::Milliseconds(10));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(20)));
+
+  InsertDelta(base::Milliseconds(10), base::Milliseconds(20));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(10)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(20)));
+}
+
+TEST_F(LazyFrameRangesTest, MultipleBidirectionalAfterKeyframe) {
+  InsertKeyframe(base::Milliseconds(0), base::Milliseconds(0));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(0)));
+
+  InsertDelta(base::Milliseconds(40), base::Milliseconds(10));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(40)));
+
+  InsertDelta(base::Milliseconds(30), base::Milliseconds(20));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(30)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(40)));
+
+  InsertDelta(base::Milliseconds(20), base::Milliseconds(30));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(20)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(30)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(40)));
+
+  InsertDelta(base::Milliseconds(10), base::Milliseconds(40));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(10)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(20)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(30)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(40)));
+}
+
+TEST_F(LazyFrameRangesTest, EosFinishesMultipleBidirectional) {
+  InsertKeyframe(base::Milliseconds(0), base::Milliseconds(0));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(0)));
+
+  InsertDelta(base::Milliseconds(40), base::Milliseconds(10));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(40)));
+
+  InsertDelta(base::Milliseconds(30), base::Milliseconds(20));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(30)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(40)));
+
+  InsertDelta(base::Milliseconds(20), base::Milliseconds(30));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(20)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(30)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(40)));
+
+  InsertDelta(base::Milliseconds(10), base::Milliseconds(40));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(20)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(30)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(40)));
+
+  InsertEOS();
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(20)));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(30)));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(40)));
+}
+
+TEST_F(LazyFrameRangesTest, KeyframeFinishesMultipleBidirectional) {
+  InsertKeyframe(base::Milliseconds(0), base::Milliseconds(0));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(0)));
+
+  InsertDelta(base::Milliseconds(40), base::Milliseconds(10));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(40)));
+
+  InsertDelta(base::Milliseconds(30), base::Milliseconds(20));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(30)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(40)));
+
+  InsertDelta(base::Milliseconds(20), base::Milliseconds(30));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(20)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(30)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(40)));
+
+  InsertDelta(base::Milliseconds(10), base::Milliseconds(40));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(20)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(30)));
+  EXPECT_FALSE(ranges.IsInRange(base::Milliseconds(40)));
+
+  InsertKeyframe(base::Milliseconds(60), base::Milliseconds(50));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(20)));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(30)));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(40)));
+  EXPECT_TRUE(ranges.IsInRange(base::Milliseconds(60)));
+}
+
+}  // namespace media

diff --git a/media/filters/tizen/ttvd_video_decoder_impl.cc b/media/filters/tizen/ttvd_video_decoder_impl.cc
index d5e1a1c2ded67cbad14cc3414e207adc48c14f5f..5d59237087945d43fc80c05e5073387060e5d7f0 100644 (file)
--- a/media/filters/tizen/ttvd_video_decoder_impl.cc
+++ b/media/filters/tizen/ttvd_video_decoder_impl.cc
@@ -292,8 +292,7 @@ void TTvdVideoDecoderImpl::Reset(base::OnceClosure closure) {
   const auto old_state = decoder_state_;
   decoder_state_ = DecoderState::kResetting;
   ClearDecodingQueue(DecoderStatus::Codes::kAborted);
-  last_dts_ = base::TimeDelta::Max();
-  dts_offset_ = kNoTimestamp;
+  ranges_.Clear();
 
   // TODO(vdwasm) Some optimization might be done here to push
   //              |TTvdDecodedFrame| back to the pool instead of deleting them.
@@ -336,6 +335,7 @@ void TTvdVideoDecoderImpl::Decode(scoped_refptr<DecoderBuffer> buffer,
                            << buffer->timestamp().InMicroseconds()
                            << ", duration: "
                            << buffer->duration().InMicroseconds()
+                           << ", dts: " << buffer->dts()
                            << ", is eos: " << buffer->end_of_stream()
                            << ", is keyframe: " << buffer->is_key_frame();
   TRACE_EVENT0("gpu", "VideoDecoder.Decode");
@@ -394,14 +394,7 @@ bool TTvdVideoDecoderImpl::ShouldTriggerLazyFrame(
   // error of 1us (internal unit of base::TimeDelta).
   // Low delay mode does not have any bi-directional frames, so decoding order
   // is also rendering order.
-  // Also when eos was already pushed to decoder, it means there won't be any
-  // more new frames, so pending one should be decodable (all the references
-  // must been pushed to decoder).
-  const bool can_decode_with_rounding_error =
-      ((decoding_request.timestamp <= last_dts_) ||
-       (decoding_request.timestamp - last_dts_) <= base::Microseconds(1) ||
-       !eos_cb_.is_null());
-  if (!low_delay_ && !can_decode_with_rounding_error) {
+  if (!low_delay_ && !ranges_.IsInRange(decoding_request.timestamp)) {
     return false;
   }
 
@@ -580,14 +573,10 @@ void TTvdVideoDecoderImpl::HandleDecodingRequest(
   // buffers are rejected in |Decode| method.
   TIZEN_MEDIA_LOG_ASSERT(buffer->timestamp() != kNoTimestamp);
 
-  // Check for EOS packet, which has |kNoTimestamp| dts.
-  if (buffer->dts() != kNoTimestamp) {
-    if (dts_offset_ == kNoTimestamp) {
-      dts_offset_ = buffer->timestamp() - buffer->dts();
-      TIZEN_MEDIA_LOG_ASSERT(dts_offset_ != kNoTimestamp);
-    }
-
-    last_dts_ = buffer->dts() + dts_offset_;
+  // Calculating ranges makes no sense for RTC, since it cannot contain
+  // bidirectional frames.
+  if (!config_.is_rtc()) {
+    ranges_.Insert(*buffer);
   }
 
   DecodingRequest decoding_request;
@@ -662,8 +651,7 @@ void TTvdVideoDecoderImpl::Initialize(VideoDecoderConfig config,
     allocated_decoder_.reset();
 
     ttvd_decoded_frame_pool_.clear();
-    last_dts_ = base::TimeDelta::Max();
-    dts_offset_ = kNoTimestamp;
+    ranges_.Clear();
     decoding_requests_.clear();
     decoding_results_.clear();
     last_processing_frame_ =

diff --git a/media/filters/tizen/ttvd_video_decoder_impl.h b/media/filters/tizen/ttvd_video_decoder_impl.h
index 788abcfbad7e273723deae59c81622bd9a39677e..5c259e9143d9f561da4044ac09efd22ad6507114 100644 (file)
--- a/media/filters/tizen/ttvd_video_decoder_impl.h
+++ b/media/filters/tizen/ttvd_video_decoder_impl.h
@@ -18,6 +18,7 @@
 #include "media/base/callback_registry.h"
 #include "media/base/cdm_context.h"
 #include "media/base/decoder_status.h"
+#include "media/base/ranges.h"
 #include "media/base/timestamp_constants.h"
 #include "media/base/video_codecs.h"
 #include "media/base/video_decoder.h"
@@ -26,6 +27,7 @@
 #include "media/filters/tizen/decoder_promotion.h"
 #include "media/filters/tizen/dummy_drm.h"
 #include "media/filters/tizen/extended_video_decoder_config.h"
+#include "media/filters/tizen/lazy_frame_ranges.h"
 #include "media/filters/tizen/ttvd_decoded_frame.h"
 #include "media/gpu/command_buffer_helper.h"
 #include "ui/gfx/geometry/rect_f.h"
@@ -401,12 +403,8 @@ class MEDIA_EXPORT TTvdVideoDecoderImpl {
   };
   std::pair<base::TimeDelta, ProcessingAction> last_processing_frame_;
 
-  // DTS of the last buffer that came through |HandleDecodingRequest|
-  base::TimeDelta last_dts_ = base::TimeDelta::Max();
-
-  // Offset used for aligning DTS with PTS based on the first PTS value
-  // received.
-  base::TimeDelta dts_offset_ = kNoTimestamp;
+  // Ranges of video corresponding data that were accepted by decoder.
+  LazyFrameRanges ranges_;
 
   // Rendering callback gathered from overlay system. It should be
   // used to draw decoded video picture when we're in overlay mode.

diff --git a/tizen_src/chromium_impl/media/test/BUILD.gn b/tizen_src/chromium_impl/media/test/BUILD.gn
index 36471a1fe7f8706286efa954d62e8a5e7c234719..1b5aacc6fc4aa6d0598546004deea754086dd52a 100644 (file)
--- a/tizen_src/chromium_impl/media/test/BUILD.gn
+++ b/tizen_src/chromium_impl/media/test/BUILD.gn
@@ -42,6 +42,7 @@ test("tizen_media_unittests") {
     "encoded_file_video_capture_device_unittest.cc",
     "file_video_capture_device_factory_unittest.cc",
     "//media/filters/tizen/decoder_promotion_test.cc",
+    "//media/filters/tizen/lazy_frame_ranges_test.cc",
     "//media/filters/tizen/omx/mock_omx_wrapper.cc",
     "//media/filters/tizen/omx/mock_omx_wrapper.h",
     "//media/filters/tizen/omx/omx_facade_video_test.cc"