From 9f4a5762d5bc82a65b2bf414ed002462e231c899 Mon Sep 17 00:00:00 2001
From: =?utf8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?=
 <vjaquez@igalia.com>
Date: Mon, 25 Sep 2017 17:04:12 +0200
Subject: [PATCH] vaapiencode: flush pending frames before set format

Flush pending frames, if any, in the internal encorder, before setting
the new negotiated format.

https://bugzilla.gnome.org/show_bug.cgi?id=786173
---
 gst/vaapi/gstvaapiencode.c | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/gst/vaapi/gstvaapiencode.c b/gst/vaapi/gstvaapiencode.c
index 27a807a..0aaa4d1 100644
--- a/gst/vaapi/gstvaapiencode.c
+++ b/gst/vaapi/gstvaapiencode.c
@@ -565,6 +565,22 @@ set_codec_state (GstVaapiEncode * encode, GstVideoCodecState * state)
 }
 
 static gboolean
+gst_vaapiencode_drain (GstVaapiEncode * encode)
+{
+  GstVaapiEncoderStatus status;
+
+  if (!encode->encoder)
+    return TRUE;
+
+  status = gst_vaapi_encoder_flush (encode->encoder);
+  if (status != GST_VAAPI_ENCODER_STATUS_SUCCESS)
+    return FALSE;
+  gst_vaapiencode_purge (encode);
+
+  return TRUE;
+}
+
+static gboolean
 gst_vaapiencode_set_format (GstVideoEncoder * venc, GstVideoCodecState * state)
 {
   GstVaapiEncode *const encode = GST_VAAPIENCODE_CAST (venc);
@@ -579,6 +595,9 @@ gst_vaapiencode_set_format (GstVideoEncoder * venc, GstVideoCodecState * state)
           state->caps, NULL))
     return FALSE;
 
+  if (!gst_vaapiencode_drain (encode))
+    return FALSE;
+
   if (encode->input_state)
     gst_video_codec_state_unref (encode->input_state);
   encode->input_state = gst_video_codec_state_ref (state);
@@ -824,17 +843,14 @@ static gboolean
 gst_vaapiencode_flush (GstVideoEncoder * venc)
 {
   GstVaapiEncode *const encode = GST_VAAPIENCODE_CAST (venc);
-  GstVaapiEncoderStatus status;
 
   if (!encode->encoder)
     return FALSE;
 
   GST_LOG_OBJECT (encode, "flushing");
 
-  status = gst_vaapi_encoder_flush (encode->encoder);
-  if (status != GST_VAAPI_ENCODER_STATUS_SUCCESS)
+  if (!gst_vaapiencode_drain (encode))
     return FALSE;
-  gst_vaapiencode_purge (encode);
 
   gst_vaapi_encoder_replace (&encode->encoder, NULL);
   if (!ensure_encoder (encode))
-- 
2.7.4