vaapiencode: Fixes deadlock in gst_vaapiencode_change_state function

This fixes a deadlock in gst_vaapiencode_change_state, which was due to
srcpad's chain function was locked waiting for available buffers. Since the
coded buffers in codedbuf_queue become available after sinkpad consume the
encoded frames, Paused -> Ready state change leads to deadlock. Coded buffers
are never consumed and marked free, hence gst_vaapiencode_handle_frame waits for
available buffers and holds the stream_lock of the srcpad.

diff --git a/gst/vaapi/gstvaapiencode.c b/gst/vaapi/gstvaapiencode.c
index af8c77a..7f26016 100644 (file)
--- a/gst/vaapi/gstvaapiencode.c
+++ b/gst/vaapi/gstvaapiencode.c
@@ -770,6 +770,9 @@ gst_vaapiencode_change_state (GstElement * element, GstStateChange transition)
   switch (transition) {
     case GST_STATE_CHANGE_PAUSED_TO_READY:
       gst_pad_stop_task (GST_VAAPI_PLUGIN_BASE_SRC_PAD (encode));
+
+      if (!gst_vaapiencode_drain (encode))
+        goto drain_error;
       break;
     default:
       break;
@@ -777,6 +780,12 @@ gst_vaapiencode_change_state (GstElement * element, GstStateChange transition)
   return
       GST_ELEMENT_CLASS (gst_vaapiencode_parent_class)->change_state (element,
       transition);
+
+drain_error:
+  {
+    GST_ERROR ("failed to drain pending encoded frames");
+    return GST_STATE_CHANGE_FAILURE;
+  }
 }
 
 static gboolean