msdk: don't share context between msdkvpp and msdkenc

msdkenc supports CSC implicitly, so it is possible that two VPP
processes are required when a pipeline contains msdkvpp and msdkenc.
Before this fix, msdkvpp and msdkenc may share the same context, hence
the same mfx session, which results in MFX_ERR_UNDEFINED_BEHAVIOR
in MSDK because a mfx session has at most one VPP process only

This fixes the broken pipelines below:

gst-launch-1.0 videotestsrc ! video/x-raw,format=I420 ! msdkh264enc ! \
msdkh264dec ! msdkvpp ! video/x-raw,format=YUY2 ! fakesink

gst-launch-1.0 videotestsrc ! msdkvpp ! video/x-raw,format=YUY2 ! \
msdkh264enc ! fakesink

diff --git a/sys/msdk/gstmsdkenc.c b/sys/msdk/gstmsdkenc.c
index d05e168..1afec08 100644 (file)
--- a/sys/msdk/gstmsdkenc.c
+++ b/sys/msdk/gstmsdkenc.c
@@ -1401,7 +1401,14 @@ gst_msdkenc_start (GstVideoEncoder * encoder)
     GST_INFO_OBJECT (thiz, "Found context %" GST_PTR_FORMAT " from neighbour",
         thiz->context);
 
-    if (gst_msdk_context_get_job_type (thiz->context) & GST_MSDK_JOB_ENCODER) {
+    /* Check GST_MSDK_JOB_VPP and GST_MSDK_JOB_ENCODER together to avoid sharing context
+     * between VPP and ENCODER
+     * Example:
+     * gst-launch-1.0 videotestsrc ! video/x-raw,format=I420 ! msdkh264enc ! \
+     * msdkh264dec ! msdkvpp ! video/x-raw,format=YUY2 ! fakesink
+     */
+    if (gst_msdk_context_get_job_type (thiz->context) & (GST_MSDK_JOB_VPP |
+            GST_MSDK_JOB_ENCODER)) {
       GstMsdkContext *parent_context, *msdk_context;
 
       parent_context = thiz->context;

diff --git a/sys/msdk/gstmsdkvpp.c b/sys/msdk/gstmsdkvpp.c
index d0e0a46..895141d 100644 (file)
--- a/sys/msdk/gstmsdkvpp.c
+++ b/sys/msdk/gstmsdkvpp.c
@@ -162,7 +162,13 @@ ensure_context (GstBaseTransform * trans)
     GST_INFO_OBJECT (thiz, "Found context from neighbour %" GST_PTR_FORMAT,
         thiz->context);
 
-    if (gst_msdk_context_get_job_type (thiz->context) & GST_MSDK_JOB_VPP) {
+    /* Check GST_MSDK_JOB_VPP and GST_MSDK_JOB_ENCODER together to avoid sharing context
+     * between VPP and ENCODER
+     * Example:
+     * gst-launch-1.0 videotestsrc ! msdkvpp ! video/x-raw,format=YUY2 ! msdkh264enc ! fakesink
+     */
+    if (gst_msdk_context_get_job_type (thiz->context) & (GST_MSDK_JOB_ENCODER |
+            GST_MSDK_JOB_VPP)) {
       GstMsdkContext *parent_context, *msdk_context;
 
       parent_context = thiz->context;