openh264: Fail gracefully if openh264 encoder/decoder creation fails

This can happen with the dummy "noopenh264" library that the freedesktop
flatpak runtime ships, and Fedora is planning on shipping as well. In
both cases the dummy implementation gets replaced with the actual
openh264 library that's downloaded directly from Cisco, but just to be
on safe side, this patch makes it careful to check the return values to
avoid crashing if the underlying library hasn't been swapped out yet.

The patch is taken from freedesktop-sdk and was originally written by
Valentin David <valentin.david@codethink.co.uk>.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5586>

diff --git a/subprojects/gst-plugins-bad/ext/openh264/gstopenh264dec.cpp b/subprojects/gst-plugins-bad/ext/openh264/gstopenh264dec.cpp
index 6d3464628f80950ab9fd25e70405e97b7a280c45..77f2b8fe34823617f65bbfb6ec26d6b147c674a1 100644 (file)
--- a/subprojects/gst-plugins-bad/ext/openh264/gstopenh264dec.cpp
+++ b/subprojects/gst-plugins-bad/ext/openh264/gstopenh264dec.cpp
@@ -159,7 +159,12 @@ gst_openh264dec_start (GstVideoDecoder * decoder)
     WelsDestroyDecoder (openh264dec->decoder);
     openh264dec->decoder = NULL;
   }
-  WelsCreateDecoder (&(openh264dec->decoder));
+
+  if (WelsCreateDecoder (&(openh264dec->decoder)) != 0) {
+    GST_ELEMENT_ERROR (openh264dec, LIBRARY, INIT, (NULL),
+        ("Failed to create OpenH264 decoder."));
+    return FALSE;
+  }
 
 #ifndef GST_DISABLE_GST_DEBUG
   {

diff --git a/subprojects/gst-plugins-bad/ext/openh264/gstopenh264enc.cpp b/subprojects/gst-plugins-bad/ext/openh264/gstopenh264enc.cpp
index 00ad6b1d650035beade4321cec4eeee85528b393..6b54b1584f857717d89f6e2c10d9ffb1f105e529 100644 (file)
--- a/subprojects/gst-plugins-bad/ext/openh264/gstopenh264enc.cpp
+++ b/subprojects/gst-plugins-bad/ext/openh264/gstopenh264enc.cpp
@@ -759,7 +759,13 @@ gst_openh264enc_set_format (GstVideoEncoder * encoder,
     WelsDestroySVCEncoder (openh264enc->encoder);
     openh264enc->encoder = NULL;
   }
-  WelsCreateSVCEncoder (&openh264enc->encoder);
+
+  if (WelsCreateSVCEncoder (&openh264enc->encoder) != 0) {
+    GST_ELEMENT_ERROR (openh264enc, LIBRARY, INIT, (NULL),
+        ("Failed to create OpenH264 encoder."));
+    return FALSE;
+  }
+
   unsigned int uiTraceLevel = WELS_LOG_ERROR;
   openh264enc->encoder->SetOption (ENCODER_OPTION_TRACE_LEVEL, &uiTraceLevel);