frontends/va: Return error in vaRenderPicture if decoder is NULL

This fixes a crash if a data slice is submitted before the decoder
is initialized. A well-behaved application shouldn't encounter this
but returning an error is still better than crashing the entire
process and the rest of the code is similarly defensive.

Signed-off-by: Lorenz Brun <lorenz@brun.one>
Reviewed-by: Leo Liu <leo.liu@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11792>

diff --git a/src/gallium/frontends/va/picture.c b/src/gallium/frontends/va/picture.c
index 2843983..9e630ee 100644 (file)
--- a/src/gallium/frontends/va/picture.c
+++ b/src/gallium/frontends/va/picture.c
@@ -278,7 +278,7 @@ handleVAProtectedSliceDataBufferType(vlVaContext *context, vlVaBuffer *buf)
        context->desc.base.protected_playback = true;
 }
 
-static void
+static VAStatus
 handleVASliceDataBufferType(vlVaContext *context, vlVaBuffer *buf)
 {
    enum pipe_video_format format = u_reduce_video_profile(context->templat.profile);
@@ -290,6 +290,9 @@ handleVASliceDataBufferType(vlVaContext *context, vlVaBuffer *buf)
    static const uint8_t start_code_vc1[] = { 0x00, 0x00, 0x01, 0x0d };
    static const uint8_t eoi_jpeg[] = { 0xff, 0xd9 };
 
+   if (!context->decoder)
+      return VA_STATUS_ERROR_INVALID_CONTEXT;
+
    format = u_reduce_video_profile(context->templat.profile);
    if (!context->desc.base.protected_playback) {
       switch (format) {
@@ -358,6 +361,7 @@ handleVASliceDataBufferType(vlVaContext *context, vlVaBuffer *buf)
    }
    context->decoder->decode_bitstream(context->decoder, context->target, &context->desc.base,
       num_buffers, (const void * const*)buffers, sizes);
+   return VA_STATUS_SUCCESS;
 }
 
 static VAStatus
@@ -607,7 +611,7 @@ vlVaRenderPicture(VADriverContextP ctx, VAContextID context_id, VABufferID *buff
          break;
 
       case VASliceDataBufferType:
-         handleVASliceDataBufferType(context, buf);
+         vaStatus = handleVASliceDataBufferType(context, buf);
          break;
 
       case VAProcPipelineParameterBufferType: