drm/vmwgfx: validate the screen formats

The kms code wasn't validating the modifiers and was letting through
unsupported formats. rgb8 was never properly supported and has no
matching svga screen target format so remove it.
This fixes format/modifier failures in kms_addfb_basic from IGT.

Signed-off-by: Zack Rusin <zackr@vmware.com>
Reviewed-by: Martin Krastev <krastevm@vmware.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20220318174332.440068-4-zack@kde.org

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 7a23f252d2121e755e764e33608d3c0a52acd213..693028c31b6b5b5987e57a6a94a0c12e11cc036d 100644 (file)
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -1171,6 +1171,15 @@ static int vmw_kms_new_framebuffer_surface(struct vmw_private *dev_priv,
         * Sanity checks.
         */
 
+       if (!drm_any_plane_has_format(&dev_priv->drm,
+                                     mode_cmd->pixel_format,
+                                     mode_cmd->modifier[0])) {
+               drm_dbg(&dev_priv->drm,
+                       "unsupported pixel format %p4cc / modifier 0x%llx\n",
+                       &mode_cmd->pixel_format, mode_cmd->modifier[0]);
+               return -EINVAL;
+       }
+
        /* Surface must be marked as a scanout. */
        if (unlikely(!surface->metadata.scanout))
                return -EINVAL;
@@ -1493,20 +1502,13 @@ static int vmw_kms_new_framebuffer_bo(struct vmw_private *dev_priv,
                return -EINVAL;
        }
 
-       /* Limited framebuffer color depth support for screen objects */
-       if (dev_priv->active_display_unit == vmw_du_screen_object) {
-               switch (mode_cmd->pixel_format) {
-               case DRM_FORMAT_XRGB8888:
-               case DRM_FORMAT_ARGB8888:
-                       break;
-               case DRM_FORMAT_XRGB1555:
-               case DRM_FORMAT_RGB565:
-                       break;
-               default:
-                       DRM_ERROR("Invalid pixel format: %p4cc\n",
-                                 &mode_cmd->pixel_format);
-                       return -EINVAL;
-               }
+       if (!drm_any_plane_has_format(&dev_priv->drm,
+                                     mode_cmd->pixel_format,
+                                     mode_cmd->modifier[0])) {
+               drm_dbg(&dev_priv->drm,
+                       "unsupported pixel format %p4cc / modifier 0x%llx\n",
+                       &mode_cmd->pixel_format, mode_cmd->modifier[0]);
+               return -EINVAL;
        }
 
        vfbd = kzalloc(sizeof(*vfbd), GFP_KERNEL);

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
index c95be95deb8d850f816cf10b97ac9e88cda56929..1d1c8b82c8986faee4bf247d3547716bb33e76b0 100644 (file)
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
@@ -247,7 +247,6 @@ struct vmw_framebuffer_bo {
 static const uint32_t __maybe_unused vmw_primary_plane_formats[] = {
        DRM_FORMAT_XRGB1555,
        DRM_FORMAT_RGB565,
-       DRM_FORMAT_RGB888,
        DRM_FORMAT_XRGB8888,
        DRM_FORMAT_ARGB8888,
 };