drm/vmwgfx: Properly check display/scanout surface size

The scanout surface size is the smaller of max texture size and
max STDU size.

Signed-off-by: Sinclair Yeh <syeh@vmware.com>
Reviewed-by: Thomas Hellstrom <thellstrom@vmware.com>

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 0832381..c18c81f 100644 (file)
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -2182,7 +2182,10 @@ int vmw_du_connector_fill_modes(struct drm_connector *connector,
 
        if (dev_priv->active_display_unit == vmw_du_screen_target) {
                max_width  = min(max_width,  dev_priv->stdu_max_width);
+               max_width  = min(max_width,  dev_priv->texture_max_width);
+
                max_height = min(max_height, dev_priv->stdu_max_height);
+               max_height = min(max_height, dev_priv->texture_max_height);
        }
 
        /* Add preferred mode */

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
index 4fc46b2..e872ffe 100644 (file)
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
@@ -1601,7 +1601,21 @@ int vmw_kms_stdu_init_display(struct vmw_private *dev_priv)
 
        dev_priv->active_display_unit = vmw_du_screen_target;
 
-       if (!(dev_priv->capabilities & SVGA_CAP_3D)) {
+       if (dev_priv->capabilities & SVGA_CAP_3D) {
+               /*
+                * For 3D VMs, display (scanout) buffer size is the smaller of
+                * max texture and max STDU
+                */
+               uint32_t max_width, max_height;
+
+               max_width = min(dev_priv->texture_max_width,
+                               dev_priv->stdu_max_width);
+               max_height = min(dev_priv->texture_max_height,
+                                dev_priv->stdu_max_height);
+
+               dev->mode_config.max_width = max_width;
+               dev->mode_config.max_height = max_height;
+       } else {
                /*
                 * Given various display aspect ratios, there's no way to
                 * estimate these using prim_bb_mem.  So just set these to

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
index 6abcf82..41b9d20 100644 (file)
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
@@ -1479,10 +1479,24 @@ int vmw_surface_gb_priv_define(struct drm_device *dev,
        *srf_out = NULL;
 
        if (for_scanout) {
+               uint32_t max_width, max_height;
+
                if (!svga3dsurface_is_screen_target_format(format)) {
                        DRM_ERROR("Invalid Screen Target surface format.");
                        return -EINVAL;
                }
+
+               max_width = min(dev_priv->texture_max_width,
+                               dev_priv->stdu_max_width);
+               max_height = min(dev_priv->texture_max_height,
+                                dev_priv->stdu_max_height);
+
+               if (size.width > max_width || size.height > max_height) {
+                       DRM_ERROR("%ux%u\n, exeeds max surface size %ux%u",
+                                 size.width, size.height,
+                                 max_width, max_height);
+                       return -EINVAL;
+               }
        } else {
                const struct svga3d_surface_desc *desc;