vga: fix byteswapping.

In case host and guest endianness differ the vga code first creates
a shared surface (using qemu_create_displaysurface_from), then goes
patch the surface format to indicate that the bytes must be swapped.

The switch to pixman broke that hack as the format patching isn't
propagated into the pixman image, so ui code using the pixman image
directly (such as vnc) uses the wrong format.

Fix that by adding a byteswap parameter to
qemu_create_displaysurface_from, so we'll use the correct format
when creating the surface (and the pixman image) and don't have
to patch the format afterwards.

[ v2: unbreak xen build ]

Cc: qemu-stable@nongnu.org
Cc: mark.cave-ayland@ilande.co.uk
Cc: agraf@suse.de
Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
Message-id: 1361349432-23884-1-git-send-email-kraxel@redhat.com
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>

diff --git a/hw/qxl-render.c b/hw/qxl-render.c
index 88e63f8085469fd220c54cb1be3b9a4b26bbd4a6..455fb91269b0d95151320523e095e413d2c6045d 100644 (file)
--- a/hw/qxl-render.c
+++ b/hw/qxl-render.c
@@ -118,7 +118,8 @@ static void qxl_render_update_area_unlocked(PCIQXLDevice *qxl)
                  qxl->guest_primary.surface.height,
                  qxl->guest_primary.bits_pp,
                  qxl->guest_primary.abs_stride,
-                 qxl->guest_primary.data);
+                 qxl->guest_primary.data,
+                 false);
         } else {
             qemu_resize_displaysurface(vga->ds,
                     qxl->guest_primary.surface.width,

diff --git a/hw/vga.c b/hw/vga.c
index e2ba7f208c6b06a25f9e3a58a031181daa63136d..1caf23d7b6561ea2d9ad47c064b56cf1b0e7a897 100644 (file)
--- a/hw/vga.c
+++ b/hw/vga.c
@@ -1643,6 +1643,11 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
     uint8_t *d;
     uint32_t v, addr1, addr;
     vga_draw_line_func *vga_draw_line;
+#if defined(HOST_WORDS_BIGENDIAN) == defined(TARGET_WORDS_BIGENDIAN)
+    static const bool byteswap = false;
+#else
+    static const bool byteswap = true;
+#endif
 
     full_update |= update_basic_params(s);
 
@@ -1685,18 +1690,11 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
         disp_width != s->last_width ||
         height != s->last_height ||
         s->last_depth != depth) {
-#if defined(HOST_WORDS_BIGENDIAN) == defined(TARGET_WORDS_BIGENDIAN)
-        if (depth == 16 || depth == 32) {
-#else
-        if (depth == 32) {
-#endif
+        if (depth == 32 || (depth == 16 && !byteswap)) {
             qemu_free_displaysurface(s->ds);
             s->ds->surface = qemu_create_displaysurface_from(disp_width, height, depth,
                     s->line_offset,
-                    s->vram_ptr + (s->start_addr * 4));
-#if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
-            s->ds->surface->pf = qemu_different_endianness_pixelformat(depth);
-#endif
+                    s->vram_ptr + (s->start_addr * 4), byteswap);
             dpy_gfx_resize(s->ds);
         } else {
             qemu_console_resize(s->ds, disp_width, height);
@@ -1715,7 +1713,7 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
         s->ds->surface = qemu_create_displaysurface_from(disp_width,
                 height, depth,
                 s->line_offset,
-                s->vram_ptr + (s->start_addr * 4));
+                s->vram_ptr + (s->start_addr * 4), byteswap);
         dpy_gfx_setdata(s->ds);
     }
 

diff --git a/hw/vmware_vga.c b/hw/vmware_vga.c
index cd15ee40a8775966e7f98dc88d5c8c5e843b14a5..8fc201bfb96ed54f1ff28aa7750f9b44095364db 100644 (file)
--- a/hw/vmware_vga.c
+++ b/hw/vmware_vga.c
@@ -1074,7 +1074,7 @@ static void vmsvga_screen_dump(void *opaque, const char *filename, bool cswitch,
                                  ds_get_height(s->vga.ds),
                                  32,
                                  ds_get_linesize(s->vga.ds),
-                                 s->vga.vram_ptr);
+                                 s->vga.vram_ptr, false);
         ppm_save(filename, ds, errp);
         g_free(ds);
     }

diff --git a/hw/xenfb.c b/hw/xenfb.c
index 903efd30736d905be7b42ebc9ff804f302a1f6eb..7f1f6b464324a677949627613ed482789719ebf6 100644 (file)
--- a/hw/xenfb.c
+++ b/hw/xenfb.c
@@ -756,7 +756,8 @@ static void xenfb_update(void *opaque)
             qemu_free_displaysurface(xenfb->c.ds);
             xenfb->c.ds->surface = qemu_create_displaysurface_from
                 (xenfb->width, xenfb->height, xenfb->depth,
-                 xenfb->row_stride, xenfb->pixels + xenfb->offset);
+                 xenfb->row_stride, xenfb->pixels + xenfb->offset,
+                 false);
             break;
         default:
             /* we must convert stuff */

diff --git a/include/ui/console.h b/include/ui/console.h
index fc23baa06b35c911b2ace381d90be92850b3c21d..18012f1dc1a2327269930e64e8ae437a07d6cb5d 100644 (file)
--- a/include/ui/console.h
+++ b/include/ui/console.h
@@ -184,7 +184,8 @@ struct DisplayState {
 void register_displaystate(DisplayState *ds);
 DisplayState *get_displaystate(void);
 DisplaySurface* qemu_create_displaysurface_from(int width, int height, int bpp,
-                                                int linesize, uint8_t *data);
+                                                int linesize, uint8_t *data,
+                                                bool byteswap);
 PixelFormat qemu_different_endianness_pixelformat(int bpp);
 PixelFormat qemu_default_pixelformat(int bpp);
 

diff --git a/ui/console.c b/ui/console.c
index 0a68836d506b45929b0df41c5a9bb7bf3e6c098f..25e06a5cb3885d764fbd4d899a62c18aa57ccffa 100644 (file)
--- a/ui/console.c
+++ b/ui/console.c
@@ -1339,11 +1339,16 @@ DisplaySurface *qemu_resize_displaysurface(DisplayState *ds,
 }
 
 DisplaySurface *qemu_create_displaysurface_from(int width, int height, int bpp,
-                                                int linesize, uint8_t *data)
+                                                int linesize, uint8_t *data,
+                                                bool byteswap)
 {
     DisplaySurface *surface = g_new0(DisplaySurface, 1);
 
-    surface->pf = qemu_default_pixelformat(bpp);
+    if (byteswap) {
+        surface->pf = qemu_different_endianness_pixelformat(bpp);
+    } else {
+        surface->pf = qemu_default_pixelformat(bpp);
+    }
 
     surface->format = qemu_pixman_get_format(&surface->pf);
     assert(surface->format != 0);