xfreerdp: fix RDP order color conversion

Note: /gdi:sw was working fine, this commit fixes /gdi:hw

* calculate color channel shifts based on X11 visual color masks
* fast path to skip conversion if visual color masks equal rdp color masks
* successfully tested 8/15/16/24/32 bpp rdp sessions on 16/24/32 bpp visuals

diff --git a/client/X11/xf_client.c b/client/X11/xf_client.c
index 71534ab..8631dcc 100644 (file)
--- a/client/X11/xf_client.c
+++ b/client/X11/xf_client.c
@@ -656,6 +656,14 @@ void xf_unlock_x11(xfContext* xfc, BOOL display)
        }
 }
 
+static void xf_calculate_color_shifts(UINT32 mask, UINT8* rsh, UINT8* lsh)
+{
+    for (*lsh = 0; !(mask & 1); mask >>= 1)
+        (*lsh)++;
+    for (*rsh = 8; mask; mask >>= 1)
+        (*rsh)--;
+}
+
 BOOL xf_get_pixmap_info(xfContext* xfc)
 {
        int i;
@@ -720,7 +728,7 @@ BOOL xf_get_pixmap_info(xfContext* xfc)
                }
        }
 
-       if (vi)
+       if (xfc->visual)
        {
                /*
                 * Detect if the server visual has an inverted colormap
@@ -730,6 +738,11 @@ BOOL xf_get_pixmap_info(xfContext* xfc)
                {
                        xfc->invert = TRUE;
                }
+
+               /* calculate color shifts required for rdp order color conversion */
+               xf_calculate_color_shifts(vi->red_mask, &xfc->red_shift_r, &xfc->red_shift_l);
+               xf_calculate_color_shifts(vi->green_mask, &xfc->green_shift_r, &xfc->green_shift_l);
+               xf_calculate_color_shifts(vi->blue_mask, &xfc->blue_shift_r, &xfc->blue_shift_l);
        }
 
        XFree(vis);
@@ -1294,7 +1307,7 @@ void* xf_input_thread(void *arg)
        while(1)
        {
                status = WaitForMultipleObjects(2, event, FALSE, INFINITE);
-               
+
                if(status == WAIT_OBJECT_0 + 1)
                {
                        do

diff --git a/client/X11/xf_gdi.c b/client/X11/xf_gdi.c
index 0b38808..131080f 100644 (file)
--- a/client/X11/xf_gdi.c
+++ b/client/X11/xf_gdi.c
@@ -219,6 +219,58 @@ BOOL xf_set_rop3(xfContext* xfc, int rop3)
        return TRUE;
 }
 
+UINT32 xf_convert_rdp_order_color(xfContext* xfc, UINT32 color)
+{
+       UINT32 r, g, b;
+
+       switch (xfc->srcBpp)
+       {
+       case 32:
+       case 24:
+               if (xfc->visual->red_mask == 0xFF0000 &&
+                   xfc->visual->green_mask == 0xFF00 &&
+                   xfc->visual->blue_mask == 0xFF)
+               {
+                       return color;
+               }
+               GetRGB32(r, g, b, color);
+               break;
+
+       case 16:
+               color = (color & 0xFF00) | ((color >> 16) & 0xFF);
+               if (xfc->visual->red_mask == 0xF800 &&
+                   xfc->visual->green_mask == 0x07E0 &&
+                   xfc->visual->blue_mask == 0x001F)
+               {
+                       return color;
+               }
+               GetRGB16(r, g, b, color);
+               break;
+
+       case 15:
+               color = (color & 0xFF00) | ((color >> 16) & 0xFF);
+               GetRGB15(r, g, b, color);
+               break;
+
+       case 8:
+               color = (color >> 16) & (UINT32) 0xFF;
+               if (xfc->palette)
+               {
+                       r = xfc->palette[(color * 4) + 2];
+                       g = xfc->palette[(color * 4) + 1];
+                       b = xfc->palette[(color * 4) + 0];
+               }
+               break;
+
+       default:
+               return color;
+       }
+
+       return ( ((r >> xfc->red_shift_r) << xfc->red_shift_l) |
+                ((g >> xfc->green_shift_r) << xfc->green_shift_l) |
+                ((b >> xfc->blue_shift_r) << xfc->blue_shift_l) );
+}
+
 Pixmap xf_brush_new(xfContext* xfc, int width, int height, int bpp, BYTE* data)
 {
        GC gc;
@@ -463,8 +515,8 @@ void xf_gdi_patblt(rdpContext* context, PATBLT_ORDER* patblt)
        brush = &patblt->brush;
        xf_set_rop3(xfc, gdi_rop3_code(patblt->bRop));
 
-       foreColor = freerdp_convert_gdi_order_color(patblt->foreColor, context->settings->ColorDepth, xfc->format, xfc->palette);
-       backColor = freerdp_convert_gdi_order_color(patblt->backColor, context->settings->ColorDepth, xfc->format, xfc->palette);
+       foreColor = xf_convert_rdp_order_color(xfc, patblt->foreColor);
+       backColor = xf_convert_rdp_order_color(xfc, patblt->backColor);
 
        if (brush->style == GDI_BS_SOLID)
        {
@@ -565,7 +617,7 @@ void xf_gdi_opaque_rect(rdpContext* context, OPAQUE_RECT_ORDER* opaque_rect)
 
        xf_lock_x11(xfc, FALSE);
 
-       color = freerdp_convert_gdi_order_color(opaque_rect->color, context->settings->ColorDepth, xfc->format, xfc->palette);
+       color = xf_convert_rdp_order_color(xfc, opaque_rect->color);
 
        XSetFunction(xfc->display, xfc->gc, GXcopy);
        XSetFillStyle(xfc->display, xfc->gc, FillSolid);
@@ -593,7 +645,7 @@ void xf_gdi_multi_opaque_rect(rdpContext* context, MULTI_OPAQUE_RECT_ORDER* mult
 
        xf_lock_x11(xfc, FALSE);
 
-       color = freerdp_convert_gdi_order_color(multi_opaque_rect->color, context->settings->ColorDepth, xfc->format, xfc->palette);
+       color = xf_convert_rdp_order_color(xfc, multi_opaque_rect->color);
 
        XSetFunction(xfc->display, xfc->gc, GXcopy);
        XSetFillStyle(xfc->display, xfc->gc, FillSolid);
@@ -629,7 +681,7 @@ void xf_gdi_line_to(rdpContext* context, LINE_TO_ORDER* line_to)
        xf_lock_x11(xfc, FALSE);
 
        xf_set_rop2(xfc, line_to->bRop2);
-       color = freerdp_convert_gdi_order_color(line_to->penColor, context->settings->ColorDepth, xfc->format, xfc->palette);
+       color = xf_convert_rdp_order_color(xfc, line_to->penColor);
 
        XSetFillStyle(xfc->display, xfc->gc, FillSolid);
        XSetForeground(xfc->display, xfc->gc, color);
@@ -705,7 +757,7 @@ void xf_gdi_polyline(rdpContext* context, POLYLINE_ORDER* polyline)
        xf_lock_x11(xfc, FALSE);
 
        xf_set_rop2(xfc, polyline->bRop2);
-       color = freerdp_convert_gdi_order_color(polyline->penColor, context->settings->ColorDepth, xfc->format, xfc->palette);
+       color = xf_convert_rdp_order_color(xfc, polyline->penColor);
 
        XSetFillStyle(xfc->display, xfc->gc, FillSolid);
        XSetForeground(xfc->display, xfc->gc, color);
@@ -773,8 +825,8 @@ void xf_gdi_mem3blt(rdpContext* context, MEM3BLT_ORDER* mem3blt)
        brush = &mem3blt->brush;
        bitmap = (xfBitmap*) mem3blt->bitmap;
        xf_set_rop3(xfc, gdi_rop3_code(mem3blt->bRop));
-       foreColor = freerdp_convert_gdi_order_color(mem3blt->foreColor, context->settings->ColorDepth, xfc->format, xfc->palette);
-       backColor = freerdp_convert_gdi_order_color(mem3blt->backColor, context->settings->ColorDepth, xfc->format, xfc->palette);
+       foreColor = xf_convert_rdp_order_color(xfc, mem3blt->foreColor);
+       backColor = xf_convert_rdp_order_color(xfc, mem3blt->backColor);
 
        if (brush->style == GDI_BS_PATTERN)
        {
@@ -840,7 +892,7 @@ void xf_gdi_polygon_sc(rdpContext* context, POLYGON_SC_ORDER* polygon_sc)
        xf_lock_x11(xfc, FALSE);
 
        xf_set_rop2(xfc, polygon_sc->bRop2);
-       brush_color = freerdp_convert_gdi_order_color(polygon_sc->brushColor, context->settings->ColorDepth, xfc->format, xfc->palette);
+       brush_color = xf_convert_rdp_order_color(xfc, polygon_sc->brushColor);
 
        npoints = polygon_sc->numPoints + 1;
        points = malloc(sizeof(XPoint) * npoints);
@@ -900,8 +952,8 @@ void xf_gdi_polygon_cb(rdpContext* context, POLYGON_CB_ORDER* polygon_cb)
 
        brush = &(polygon_cb->brush);
        xf_set_rop2(xfc, polygon_cb->bRop2);
-       foreColor = freerdp_convert_gdi_order_color(polygon_cb->foreColor, context->settings->ColorDepth, xfc->format, xfc->palette);
-       backColor = freerdp_convert_gdi_order_color(polygon_cb->backColor, context->settings->ColorDepth, xfc->format, xfc->palette);
+       foreColor = xf_convert_rdp_order_color(xfc, polygon_cb->foreColor);
+       backColor = xf_convert_rdp_order_color(xfc, polygon_cb->backColor);
 
        npoints = polygon_cb->numPoints + 1;
        points = malloc(sizeof(XPoint) * npoints);

diff --git a/client/X11/xf_gdi.h b/client/X11/xf_gdi.h
index eefd97c..bd36d58 100644 (file)
--- a/client/X11/xf_gdi.h
+++ b/client/X11/xf_gdi.h
@@ -27,5 +27,6 @@
 
 void xf_gdi_register_update_callbacks(rdpUpdate* update);
 void xf_gdi_bitmap_update(rdpContext* context, BITMAP_UPDATE* bitmapUpdate);
+UINT32 xf_convert_rdp_order_color(xfContext* xfc, UINT32 color);
 
 #endif /* __XF_GDI_H */

diff --git a/client/X11/xf_graphics.c b/client/X11/xf_graphics.c
index e73833a..b4c6e00 100644 (file)
--- a/client/X11/xf_graphics.c
+++ b/client/X11/xf_graphics.c
@@ -35,6 +35,7 @@
 #include <freerdp/codec/jpeg.h>
 
 #include "xf_graphics.h"
+#include "xf_gdi.h"
 
 #include <freerdp/log.h>
 #define TAG CLIENT_TAG("x11")
@@ -381,8 +382,8 @@ void xf_Glyph_BeginDraw(rdpContext* context, int x, int y, int width, int height
 {
        xfContext* xfc = (xfContext*) context;
 
-       bgcolor = freerdp_convert_gdi_order_color(bgcolor, context->settings->ColorDepth, xfc->format, xfc->palette);
-       fgcolor = freerdp_convert_gdi_order_color(fgcolor, context->settings->ColorDepth, xfc->format, xfc->palette);
+       bgcolor = xf_convert_rdp_order_color(xfc, bgcolor);
+       fgcolor = xf_convert_rdp_order_color(xfc, fgcolor);
 
        xf_lock_x11(xfc, FALSE);
 

diff --git a/client/X11/xfreerdp.h b/client/X11/xfreerdp.h
index 75dcd84..8a49ae2 100644 (file)
--- a/client/X11/xfreerdp.h
+++ b/client/X11/xfreerdp.h
@@ -132,6 +132,13 @@ struct xf_context
        UINT16 frame_x2;
        UINT16 frame_y2;
 
+       UINT8 red_shift_l;
+       UINT8 red_shift_r;
+       UINT8 green_shift_l;
+       UINT8 green_shift_r;
+       UINT8 blue_shift_l;
+       UINT8 blue_shift_r;
+
        int XInputOpcode;
 
 #ifdef WITH_XRENDER