BR 3109604: Fix C4 vs C5 VEX form selection in calcsize()

calcsize() had the wrong criterion for when C5 prefixes are permitted
(REX.R is permitted, REX.X is forbidden.)  assemble() had the right
test already.  This caused symbol value errors.

diff --git a/assemble.c b/assemble.c
index 6fb5de9..99b833c 100644 (file)
--- a/assemble.c
+++ b/assemble.c
@@ -1152,7 +1152,7 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
             errfunc(ERR_NONFATAL, "invalid operands in non-64-bit mode");
             return -1;
         }
-        if (ins->vex_cm != 1 || (ins->rex & (REX_W|REX_R|REX_B)))
+        if (ins->vex_cm != 1 || (ins->rex & (REX_W|REX_X|REX_B)))
             length += 3;
         else
             length += 2;

diff --git a/test/br3109604.asm b/test/br3109604.asm
new file mode 100644 (file)
index 0000000..9032e8d
--- /dev/null
+++ b/test/br3109604.asm
@@ -0,0 +1,9 @@
+;Testname=unoptimized; Arguments=-O0 -fbin -obr3109604.bin; Files=stdout stderr br3109604.bin
+;Testname=optimized;   Arguments=-Ox -fbin -obr3109604.bin; Files=stdout stderr br3109604.bin
+
+       bits 64
+b0:    vmovd xmm2, [rdx+r9]
+e0:    
+       
+       section .data
+len:   dd e0 - b0              ; Should be 6