[x86/Linux] Fix "Bad opcode" assert in unwindLazyState (#8609)

* [x86/Linux] Fix "Bad opcode" assert in unwindLazyState

This commit suppresses "Bad opcode" assert while runing "Hello, World" example.

This commit address the following three code patterns discovered while
digging the assert failure:
 - and $0x1, %al
 - xor $0xff, %al
 - stack protection code:
   mov %gs:<off>, <reg>
   cmp <off>(%esp), <reg>
   mov <reg>, <off>($esp)
   jne <disp32>

This commit revises LazyMachState::unwindLazyState to handle the first two patterns,
and revises compile options not to emit the third pattern.

diff --git a/compileoptions.cmake b/compileoptions.cmake
index 75d51fd..7baa86b 100644 (file)
--- a/compileoptions.cmake
+++ b/compileoptions.cmake
@@ -15,7 +15,11 @@ if (CLR_CMAKE_PLATFORM_UNIX)
     # We cannot enable "stack-protector-strong" on OS X due to a bug in clang compiler (current version 7.0.2)
     add_compile_options(-fstack-protector)
   else()
-    add_compile_options(-fstack-protector-strong)
+    if(NOT CLR_CMAKE_PLATFORM_ARCH_I386)
+      # x86 unwinder cannot handle stack protection code, yet
+      # see https://github.com/dotnet/coreclr/issues/8625 for details
+      add_compile_options(-fstack-protector-strong)
+    endif(NOT CLR_CMAKE_PLATFORM_ARCH_I386)
   endif(CLR_CMAKE_PLATFORM_DARWIN)
 
   add_definitions(-DDISABLE_CONTRACTS)

diff --git a/src/vm/i386/gmsx86.cpp b/src/vm/i386/gmsx86.cpp
index 75d37ba..ae4b9dd 100644 (file)
--- a/src/vm/i386/gmsx86.cpp
+++ b/src/vm/i386/gmsx86.cpp
@@ -683,6 +683,10 @@ void LazyMachState::unwindLazyState(LazyMachState* baseState,
                 ip += 2;
                 break;
 
+            case 0x34:                            // XOR AL, imm8
+                ip += 2;
+                break;
+
             case 0x31:
             case 0x32:
             case 0x33:
@@ -879,6 +883,10 @@ void LazyMachState::unwindLazyState(LazyMachState* baseState,
                 datasize = b16bit?2:4;
                 goto decodeRM;
 
+            case 0x24:                           // AND AL, imm8
+                ip += 2;
+                break;
+
             case 0x01:                           // ADD mod/rm
             case 0x03:
             case 0x29:                           // SUB mod/rm