[x86/Linux] Fix "Bad opcode" assert in unwindLazyState (#8609)

* [x86/Linux] Fix "Bad opcode" assert in unwindLazyState

This commit suppresses "Bad opcode" assert while runing "Hello, World" example.

This commit address the following three code patterns discovered while
digging the assert failure:
 - and $0x1, %al
 - xor $0xff, %al
 - stack protection code:
   mov %gs:<off>, <reg>
   cmp <off>(%esp), <reg>
   mov <reg>, <off>($esp)
   jne <disp32>

This commit revises LazyMachState::unwindLazyState to handle the first two patterns,
and revises compile options not to emit the third pattern.

diff --git a/compileoptions.cmake b/compileoptions.cmake
index 75d51fd5bb125d2db2c7444b29de65ba164d9291..7baa86baf1d9c4de0051b4bcf12aeecdb9d89239 100644 (file)
--- a/compileoptions.cmake
+++ b/compileoptions.cmake
@@ -15,7 +15,11 @@ if (CLR_CMAKE_PLATFORM_UNIX)
     # We cannot enable "stack-protector-strong" on OS X due to a bug in clang compiler (current version 7.0.2)
     add_compile_options(-fstack-protector)
   else()
-    add_compile_options(-fstack-protector-strong)
+    if(NOT CLR_CMAKE_PLATFORM_ARCH_I386)
+      # x86 unwinder cannot handle stack protection code, yet
+      # see https://github.com/dotnet/coreclr/issues/8625 for details
+      add_compile_options(-fstack-protector-strong)
+    endif(NOT CLR_CMAKE_PLATFORM_ARCH_I386)
   endif(CLR_CMAKE_PLATFORM_DARWIN)
 
   add_definitions(-DDISABLE_CONTRACTS)

diff --git a/src/vm/i386/gmsx86.cpp b/src/vm/i386/gmsx86.cpp
index 75d37ba0d3cdcb266658ff9c067ec1219c76c0c7..ae4b9dda9818a7b149e978c261f0793b8d747784 100644 (file)
--- a/src/vm/i386/gmsx86.cpp
+++ b/src/vm/i386/gmsx86.cpp
@@ -683,6 +683,10 @@ void LazyMachState::unwindLazyState(LazyMachState* baseState,
                 ip += 2;
                 break;
 
+            case 0x34:                            // XOR AL, imm8
+                ip += 2;
+                break;
+
             case 0x31:
             case 0x32:
             case 0x33:
@@ -879,6 +883,10 @@ void LazyMachState::unwindLazyState(LazyMachState* baseState,
                 datasize = b16bit?2:4;
                 goto decodeRM;
 
+            case 0x24:                           // AND AL, imm8
+                ip += 2;
+                break;
+
             case 0x01:                           // ADD mod/rm
             case 0x03:
             case 0x29:                           // SUB mod/rm