[PowerPC][lld] Account for additional X-Forms -> D-Form/DS-Forms load/stores when...

D153645 added additional X-Form load/stores that can be generated for TLS accesses.
However, these added instructions have not been accounted for in lld. As a result,
lld does not know how to handle them and cannot relax initial-exec to local-exec
when the initial-exec sequence contains these additional load/stores.

This patch aims to resolve https://github.com/llvm/llvm-project/issues/64424.

Differential Revision: https://reviews.llvm.org/D158197

(cherry picked from commit 698b45aa902de4d30c798e8d6bd080c8e31bade8)

diff --git a/lld/ELF/Arch/PPC.cpp b/lld/ELF/Arch/PPC.cpp
index 87942c1..3d21edb 100644 (file)
--- a/lld/ELF/Arch/PPC.cpp
+++ b/lld/ELF/Arch/PPC.cpp
@@ -471,10 +471,14 @@ void PPC::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
     if (insn >> 26 != 31)
       error("unrecognized instruction for IE to LE R_PPC_TLS");
     // addi rT, rT, x@tls --> addi rT, rT, x@tprel@l
-    uint32_t dFormOp = getPPCDFormOp((read32(loc) & 0x000007fe) >> 1);
-    if (dFormOp == 0)
-      error("unrecognized instruction for IE to LE R_PPC_TLS");
-    write32(loc, (dFormOp << 26) | (insn & 0x03ff0000) | lo(val));
+    unsigned secondaryOp = (read32(loc) & 0x000007fe) >> 1;
+    uint32_t dFormOp = getPPCDFormOp(secondaryOp);
+    if (dFormOp == 0) { // Expecting a DS-Form instruction.
+      dFormOp = getPPCDSFormOp(secondaryOp);
+      if (dFormOp == 0)
+        error("unrecognized instruction for IE to LE R_PPC_TLS");
+    }
+    write32(loc, (dFormOp | (insn & 0x03ff0000) | lo(val)));
     break;
   }
   default:

diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp
index 36b1d0e..0b6459f 100644 (file)
--- a/lld/ELF/Arch/PPC64.cpp
+++ b/lld/ELF/Arch/PPC64.cpp
@@ -37,6 +37,12 @@ enum XFormOpcd {
   STHX = 407,
   STWX = 151,
   STDX = 149,
+  LHAX = 343,
+  LWAX = 341,
+  LFSX = 535,
+  LFDX = 599,
+  STFSX = 663,
+  STFDX = 727,
   ADD = 266,
 };
 
@@ -49,7 +55,6 @@ enum DFormOpcd {
   LWZ = 32,
   LWZU = 33,
   LFSU = 49,
-  LD = 58,
   LFDU = 51,
   STB = 38,
   STBU = 39,
@@ -59,10 +64,20 @@ enum DFormOpcd {
   STWU = 37,
   STFSU = 53,
   STFDU = 55,
-  STD = 62,
+  LHA = 42,
+  LFS = 48,
+  LFD = 50,
+  STFS = 52,
+  STFD = 54,
   ADDI = 14
 };
 
+enum DSFormOpcd {
+  LD = 58,
+  LWA = 58,
+  STD = 62
+};
+
 constexpr uint32_t NOP = 0x60000000;
 
 enum class PPCLegacyInsn : uint32_t {
@@ -825,26 +840,48 @@ void PPC64::relaxTlsLdToLe(uint8_t *loc, const Relocation &rel,
   }
 }
 
+// Map X-Form instructions to their DS-Form counterparts, if applicable.
+// The full encoding is returned here to distinguish between the different
+// DS-Form instructions.
+unsigned elf::getPPCDSFormOp(unsigned secondaryOp) {
+  switch (secondaryOp) {
+  case LWAX:
+    return (LWA << 26) | 0x2;
+  case LDX:
+    return LD << 26;
+  case STDX:
+    return STD << 26;
+  default:
+    return 0;
+  }
+}
+
 unsigned elf::getPPCDFormOp(unsigned secondaryOp) {
   switch (secondaryOp) {
   case LBZX:
-    return LBZ;
+    return LBZ << 26;
   case LHZX:
-    return LHZ;
+    return LHZ << 26;
   case LWZX:
-    return LWZ;
-  case LDX:
-    return LD;
+    return LWZ << 26;
   case STBX:
-    return STB;
+    return STB << 26;
   case STHX:
-    return STH;
+    return STH << 26;
   case STWX:
-    return STW;
-  case STDX:
-    return STD;
+    return STW << 26;
+  case LHAX:
+    return LHA << 26;
+  case LFSX:
+    return LFS << 26;
+  case LFDX:
+    return LFD << 26;
+  case STFSX:
+    return STFS << 26;
+  case STFDX:
+    return STFD << 26;
   case ADD:
-    return ADDI;
+    return ADDI << 26;
   default:
     return 0;
   }
@@ -898,10 +935,16 @@ void PPC64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
         error("unrecognized instruction for IE to LE R_PPC64_TLS");
       uint32_t secondaryOp = (read32(loc) & 0x000007FE) >> 1; // bits 21-30
       uint32_t dFormOp = getPPCDFormOp(secondaryOp);
-      if (dFormOp == 0)
-        error("unrecognized instruction for IE to LE R_PPC64_TLS");
-      write32(loc, ((dFormOp << 26) | (read32(loc) & 0x03FFFFFF)));
-      relocateNoSym(loc + offset, R_PPC64_TPREL16_LO, val);
+      uint32_t finalReloc;
+      if (dFormOp == 0) { // Expecting a DS-Form instruction.
+        dFormOp = getPPCDSFormOp(secondaryOp);
+        if (dFormOp == 0)
+          error("unrecognized instruction for IE to LE R_PPC64_TLS");
+        finalReloc = R_PPC64_TPREL16_LO_DS;
+      } else
+        finalReloc = R_PPC64_TPREL16_LO;
+      write32(loc, dFormOp | (read32(loc) & 0x03ff0000));
+      relocateNoSym(loc + offset, finalReloc, val);
     } else if (locAsInt % 4 == 1) {
       // If the offset is not 4 byte aligned then we have a PCRel type reloc.
       // This version of the relocation is offset by one byte from the
@@ -926,9 +969,12 @@ void PPC64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
         }
       } else {
         uint32_t dFormOp = getPPCDFormOp(secondaryOp);
-        if (dFormOp == 0)
-          errorOrWarn("unrecognized instruction for IE to LE R_PPC64_TLS");
-        write32(loc - 1, ((dFormOp << 26) | (tlsInstr & 0x03FF0000)));
+        if (dFormOp == 0) { // Expecting a DS-Form instruction.
+          dFormOp = getPPCDSFormOp(secondaryOp);
+          if (dFormOp == 0)
+            errorOrWarn("unrecognized instruction for IE to LE R_PPC64_TLS");
+        }
+        write32(loc - 1, (dFormOp | (tlsInstr & 0x03ff0000)));
       }
     } else {
       errorOrWarn("R_PPC64_TLS must be either 4 byte aligned or one byte "

diff --git a/lld/ELF/Target.h b/lld/ELF/Target.h
index 9d4f22d..47dbe6b 100644 (file)
--- a/lld/ELF/Target.h
+++ b/lld/ELF/Target.h
@@ -207,6 +207,7 @@ void processArmCmseSymbols();
 void writePPC32GlinkSection(uint8_t *buf, size_t numEntries);
 
 unsigned getPPCDFormOp(unsigned secondaryOp);
+unsigned getPPCDSFormOp(unsigned secondaryOp);
 
 // In the PowerPC64 Elf V2 abi a function can have 2 entry points.  The first
 // is a global entry point (GEP) which typically is used to initialize the TOC

diff --git a/lld/test/ELF/ppc32-tls-ie.s b/lld/test/ELF/ppc32-tls-ie.s
index f9f4645..84a105c 100644 (file)
--- a/lld/test/ELF/ppc32-tls-ie.s
+++ b/lld/test/ELF/ppc32-tls-ie.s
@@ -12,8 +12,8 @@
 # IE-REL:      FLAGS STATIC_TLS
 ## A non-preemptable symbol (b) has 0 st_shndx.
 # IE-REL:      .rela.dyn {
-# IE-REL-NEXT:   0x20238 R_PPC_TPREL32 - 0xC
-# IE-REL-NEXT:   0x20234 R_PPC_TPREL32 a 0x0
+# IE-REL-NEXT:   0x20258 R_PPC_TPREL32 - 0xC
+# IE-REL-NEXT:   0x20254 R_PPC_TPREL32 a 0x0
 # IE-REL-NEXT: }
 
 ## &.got[3] - _GLOBAL_OFFSET_TABLE_ = 12
@@ -44,6 +44,12 @@ lbzx 10, 8, c@tls
 # IE-NEXT: stbx 14, 4, 2
 # IE-NEXT: sthx 15, 5, 2
 # IE-NEXT: stwx 16, 6, 2
+# IE-NEXT: lhax 17, 7, 2
+# IE-NEXT: lwax 18, 8, 2
+# IE-NEXT: lfsx 19, 9, 2
+# IE-NEXT: lfdx 20, 10, 2
+# IE-NEXT: stfsx 21, 11, 2
+# IE-NEXT: stfdx 22, 12, 2
 
 ## In LE, these X-Form instructions are changed to their corresponding D-Form.
 # LE-NEXT: lhz 12, -28660(2)
@@ -51,12 +57,26 @@ lbzx 10, 8, c@tls
 # LE-NEXT: stb 14, -28660(4)
 # LE-NEXT: sth 15, -28660(5)
 # LE-NEXT: stw 16, -28660(6)
+# LE-NEXT: lha 17, -28660(7)
+# LE-NEXT: lwa 18, -28660(8)
+# LE-NEXT: lfs 19, -28660(9)
+# LE-NEXT: lfd 20, -28660(10)
+# LE-NEXT: stfs 21, -28660(11)
+# LE-NEXT: stfd 22, -28660(12)
 
 lhzx 12, 2, s@tls
 lwzx 13, 3, i@tls
 stbx 14, 4, c@tls
 sthx 15, 5, s@tls
 stwx 16, 6, i@tls
+lhax 17, 7, s@tls
+lwax 18, 8, i@tls
+lfsx 19, 9, f@tls
+lfdx 20, 10, d@tls
+stfsx 21, 11, f@tls
+stfdx 22, 12, d@tls
+ldx 23, 13, l@tls
+stdx 24, 14, l@tls
 
 .section .tbss
 .globl a
@@ -66,3 +86,6 @@ a:
 c:
 s:
 i:
+f:
+d:
+l:

diff --git a/lld/test/ELF/ppc64-tls-ie.s b/lld/test/ELF/ppc64-tls-ie.s
index 8da808b..8855e8c 100644 (file)
--- a/lld/test/ELF/ppc64-tls-ie.s
+++ b/lld/test/ELF/ppc64-tls-ie.s
@@ -24,10 +24,12 @@
 
 # IE-REL:      FLAGS STATIC_TLS
 # IE-REL:      .rela.dyn {
-# IE-REL-NEXT:   0x204C8 R_PPC64_TPREL64 c 0x0
-# IE-REL-NEXT:   0x204D0 R_PPC64_TPREL64 s 0x0
-# IE-REL-NEXT:   0x204D8 R_PPC64_TPREL64 i 0x0
-# IE-REL-NEXT:   0x204E0 R_PPC64_TPREL64 l 0x0
+# IE-REL-NEXT:   0x205A8 R_PPC64_TPREL64 c 0x0
+# IE-REL-NEXT:   0x205B0 R_PPC64_TPREL64 s 0x0
+# IE-REL-NEXT:   0x205B8 R_PPC64_TPREL64 i 0x0
+# IE-REL-NEXT:   0x205C0 R_PPC64_TPREL64 l 0x0
+# IE-REL-NEXT:   0x205C8 R_PPC64_TPREL64 f 0x0
+# IE-REL-NEXT:   0x205D0 R_PPC64_TPREL64 d 0x0
 # IE-REL-NEXT: }
 
 # INPUT-REL: R_PPC64_GOT_TPREL16_HA c 0x0
@@ -152,10 +154,64 @@ test_ds:
   ld 4, l@got@tprel(2)
   stdx 3, 4, l@tls
 
+# LE-LABEL: <test_lhax>:
+# LE-NEXT:    nop
+# LE-NEXT:    addis 3, 13, 0
+# LE-NEXT:    lha 3, -28670(3)
+test_lhax:
+  addis 3, 2, s@got@tprel@ha
+  ld 3, s@got@tprel@l(3)
+  lhax 3, 3, s@tls
+
+# LE-LABEL: <test_lwax>:
+# LE-NEXT:    nop
+# LE-NEXT:    addis 3, 13, 0
+# LE-NEXT:    lwa 3, -28668(3)
+test_lwax:
+  addis 3, 2, i@got@tprel@ha
+  ld 3, i@got@tprel@l(3)
+  lwax 3, 3, i@tls
+
+# LE-LABEL: <test_lfsx>:
+# LE-NEXT:    nop
+# LE-NEXT:    addis 3, 13, 0
+# LE-NEXT:    lfs 3, -28656(3)
+test_lfsx:
+  addis 3, 2, f@got@tprel@ha
+  ld 3, f@got@tprel@l(3)
+  lfsx 3, 3, f@tls
+
+# LE-LABEL: <test_lfdx>:
+# LE-NEXT:    nop
+# LE-NEXT:    addis 3, 13, 0
+# LE-NEXT:    lfd 3, -28648(3)
+test_lfdx:
+  addis 3, 2, d@got@tprel@ha
+  ld 3, d@got@tprel@l(3)
+  lfdx 3, 3, d@tls
+
+# LE-LABEL: <test_stfsx>:
+# LE-NEXT:    nop
+# LE-NEXT:    addis 4, 13, 0
+# LE-NEXT:    stfs 3, -28656(4)
+test_stfsx:
+  addis 4, 2, f@got@tprel@ha
+  ld 4, f@got@tprel@l(4)
+  stfsx 3, 4, f@tls
+
+# LE-LABEL: <test_stfdx>:
+# LE-NEXT:    nop
+# LE-NEXT:    addis 4, 13, 0
+# LE-NEXT:    stfd 3, -28648(4)
+test_stfdx:
+  addis 4, 2, d@got@tprel@ha
+  ld 4, d@got@tprel@l(4)
+  stfdx 3, 4, d@tls
+
 # NOREL: There are no relocations in this file.
 
 .section .tdata,"awT",@progbits
-.globl c, s, i, l
+.globl c, s, i, l, f, d
 c:
 .byte 97
 
@@ -170,3 +226,9 @@ i:
 .p2align 3
 l:
 .quad 55
+f:
+.long 55
+
+.p2align 3
+d:
+.quad 55

diff --git a/lld/test/ELF/ppc64-tls-pcrel-ie.s b/lld/test/ELF/ppc64-tls-pcrel-ie.s
index f7a828d..38c081f 100644 (file)
--- a/lld/test/ELF/ppc64-tls-pcrel-ie.s
+++ b/lld/test/ELF/ppc64-tls-pcrel-ie.s
@@ -29,6 +29,12 @@ SECTIONS {
   .text_val 0x1002000 : { *(.text_val) }
   .text_twoval 0x1003000 : { *(.text_twoval) }
   .text_incrval 0x1004000 : { *(.text_incrval) }
+  .text_incrval_half 0x1005000 : { *(.text_incrval_half) }
+  .text_incrval_word 0x1006000 : { *(.text_incrval_word) }
+  .text_incrval_float 0x1007000 : { *(.text_incrval_float) }
+  .text_incrval_double 0x1008000 : { *(.text_incrval_double) }
+  .text_incrval_dword 0x1009000 : { *(.text_incrval_dword) }
+  .text_incrval_half_zero 0x1010000 : { *(.text_incrval_half_zero) }
 }
 
 #--- defs
@@ -42,26 +48,26 @@ y:
 
 #--- asm
 # IE-RELOC: Relocation section '.rela.dyn' at offset 0x10090 contains 2 entries:
-# IE-RELOC: 00000000010040f0  0000000100000049 R_PPC64_TPREL64        0000000000000000 x + 0
-# IE-RELOC: 00000000010040f8  0000000200000049 R_PPC64_TPREL64        0000000000000000 y + 0
+# IE-RELOC: 00000000010100f0  0000000100000049 R_PPC64_TPREL64        0000000000000000 x + 0
+# IE-RELOC-NEXT: 00000000010100f8  0000000200000049 R_PPC64_TPREL64        0000000000000000 y + 0
 
 # IE-SYM:   Symbol table '.dynsym' contains 3 entries:
 # IE-SYM:   1: 0000000000000000     0 TLS     GLOBAL DEFAULT   UND x
 # IE-SYM:   2: 0000000000000000     0 TLS     GLOBAL DEFAULT   UND y
 
 # IE-GOT:      Hex dump of section '.got':
-# IE-GOT-NEXT: 0x010040e8 e8c00001 00000000 00000000 00000000
+# IE-GOT-NEXT: 0x010100e8 e8800101 00000000 00000000 00000000
 
 # LE-RELOC: There are no relocations in this file.
 
-# LE-SYM: Symbol table '.symtab' contains 8 entries:
-# LE-SYM: 6: 0000000000000000     0 TLS     GLOBAL DEFAULT     6 x
-# LE-SYM: 7: 0000000000000004     0 TLS     GLOBAL DEFAULT     6 y
+# LE-SYM: Symbol table '.symtab' contains 14 entries:
+# LE-SYM: 0000000000000000     0 TLS     GLOBAL DEFAULT     [[#]] x
+# LE-SYM: 0000000000000004     0 TLS     GLOBAL DEFAULT     [[#]] y
 
 # LE-GOT: could not find section '.got'
 
 # IE-LABEL: <IEAddr>:
-# IE-NEXT:    pld 3, 12528(0), 1
+# IE-NEXT:    pld 3, 61680(0), 1
 # IE-NEXT:    add 3, 3, 13
 # IE-NEXT:    blr
 # LE-LABEL: <IEAddr>:
@@ -75,7 +81,7 @@ IEAddr:
        blr
 
 # IE-LABEL: <IEAddrCopy>:
-# IE-NEXT:    pld 3, 12512(0), 1
+# IE-NEXT:    pld 3, 61664(0), 1
 # IE-NEXT:    add 4, 3, 13
 # IE-NEXT:    blr
 # LE-LABEL: <IEAddrCopy>:
@@ -89,7 +95,7 @@ IEAddrCopy:
        blr
 
 # IE-LABEL: <IEVal>:
-# IE-NEXT:    pld 3, 8432(0), 1
+# IE-NEXT:    pld 3, 57584(0), 1
 # IE-NEXT:    lwzx 3, 3, 13
 # IE-NEXT:    blr
 # LE-LABEL: <IEVal>:
@@ -103,8 +109,8 @@ IEVal:
        blr
 
 # IE-LABEL: <IETwoVal>:
-# IE-NEXT:    pld 3, 4336(0), 1
-# IE-NEXT:    pld 4, 4336(0), 1
+# IE-NEXT:    pld 3, 53488(0), 1
+# IE-NEXT:    pld 4, 53488(0), 1
 # IE-NEXT:    lwzx 3, 3, 13
 # IE-NEXT:    lwzx 4, 4, 13
 # IE-NEXT:    blr
@@ -123,7 +129,7 @@ IETwoVal:
        blr
 
 # IE-LABEL: <IEIncrementVal>:
-# IE-NEXT:    pld 4, 248(0), 1
+# IE-NEXT:    pld 4, 49400(0), 1
 # IE-NEXT:    lwzx 3, 4, 13
 # IE-NEXT:    stwx 3, 4, 13
 # IE-NEXT:    blr
@@ -138,3 +144,105 @@ IEIncrementVal:
        lwzx 3, 4, y@tls@pcrel
        stwx 3, 4, y@tls@pcrel
        blr
+
+# IE-LABEL: <IEIncrementValHalf>:
+# IE-NEXT:    pld 4, 45304(0), 1
+# IE-NEXT:    lhax 3, 4, 13
+# IE-NEXT:    sthx 3, 4, 13
+# IE-NEXT:    blr
+# LE-LABEL: <IEIncrementValHalf>:
+# LE-NEXT:    paddi 4, 13, -28668, 0
+# LE-NEXT:    lha 3, 0(4)
+# LE-NEXT:    sth 3, 0(4)
+# LE-NEXT:    blr
+.section .text_incrval_half, "ax", %progbits
+IEIncrementValHalf:
+       pld 4, y@got@tprel@pcrel(0), 1
+       lhax 3, 4, y@tls@pcrel
+       sthx 3, 4, y@tls@pcrel
+       blr
+
+# IE-LABEL: <IEIncrementValWord>:
+# IE-NEXT:    pld 4, 41208(0), 1
+# IE-NEXT:    lwax 3, 4, 13
+# IE-NEXT:    stwx 3, 4, 13
+# IE-NEXT:    blr
+# LE-LABEL: <IEIncrementValWord>:
+# LE-NEXT:    paddi 4, 13, -28668, 0
+# LE-NEXT:    lwa 3, 0(4)
+# LE-NEXT:    stw 3, 0(4)
+# LE-NEXT:    blr
+.section .text_incrval_word, "ax", %progbits
+IEIncrementValWord:
+       pld 4, y@got@tprel@pcrel(0), 1
+       lwax 3, 4, y@tls@pcrel
+       stwx 3, 4, y@tls@pcrel
+       blr
+
+# IE-LABEL: <IEIncrementValFloat>:
+# IE-NEXT:    pld 4, 37112(0), 1
+# IE-NEXT:    lfsx 3, 4, 13
+# IE-NEXT:    stfsx 3, 4, 13
+# IE-NEXT:    blr
+# LE-LABEL: <IEIncrementValFloat>:
+# LE-NEXT:    paddi 4, 13, -28668, 0
+# LE-NEXT:    lfs 3, 0(4)
+# LE-NEXT:    stfs 3, 0(4)
+# LE-NEXT:    blr
+.section .text_incrval_float, "ax", %progbits
+IEIncrementValFloat:
+       pld 4, y@got@tprel@pcrel(0), 1
+       lfsx 3, 4, y@tls@pcrel
+       stfsx 3, 4, y@tls@pcrel
+       blr
+
+# IE-LABEL: <IEIncrementValDouble>:
+# IE-NEXT:    pld 4, 33016(0), 1
+# IE-NEXT:    lfdx 3, 4, 13
+# IE-NEXT:    stfdx 3, 4, 13
+# IE-NEXT:    blr
+# LE-LABEL: <IEIncrementValDouble>:
+# LE-NEXT:    paddi 4, 13, -28668, 0
+# LE-NEXT:    lfd 3, 0(4)
+# LE-NEXT:    stfd 3, 0(4)
+# LE-NEXT:    blr
+.section .text_incrval_double, "ax", %progbits
+IEIncrementValDouble:
+       pld 4, y@got@tprel@pcrel(0), 1
+       lfdx 3, 4, y@tls@pcrel
+       stfdx 3, 4, y@tls@pcrel
+       blr
+
+# IE-LABEL: <IEIncrementValDword>:
+# IE-NEXT:    pld 4, 28920(0), 1
+# IE-NEXT:    ldx 3, 4, 13
+# IE-NEXT:    stdx 3, 4, 13
+# IE-NEXT:    blr
+# LE-LABEL: <IEIncrementValDword>:
+# LE-NEXT:    paddi 4, 13, -28668, 0
+# LE-NEXT:    ld 3, 0(4)
+# LE-NEXT:    std 3, 0(4)
+# LE-NEXT:    blr
+.section .text_incrval_dword, "ax", %progbits
+IEIncrementValDword:
+       pld 4, y@got@tprel@pcrel(0), 1
+       ldx 3, 4, y@tls@pcrel
+       stdx 3, 4, y@tls@pcrel
+       blr
+
+# IE-LABEL: <IEIncrementValHalfZero>:
+# IE-NEXT:    pld 4, 248(0), 1
+# IE-NEXT:    lhzx 3, 4, 13
+# IE-NEXT:    sthx 3, 4, 13
+# IE-NEXT:    blr
+# LE-LABEL: <IEIncrementValHalfZero>:
+# LE-NEXT:    paddi 4, 13, -28668, 0
+# LE-NEXT:    lhz 3, 0(4)
+# LE-NEXT:    sth 3, 0(4)
+# LE-NEXT:    blr
+.section .text_incrval_half_zero, "ax", %progbits
+IEIncrementValHalfZero:
+       pld 4, y@got@tprel@pcrel(0), 1
+       lhzx 3, 4, y@tls@pcrel
+       sthx 3, 4, y@tls@pcrel
+       blr