[XRay] Create an Index of sleds per function

Summary:
This change adds a new section to the xray-instrumented binary that
stores an index into ranges of the instrumentation map, where sleds
associated with the same function can be accessed as an array. At
runtime, we can get access to this index by function ID offset allowing
for selective patching and unpatching by function ID.

Each entry in this new section (xray_fn_idx) will include two pointers
indicating the start and one past the end of the sleds associated with
the same function. These entries will be 16 bytes long on x86 and
aarch64. On arm, we align to 16 bytes anyway so the runtime has to take
that into consideration.

__{start,stop}_xray_fn_idx will be the symbols that the runtime will
look for when we implement the selective patching/unpatching by function
id APIs. Because XRay synthesizes the function id's in a monotonically
increasing manner at runtime now, implementations (and users) can use
this table to look up the sleds associated with a specific function.
This is useful in implementations that want to do things like:

  - Implement coverage mode for functions by patching everything
    pre-main, then as functions are encountered, the installed handler
    can unpatch the function that's been encountered after recording
    that it's been called.
  - Do "learning mode", so that the implementation can figure out some
    statistical information about function calls by function id for a
    time being, and then determine which functions are worth
    uninstrumenting at runtime.
  - Do "selective instrumentation" where an implementation can
    specifically instrument only certain function id's at runtime
    (either based on some external data, or through some other
    heuristics) instead of patching all the instrumented functions at
    runtime.

Reviewers: dblaikie, echristo, chandlerc, javed.absar

Subscribers: pelikan, aemerson, kpw, llvm-commits, rengolin

Differential Revision: https://reviews.llvm.org/D32693

llvm-svn: 302109

diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h
index fb8c8408fc7794b7418e3732fab17f05a3c947a0..e008c7ceb066e50cb4a0e31e5965754ff4ada8f0 100644 (file)
--- a/llvm/include/llvm/CodeGen/AsmPrinter.h
+++ b/llvm/include/llvm/CodeGen/AsmPrinter.h
@@ -242,7 +242,7 @@ public:
   };
 
   // All the sleds to be emitted.
-  std::vector<XRayFunctionEntry> Sleds;
+  SmallVector<XRayFunctionEntry, 4> Sleds;
 
   // Helper function to record a given XRay sled.
   void recordSled(MCSymbol *Sled, const MachineInstr &MI, SledKind Kind);

diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index b11e30c359b33005140e11421a5d601d7029813f..7f38c27f8a9891d0cd1ad49a6c74cf56dd23c477 100644 (file)
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -2761,37 +2761,63 @@ void AsmPrinter::emitXRayTable() {
 
   auto PrevSection = OutStreamer->getCurrentSectionOnly();
   auto Fn = MF->getFunction();
-  MCSection *Section = nullptr;
+  MCSection *InstMap = nullptr;
+  MCSection *FnSledIndex = nullptr;
   if (MF->getSubtarget().getTargetTriple().isOSBinFormatELF()) {
     if (Fn->hasComdat()) {
-      Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS,
+      InstMap = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS,
                                          ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
                                          Fn->getComdat()->getName());
+      FnSledIndex = OutContext.getELFSection("xray_fn_idx", ELF::SHT_PROGBITS,
+                                             ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
+                                             Fn->getComdat()->getName());
     } else {
-      Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS,
+      InstMap = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS,
                                          ELF::SHF_ALLOC);
+      FnSledIndex = OutContext.getELFSection("xray_fn_idx", ELF::SHT_PROGBITS,
+                                             ELF::SHF_ALLOC);
     }
   } else if (MF->getSubtarget().getTargetTriple().isOSBinFormatMachO()) {
-    Section = OutContext.getMachOSection("__DATA", "xray_instr_map", 0,
+    InstMap = OutContext.getMachOSection("__DATA", "xray_instr_map", 0,
                                          SectionKind::getReadOnlyWithRel());
+    FnSledIndex = OutContext.getMachOSection("__DATA", "xray_fn_idx", 0,
+                                             SectionKind::getReadOnlyWithRel());
   } else {
     llvm_unreachable("Unsupported target");
   }
 
   // Before we switch over, we force a reference to a label inside the
-  // xray_instr_map section. Since this function is always called just
-  // before the function's end, we assume that this is happening after
-  // the last return instruction.
-
+  // xray_instr_map and xray_fn_idx sections. Since this function is always
+  // called just before the function's end, we assume that this is happening
+  // after the last return instruction. We also use the synthetic label in the
+  // xray_inster_map as a delimeter for the range of sleds for this function in
+  // the index.
   auto WordSizeBytes = MAI->getCodePointerSize();
-  MCSymbol *Tmp = OutContext.createTempSymbol("xray_synthetic_", true);
+  MCSymbol *SledsStart = OutContext.createTempSymbol("xray_synthetic_", true);
+  MCSymbol *IdxRef = OutContext.createTempSymbol("xray_fn_idx_synth_", true);
   OutStreamer->EmitCodeAlignment(16);
-  OutStreamer->EmitSymbolValue(Tmp, WordSizeBytes, false);
-  OutStreamer->SwitchSection(Section);
-  OutStreamer->EmitLabel(Tmp);
+  OutStreamer->EmitSymbolValue(SledsStart, WordSizeBytes, false);
+  OutStreamer->EmitSymbolValue(IdxRef, WordSizeBytes, false);
+
+  // Now we switch to the instrumentation map section. Because this is done
+  // per-function, we are able to create an index entry that will represent the
+  // range of sleds associated with a function.
+  OutStreamer->SwitchSection(InstMap);
+  OutStreamer->EmitLabel(SledsStart);
   for (const auto &Sled : Sleds)
     Sled.emit(WordSizeBytes, OutStreamer.get(), CurrentFnSym);
-
+  MCSymbol *SledsEnd = OutContext.createTempSymbol("xray_synthetic_end", true);
+  OutStreamer->EmitLabel(SledsEnd);
+
+  // We then emit a single entry in the index per function. We use the symbols
+  // that bound the instrumentation map as the range for a specific function.
+  // Each entry here will be 16-byte aligned, as we're writing down two
+  // pointers.
+  OutStreamer->SwitchSection(FnSledIndex);
+  OutStreamer->EmitCodeAlignment(16);
+  OutStreamer->EmitLabel(IdxRef);
+  OutStreamer->EmitSymbolValue(SledsStart, WordSizeBytes);
+  OutStreamer->EmitSymbolValue(SledsEnd, WordSizeBytes);
   OutStreamer->SwitchSection(PrevSection);
   Sleds.clear();
 }

diff --git a/llvm/test/CodeGen/AArch64/xray-attribute-instrumentation.ll b/llvm/test/CodeGen/AArch64/xray-attribute-instrumentation.ll
index d0f5f40e156c9e6c96a819e3937c83907c002274..38b62a72a20f5ac7eaf8fe0c2b9c9be5d85db905 100644 (file)
--- a/llvm/test/CodeGen/AArch64/xray-attribute-instrumentation.ll
+++ b/llvm/test/CodeGen/AArch64/xray-attribute-instrumentation.ll
@@ -26,6 +26,7 @@ define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always"
 }
 ; CHECK:       .p2align 4
 ; CHECK-NEXT:  .xword .Lxray_synthetic_0
+; CHECK-NEXT:  .xword .Lxray_fn_idx_synth_0
 ; CHECK-NEXT:  .section xray_instr_map,{{.*}}
 ; CHECK-LABEL: Lxray_synthetic_0:
 ; CHECK:       .xword .Lxray_sled_0

diff --git a/llvm/test/CodeGen/AArch64/xray-tail-call-sled.ll b/llvm/test/CodeGen/AArch64/xray-tail-call-sled.ll
index 6ada3ce8d551b0ec72056a5d203ed84ebaca6b30..fb89950b99c844f0ee51a86a7bab3f4da87f7a4d 100644 (file)
--- a/llvm/test/CodeGen/AArch64/xray-tail-call-sled.ll
+++ b/llvm/test/CodeGen/AArch64/xray-tail-call-sled.ll
@@ -29,10 +29,16 @@ define i32 @callee() nounwind noinline uwtable "function-instrument"="xray-alway
 }\r
 ; CHECK:       .p2align 4\r
 ; CHECK-NEXT:  .xword .Lxray_synthetic_0\r
+; CHECK-NEXT:  .xword .Lxray_fn_idx_synth_0\r
 ; CHECK-NEXT:  .section xray_instr_map,{{.*}}\r
 ; CHECK-LABEL: Lxray_synthetic_0:\r
 ; CHECK:       .xword .Lxray_sled_0\r
 ; CHECK:       .xword .Lxray_sled_1\r
+; CHECK-LABEL: Lxray_synthetic_end0:\r
+; CHECK:       .section xray_fn_idx,{{.*}}\r
+; CHECK-LABEL: Lxray_fn_idx_synth_0:\r
+; CHECK:       .xword .Lxray_synthetic_0\r
+; CHECK-NEXT:  .xword .Lxray_synthetic_end0\r
 \r
 define i32 @caller() nounwind noinline uwtable "function-instrument"="xray-always" {\r
 ; CHECK:       .p2align        2\r
@@ -63,7 +69,13 @@ define i32 @caller() nounwind noinline uwtable "function-instrument"="xray-alway
 }\r
 ; CHECK:       .p2align 4\r
 ; CHECK-NEXT:  .xword .Lxray_synthetic_1\r
+; CHECK-NEXT:  .xword .Lxray_fn_idx_synth_1\r
 ; CHECK-NEXT:  .section xray_instr_map,{{.*}}\r
 ; CHECK-LABEL: Lxray_synthetic_1:\r
 ; CHECK:       .xword .Lxray_sled_2\r
 ; CHECK:       .xword .Lxray_sled_3\r
+; CHECK-LABEL: Lxray_synthetic_end1:\r
+; CHECK:       .section xray_fn_idx,{{.*}}\r
+; CHECK-LABEL: Lxray_fn_idx_synth_1:\r
+; CHECK:       .xword .Lxray_synthetic_1\r
+; CHECK-NEXT:  .xword .Lxray_synthetic_end1\r

diff --git a/llvm/test/CodeGen/ARM/xray-armv6-attribute-instrumentation.ll b/llvm/test/CodeGen/ARM/xray-armv6-attribute-instrumentation.ll
index 93c3cb14fb73848e00bb33c8f02dafe67af5594e..5e3c45c3454d850ca25a80434c6ddaaf0bebb16f 100644 (file)
--- a/llvm/test/CodeGen/ARM/xray-armv6-attribute-instrumentation.ll
+++ b/llvm/test/CodeGen/ARM/xray-armv6-attribute-instrumentation.ll
@@ -25,7 +25,13 @@ define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always"
 }
 ; CHECK:       .p2align 4
 ; CHECK-NEXT:  .long {{.*}}Lxray_synthetic_0
+; CHECK-NEXT:  .long {{.*}}Lxray_fn_idx_synth_0
 ; CHECK-NEXT:  .section {{.*}}xray_instr_map{{.*}}
 ; CHECK-LABEL: Lxray_synthetic_0:
 ; CHECK:       .long {{.*}}Lxray_sled_0
 ; CHECK:       .long {{.*}}Lxray_sled_1
+; CHECK-LABEL: Lxray_synthetic_end0:
+; CHECK:       .section {{.*}}xray_fn_idx{{.*}}
+; CHECK-LABEL: Lxray_fn_idx_synth_0:
+; CHECK:       .long {{.*}}Lxray_synthetic_0
+; CHECK-NEXT:  .long {{.*}}Lxray_synthetic_end0

diff --git a/llvm/test/CodeGen/ARM/xray-armv7-attribute-instrumentation.ll b/llvm/test/CodeGen/ARM/xray-armv7-attribute-instrumentation.ll
index d14590b88679463bfd76422d70d5e494c5947384..739151fbdd5e541f4e4b85d458b0b16e5de1ae2e 100644 (file)
--- a/llvm/test/CodeGen/ARM/xray-armv7-attribute-instrumentation.ll
+++ b/llvm/test/CodeGen/ARM/xray-armv7-attribute-instrumentation.ll
@@ -25,7 +25,14 @@ define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always"
 }
 ; CHECK:       .p2align 4
 ; CHECK-NEXT:  .long {{.*}}Lxray_synthetic_0
+; CHECK-NEXT:  .long {{.*}}Lxray_fn_idx_synth_0
 ; CHECK-NEXT:  .section {{.*}}xray_instr_map{{.*}}
 ; CHECK-LABEL: Lxray_synthetic_0:
 ; CHECK:       .long {{.*}}Lxray_sled_0
 ; CHECK:       .long {{.*}}Lxray_sled_1
+; CHECK-LABEL: Lxray_synthetic_end0:
+; CHECK:       .section {{.*}}xray_fn_idx{{.*}}
+; CHECK-LABEL: Lxray_fn_idx_synth_0:
+; CHECK:       .long {{.*}}xray_synthetic_0
+; CHECK-NEXT:  .long {{.*}}xray_synthetic_end0
+

diff --git a/llvm/test/CodeGen/X86/xray-attribute-instrumentation.ll b/llvm/test/CodeGen/X86/xray-attribute-instrumentation.ll
index c52ccf9356bc52833ea1ffa1117c11052d80bfce..7c60327d2c3048bf1de7978c525f8730741ab144 100644 (file)
--- a/llvm/test/CodeGen/X86/xray-attribute-instrumentation.ll
+++ b/llvm/test/CodeGen/X86/xray-attribute-instrumentation.ll
@@ -15,10 +15,17 @@ define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always"
 }
 ; CHECK:       .p2align 4, 0x90
 ; CHECK-NEXT:  .quad {{.*}}xray_synthetic_0
+; CHECK-NEXT:  .quad {{.*}}xray_fn_idx_synth_0
 ; CHECK-NEXT:  .section {{.*}}xray_instr_map
 ; CHECK-LABEL: Lxray_synthetic_0:
 ; CHECK:       .quad {{.*}}xray_sled_0
 ; CHECK:       .quad {{.*}}xray_sled_1
+; CHECK-LABEL: Lxray_synthetic_end0:
+; CHECK:       .section {{.*}}xray_fn_idx
+; CHECK-LABEL: Lxray_fn_idx_synth_0:
+; CHECK:       .quad {{.*}}xray_synthetic_0
+; CHECK-NEXT:  .quad {{.*}}xray_synthetic_end0
+
 
 ; We test multiple returns in a single function to make sure we're getting all
 ; of them with XRay instrumentation.
@@ -46,8 +53,14 @@ NotEqual:
 }
 ; CHECK:       .p2align 4, 0x90
 ; CHECK-NEXT:  .quad {{.*}}xray_synthetic_1
+; CHECK-NEXT:  .quad {{.*}}xray_fn_idx_synth_1
 ; CHECK-NEXT:  .section {{.*}}xray_instr_map
 ; CHECK-LABEL: Lxray_synthetic_1:
 ; CHECK:       .quad {{.*}}xray_sled_2
 ; CHECK:       .quad {{.*}}xray_sled_3
 ; CHECK:       .quad {{.*}}xray_sled_4
+; CHECK-LABEL: Lxray_synthetic_end1:
+; CHECK:       .section {{.*}}xray_fn_idx
+; CHECK-LABEL: Lxray_fn_idx_synth_1:
+; CHECK:       .quad {{.*}}xray_synthetic_1
+; CHECK-NEXT:  .quad {{.*}}xray_synthetic_end1

diff --git a/llvm/test/CodeGen/X86/xray-tail-call-sled.ll b/llvm/test/CodeGen/X86/xray-tail-call-sled.ll
index ece786a5e809bd6d323e4483b03429d71db36543..b12c78a77b203f1537ac16e1a8cafe7fe43041ab 100644 (file)
--- a/llvm/test/CodeGen/X86/xray-tail-call-sled.ll
+++ b/llvm/test/CodeGen/X86/xray-tail-call-sled.ll
@@ -14,11 +14,17 @@ define i32 @callee() nounwind noinline uwtable "function-instrument"="xray-alway
 ; CHECK-NEXT:  nopw %cs:512(%rax,%rax)
 }
 ; CHECK:       .p2align 4, 0x90
-; CHECK-NEXT:  .quad {{.*}}xray_synthetic_0
+; CHECK-NEXT:  .quad {{.*}}xray_synthetic_0{{.*}}
+; CHECK-NEXT:  .quad {{.*}}xray_fn_idx_synth_0{{.*}}
 ; CHECK-NEXT:  .section {{.*}}xray_instr_map
 ; CHECK-LABEL: Lxray_synthetic_0:
 ; CHECK:       .quad {{.*}}xray_sled_0
 ; CHECK:       .quad {{.*}}xray_sled_1
+; CHECK-LABEL: Lxray_synthetic_end0:
+; CHECK-NEXT:  .section {{.*}}xray_fn_idx
+; CHECK-LABEL: Lxray_fn_idx_synth_0:
+; CHECK:       .quad {{.*}}xray_synthetic_0
+; CHECK-NEXT:  .quad {{.*}}xray_synthetic_end0
 
 define i32 @caller() nounwind noinline uwtable "function-instrument"="xray-always" {
 ; CHECK:       .p2align 1, 0x90
@@ -36,7 +42,13 @@ define i32 @caller() nounwind noinline uwtable "function-instrument"="xray-alway
   ret i32 %retval
 }
 ; CHECK:       .p2align 4, 0x90
-; CHECK-NEXT:  .quad {{.*}}xray_synthetic_1
+; CHECK-NEXT:  .quad {{.*}}xray_synthetic_1{{.*}}
+; CHECK-NEXT:  .quad {{.*}}xray_fn_idx_synth_1{{.*}}
 ; CHECK-LABEL: Lxray_synthetic_1:
 ; CHECK:       .quad {{.*}}xray_sled_2
 ; CHECK:       .quad {{.*}}xray_sled_3
+; CHECK-LABEL: Lxray_synthetic_end1:
+; CHECK:       .section {{.*}}xray_fn_idx
+; CHECK-LABEL: Lxray_fn_idx_synth_1:
+; CHECK:       .quad {{.*}}xray_synthetic_1
+; CHECK:       .quad {{.*}}xray_synthetic_end1