From c53cb2bdc78ee3d3eec8c30821480b390b0eb145 Mon Sep 17 00:00:00 2001 From: Sergey Dmitriev Date: Thu, 30 Jan 2020 08:00:20 -0800 Subject: [PATCH] [Clang][Bundler] Reduce fat object size Summary: Fat object size has significantly increased after D65819 which changed bundler tool to add host object as a normal bundle to the fat output which almost doubled its size. That patch was fixing the following issues 1. Problems associated with the partial linking - global constructors were not called for partially linking objects which clearly resulted in incorrect behavior. 2. Eliminating "junk" target object sections from the linked binary on the host side. The first problem is no longer relevant because we do not use partial linking for creating fat objects anymore. Target objects sections are now inserted into the resulting fat object with a help of llvm-objcopy tool. The second issue, "junk" sections in the linked host binary, has been fixed in D73408 by adding "exclude" flag to the fat object's sections which contain target objects. This flag tells linker to drop section from the inputs when linking executable or shared library, therefore these sections will not be propagated in the linked binary. Since both problems have been solved, we can revert D65819 changes to reduce fat object size and this patch essentially is doing that. Reviewers: ABataev, alexshap, jdoerfert Reviewed By: ABataev Subscribers: cfe-commits Tags: #clang Differential Revision: https://reviews.llvm.org/D73642 --- clang/test/Driver/clang-offload-bundler.c | 6 +- .../clang-offload-bundler/ClangOffloadBundler.cpp | 78 ++++++++++++++++++---- 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/clang/test/Driver/clang-offload-bundler.c b/clang/test/Driver/clang-offload-bundler.c index 99d638c..a0724b3 100644 --- a/clang/test/Driver/clang-offload-bundler.c +++ b/clang/test/Driver/clang-offload-bundler.c @@ -253,16 +253,16 @@ // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.o -### 2>&1 \ // RUN: | FileCheck %s -DHOST=%itanium_abi_triple -DINOBJ1=%t.o -DINOBJ2=%t.tgt1 -DINOBJ3=%t.tgt2 -DOUTOBJ=%t.bundle3.o --check-prefix CK-OBJ-CMD -// CK-OBJ-CMD: llvm-objcopy{{(.exe)?}}" "--add-section=__CLANG_OFFLOAD_BUNDLE__host-[[HOST]]=[[INOBJ1]]" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu=[[INOBJ2]]" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu=[[INOBJ3]]" "[[INOBJ1]]" "[[TEMPOBJ:.*]]" +// CK-OBJ-CMD: llvm-objcopy{{(.exe)?}}" "--add-section=__CLANG_OFFLOAD_BUNDLE__host-[[HOST]]={{.*}}" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu=[[INOBJ2]]" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu=[[INOBJ3]]" "[[INOBJ1]]" "[[TEMPOBJ:.*]]" // CK-OBJ-CMD: llvm-objcopy{{(.exe)?}}" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__host-[[HOST]]=readonly,exclude" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu=readonly,exclude" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu=readonly,exclude" "[[TEMPOBJ]]" "[[OUTOBJ]]" // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.o // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.o,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.o -unbundle -// RUN: diff %t.o %t.res.o +// RUN: diff %t.bundle3.o %t.res.o // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 // RUN: clang-offload-bundler -type=o -targets=openmp-powerpc64le-ibm-linux-gnu,host-%itanium_abi_triple,openmp-x86_64-pc-linux-gnu -outputs=%t.res.tgt1,%t.res.o,%t.res.tgt2 -inputs=%t.bundle3.o -unbundle -// RUN: diff %t.o %t.res.o +// RUN: diff %t.bundle3.o %t.res.o // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 // RUN: clang-offload-bundler -type=o -targets=openmp-powerpc64le-ibm-linux-gnu -outputs=%t.res.tgt1 -inputs=%t.bundle3.o -unbundle diff --git a/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp b/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp index c215cff..2a99a715 100644 --- a/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp +++ b/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -30,7 +30,6 @@ #include "llvm/Support/Error.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/FileSystem.h" -#include "llvm/Support/FileUtilities.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/Program.h" @@ -365,6 +364,41 @@ public: } }; +namespace { + +// This class implements a list of temporary files that are removed upon +// object destruction. +class TempFileHandlerRAII { +public: + ~TempFileHandlerRAII() { + for (const auto &File : Files) + sys::fs::remove(File); + } + + // Creates temporary file with given contents. + Expected Create(Optional> Contents) { + SmallString<128u> File; + if (std::error_code EC = + sys::fs::createTemporaryFile("clang-offload-bundler", "tmp", File)) + return createFileError(File, EC); + Files.push_back(File); + + if (Contents) { + std::error_code EC; + raw_fd_ostream OS(File, EC); + if (EC) + return createFileError(File, EC); + OS.write(Contents->data(), Contents->size()); + } + return Files.back(); + } + +private: + SmallVector, 4u> Files; +}; + +} // end anonymous namespace + /// Handler for object files. The bundles are organized by sections with a /// designated name. /// @@ -433,11 +467,16 @@ public: Error ReadBundleEnd(MemoryBuffer &Input) final { return Error::success(); } Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final { - Expected Content = CurrentSection->getContents(); - if (!Content) - return Content.takeError(); + Expected ContentOrErr = CurrentSection->getContents(); + if (!ContentOrErr) + return ContentOrErr.takeError(); + StringRef Content = *ContentOrErr; + + // Copy fat object contents to the output when extracting host bundle. + if (Content.size() == 1u && Content.front() == 0) + Content = StringRef(Input.getBufferStart(), Input.getBufferSize()); - OS.write(Content->data(), Content->size()); + OS.write(Content.data(), Content.size()); return Error::success(); } @@ -486,22 +525,37 @@ public: // to pass down to llvm-objcopy. OS.close(); + // Temporary files that need to be removed. + TempFileHandlerRAII TempFiles; + // Create an intermediate temporary file to save object after the first // llvm-objcopy run. - SmallString<128u> IntermediateObj; - if (std::error_code EC = sys::fs::createTemporaryFile( - "clang-offload-bundler", "tmp", IntermediateObj)) - return createFileError(IntermediateObj, EC); - FileRemover IntermediateObjRemover(IntermediateObj); + Expected> IntermediateObjOrErr = TempFiles.Create(None); + if (!IntermediateObjOrErr) + return IntermediateObjOrErr.takeError(); + const SmallString<128u> &IntermediateObj = *IntermediateObjOrErr; // Compose llvm-objcopy command line for add target objects' sections. BumpPtrAllocator Alloc; StringSaver SS{Alloc}; SmallVector ObjcopyArgs{"llvm-objcopy"}; - for (unsigned I = 0; I < NumberOfInputs; ++I) + for (unsigned I = 0; I < NumberOfInputs; ++I) { + StringRef InputFile = InputFileNames[I]; + if (I == HostInputIndex) { + // Special handling for the host bundle. We do not need to add a + // standard bundle for the host object since we are going to use fat + // object as a host object. Therefore use dummy contents (one zero byte) + // when creating section for the host bundle. + Expected TempFileOrErr = TempFiles.Create(ArrayRef(0)); + if (!TempFileOrErr) + return TempFileOrErr.takeError(); + InputFile = *TempFileOrErr; + } + ObjcopyArgs.push_back(SS.save(Twine("--add-section=") + OFFLOAD_BUNDLER_MAGIC_STR + TargetNames[I] + - "=" + InputFileNames[I])); + "=" + InputFile)); + } ObjcopyArgs.push_back(InputFileNames[HostInputIndex]); ObjcopyArgs.push_back(IntermediateObj); -- 2.7.4