From af69947e7028274573cfc927aabead8326b63367 Mon Sep 17 00:00:00 2001 From: Noah Shutty Date: Mon, 6 Dec 2021 04:27:53 +0000 Subject: [PATCH] [llvm] [Debuginfo] Debuginfod client library. This adds a Debuginfod library containing the `fetchDebuginfo` function which queries servers specified by the `DEBUGINFOD_URLS` environment variable for the debuginfo, executable, or a specified source file associated with a given build id. This diff was split out from D111252. Reviewed By: dblaikie Differential Revision: https://reviews.llvm.org/D112758 --- llvm/include/llvm/Debuginfod/Debuginfod.h | 71 +++++++++++ llvm/include/llvm/Support/Caching.h | 7 +- llvm/lib/CMakeLists.txt | 1 + llvm/lib/Debuginfod/CMakeLists.txt | 9 ++ llvm/lib/Debuginfod/Debuginfod.cpp | 176 ++++++++++++++++++++++++++ llvm/unittests/CMakeLists.txt | 1 + llvm/unittests/Debuginfod/CMakeLists.txt | 9 ++ llvm/unittests/Debuginfod/DebuginfodTests.cpp | 45 +++++++ 8 files changed, 316 insertions(+), 3 deletions(-) create mode 100644 llvm/include/llvm/Debuginfod/Debuginfod.h create mode 100644 llvm/lib/Debuginfod/CMakeLists.txt create mode 100644 llvm/lib/Debuginfod/Debuginfod.cpp create mode 100644 llvm/unittests/Debuginfod/CMakeLists.txt create mode 100644 llvm/unittests/Debuginfod/DebuginfodTests.cpp diff --git a/llvm/include/llvm/Debuginfod/Debuginfod.h b/llvm/include/llvm/Debuginfod/Debuginfod.h new file mode 100644 index 0000000..fcb8ed3 --- /dev/null +++ b/llvm/include/llvm/Debuginfod/Debuginfod.h @@ -0,0 +1,71 @@ +//===-- llvm/Debuginfod/Debuginfod.h - Debuginfod client --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the declarations of getCachedOrDownloadArtifact and +/// several convenience functions for specific artifact types: +/// getCachedOrDownloadSource, getCachedOrDownloadExecutable, and +/// getCachedOrDownloadDebuginfo. This file also declares +/// getDefaultDebuginfodUrls and getDefaultDebuginfodCacheDirectory. +/// +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFOD_DEBUGINFOD_H +#define LLVM_DEBUGINFOD_DEBUGINFOD_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { + +typedef ArrayRef BuildIDRef; + +typedef SmallVector BuildID; + +/// Finds default array of Debuginfod server URLs by checking DEBUGINFOD_URLS +/// environment variable. +Expected> getDefaultDebuginfodUrls(); + +/// Finds a default local file caching directory for the debuginfod client, +/// first checking DEBUGINFOD_CACHE_PATH. +Expected getDefaultDebuginfodCacheDirectory(); + +/// Finds a default timeout for debuginfod HTTP requests. Checks +/// DEBUGINFOD_TIMEOUT environment variable, default is 90 seconds (90000 ms). +std::chrono::milliseconds getDefaultDebuginfodTimeout(); + +/// Fetches a specified source file by searching the default local cache +/// directory and server URLs. +Expected getCachedOrDownloadSource(BuildIDRef ID, + StringRef SourceFilePath); + +/// Fetches an executable by searching the default local cache directory and +/// server URLs. +Expected getCachedOrDownloadExecutable(BuildIDRef ID); + +/// Fetches a debug binary by searching the default local cache directory and +/// server URLs. +Expected getCachedOrDownloadDebuginfo(BuildIDRef ID); + +/// Fetches any debuginfod artifact using the default local cache directory and +/// server URLs. +Expected getCachedOrDownloadArtifact(StringRef UniqueKey, + StringRef UrlPath); + +/// Fetches any debuginfod artifact using the specified local cache directory, +/// server URLs, and request timeout (in milliseconds). If the artifact is +/// found, uses the UniqueKey for the local cache file. +Expected getCachedOrDownloadArtifact( + StringRef UniqueKey, StringRef UrlPath, StringRef CacheDirectoryPath, + ArrayRef DebuginfodUrls, std::chrono::milliseconds Timeout); + +} // end namespace llvm + +#endif diff --git a/llvm/include/llvm/Support/Caching.h b/llvm/include/llvm/Support/Caching.h index 1e5fea1..fbf5f3b 100644 --- a/llvm/include/llvm/Support/Caching.h +++ b/llvm/include/llvm/Support/Caching.h @@ -63,9 +63,10 @@ using AddBufferFn = /// the cache directory if it does not already exist. The cache name appears in /// error messages for errors during caching. The temporary file prefix is used /// in the temporary file naming scheme used when writing files atomically. -Expected localCache(Twine CacheNameRef, Twine TempFilePrefixRef, - Twine CacheDirectoryPathRef, - AddBufferFn AddBuffer); +Expected localCache( + Twine CacheNameRef, Twine TempFilePrefixRef, Twine CacheDirectoryPathRef, + AddBufferFn AddBuffer = [](size_t Task, std::unique_ptr MB) { + }); } // namespace llvm #endif diff --git a/llvm/lib/CMakeLists.txt b/llvm/lib/CMakeLists.txt index d88bb15..68236c3 100644 --- a/llvm/lib/CMakeLists.txt +++ b/llvm/lib/CMakeLists.txt @@ -25,6 +25,7 @@ add_subdirectory(Object) add_subdirectory(ObjectYAML) add_subdirectory(Option) add_subdirectory(Remarks) +add_subdirectory(Debuginfod) add_subdirectory(DebugInfo) add_subdirectory(DWP) add_subdirectory(ExecutionEngine) diff --git a/llvm/lib/Debuginfod/CMakeLists.txt b/llvm/lib/Debuginfod/CMakeLists.txt new file mode 100644 index 0000000..96dc8c3 --- /dev/null +++ b/llvm/lib/Debuginfod/CMakeLists.txt @@ -0,0 +1,9 @@ +add_llvm_component_library(LLVMDebuginfod + Debuginfod.cpp + + ADDITIONAL_HEADER_DIRS + ${LLVM_MAIN_INCLUDE_DIR}/llvm/Debuginfod + + LINK_COMPONENTS + Support + ) diff --git a/llvm/lib/Debuginfod/Debuginfod.cpp b/llvm/lib/Debuginfod/Debuginfod.cpp new file mode 100644 index 0000000..e71db2e --- /dev/null +++ b/llvm/lib/Debuginfod/Debuginfod.cpp @@ -0,0 +1,176 @@ +//===-- llvm/Debuginfod/Debuginfod.cpp - Debuginfod client library --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// +/// This file defines the fetchInfo function, which retrieves +/// any of the three supported artifact types: (executable, debuginfo, source +/// file) associated with a build-id from debuginfod servers. If a source file +/// is to be fetched, its absolute path must be specified in the Description +/// argument to fetchInfo. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/Debuginfod/Debuginfod.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CachePruning.h" +#include "llvm/Support/Caching.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FileUtilities.h" +#include "llvm/Support/HTTPClient.h" +#include "llvm/Support/xxhash.h" + +namespace llvm { +static std::string uniqueKey(llvm::StringRef S) { return utostr(xxHash64(S)); } + +// Returns a binary BuildID as a normalized hex string. +// Uses lowercase for compatibility with common debuginfod servers. +static std::string buildIDToString(BuildIDRef ID) { + return llvm::toHex(ID, /*LowerCase=*/true); +} + +Expected> getDefaultDebuginfodUrls() { + const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS"); + if (DebuginfodUrlsEnv == NULL) + return SmallVector(); + + SmallVector DebuginfodUrls; + StringRef(DebuginfodUrlsEnv).split(DebuginfodUrls, " "); + return DebuginfodUrls; +} + +Expected getDefaultDebuginfodCacheDirectory() { + if (const char *CacheDirectoryEnv = std::getenv("DEBUGINFOD_CACHE_PATH")) + return CacheDirectoryEnv; + + SmallString<64> CacheDirectory; + if (!sys::path::cache_directory(CacheDirectory)) + return createStringError( + errc::io_error, "Unable to determine appropriate cache directory."); + return std::string(CacheDirectory); +} + +std::chrono::milliseconds getDefaultDebuginfodTimeout() { + long Timeout; + const char *DebuginfodTimeoutEnv = std::getenv("DEBUGINFOD_TIMEOUT"); + if (DebuginfodTimeoutEnv && + to_integer(StringRef(DebuginfodTimeoutEnv).trim(), Timeout, 10)) + return std::chrono::milliseconds(Timeout * 1000); + + return std::chrono::milliseconds(90 * 1000); +} + +/// The following functions fetch a debuginfod artifact to a file in a local +/// cache and return the cached file path. They first search the local cache, +/// followed by the debuginfod servers. + +Expected getCachedOrDownloadSource(BuildIDRef ID, + StringRef SourceFilePath) { + SmallString<64> UrlPath; + sys::path::append(UrlPath, sys::path::Style::posix, "buildid", + buildIDToString(ID), "source", + sys::path::convert_to_slash(SourceFilePath)); + return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath); +} + +Expected getCachedOrDownloadExecutable(BuildIDRef ID) { + SmallString<64> UrlPath; + sys::path::append(UrlPath, sys::path::Style::posix, "buildid", + buildIDToString(ID), "executable"); + return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath); +} + +Expected getCachedOrDownloadDebuginfo(BuildIDRef ID) { + SmallString<64> UrlPath; + sys::path::append(UrlPath, sys::path::Style::posix, "buildid", + buildIDToString(ID), "debuginfo"); + return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath); +} + +// General fetching function. +Expected getCachedOrDownloadArtifact(StringRef UniqueKey, + StringRef UrlPath) { + SmallString<10> CacheDir; + + Expected CacheDirOrErr = getDefaultDebuginfodCacheDirectory(); + if (Error Err = CacheDirOrErr.takeError()) + return Err; + CacheDir = *CacheDirOrErr; + + Expected> DebuginfodUrlsOrErr = + getDefaultDebuginfodUrls(); + if (Error Err = DebuginfodUrlsOrErr.takeError()) + return Err; + SmallVector &DebuginfodUrls = *DebuginfodUrlsOrErr; + return getCachedOrDownloadArtifact(UniqueKey, UrlPath, CacheDir, + DebuginfodUrls, + getDefaultDebuginfodTimeout()); +} + +Expected getCachedOrDownloadArtifact( + StringRef UniqueKey, StringRef UrlPath, StringRef CacheDirectoryPath, + ArrayRef DebuginfodUrls, std::chrono::milliseconds Timeout) { + SmallString<64> AbsCachedArtifactPath; + sys::path::append(AbsCachedArtifactPath, CacheDirectoryPath, + "llvmcache-" + UniqueKey); + + Expected CacheOrErr = + localCache("Debuginfod-client", ".debuginfod-client", CacheDirectoryPath); + if (Error Err = CacheOrErr.takeError()) + return Err; + + FileCache Cache = *CacheOrErr; + // We choose an arbitrary Task parameter as we do not make use of it. + unsigned Task = 0; + Expected CacheAddStreamOrErr = Cache(Task, UniqueKey); + if (Error Err = CacheAddStreamOrErr.takeError()) + return Err; + AddStreamFn &CacheAddStream = *CacheAddStreamOrErr; + if (!CacheAddStream) + return std::string(AbsCachedArtifactPath); + // The artifact was not found in the local cache, query the debuginfod + // servers. + if (!HTTPClient::isAvailable()) + return createStringError(errc::io_error, + "No working HTTP client is available."); + + HTTPClient Client; + Client.setTimeout(Timeout); + for (StringRef ServerUrl : DebuginfodUrls) { + SmallString<64> ArtifactUrl; + sys::path::append(ArtifactUrl, sys::path::Style::posix, ServerUrl, UrlPath); + + Expected ResponseOrErr = Client.get(ArtifactUrl); + if (Error Err = ResponseOrErr.takeError()) + return Err; + + HTTPResponseBuffer &Response = *ResponseOrErr; + if (Response.Code != 200) + continue; + + // We have retrieved the artifact from this server, and now add it to the + // file cache. + Expected> FileStreamOrErr = + CacheAddStream(Task); + if (Error Err = FileStreamOrErr.takeError()) + return Err; + std::unique_ptr &FileStream = *FileStreamOrErr; + if (!Response.Body) + return createStringError( + errc::io_error, "Unallocated MemoryBuffer in HTTPResponseBuffer."); + + *FileStream->OS << StringRef(Response.Body->getBufferStart(), + Response.Body->getBufferSize()); + + // Return the path to the artifact on disk. + return std::string(AbsCachedArtifactPath); + } + + return createStringError(errc::argument_out_of_domain, "build id not found"); +} +} // namespace llvm diff --git a/llvm/unittests/CMakeLists.txt b/llvm/unittests/CMakeLists.txt index 7b3804f..6f4df5b 100644 --- a/llvm/unittests/CMakeLists.txt +++ b/llvm/unittests/CMakeLists.txt @@ -22,6 +22,7 @@ add_subdirectory(Bitcode) add_subdirectory(Bitstream) add_subdirectory(CodeGen) add_subdirectory(DebugInfo) +add_subdirectory(Debuginfod) add_subdirectory(Demangle) add_subdirectory(ExecutionEngine) add_subdirectory(FileCheck) diff --git a/llvm/unittests/Debuginfod/CMakeLists.txt b/llvm/unittests/Debuginfod/CMakeLists.txt new file mode 100644 index 0000000..439a48a --- /dev/null +++ b/llvm/unittests/Debuginfod/CMakeLists.txt @@ -0,0 +1,9 @@ +set(LLVM_LINK_COMPONENTS + Debuginfod + ) + +add_llvm_unittest(DebuginfodTests + DebuginfodTests.cpp + ) + +target_link_libraries(DebuginfodTests PRIVATE LLVMTestingSupport) diff --git a/llvm/unittests/Debuginfod/DebuginfodTests.cpp b/llvm/unittests/Debuginfod/DebuginfodTests.cpp new file mode 100644 index 0000000..4187122 --- /dev/null +++ b/llvm/unittests/Debuginfod/DebuginfodTests.cpp @@ -0,0 +1,45 @@ +//===-- llvm/unittest/Support/DebuginfodTests.cpp - unit tests --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Debuginfod/Debuginfod.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/HTTPClient.h" +#include "llvm/Support/Path.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; + +// Check that the Debuginfod client can find locally cached artifacts. +TEST(DebuginfodClient, CacheHit) { + int FD; + SmallString<64> CachedFilePath; + sys::fs::createTemporaryFile("llvmcache-key", "temp", FD, CachedFilePath); + StringRef CacheDir = sys::path::parent_path(CachedFilePath); + StringRef UniqueKey = sys::path::filename(CachedFilePath); + EXPECT_TRUE(UniqueKey.consume_front("llvmcache-")); + raw_fd_ostream OF(FD, true, /*unbuffered=*/true); + OF << "contents\n"; + OF << CacheDir << "\n"; + OF.close(); + Expected PathOrErr = getCachedOrDownloadArtifact( + UniqueKey, /*UrlPath=*/"/null", CacheDir, + /*DebuginfodUrls=*/{}, /*Timeout=*/std::chrono::milliseconds(1)); + EXPECT_THAT_EXPECTED(PathOrErr, HasValue(CachedFilePath)); +} + +// Check that the Debuginfod client returns an Error when it fails to find an +// artifact. +TEST(DebuginfodClient, CacheMiss) { + // Ensure there are no urls to guarantee a cache miss. + setenv("DEBUGINFOD_URLS", "", /*replace=*/1); + HTTPClient::initialize(); + Expected PathOrErr = getCachedOrDownloadArtifact( + /*UniqueKey=*/"nonexistent-key", /*UrlPath=*/"/null"); + EXPECT_THAT_EXPECTED(PathOrErr, Failed()); +} -- 2.7.4