From: TizenOpenSource Date: Tue, 27 Dec 2022 02:45:37 +0000 (+0900) Subject: Imported Upstream version 4.7.4 X-Git-Tag: upstream/4.7.4^0 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=63c4f286735e1cae23370e928b51e9a2e582f9a8;p=platform%2Fupstream%2Fccache.git Imported Upstream version 4.7.4 --- diff --git a/.clang-format b/.clang-format index c5ffe56..67f0665 100644 --- a/.clang-format +++ b/.clang-format @@ -26,12 +26,18 @@ IncludeBlocks: Regroup IncludeCategories: - Regex: '^"system.hpp"$' Priority: 1 - - Regex: '^"third_party/' - Priority: 3 + - Regex: '^["<]third_party/' + Priority: 4 + # System headers: + - Regex: '\.h>$' + Priority: 5 + # C++ headers: + - Regex: '^<[^.]+>$' + Priority: 6 - Regex: '^"' Priority: 2 - Regex: '.*' - Priority: 4 + Priority: 3 IndentPPDirectives: AfterHash KeepEmptyLinesAtTheStartOfBlocks: false PointerAlignment: Left diff --git a/.github/no-response.yml b/.github/no-response.yml deleted file mode 100644 index 4a5ab95..0000000 --- a/.github/no-response.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Configuration for probot-no-response - https://github.com/probot/no-response - -# Number of days of inactivity before an issue is closed for lack of response. -daysUntilClose: 30 -# Label requiring a response. -responseRequiredLabel: awaiting feedback -# Comment to post when closing an issue for lack of response. Set to `false` to -# disable. -closeComment: > - This issue has been automatically closed due to no response to our request for - more information from the original author. Please feel free to reopen the - issue or leave a comment if you have more information. diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 07f2904..daea64b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -11,6 +11,9 @@ defaults: run: shell: bash +permissions: + contents: read + jobs: build_and_test: env: @@ -22,27 +25,11 @@ jobs: fail-fast: false matrix: config: - - os: ubuntu-16.04 - compiler: gcc - version: "4.8" # results in 4.8.5 - - - os: ubuntu-16.04 - compiler: gcc - version: "5" - - - os: ubuntu-18.04 - compiler: gcc - version: "6" - - - os: ubuntu-18.04 - compiler: gcc - version: "7" - - - os: ubuntu-18.04 + - os: ubuntu-20.04 compiler: gcc version: "8" - - os: ubuntu-18.04 + - os: ubuntu-20.04 compiler: gcc version: "9" @@ -50,52 +37,49 @@ jobs: compiler: gcc version: "10" - - os: ubuntu-16.04 - compiler: clang - version: "5.0" - - - os: ubuntu-16.04 - compiler: clang - version: "6.0" - - - os: ubuntu-18.04 + - os: ubuntu-20.04 compiler: clang - version: "7" + version: "9" - - os: ubuntu-18.04 + - os: ubuntu-20.04 compiler: clang - version: "8" + version: "10" - os: ubuntu-20.04 compiler: clang - version: "9" + version: "11" - os: ubuntu-20.04 compiler: clang - version: "10" + version: "12" - - os: macOS-latest - compiler: xcode - version: "10.3" + - os: ubuntu-22.04 + compiler: gcc + version: "11" + + - os: ubuntu-22.04 + compiler: gcc + version: "12" - - os: macOS-latest + - os: macOS-11 compiler: xcode version: "11.7" - - os: macOS-latest + - os: macOS-11 compiler: xcode - version: "12.2" + version: "12.4" steps: - name: Install dependencies run: | if [ "${{ runner.os }}" = "Linux" ]; then sudo apt-get update - # Install ld.gold (binutils) and ld.lld on different runs. - if [ "${{ matrix.config.os }}" = "ubuntu-16.04" ]; then - sudo apt-get install -y ninja-build elfutils libzstd1-dev binutils + packages="elfutils libhiredis-dev libzstd-dev ninja-build pkg-config python3 redis-server redis-tools" + # Install ld.gold (binutils) and ld.lld (lld) on different runs. + if [ "${{ matrix.config.os }}" = "ubuntu-22.04" ]; then + sudo apt-get install -y $packages binutils else - sudo apt-get install -y ninja-build elfutils libzstd-dev lld + sudo apt-get install -y $packages lld fi if [ "${{ matrix.config.compiler }}" = "gcc" ]; then @@ -103,15 +87,22 @@ jobs: echo "CXX=g++-${{ matrix.config.version }}" >> $GITHUB_ENV sudo apt install -y g++-${{ matrix.config.version }} g++-${{ matrix.config.version }}-multilib + if [ "${{ matrix.config.version }}" = 8 ]; then + # The compilation test in StdFilesystem.cmake doesn't work when + # GCC 9 is installed as well, so need to force linking with + # libstdc++fs for GCC 8. Note: This requires using -fuse-ld=lld to + # work. + echo "LDFLAGS=-lstdc++fs" >> $GITHUB_ENV + fi else echo "CC=clang-${{ matrix.config.version }}" >> $GITHUB_ENV echo "CXX=clang++-${{ matrix.config.version }}" >> $GITHUB_ENV - sudo apt install -y clang-${{ matrix.config.version }} g++-multilib + sudo apt install -y clang-${{ matrix.config.version }} g++-multilib lld-${{ matrix.config.version }} fi elif [ "${{ runner.os }}" = "macOS" ]; then HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 \ - brew install ninja + brew install ninja pkg-config hiredis redis if [ "${{ matrix.config.compiler }}" = "gcc" ]; then brew install gcc@${{ matrix.config.version }} @@ -144,6 +135,75 @@ jobs: name: ${{ matrix.config.os }}-${{ matrix.config.compiler }}-${{ matrix.config.version }}-testdir.tar.xz path: testdir.tar.xz + build_and_test_msys: + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + config: + - sys: mingw64 + env: x86_64 + compiler: gcc + + - sys: mingw64 + env: x86_64 + compiler: clang + + name: 'Windows ${{ matrix.config.sys }} ${{ matrix.config.compiler }}' + defaults: + run: + shell: msys2 {0} + steps: + - name: '${{ matrix.config.sys }} Setup MSYS2' + uses: msys2/setup-msys2@v2 + with: + msystem: ${{matrix.config.sys}} + update: true + install: >- + git + diffutils + tar + mingw-w64-${{matrix.config.env}}-toolchain + mingw-w64-${{matrix.config.env}}-cmake + mingw-w64-${{matrix.config.env}}-ninja + mingw-w64-${{matrix.config.env}}-hiredis + mingw-w64-${{matrix.config.env}}-lld + mingw-w64-${{matrix.config.env}}-${{matrix.config.compiler}} + + - name: setup env + run: | + if [ "${{ matrix.config.compiler }}" = "gcc" ]; then + echo "CC=gcc" >> $GITHUB_ENV + echo "CXX=g++" >> $GITHUB_ENV + else + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV + fi + + - name: Get source + uses: actions/checkout@v2 + + - name: Build and test + run: ci/build + continue-on-error: ${{ matrix.config.allow_test_failures == true && + steps.build-and-test.outputs.exit_status == 8 }} + env: + ENABLE_CACHE_CLEANUP_TESTS: 1 + CMAKE_GENERATOR: Ninja + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI + TEST_CC: gcc + + - name: Collect testdir from failed tests + if: failure() + run: ci/collect-testdir + + - name: Upload testdir from failed tests + if: failure() + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.config.sys}}-${{ matrix.config.env }}-${{ matrix.config.compiler }}-testdir.tar.xz + path: testdir.tar.xz + specific_tests: name: ${{ matrix.config.name }} runs-on: ${{ matrix.config.os }} @@ -151,41 +211,41 @@ jobs: fail-fast: false matrix: config: - - name: Linux GCC debug + C++14 + in source + tracing - os: ubuntu-18.04 + - name: Linux GCC debug + in source + tracing + os: ubuntu-20.04 CC: gcc CXX: g++ ENABLE_CACHE_CLEANUP_TESTS: 1 BUILDDIR: . CCACHE_LOC: . - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=Debug -DENABLE_TRACING=1 -DCMAKE_CXX_STANDARD=14 - apt_get: elfutils libzstd-dev + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=Debug -DENABLE_TRACING=1 + apt_get: elfutils libzstd-dev pkg-config libhiredis-dev - name: Linux GCC 32-bit - os: ubuntu-18.04 + os: ubuntu-20.04 CC: gcc CXX: g++ CFLAGS: -m32 -g -O2 CXXFLAGS: -m32 -g -O2 LDFLAGS: -m32 - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON ENABLE_CACHE_CLEANUP_TESTS: 1 - apt_get: elfutils gcc-multilib g++-multilib lib32stdc++-5-dev + apt_get: elfutils gcc-multilib g++-multilib lib32stdc++-10-dev - name: Linux GCC CUDA - os: ubuntu-18.04 + os: ubuntu-20.04 CC: gcc CXX: g++ - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON ENABLE_CACHE_CLEANUP_TESTS: 1 - CUDA: 10.1.243-1 + CUDA: 11.7.0-1 apt_get: elfutils libzstd-dev - name: Linux MinGW 32-bit - os: ubuntu-18.04 + os: ubuntu-20.04 CC: i686-w64-mingw32-gcc-posix CXX: i686-w64-mingw32-g++-posix - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON RUN_TESTS: none apt_get: elfutils mingw-w64 @@ -194,30 +254,94 @@ jobs: CC: x86_64-w64-mingw32-gcc-posix CXX: x86_64-w64-mingw32-g++-posix ENABLE_CACHE_CLEANUP_TESTS: 1 - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DCMAKE_SYSTEM_NAME=Windows -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON RUN_TESTS: unittest-in-wine apt_get: elfutils mingw-w64 wine - - name: Windows VS2019 32-bit + - name: Windows VS2019 32-bit Ninja os: windows-2019 msvc_arch: x64_x86 - allow_test_failures: true # For now, don't fail the build on failure CC: cl CXX: cl ENABLE_CACHE_CLEANUP_TESTS: 1 CMAKE_GENERATOR: Ninja - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON + TEST_CC: clang -target i686-pc-windows-msvc + + - name: Windows VS2019 32-bit MSBuild + os: windows-2019 + msvc_arch: x64_x86 + allow_test_failures: true # For now, don't fail the build on failure + CC: cl + CXX: cl + ENABLE_CACHE_CLEANUP_TESTS: 1 + CMAKE_GENERATOR: Visual Studio 16 2019 + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON -A Win32 TEST_CC: clang -target i686-pc-windows-msvc - - name: Windows VS2019 64-bit + - name: Windows VS2019 64-bit Ninja + os: windows-2019 + msvc_arch: x64 + CC: cl + CXX: cl + ENABLE_CACHE_CLEANUP_TESTS: 1 + CMAKE_GENERATOR: Ninja + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON + TEST_CC: clang -target x86_64-pc-windows-msvc + + - name: Windows VS2019 64-bit MSBuild os: windows-2019 msvc_arch: x64 allow_test_failures: true # For now, don't fail the build on failure CC: cl CXX: cl ENABLE_CACHE_CLEANUP_TESTS: 1 + CMAKE_GENERATOR: Visual Studio 16 2019 + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON -A x64 + TEST_CC: clang -target x86_64-pc-windows-msvc + + - name: Windows VS2022 32-bit Ninja + os: windows-2022 + msvc_arch: x64_x86 + allow_test_failures: true # For now, don't fail the build on failure + CC: cl + CXX: cl + ENABLE_CACHE_CLEANUP_TESTS: 1 + CMAKE_GENERATOR: Ninja + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON + TEST_CC: clang -target i686-pc-windows-msvc + + - name: Windows VS2022 32-bit MSBuild + os: windows-2022 + msvc_arch: x64_x86 + allow_test_failures: true # For now, don't fail the build on failure + CC: cl + CXX: cl + ENABLE_CACHE_CLEANUP_TESTS: 1 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON -A Win32 + TEST_CC: clang -target i686-pc-windows-msvc + + - name: Windows VS2022 64-bit Ninja + os: windows-2022 + msvc_arch: x64 + allow_test_failures: true # For now, don't fail the build on failure + CC: cl + CXX: cl + ENABLE_CACHE_CLEANUP_TESTS: 1 CMAKE_GENERATOR: Ninja - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON + TEST_CC: clang -target x86_64-pc-windows-msvc + + - name: Windows VS2022 64-bit MSBuild + os: windows-2022 + msvc_arch: x64 + allow_test_failures: true # For now, don't fail the build on failure + CC: cl + CXX: cl + ENABLE_CACHE_CLEANUP_TESTS: 1 + CMAKE_GENERATOR: Visual Studio 17 2022 + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON -A x64 TEST_CC: clang -target x86_64-pc-windows-msvc - name: Clang address & UB sanitizer @@ -227,7 +351,7 @@ jobs: ENABLE_CACHE_CLEANUP_TESTS: 1 CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DENABLE_SANITIZER_ADDRESS=ON -DENABLE_SANITIZER_UNDEFINED_BEHAVIOR=ON ASAN_OPTIONS: detect_leaks=0 - apt_get: elfutils libzstd-dev + apt_get: elfutils libzstd-dev pkg-config libhiredis-dev - name: Clang static analyzer os: ubuntu-20.04 @@ -236,7 +360,7 @@ jobs: ENABLE_CACHE_CLEANUP_TESTS: 1 CMAKE_PREFIX: scan-build RUN_TESTS: none - apt_get: libzstd-dev + apt_get: libzstd-dev pkg-config libhiredis-dev - name: Linux binary os: ubuntu-20.04 @@ -244,34 +368,34 @@ jobs: CXX: g++ SPECIAL: build-and-verify-package CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=Release - apt_get: elfutils libzstd-dev ninja-build + apt_get: elfutils libzstd-dev pkg-config libhiredis-dev ninja-build - name: Source package os: ubuntu-20.04 CC: gcc CXX: g++ SPECIAL: build-and-verify-source-package - apt_get: elfutils libzstd-dev ninja-build asciidoc xsltproc docbook-xml docbook-xsl + apt_get: elfutils libzstd-dev pkg-config libhiredis-dev ninja-build asciidoctor - name: HTML documentation - os: ubuntu-18.04 + os: ubuntu-20.04 EXTRA_CMAKE_BUILD_FLAGS: --target doc-html RUN_TESTS: none - apt_get: libzstd-dev asciidoc docbook-xml docbook-xsl + apt_get: libzstd-dev pkg-config libhiredis-dev asciidoctor - name: Manual page - os: ubuntu-18.04 + os: ubuntu-20.04 EXTRA_CMAKE_BUILD_FLAGS: --target doc-man-page RUN_TESTS: none - apt_get: libzstd-dev asciidoc xsltproc docbook-xml docbook-xsl + apt_get: libzstd-dev pkg-config libhiredis-dev asciidoctor - name: Clang-Tidy - os: ubuntu-18.04 - CC: clang-9 - CXX: clang++-9 + os: ubuntu-20.04 + CC: clang-12 + CXX: clang++-12 RUN_TESTS: none - CMAKE_PARAMS: -DENABLE_CLANG_TIDY=ON -DCLANGTIDY=/usr/bin/clang-tidy-9 - apt_get: libzstd-dev clang-9 clang-tidy-9 + CMAKE_PARAMS: -DENABLE_CLANG_TIDY=ON -DCLANGTIDY=/usr/bin/clang-tidy-12 + apt_get: libzstd-dev pkg-config libhiredis-dev clang-12 clang-tidy-12 steps: - name: Get source @@ -331,7 +455,7 @@ jobs: run: | rc=0 ci/build || rc=$? - echo "::set-output name=exit_status::$rc" + echo "exit_status=$rc" >> $GITHUB_OUTPUT exit $rc # CTest exits with return code 8 on test failure. continue-on-error: ${{ matrix.config.allow_test_failures == true && @@ -371,7 +495,10 @@ jobs: uses: actions/checkout@v2 - name: Install codespell - run: sudo apt-get update && sudo apt-get install codespell + run: | + sudo apt-get update + sudo apt-get install python3-pip + pip3 install codespell==2.1.0 - name: Run codespell - run: codespell -q 7 -S ".git,LICENSE.adoc,./src/third_party/*" -I misc/codespell-allowlist.txt + run: codespell -q 7 -S ".git,build*,./src/third_party/*" -I misc/codespell-allowlist.txt diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml index b19e807..5bd772b 100644 --- a/.github/workflows/codeql-analysis.yaml +++ b/.github/workflows/codeql-analysis.yaml @@ -17,10 +17,17 @@ on: # Full scan once a week - cron: '0 14 * * 3' +permissions: + contents: read + jobs: analyze: + permissions: + actions: read # for github/codeql-action/init to get workflow details + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/analyze to upload SARIF results name: Analyze - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Checkout repository @@ -31,10 +38,10 @@ jobs: fetch-depth: 2 - name: Install dependencies - run: sudo apt-get update && sudo apt-get install ninja-build elfutils libzstd-dev + run: sudo apt-get update && sudo apt-get install ninja-build elfutils libzstd-dev pkg-config libhiredis-dev - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: cpp queries: +security-and-quality @@ -44,6 +51,7 @@ jobs: env: RUN_TESTS: none CMAKE_GENERATOR: Ninja + EXTRA_CMAKE_BUILD_FLAGS: --target ccache - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 diff --git a/.gitignore b/.gitignore index 74db15f..d8e547f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ # Typical build directories /build*/ +# CLion +/.idea/ +/cmake-build-*/ + # Downloaded tools misc/.clang-format-exe diff --git a/.mailmap b/.mailmap index 7e59d6e..9006713 100644 --- a/.mailmap +++ b/.mailmap @@ -10,14 +10,18 @@ Clemens Rabe Clemens Rabe Doug Anderson Erik Flodin +Florin Trofin Hongli Lai +Jacob Young Jonny Yu Ka Ho Ng Kona Blend Leanid Chaika +Louis Caron Luboš Luňák Martin Ettl Mizuha Himuraki +Oleg Sidorkin Paul Bunch Pawel Krysiak Per Nordlöw @@ -25,8 +29,10 @@ Peter Budai Ramiro Polla Ramiro Polla Ryan Brown +Ryan Burns <52847440+r-burns@users.noreply.github.com> Thomas Otto <39962140+totph@users.noreply.github.com> Thomas Röfer Tor Arne Vestbø +Vili Väinölä Ville Skyttä Xavier René-Corail diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..26ecdd8 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,34 @@ +# Ccache architecture + +## Code structure + +### Top-level directories + +* `ci`: Utility scripts used in CI. +* `cmake`: CMake scripts. +* `doc`: Documentation. +* `dockerfiles`: Dockerfiles that specify different environments of interest for + ccache. +* `misc`: Miscellaneous utility scripts, example files, etc. +* `src`: Source code. See below. +* `test`: Integration test suite which tests the ccache binary in different + scenarios. +* `unittest`: Unit test suite which typically tests individual functions. + +### Subdirectories of `src` + +This section describes the directory structure that the project aims to +transform the `src` directory into in the long run to make the code base easier +to understand and work with. In other words, this is work in progress. + +* `compiler`: Knowledge about things like compiler options, compiler behavior, + preprocessor output format, etc. Ideally this code should in the future be + refactored into compiler-specific frontends, such as GCC, Clang, NVCC, MSVC, + etc. +* `core`: Everything not part of other directories. +* `storage`: Storage backends. +* `storage/local`: Code for the local storage backend. +* `storage/remote`: Code for remote storage backends. +* `third_party`: Bundled third party code. +* `util`: Generic utility functionality that does not depend on ccache-specific + things. diff --git a/CMakeLists.txt b/CMakeLists.txt index e186e81..6407690 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ -cmake_minimum_required(VERSION 3.4.3) +cmake_minimum_required(VERSION 3.15) -project(ccache LANGUAGES C CXX) +project(ccache LANGUAGES C CXX ASM ASM_MASM) if(MSVC) enable_language(ASM_MASM) else() @@ -8,9 +8,7 @@ else() endif() set(CMAKE_PROJECT_DESCRIPTION "a fast C/C++ compiler cache") -if(NOT "${CMAKE_CXX_STANDARD}") - set(CMAKE_CXX_STANDARD 11) -endif() +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED YES) set(CMAKE_CXX_EXTENSIONS NO) @@ -28,10 +26,8 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") # C++ error messages) # -# Clang 3.4 and AppleClang 6.0 fail to compile doctest. -# GCC 4.8.4 is known to work OK but give warnings. -if((CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.5) - OR (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8.4) +if((CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.8) + OR (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5) OR (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0)) message( FATAL_ERROR @@ -39,10 +35,8 @@ if((CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSIO "You need one listed here: https://ccache.dev/platform-compiler-language-support.html") endif() -# All Clang problems / special handling ccache has are because of version 3.5. -# All GCC problems / special handling ccache has are because of version 4.8.4. -if((CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.6) - OR (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)) +if((CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4) + OR (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6)) message( WARNING "The compiler you are using is rather old.\n" @@ -58,53 +52,78 @@ endif() # include(CcacheVersion) -if("${CCACHE_VERSION_ORIGIN}" STREQUAL git OR DEFINED ENV{CI}) - set(CCACHE_DEV_MODE ON) -else() - set(CCACHE_DEV_MODE OFF) +if(NOT DEFINED CCACHE_DEV_MODE) + if("${CCACHE_VERSION_ORIGIN}" STREQUAL git OR DEFINED ENV{CI}) + set(CCACHE_DEV_MODE ON) + else() + set(CCACHE_DEV_MODE OFF) + endif() endif() message(STATUS "Ccache dev mode: ${CCACHE_DEV_MODE}") -include(UseCcache) -if(NOT MSVC) - include(UseFastestLinker) +option(ENABLE_IPO "Enable interprocedural (link time, LTO) optimization" OFF) +if(ENABLE_IPO AND NOT MINGW) + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON) endif() + +include(UseFastestLinker) include(StandardSettings) include(StandardWarnings) include(CIBuildType) include(DefaultBuildType) -if(NOT ${CMAKE_VERSION} VERSION_LESS "3.9") - cmake_policy(SET CMP0069 NEW) - option(ENABLE_IPO "Enable interprocedural (link time, LTO) optimization" OFF) - if(ENABLE_IPO) - set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON) - endif() -endif() - # # Configuration # -include(GNUInstallDirs) + +include(InstallDirs) include(GenerateConfigurationFile) include(GenerateVersionFile) -if(HAVE_SYS_MMAN_H AND HAVE_PTHREAD_MUTEXATTR_SETPSHARED) - set(INODE_CACHE_SUPPORTED 1) +# +# Static link configuration +# + +set(STATIC_LINK_DEFAULT OFF) +if(WIN32) + set(STATIC_LINK_DEFAULT ON) +endif() + +option(STATIC_LINK "Prefer linking libraries statically" ${STATIC_LINK_DEFAULT}) + +if(STATIC_LINK) + list(INSERT CMAKE_FIND_LIBRARY_SUFFIXES 0 "${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(CMAKE_LINK_SEARCH_START_STATIC ON) + + # Link MSVC runtime statically. + if(MSVC) + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") + # Link MINGW runtime statically. + elseif(WIN32) + if((CMAKE_CXX_COMPILER_ID STREQUAL GNU) OR (CMAKE_CXX_COMPILER_ID STREQUAL Clang)) + list(APPEND CCACHE_EXTRA_LIBS -static-libgcc -static-libstdc++ -static -lwinpthread -dynamic) + endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL Clang) + list(APPEND CCACHE_EXTRA_LIBS -fuse-ld=lld) + endif() + endif() endif() # # Third party # -set(ZSTD_FROM_INTERNET_DEFAULT OFF) +option(ZSTD_FROM_INTERNET + "Download and use libzstd from the Internet if not available" ON) +option(HIREDIS_FROM_INTERNET + "Download and use libhiredis from the Internet if not available" ON) -# Default to downloading deps for Visual Studio, unless using a package manager. -if(MSVC AND NOT CMAKE_TOOLCHAIN_FILE MATCHES "vcpkg|conan") - set(ZSTD_FROM_INTERNET_DEFAULT ON) -endif() +find_package(zstd 1.1.2 MODULE REQUIRED) -option(ZSTD_FROM_INTERNET "Download and use libzstd from the Internet" ${ZSTD_FROM_INTERNET_DEFAULT}) -find_package(zstd 1.1.2 REQUIRED) +option(REDIS_STORAGE_BACKEND "Enable Redis remote storage" ON) + +if(REDIS_STORAGE_BACKEND) + find_package(hiredis 0.13.3 MODULE REQUIRED) +endif() # # Special flags @@ -115,10 +134,6 @@ find_package(zstd 1.1.2 REQUIRED) include(CodeAnalysis) option(ENABLE_TRACING "Enable possibility to use internal ccache tracing" OFF) -if(WIN32 AND CMAKE_CXX_COMPILER_ID MATCHES "GNU") - option(STATIC_LINK "Link statically with system libraries" ON) -endif() - # # Source code # @@ -128,7 +143,7 @@ add_subdirectory(src) # ccache executable # add_executable(ccache src/main.cpp) -target_link_libraries(ccache PRIVATE standard_settings standard_warnings ccache_lib) +target_link_libraries(ccache PRIVATE standard_settings standard_warnings ccache_framework) # # Documentation diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ad4b8aa..a285535 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,6 +33,7 @@ proposal(s) on [GitHub](https://github.com/ccache/ccache). Here are some hints to make the process smoother: +* Have a look in `ARCHITECTURE.md` for an overview of the source code tree. * If you plan to implement major changes it is wise to open an issue on GitHub (or ask in the Gitter room, or send a mail to the mailing list) asking for comments on your plans before doing the bulk of the work. That way you can @@ -42,17 +43,42 @@ Here are some hints to make the process smoother: for merging yet but you want early comments and CI test results? Then create a draft pull request as described in [this Github blog post](https://github.blog/2019-02-14-introducing-draft-pull-requests/). +* Please add test cases for your new changes if applicable. * Please follow the ccache's code style (see the section below). -* Consider [A Note About Git Commit - Messages](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html) - when writing commit messages. -## Code style +## Commit message conventions + +It is preferable, but not mandatory, to format commit messages in the spirit of +[Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). The +commit message subject should look like this: + + : + (): + +`` is a succinct description of the change: -Ccache was written in C99 until 2019 when it started being converted to C++11. -The conversion is a slow work in progress, which is why there is some C-style -code left. Please refrain from doing large C to C++ conversions; do it little by -little. +* Use the imperative, present tense: "Change", not "Changed" nor "Changes". +* Capitalize the first letter. +* No dot (`.`) at the end. + +Here is a summary of types used for ccache: + +| Type | Explanation | +| ------------ | ----------- | +| **build** | A change of the build system or build configuration. | +| **bump** | An increase of the version of an external dependency or an update of a bundled third party package. | +| **chore** | A change that doesn't match any other type. | +| **ci** | A change of CI scripts or configuration. | +| **docs** | A change of documentation only. | +| **enhance** | An enhancement of the code without adding a user-visible feature, for example adding a new utility class to be used by a future feature or refactoring. | +| **feat** | An addition or improvement of a user-visible feature. | +| **fix** | A bug fix (not necessarily user-visible). | +| **perf** | A performance improvement. | +| **refactor** | A restructuring of the existing code without changing its external behavior. | +| **style** | A change of code style. | +| **test** | An addition or modification of tests or test framework. | + +## Code style Source code formatting is defined by `.clang-format` in the root directory. The format is loosely based on [LLVM's code formatting @@ -65,10 +91,11 @@ fine. Please follow these conventions: -* Use `UpperCamelCase` for types (e.g. classes and structs) and namespaces. -* Use `UPPER_CASE` names for macros and (non-class )enum values. -* Use `snake_case` for other names (functions, variables, enum class values, - etc.). +* Use `UpperCamelCase` for types (e.g. classes and structs). +* Use `UPPER_CASE` names for macros and (non-class) enum values. +* Use `snake_case` for other names (namespaces, functions, variables, enum class + values, etc.). (Namespaces used to be in `UpperCamelCase`; transition is work + in progress.) * Use an `m_` prefix for non-public member variables. * Use a `g_` prefix for global mutable variables. * Use a `k_` prefix for global constants. diff --git a/LICENSE.adoc b/LICENSE.adoc index 4c6df86..b863cec 100644 --- a/LICENSE.adoc +++ b/LICENSE.adoc @@ -1,12 +1,10 @@ -Ccache copyright and license -============================ += Ccache copyright and license -Overall license ---------------- +== Overall license The license for ccache as a whole is as follows: -------------------------------------------------------------------------------- +---- This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later @@ -19,14 +17,13 @@ PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -------------------------------------------------------------------------------- +---- The full license text can be found in GPL-3.0.txt and at https://www.gnu.org/licenses/gpl-3.0.html. -Copyright and authors ---------------------- +== Copyright and authors Ccache is a collective work with contributions from many people, listed in AUTHORS.adoc and at https://ccache.dev/credits.html. Subsequent additions by @@ -36,14 +33,13 @@ their portions of the work. The copyright for ccache as a whole is as follows: -------------------------------------------------------------------------------- +---- Copyright (C) 2002-2007 Andrew Tridgell -Copyright (C) 2009-2021 Joel Rosdahl and other contributors -------------------------------------------------------------------------------- +Copyright (C) 2009-2022 Joel Rosdahl and other contributors +---- -Files derived from other sources --------------------------------- +== Files derived from other sources The ccache distribution contain some files from other sources and some have been modified for use in ccache. These files all carry attribution notices, and @@ -52,13 +48,12 @@ the GPL: that is, if separated from the ccache sources, they may be usable under less restrictive terms. -src/third_party/base32hex.* -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/base32hex.* This base32hex implementation comes from . -------------------------------------------------------------------------------- +---- (C) 2012 Peter Conrad This program is free software: you can redistribute it and/or modify @@ -72,20 +67,18 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . -------------------------------------------------------------------------------- +---- -src/third_party/blake3/blake3_* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/blake3/blake3_* -This is a subset of https://github.com/BLAKE3-team/BLAKE3[BLAKE3] 0.3.7 with +This is a subset of https://github.com/BLAKE3-team/BLAKE3[BLAKE3] 1.3.1 with the following license: -------------------------------------------------------------------------------- +---- This work is released into the public domain with CC0 1.0. Alternatively, it is licensed under the Apache License 2.0. -------------------------------------------------------------------------------- ------------------------------------------------------------------------------- Creative Commons Legal Code @@ -210,7 +203,6 @@ express Statement of Purpose. party to this document and has no duty or obligation with respect to this CC0 or use of the Work. -------------------------------------------------------------------------------- ------------------------------------------------------------------------------- Apache License @@ -414,16 +406,15 @@ express Statement of Purpose. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -------------------------------------------------------------------------------- +---- -src/third_party/doctest.h -~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/doctest.h This is the single header version of https://github.com/onqtam/doctest[doctest] -2.4.6 with the following license: +2.4.8 with the following license: -------------------------------------------------------------------------------- +---- The MIT License (MIT) Copyright (c) 2016-2021 Viktor Kirilov @@ -445,15 +436,14 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- +---- -src/third_party/fmt/*.h and src/third_party/format.cpp -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/fmt/*.h and src/third_party/format.cpp -This is a subset of https://fmt.dev[fmt] 7.1.3 with the following license: +This is a subset of https://fmt.dev[fmt] 8.1.1 with the following license: -------------------------------------------------------------------------------- +---- Formatting library for C++ Copyright (c) 2012 - present, Victor Zverovich @@ -482,16 +472,15 @@ As an exception, if, as a result of your compiling your source code, portions of this Software are embedded into a machine-executable object form of such source code, you may redistribute such embedded portions in such object form without including the above copyright and permission notices. -------------------------------------------------------------------------------- +---- -src/third_party/getopt_long.* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/getopt_long.* This implementation of `getopt_long()` was copied from https://www.postgresql.org[PostgreSQL] and has the following license text: -------------------------------------------------------------------------------- +---- Portions Copyright (c) 1987, 1993, 1994 The Regents of the University of California. All rights reserved. @@ -521,16 +510,46 @@ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- +---- + + +=== src/third_party/httplib.* + +cpp-httplib - A C++11 cross-platform HTTP/HTTPS library. Copied from cpp-httplib +v0.11.1 downloaded from https://github.com/yhirose/cpp-httplib[cpp-httplib]. The +library has the following license: + +---- +The MIT License (MIT) + +Copyright (c) 2022 yhirose + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +---- -src/third_party/minitrace.* -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/minitrace.* A library for producing JSON traces suitable for Chrome's built-in trace viewer (chrome://tracing). Downloaded from . -------------------------------------------------------------------------------- +---- The MIT License (MIT) Copyright (c) 2014 Henrik Rydgård @@ -552,18 +571,17 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- +---- -src/third_party/nonstd/optional.hpp -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/nonstd/expected.hpp This is the single header version of -https://github.com/martinmoene/optional-lite[optional-lite] 3.4.0 with the +https://github.com/martinmoene/expected-lite[expected-lite] 0.6.2 with the following license: -------------------------------------------------------------------------------- -Copyright (c) 2014-2018 Martin Moene +---- +Copyright (c) 2016-2020 Martin Moene Boost Software License - Version 1.0 - August 17th, 2003 @@ -588,18 +606,17 @@ SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- +---- -src/third_party/nonstd/string_view.hpp -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/nonstd/span.hpp -This alternative implementation of `std::string_view` was downloaded from - and has the following license -text: +This is the single header version of +https://github.com/martinmoene/span-lite[expected-lite] 0.10.3 with the +following license: -------------------------------------------------------------------------------- -Copyright 2017-2020 by Martin Moene +---- +Copyright (c) 2018-2021 Martin Moene Boost Software License - Version 1.0 - August 17th, 2003 @@ -624,11 +641,40 @@ SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- +---- + + +=== src/third_party/url.* + +CxxUrl - A simple C++ URL class. Copied from CxxUrl v0.3 downloaded from +. It has the following license text: + +---- +The MIT License (MIT) + +Copyright (c) 2015 Christophe Meessen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +---- -src/third_party/win32/getopt.* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/win32/getopt.* This implementation of `getopt_long()` for Win32 was taken from https://www.codeproject.com/Articles/157001/Full-getopt-Port-for-Unicode-and-Multibyte-Microso @@ -638,14 +684,13 @@ The full license text can be found in LGPL-3.0.txt and at https://www.gnu.org/licenses/lgpl-3.0.html. -src/third_party/win32/mktemp.* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/win32/mktemp.* This implementation of `mkstemp()` for Win32 was adapted from -and has the folowing license text: +and has the following license text: -------------------------------------------------------------------------------- +---- Copyright (c) 1996-1998, 2008 Theo de Raadt Copyright (c) 1997, 2008-2009 Todd C. Miller @@ -660,16 +705,16 @@ ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -------------------------------------------------------------------------------- +---- + -src/third_party/win32/winerror_to_errno.h -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/win32/winerror_to_errno.h The implementation of `winerror_to_errno()` was adapted from -and has the folowing license text: +and has the following license text: -------------------------------------------------------------------------------- +---- PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 1. This LICENSE AGREEMENT is between the Python Software Foundation @@ -717,15 +762,15 @@ products or services of Licensee, or any third party. 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this License Agreement. -------------------------------------------------------------------------------- +---- + -src/third_party/xxh* -~~~~~~~~~~~~~~~~~~~~ +=== src/third_party/xxh* -xxHash - Extremely Fast Hash algorithm. Copied from xxHash v0.8.0 downloaded +xxHash - Extremely Fast Hash algorithm. Copied from xxHash v0.8.1 downloaded from . -------------------------------------------------------------------------------- +---- Copyright (c) 2012-2020 Yann Collet All rights reserved. @@ -753,4 +798,4 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- +---- diff --git a/README.md b/README.md index 57ae2ca..efe4686 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,10 @@ Ccache – a fast compiler cache [![Build Status](https://github.com/ccache/ccache/workflows/Build/badge.svg)](https://github.com/ccache/ccache/actions?query=workflow%3A%22Build%22) [![Code Quality: Cpp](https://img.shields.io/lgtm/grade/cpp/g/ccache/ccache.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/ccache/ccache/context:cpp) [![Total Alerts](https://img.shields.io/lgtm/alerts/g/ccache/ccache.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/ccache/ccache/alerts) +[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5057/badge)](https://bestpractices.coreinfrastructure.org/projects/5057) [![Gitter](https://img.shields.io/gitter/room/ccache/ccache.svg)](https://gitter.im/ccache/ccache) -Ccache (or “ccache”) is a compiler cache. It [speeds up +Ccache is a compiler cache. It [speeds up recompilation](https://ccache.dev/performance.html) by caching previous compilations and detecting when the same compilation is being done again. diff --git a/ci/build b/ci/build index ff50bbf..9001bfb 100755 --- a/ci/build +++ b/ci/build @@ -4,23 +4,40 @@ set -eu -if [ -n "${VERBOSE:-}" ]; then +# Set default values. +: ${BUILDDIR:=build} +: ${CCACHE_LOC:=..} +: ${CMAKE_PARAMS:=} +: ${CMAKE_PREFIX:=} +: ${EXTRA_CMAKE_BUILD_FLAGS:=} +: ${JOBS:=$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)} +: ${SPECIAL:=} +: ${TEST_CC:=${CC:-}} +: ${VERBOSE:=} + +if [ -n "${VERBOSE}" ]; then set -x fi -if [ -n "${SPECIAL:-}" ]; then +if [ -n "${SPECIAL}" ]; then exec "ci/$SPECIAL" else - [ -z ${JOBS:+x} ] && JOBS=$(getconf _NPROCESSORS_ONLN 2>/dev/null) - [ -z ${JOBS:+x} ] && JOBS=2 + mkdir -p "${BUILDDIR}" + cd "${BUILDDIR}" + ${CMAKE_PREFIX} cmake ${CMAKE_PARAMS} ${CCACHE_LOC} + + case "${CMAKE_GENERATOR}" in + [Vv]isual" "[Ss]tudio*) # MSBuild, use all CPUs. + ${CMAKE_PREFIX} cmake --build . ${EXTRA_CMAKE_BUILD_FLAGS} -- -m + ;; + *) # Ninja automatically uses all available CPUs. + ${CMAKE_PREFIX} cmake --build . ${EXTRA_CMAKE_BUILD_FLAGS} + ;; + esac - mkdir -p ${BUILDDIR:-build} - cd ${BUILDDIR:-build} - ${CMAKE_PREFIX:-} cmake ${CMAKE_PARAMS:-} ${CCACHE_LOC:-..} - ${CMAKE_PREFIX:-} cmake --build . ${EXTRA_CMAKE_BUILD_FLAGS:-} -- -j$JOBS case "${RUN_TESTS:-all}" in all) - CC=${TEST_CC:-${CC}} ctest --output-on-failure -j$JOBS "$@" + CC="${TEST_CC}" ctest --output-on-failure -j"${JOBS}" "$@" ;; unittest-in-wine) wine ccache.exe --version diff --git a/ci/collect-testdir b/ci/collect-testdir index 21a3971..fa08bdd 100755 --- a/ci/collect-testdir +++ b/ci/collect-testdir @@ -6,7 +6,7 @@ elif [ -d build/testdir ]; then testdir=build/testdir else echo "No testdir found" >&2 - exit 1 + exit 0 fi tar -caf testdir.tar.xz $testdir diff --git a/ci/install-cuda b/ci/install-cuda index 169f2a8..daba29c 100755 --- a/ci/install-cuda +++ b/ci/install-cuda @@ -17,9 +17,8 @@ retry() { echo "Installing CUDA support" -retry wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_${CUDA}_amd64.deb -retry sudo dpkg -i cuda-repo-ubuntu1804_${CUDA}_amd64.deb -retry sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub +retry wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb +sudo dpkg -i cuda-keyring_1.0-1_all.deb retry sudo apt-get update -qq cuda_prefix=${CUDA:0:4} diff --git a/cmake/CcachePackConfig.cmake b/cmake/CcachePackConfig.cmake index a35949d..c41aeca 100644 --- a/cmake/CcachePackConfig.cmake +++ b/cmake/CcachePackConfig.cmake @@ -1,10 +1,6 @@ # Note: This is part of CMakeLists.txt file, not to be confused with # CPackConfig.cmake. -if(${CMAKE_VERSION} VERSION_LESS "3.9") - set(CPACK_PACKAGE_DESCRIPTION "${CMAKE_PROJECT_DESCRIPTION}") -endif() - # From CcacheVersion.cmake. set(CPACK_PACKAGE_VERSION ${CCACHE_VERSION}) diff --git a/cmake/CcacheVersion.cmake b/cmake/CcacheVersion.cmake index c0cbd91..1181651 100644 --- a/cmake/CcacheVersion.cmake +++ b/cmake/CcacheVersion.cmake @@ -22,7 +22,8 @@ # CCACHE_VERSION_ORIGIN is set to "archive" in scenario 1 and "git" in scenario # 3. -set(version_info "2c13bce2609a02b70c92e84cdd177a6072f44d5e HEAD, tag: v4.3, origin/master, origin/HEAD, master") +set(version_info "1527040bc2a278b9d3d51badb732ecf5841d8bb5 HEAD, tag: v4.7.4, origin/master, origin/HEAD, master") +set(CCACHE_VERSION "unknown") if(version_info MATCHES "^([0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f])[0-9a-f]* (.*)") # Scenario 1. @@ -43,12 +44,12 @@ elseif(EXISTS "${CMAKE_SOURCE_DIR}/.git") find_package(Git QUIET) if(NOT GIT_FOUND) - set(CCACHE_VERSION "unknown") message(WARNING "Could not find git") else() macro(git) execute_process( COMMAND "${GIT_EXECUTABLE}" ${ARGN} + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" OUTPUT_VARIABLE git_stdout OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_VARIABLE git_stderr ERROR_STRIP_TRAILING_WHITESPACE) endmacro() @@ -72,9 +73,9 @@ elseif(EXISTS "${CMAKE_SOURCE_DIR}/.git") endif() endif() -if(CCACHE_VERSION STREQUAL "") +if("${CCACHE_VERSION}" STREQUAL "unknown") # Scenario 2 or unexpected error. - message(SEND_ERROR "Cannot determine Ccache version") + message(WARNING "Could not determine ccache version") endif() message(STATUS "Ccache version: ${CCACHE_VERSION}") diff --git a/cmake/CodeAnalysis.cmake b/cmake/CodeAnalysis.cmake index 4e639a7..7bba40d 100644 --- a/cmake/CodeAnalysis.cmake +++ b/cmake/CodeAnalysis.cmake @@ -1,38 +1,30 @@ option(ENABLE_CPPCHECK "Enable static analysis with Cppcheck" OFF) if(ENABLE_CPPCHECK) - if(${CMAKE_VERSION} VERSION_LESS "3.10") - message(WARNING "Cppcheck requires CMake 3.10") + find_program(CPPCHECK_EXE cppcheck) + mark_as_advanced(CPPCHECK_EXE) # Don't show in CMake UIs + if(CPPCHECK_EXE) + set(CMAKE_CXX_CPPCHECK + ${CPPCHECK_EXE} + --suppressions-list=${CMAKE_SOURCE_DIR}/misc/cppcheck-suppressions.txt + --inline-suppr + -q + --enable=all + --force + --std=c++17 + -I ${CMAKE_SOURCE_DIR} + --template="cppcheck: warning: {id}:{file}:{line}: {message}" + -i src/third_party) else() - find_program(CPPCHECK_EXE cppcheck) - mark_as_advanced(CPPCHECK_EXE) # Don't show in CMake UIs - if(CPPCHECK_EXE) - set(CMAKE_CXX_CPPCHECK - ${CPPCHECK_EXE} - --suppressions-list=${CMAKE_SOURCE_DIR}/misc/cppcheck-suppressions.txt - --inline-suppr - -q - --enable=all - --force - --std=c++11 - -I ${CMAKE_SOURCE_DIR} - --template="cppcheck: warning: {id}:{file}:{line}: {message}" - -i src/third_party) - else() - message(WARNING "Cppcheck requested but executable not found") - endif() + message(WARNING "Cppcheck requested but executable not found") endif() endif() option(ENABLE_CLANG_TIDY "Enable static analysis with Clang-Tidy" OFF) if(ENABLE_CLANG_TIDY) - if(${CMAKE_VERSION} VERSION_LESS "3.6") - message(WARNING "Clang-Tidy requires CMake 3.6") + find_program(CLANGTIDY clang-tidy) + if(CLANGTIDY) + set(CMAKE_CXX_CLANG_TIDY ${CLANGTIDY}) else() - find_program(CLANGTIDY clang-tidy) - if(CLANGTIDY) - set(CMAKE_CXX_CLANG_TIDY ${CLANGTIDY}) - else() - message(SEND_ERROR "Clang-Tidy requested but executable not found") - endif() + message(SEND_ERROR "Clang-Tidy requested but executable not found") endif() endif() diff --git a/cmake/DevModeWarnings.cmake b/cmake/DevModeWarnings.cmake index 7ff5411..acc32b7 100644 --- a/cmake/DevModeWarnings.cmake +++ b/cmake/DevModeWarnings.cmake @@ -82,8 +82,6 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") CCACHE_COMPILER_WARNINGS "-Wno-zero-as-null-pointer-constant") add_compile_flag_if_supported( CCACHE_COMPILER_WARNINGS "-Wno-undefined-func-template") - add_compile_flag_if_supported( - CCACHE_COMPILER_WARNINGS "-Wno-return-std-move-in-c++11") elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") list( APPEND @@ -99,15 +97,6 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # -Wduplicated-branches # -Wuseless-cast ) - - # TODO: Exact version or reason unknown, discovered in Ubuntu 14 Docker test - # with GCC 4.8.4 - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8.5) - add_compile_flag_if_supported( - CCACHE_COMPILER_WARNINGS "-Wno-missing-field-initializers") - add_compile_flag_if_supported( - CCACHE_COMPILER_WARNINGS "-Wno-unused-variable") - endif() elseif(MSVC) # Remove any warning level flags added by CMake. string(REGEX REPLACE "/W[0-4]" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") diff --git a/cmake/Findhiredis.cmake b/cmake/Findhiredis.cmake new file mode 100644 index 0000000..b8045f6 --- /dev/null +++ b/cmake/Findhiredis.cmake @@ -0,0 +1,99 @@ +if(hiredis_FOUND) + return() +endif() + +set(hiredis_FOUND FALSE) + +find_package(PkgConfig) +if(PKG_CONFIG_FOUND) + pkg_check_modules(HIREDIS hiredis>=${hiredis_FIND_VERSION}) + find_library(HIREDIS_LIBRARY ${HIREDIS_LIBRARIES} HINTS ${HIREDIS_LIBDIR}) + find_path(HIREDIS_INCLUDE_DIR hiredis/hiredis.h HINTS ${HIREDIS_PREFIX}/include) +else() + find_library(HIREDIS_LIBRARY hiredis) + find_path(HIREDIS_INCLUDE_DIR hiredis/hiredis.h) +endif() + +if(HIREDIS_INCLUDE_DIR AND HIREDIS_LIBRARY) + mark_as_advanced(HIREDIS_INCLUDE_DIR HIREDIS_LIBRARY) + + add_library(HIREDIS::HIREDIS UNKNOWN IMPORTED) + set_target_properties( + HIREDIS::HIREDIS + PROPERTIES + IMPORTED_LOCATION "${HIREDIS_LIBRARY}" + INTERFACE_COMPILE_OPTIONS "${HIREDIS_CFLAGS_OTHER}" + INTERFACE_INCLUDE_DIRECTORIES "${HIREDIS_INCLUDE_DIR}") + if(WIN32 AND STATIC_LINK) + target_link_libraries(HIREDIS::HIREDIS INTERFACE ws2_32) + endif() + + set(hiredis_FOUND TRUE) + set(target HIREDIS::HIREDIS) +elseif(HIREDIS_FROM_INTERNET) + message(STATUS "*** WARNING ***: Using hiredis from the internet because it was NOT found and HIREDIS_FROM_INTERNET is TRUE") + + set(hiredis_version "1.1.0") + + set(hiredis_dir ${CMAKE_BINARY_DIR}/hiredis-${hiredis_version}) + set(hiredis_build ${CMAKE_BINARY_DIR}/hiredis-build) + + include(FetchContent) + + FetchContent_Declare( + hiredis + URL https://github.com/redis/hiredis/archive/v${hiredis_version}.tar.gz + URL_HASH SHA256=fe6d21741ec7f3fc9df409d921f47dfc73a4d8ff64f4ac6f1d95f951bf7f53d6 + SOURCE_DIR ${hiredis_dir} + BINARY_DIR ${hiredis_build} + ) + + FetchContent_GetProperties(hiredis) + + if(NOT hiredis_POPULATED) + FetchContent_Populate(hiredis) + endif() + + set( + hiredis_sources + "${hiredis_dir}/alloc.c" + "${hiredis_dir}/async.c" + "${hiredis_dir}/dict.c" + "${hiredis_dir}/hiredis.c" + "${hiredis_dir}/net.c" + "${hiredis_dir}/read.c" + "${hiredis_dir}/sds.c" + "${hiredis_dir}/sockcompat.c" + ) + add_library(libhiredis_static STATIC EXCLUDE_FROM_ALL ${hiredis_sources}) + add_library(HIREDIS::HIREDIS ALIAS libhiredis_static) + + if(WIN32) + target_compile_definitions(libhiredis_static PRIVATE _CRT_SECURE_NO_WARNINGS) + endif() + + make_directory("${hiredis_dir}/include") + make_directory("${hiredis_dir}/include/hiredis") + file(GLOB hiredis_headers "${hiredis_dir}/*.h") + file(COPY ${hiredis_headers} DESTINATION "${hiredis_dir}/include/hiredis") + set_target_properties( + libhiredis_static + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "$") + + set(hiredis_FOUND TRUE) + set(target libhiredis_static) +endif() + +if(WIN32 AND hiredis_FOUND) + target_link_libraries(${target} INTERFACE ws2_32) +endif() +unset(target) + +include(FeatureSummary) +set_package_properties( + hiredis + PROPERTIES + URL "https://github.com/redis/hiredis" + DESCRIPTION "Hiredis is a minimalistic C client library for the Redis database" +) diff --git a/cmake/Findzstd.cmake b/cmake/Findzstd.cmake index e889a68..a6d30a4 100644 --- a/cmake/Findzstd.cmake +++ b/cmake/Findzstd.cmake @@ -2,32 +2,54 @@ if(zstd_FOUND) return() endif() -if(ZSTD_FROM_INTERNET) +set(zstd_FOUND FALSE) + +find_package(PkgConfig) +if(PKG_CONFIG_FOUND) + pkg_search_module(PC_ZSTD libzstd) + find_library(ZSTD_LIBRARY zstd HINTS ${PC_ZSTD_LIBDIR} ${PC_ZSTD_LIBRARY_DIRS}) + find_path(ZSTD_INCLUDE_DIR zstd.h HINTS ${PC_ZSTD_INCLUDEDIR} ${PC_ZSTD_INCLUDE_DIRS}) +else() + find_library(ZSTD_LIBRARY zstd) + find_path(ZSTD_INCLUDE_DIR zstd.h) +endif() + +if(ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR) + mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY) + + add_library(ZSTD::ZSTD UNKNOWN IMPORTED) + set_target_properties( + ZSTD::ZSTD + PROPERTIES + IMPORTED_LOCATION "${ZSTD_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}") + + set(zstd_FOUND TRUE) +elseif(ZSTD_FROM_INTERNET) + message(STATUS "*** WARNING ***: Using zstd from the internet because it was NOT found and ZSTD_FROM_INTERNET is TRUE") + # Although ${zstd_FIND_VERSION} was requested, let's download a newer version. # Note: The directory structure has changed in 1.3.0; we only support 1.3.0 # and newer. - set(zstd_version "1.4.9") - set(zstd_url https://github.com/facebook/zstd/archive/v${zstd_version}.tar.gz) + set(zstd_version "1.5.2") - set(zstd_dir ${CMAKE_BINARY_DIR}/zstd-${zstd_version}) + set(zstd_dir ${CMAKE_BINARY_DIR}/zstd-${zstd_version}) set(zstd_build ${CMAKE_BINARY_DIR}/zstd-build) - if(NOT EXISTS "${zstd_dir}.tar.gz") - file(DOWNLOAD "${zstd_url}" "${zstd_dir}.tar.gz" STATUS download_status) - list(GET download_status 0 error_code) - if(error_code) - file(REMOVE "${zstd_dir}.tar.gz") - list(GET download_status 1 error_message) - message(FATAL "Failed to download zstd: ${error_message}") - endif() - endif() + include(FetchContent) + + FetchContent_Declare( + zstd + URL https://github.com/facebook/zstd/archive/v${zstd_version}.tar.gz + URL_HASH SHA256=f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e + SOURCE_DIR ${zstd_dir} + BINARY_DIR ${zstd_build} + ) + + FetchContent_GetProperties(zstd) - execute_process( - COMMAND tar xf "${zstd_dir}.tar.gz" - WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" - RESULT_VARIABLE tar_error) - if(NOT tar_error EQUAL 0) - message(FATAL "extracting ${zstd_dir}.tar.gz failed") + if(NOT zstd_POPULATED) + FetchContent_Populate(zstd) endif() set(ZSTD_BUILD_SHARED OFF) @@ -37,25 +59,10 @@ if(ZSTD_FROM_INTERNET) set_target_properties( libzstd_static PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "$") + INTERFACE_INCLUDE_DIRECTORIES "$" + ) set(zstd_FOUND TRUE) -else() - find_library(ZSTD_LIBRARY zstd) - find_path(ZSTD_INCLUDE_DIR zstd.h) - - include(FindPackageHandleStandardArgs) - find_package_handle_standard_args( - zstd "please install libzstd or use -DZSTD_FROM_INTERNET=ON" - ZSTD_INCLUDE_DIR ZSTD_LIBRARY) - mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY) - - add_library(ZSTD::ZSTD UNKNOWN IMPORTED) - set_target_properties( - ZSTD::ZSTD - PROPERTIES - IMPORTED_LOCATION "${ZSTD_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}") endif() include(FeatureSummary) diff --git a/cmake/GenerateConfigurationFile.cmake b/cmake/GenerateConfigurationFile.cmake index a460529..858567b 100644 --- a/cmake/GenerateConfigurationFile.cmake +++ b/cmake/GenerateConfigurationFile.cmake @@ -26,26 +26,26 @@ endforeach() include(CheckFunctionExists) set(functions asctime_r - geteuid getopt_long getpwuid - gettimeofday posix_fallocate realpath setenv strndup syslog unsetenv - utimes) + utimensat + utimes +) foreach(func IN ITEMS ${functions}) string(TOUPPER ${func} func_var) set(func_var HAVE_${func_var}) check_function_exists(${func} ${func_var}) endforeach() -include(CheckCSourceCompiles) +include(CheckCXXSourceCompiles) set(CMAKE_REQUIRED_FLAGS -pthread) -check_c_source_compiles( +check_cxx_source_compiles( [=[ #include int main() @@ -60,18 +60,28 @@ check_function_exists(pthread_mutexattr_setpshared HAVE_PTHREAD_MUTEXATTR_SETPSH set(CMAKE_REQUIRED_FLAGS) include(CheckStructHasMember) +check_struct_has_member("struct stat" st_atim sys/stat.h + HAVE_STRUCT_STAT_ST_ATIM LANGUAGE CXX) check_struct_has_member("struct stat" st_ctim sys/stat.h - HAVE_STRUCT_STAT_ST_CTIM) + HAVE_STRUCT_STAT_ST_CTIM LANGUAGE CXX) check_struct_has_member("struct stat" st_mtim sys/stat.h - HAVE_STRUCT_STAT_ST_MTIM) + HAVE_STRUCT_STAT_ST_MTIM LANGUAGE CXX) +check_struct_has_member("struct stat" st_atimespec sys/stat.h + HAVE_STRUCT_STAT_ST_ATIMESPEC LANGUAGE CXX) +check_struct_has_member("struct stat" st_ctimespec sys/stat.h + HAVE_STRUCT_STAT_ST_CTIMESPEC LANGUAGE CXX) +check_struct_has_member("struct stat" st_mtimespec sys/stat.h + HAVE_STRUCT_STAT_ST_MTIMESPEC LANGUAGE CXX) check_struct_has_member("struct statfs" f_fstypename sys/mount.h - HAVE_STRUCT_STATFS_F_FSTYPENAME) + HAVE_STRUCT_STATFS_F_FSTYPENAME LANGUAGE CXX) include(CheckCXXSourceCompiles) check_cxx_source_compiles( [=[ #include + #ifndef _MSC_VER // MSVC does not need explicit enabling of AVX2. void func() __attribute__((target("avx2"))); + #endif void func() { _mm256_abs_epi8(_mm256_set1_epi32(42)); } int main() { @@ -81,12 +91,6 @@ check_cxx_source_compiles( ]=] HAVE_AVX2) -list(APPEND CMAKE_REQUIRED_LIBRARIES ws2_32) -list(REMOVE_ITEM CMAKE_REQUIRED_LIBRARIES ws2_32) - -include(CheckTypeSize) -check_type_size("long long" HAVE_LONG_LONG) - if(WIN32) set(_WIN32_WINNT 0x0600) endif() @@ -98,5 +102,16 @@ endif() # alias set(MTR_ENABLED "${ENABLE_TRACING}") +if(HAVE_SYS_MMAN_H + AND HAVE_PTHREAD_MUTEXATTR_SETPSHARED + AND (HAVE_STRUCT_STAT_ST_MTIM OR HAVE_STRUCT_STAT_ST_MTIMESPEC) + AND (HAVE_LINUX_FS_H OR HAVE_STRUCT_STATFS_F_FSTYPENAME)) + set(INODE_CACHE_SUPPORTED 1) +endif() + +# Escape backslashes in SYSCONFDIR for C. +file(TO_NATIVE_PATH "${CMAKE_INSTALL_FULL_SYSCONFDIR}" CONFIG_SYSCONFDIR_C_ESCAPED) +string(REPLACE "\\" "\\\\" CONFIG_SYSCONFDIR_C_ESCAPED "${CONFIG_SYSCONFDIR_C_ESCAPED}") + configure_file(${CMAKE_SOURCE_DIR}/cmake/config.h.in ${CMAKE_BINARY_DIR}/config.h @ONLY) diff --git a/cmake/InstallDirs.cmake b/cmake/InstallDirs.cmake new file mode 100644 index 0000000..f258a19 --- /dev/null +++ b/cmake/InstallDirs.cmake @@ -0,0 +1,41 @@ +if(WIN32) + if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + set(program_files "$ENV{ProgramFiles}") + + # For 32 bit builds. + if(CMAKE_SIZEOF_VOID_P EQUAL 4 AND ENV{ProgramFiles\(x86\)}) + set(program_files "$ENV{ProgramFiles\(x86\)}") + endif() + + if(NOT program_files) + if(NOT CMAKE_SIZEOF_VOID_P EQUAL 4) + set(program_files "/Program Files") + else() + set(program_files "/Program Files (x86)") + endif() + endif() + + file(TO_CMAKE_PATH "${program_files}/ccache" CMAKE_INSTALL_PREFIX) + + set(CMAKE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" CACHE STRING "System-wide installation prefix" FORCE) + endif() + + if(NOT CMAKE_INSTALL_SYSCONFDIR) + set(program_data "$ENV{ALLUSERSPROFILE}") + + if(NOT program_data) + set(program_data "/ProgramData") + endif() + + file(TO_CMAKE_PATH "${program_data}/ccache" CMAKE_INSTALL_SYSCONFDIR) + + set(CMAKE_INSTALL_SYSCONFDIR "${CMAKE_INSTALL_SYSCONFDIR}" CACHE PATH "System-wide config file location" FORCE) + endif() + + set(CMAKE_INSTALL_BINDIR "" CACHE PATH "user executables directory" FORCE) + set(CMAKE_INSTALL_SBINDIR "" CACHE PATH "system administration executables directory" FORCE) + set(CMAKE_INSTALL_LIBEXECDIR "" CACHE PATH "program executables directory" FORCE) + set(CMAKE_INSTALL_LIBDIR "" CACHE PATH "object libraries directory" FORCE) +endif() + +include(GNUInstallDirs) diff --git a/cmake/StandardSettings.cmake b/cmake/StandardSettings.cmake index a0f0189..9e09ea6 100644 --- a/cmake/StandardSettings.cmake +++ b/cmake/StandardSettings.cmake @@ -3,8 +3,14 @@ add_library(standard_settings INTERFACE) -# Not supported in CMake 3.4: target_compile_features(project_options INTERFACE -# c_std_11 cxx_std_11) +if(MSVC) + target_compile_options(standard_settings INTERFACE "/FI${CMAKE_BINARY_DIR}/config.h") +else() + target_compile_options( + standard_settings + INTERFACE -include ${CMAKE_BINARY_DIR}/config.h + ) +endif() if(CMAKE_CXX_COMPILER_ID MATCHES "^GNU|(Apple)?Clang$") option(ENABLE_COVERAGE "Enable coverage reporting for GCC/Clang" FALSE) @@ -48,7 +54,18 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "^GNU|(Apple)?Clang$") endforeach() include(StdAtomic) + include(StdFilesystem) elseif(MSVC) - target_compile_options(standard_settings INTERFACE /std:c++latest /Zc:preprocessor /Zc:__cplusplus /D_CRT_SECURE_NO_WARNINGS) + target_compile_options( + standard_settings + INTERFACE /Zc:preprocessor /Zc:__cplusplus /D_CRT_SECURE_NO_WARNINGS + ) +endif() + +if(WIN32) + target_compile_definitions( + standard_settings + INTERFACE WIN32_LEAN_AND_MEAN + ) endif() diff --git a/cmake/StdAtomic.cmake b/cmake/StdAtomic.cmake index 8cebe51..119518a 100644 --- a/cmake/StdAtomic.cmake +++ b/cmake/StdAtomic.cmake @@ -2,7 +2,6 @@ include(CheckCXXSourceCompiles) -set(CMAKE_REQUIRED_FLAGS ${CMAKE_CXX11_STANDARD_COMPILE_OPTION}) set( check_std_atomic_source_code [=[ @@ -28,5 +27,3 @@ if(NOT std_atomic_without_libatomic) target_link_libraries(standard_settings INTERFACE atomic) endif() endif() - -set(CMAKE_REQUIRED_FLAGS) diff --git a/cmake/StdFilesystem.cmake b/cmake/StdFilesystem.cmake new file mode 100644 index 0000000..ca451ce --- /dev/null +++ b/cmake/StdFilesystem.cmake @@ -0,0 +1,26 @@ +# Check if std::filesystem needs -lstdc++fs + +include(CheckCXXSourceCompiles) + +set( + check_std_filesystem_source_code + [=[ + #include + int main(void) + { + return std::filesystem::is_regular_file(\"/\") ? 0 : 1; + } + ]=]) + +check_cxx_source_compiles("${check_std_filesystem_source_code}" std_filesystem_without_libfs) + +if(NOT std_filesystem_without_libfs) + set(CMAKE_REQUIRED_LIBRARIES stdc++fs) + check_cxx_source_compiles("${check_std_filesystem_source_code}" std_filesystem_with_libfs) + set(CMAKE_REQUIRED_LIBRARIES) + if(NOT std_filesystem_with_libfs) + message(FATAL_ERROR "Toolchain doesn't support std::filesystem with nor without -lstdc++fs") + else() + target_link_libraries(standard_settings INTERFACE stdc++fs) + endif() +endif() diff --git a/cmake/UseCcache.cmake b/cmake/UseCcache.cmake deleted file mode 100644 index 3137d85..0000000 --- a/cmake/UseCcache.cmake +++ /dev/null @@ -1,74 +0,0 @@ -# Note: Compiling ccache via ccache is fine because the ccache version installed -# in the system is used. - -# Calls `message(VERBOSE msg)` if and only if VERBOSE is available (since CMake -# 3.15). Call CMake with --log-level=VERBOSE to view verbose messages. -function(message_verbose msg) - if(NOT ${CMAKE_VERSION} VERSION_LESS "3.15") - message(VERBOSE ${msg}) - endif() -endfunction() - -function(use_ccache) - if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) - message(WARNING "use_ccache() disabled, as it is not called from the project top level") - return() - endif() - - find_program(CCACHE_PROGRAM ccache) - if(NOT CCACHE_PROGRAM) - message_verbose("Ccache program not found, not enabling ccache for faster recompilation") - return() - endif() - - message_verbose("Ccache enabled for faster recompilation") - - # Note: This will override any config and environment settings. - set(ccache_env - # Another option would be CMAKE_BINARY_DIR, but currently only one base - # directory is supported. - CCACHE_BASEDIR=${CMAKE_SOURCE_DIR} - - # In case of very old ccache versions (pre 3.3). - CCACHE_CPP2=true - ) - - if(CMAKE_GENERATOR MATCHES "Ninja|Makefiles") - find_program(ENV_PROGRAM env) - if(ENV_PROGRAM) - set(env_program ${ENV_PROGRAM}) # faster than "cmake -E env" - else() - set(env_program ${CMAKE_COMMAND} -E env) - endif() - foreach(lang IN ITEMS C CXX OBJC OBJCXX CUDA) - set(CMAKE_${lang}_COMPILER_LAUNCHER - ${env_program} ${ccache_env} ${CCACHE_PROGRAM} - PARENT_SCOPE) - endforeach() - elseif(CMAKE_GENERATOR STREQUAL Xcode) - foreach(lang IN ITEMS C CXX) - set(launcher ${CMAKE_BINARY_DIR}/launch-${lang}) - file(WRITE ${launcher} "#!/bin/bash\n\n") - foreach(key_val IN LISTS ccache_env) - file(APPEND ${launcher} "export ${key_val}\n") - endforeach() - file(APPEND ${launcher} - "exec \"${CCACHE_PROGRAM}\" \"${CMAKE_${lang}_COMPILER}\" \"$@\"\n") - execute_process(COMMAND chmod a+rx ${launcher}) - endforeach() - set(CMAKE_XCODE_ATTRIBUTE_CC ${CMAKE_BINARY_DIR}/launch-C PARENT_SCOPE) - set(CMAKE_XCODE_ATTRIBUTE_CXX ${CMAKE_BINARY_DIR}/launch-CXX PARENT_SCOPE) - set(CMAKE_XCODE_ATTRIBUTE_LD ${CMAKE_BINARY_DIR}/launch-C PARENT_SCOPE) - set(CMAKE_XCODE_ATTRIBUTE_LDPLUSPLUS ${CMAKE_BINARY_DIR}/launch-CXX PARENT_SCOPE) - endif() -endfunction() - -if(MSVC) - # Ccache does not support cl.exe-style arguments at this time. - return() -endif() - -option(USE_CCACHE "Use ccache to speed up recompilation time" TRUE) -if(USE_CCACHE) - use_ccache() -endif() diff --git a/cmake/UseFastestLinker.cmake b/cmake/UseFastestLinker.cmake index ffc26ea..345ef1d 100644 --- a/cmake/UseFastestLinker.cmake +++ b/cmake/UseFastestLinker.cmake @@ -1,3 +1,33 @@ +if(NOT CCACHE_DEV_MODE) + # For ccache, using a faster linker is in practice only relevant to reduce the + # compile-link-test cycle for developers, so use the standard linker for + # non-developer builds. + return() +endif() + +if(DISABLE_FASTEST_LINKER) + message(STATUS "Not probing for fastest linker") + return() +endif() + +if(MSVC) + message(STATUS "Using standard linker for MSVC") + return() +endif() + +if(ENABLE_IPO) + message(STATUS "Using standard linker for IPO") + return() +endif() + +if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64) + # Be conservative and only probe for a faster linker on platforms that likely + # don't have toolchain bugs. See for example + # . + message(STATUS "Not probing for faster linker on ${CMAKE_SYSTEM_PROCESSOR}") + return() +endif() + function(check_linker linker) string(TOUPPER ${linker} upper_linker) file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/CMakefiles/CMakeTmp/main.c" "int main() { return 0; }") @@ -15,23 +45,31 @@ function(use_fastest_linker) return() endif() - set(use_default_linker 1) + # prefer an lld that matches the clang version + if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND CMAKE_C_COMPILER_VERSION MATCHES "^([0-9]*)\\.") + check_linker(lld-${CMAKE_MATCH_1}) + if(HAVE_LD_LLD-${CMAKE_MATCH_1}) + link_libraries("-fuse-ld=lld-${CMAKE_MATCH_1}") + message(STATUS "Using lld-${CMAKE_MATCH_1} linker") + return() + endif() + endif() + check_linker(lld) if(HAVE_LD_LLD) link_libraries("-fuse-ld=lld") - set(use_default_linker 0) message(STATUS "Using lld linker") - else() - check_linker(gold) - if(HAVE_LD_GOLD) - link_libraries("-fuse-ld=gold") - set(use_default_linker 0) - message(STATUS "Using gold linker") - endif() + return() endif() - if(use_default_linker) - message(STATUS "Using default linker") + + check_linker(gold) + if(HAVE_LD_GOLD) + link_libraries("-fuse-ld=gold") + message(STATUS "Using gold linker") + return() endif() + + message(STATUS "Using default linker") endfunction() option(USE_FASTER_LINKER "Use the lld or gold linker instead of the default for faster linking" TRUE) diff --git a/cmake/config.h.in b/cmake/config.h.in index d10e164..1537119 100644 --- a/cmake/config.h.in +++ b/cmake/config.h.in @@ -1,3 +1,6 @@ +// This file is included by all compilation units, including those in +// src/third_party. It should only contain macros and typedefs. + #pragma once #ifdef __clang__ # pragma clang diagnostic push @@ -6,6 +9,11 @@ # endif #endif +#ifdef __MINGW32__ +# define __USE_MINGW_ANSI_STDIO 1 +# define __STDC_FORMAT_MACROS 1 +#endif + // For example for vasprintf under i686-w64-mingw32-g++-posix. The later // definition of _XOPEN_SOURCE disables certain features on Linux, so we need // _GNU_SOURCE to re-enable them (makedev, tm_zone). @@ -56,8 +64,6 @@ #cmakedefine _WIN32_WINNT @_WIN32_WINNT@ // clang-format on -#define SYSCONFDIR "@CMAKE_INSTALL_FULL_SYSCONFDIR@" - #ifdef __clang__ # pragma clang diagnostic pop #endif @@ -70,18 +76,12 @@ // Define if your compiler supports AVX2. #cmakedefine HAVE_AVX2 -// Define if you have the "geteuid" function. -#cmakedefine HAVE_GETEUID - // Define if you have the "getopt_long" function. #cmakedefine HAVE_GETOPT_LONG // Define if you have the "getpwuid" function. #cmakedefine HAVE_GETPWUID -// Define if you have the "gettimeofday" function. -#cmakedefine HAVE_GETTIMEOFDAY - // Define if you have the header file. #cmakedefine HAVE_LINUX_FS_H @@ -109,12 +109,24 @@ // Define if "f_fstypename" is a member of "struct statfs". #cmakedefine HAVE_STRUCT_STATFS_F_FSTYPENAME +// Define if "st_atim" is a member of "struct stat". +#cmakedefine HAVE_STRUCT_STAT_ST_ATIM + // Define if "st_ctim" is a member of "struct stat". #cmakedefine HAVE_STRUCT_STAT_ST_CTIM // Define if "st_mtim" is a member of "struct stat". #cmakedefine HAVE_STRUCT_STAT_ST_MTIM +// Define if "st_atimespec" is a member of "struct stat". +#cmakedefine HAVE_STRUCT_STAT_ST_ATIMESPEC + +// Define if "st_ctimespec" is a member of "struct stat". +#cmakedefine HAVE_STRUCT_STAT_ST_CTIMESPEC + +// Define if "st_mtimespec" is a member of "struct stat". +#cmakedefine HAVE_STRUCT_STAT_ST_MTIMESPEC + // Define if you have the "syslog" function. #cmakedefine HAVE_SYSLOG @@ -163,6 +175,9 @@ // Define if you have the "unsetenv" function. #cmakedefine HAVE_UNSETENV +// Define if you have the "utimensat" function. +#cmakedefine HAVE_UTIMENSAT + // Define if you have the "utimes" function. #cmakedefine HAVE_UTIMES @@ -174,3 +189,32 @@ # undef HAVE_STRUCT_STAT_ST_CTIM # undef HAVE_STRUCT_STAT_ST_MTIM #endif + +// Typedefs that make it possible to use common types in ccache header files +// without including core/wincompat.hpp. +#ifdef _WIN32 +# ifdef _MSC_VER +typedef unsigned __int32 mode_t; +typedef int pid_t; +# endif // _MSC_VER +#endif // _WIN32 + +// O_BINARY is needed when reading binary data on Windows, so use it everywhere +// with a compatibility define for Unix platforms. +#if !defined(_WIN32) && !defined(O_BINARY) +# define O_BINARY 0 +#endif +#if !defined(_WIN32) && !defined(O_TEXT) +# define O_TEXT 0 +#endif + +#ifndef ESTALE +# define ESTALE -1 +#endif + +#define SYSCONFDIR "@CONFIG_SYSCONFDIR_C_ESCAPED@" + +#cmakedefine INODE_CACHE_SUPPORTED + +// Buffer size for I/O operations. Should be a multiple of 4 KiB. +#define CCACHE_READ_BUFFER_SIZE 65536 diff --git a/doc/AUTHORS.adoc b/doc/AUTHORS.adoc index 2a8363a..8d8f12c 100644 --- a/doc/AUTHORS.adoc +++ b/doc/AUTHORS.adoc @@ -1,5 +1,4 @@ -Ccache authors -============== += Ccache authors Ccache was originally written by Andrew Tridgell and is currently developed and maintained by Joel Rosdahl. @@ -8,8 +7,11 @@ Ccache is a collective work with contributions from many people, including: * Abubakar Nur Khalil * Aleksander Salwa +* Alexander Falkenstern * Alexander Korsunsky * Alexander Lanin +* Alexey Sheplyakov +* Alexey Telishev * Alexey Tourbin * Alfred Landrum * Anders F Björklund @@ -30,6 +32,7 @@ Ccache is a collective work with contributions from many people, including: * Chris Burr * Clemens Rabe * Cristian Adam +* Daniel Richtmann * David Givone * Deepak Yadav * Doug Anderson @@ -37,6 +40,7 @@ Ccache is a collective work with contributions from many people, including: * Enrico Sorichetti * Erik Flodin * Evangelos Foutras +* Florin Trofin * Francois Marier * Gabriel Scherer * Geert Bosch @@ -49,6 +53,7 @@ Ccache is a collective work with contributions from many people, including: * Igor Pylypiv * Ivan Vaigult * Ivan Volnov +* Jacob Young * Jiang Jiang * Joel Galenson * Joel Rosdahl @@ -64,15 +69,17 @@ Ccache is a collective work with contributions from many people, including: * Ka Ho Ng * Karl Chen * Khem Raj +* Kira Bruneau * Kona Blend * Kovarththanan Rajaratnam * Lalit Chhabra * Lars Gustäbel * Leanid Chaika * Loïc Yhuel +* Louis Caron * Luboš Luňák -* luzpaz * Maarten Maathuis +* Marius Zwicker * Mark Starovoytov * Martin Ettl * Martin Pool @@ -80,6 +87,7 @@ Ccache is a collective work with contributions from many people, including: * Matthias Kretz * Matt Whitlock * Melven Roehrig-Zoellner +* Michael Kruse * Michael Marineau * Michael Meeks * Michał Mirosław @@ -95,6 +103,7 @@ Ccache is a collective work with contributions from many people, including: * Nick Schultz * Norbert Lange * Oded Shimon +* Oleg Sidorkin * Olle Liljenzin * Orgad Shaneh * Orion Poplawski @@ -111,18 +120,24 @@ Ccache is a collective work with contributions from many people, including: * Peter Steinberger * Petr Štetiar * Philippe Proulx +* Philipp Gortan * Philipp Storz * Rafael Kitover +* Raihaan Shouhell * Ramiro Polla * Robert Yang * Robin H. Johnson * Rolf Bjarne Kvinge -* RW +* Rosen Penev +* R. Voggenauer * Ryan Brown +* Ryan Burns * Ryan Egesdahl * Sam Gross * Sergei Trofimovich +* Sergey Semushin * Steffen Dettmer +* Steve Mokris * Stuart Henderson * Sumit Jamgade * Thomas Otto @@ -131,8 +146,11 @@ Ccache is a collective work with contributions from many people, including: * Tim Potter * Tomasz Miąsko * Tom Hughes +* Tom Stellard * Tor Arne Vestbø * Vadim Petrochenkov +* Varun Sharma +* Vili Väinölä * Ville Skyttä * William S Fulton * Wilson Snyder diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index c5ce224..997e49d 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,70 +1,54 @@ -find_program(ASCIIDOC_EXE asciidoc) -mark_as_advanced(ASCIIDOC_EXE) # Don't show in CMake UIs +find_program(ASCIIDOCTOR_EXE asciidoctor) +mark_as_advanced(ASCIIDOCTOR_EXE) # Don't show in CMake UIs -if(NOT ASCIIDOC_EXE) - message(WARNING "Could not find asciidoc; documentation will not be generated") +if(NOT ASCIIDOCTOR_EXE) + message(WARNING "Could not find asciidoctor; documentation will not be generated") else() - # - # HTML documentation - # - function(generate_html adoc_file) + function(generate_doc backend adoc_file output_file) get_filename_component(base_name "${adoc_file}" NAME_WE) - set(html_file "${base_name}.html") add_custom_command( - OUTPUT "${html_file}" + OUTPUT "${output_file}" COMMAND - ${ASCIIDOC_EXE} - -o "${html_file}" + ${ASCIIDOCTOR_EXE} + -o "${output_file}" -a revnumber="${CCACHE_VERSION}" - -a toc - -b xhtml11 + -a icons=font + -a toc=left + -a sectanchors + -a stylesheet="${CMAKE_CURRENT_SOURCE_DIR}/ccache-doc.css" + -b "${backend}" "${CMAKE_SOURCE_DIR}/${adoc_file}" MAIN_DEPENDENCY "${CMAKE_SOURCE_DIR}/${adoc_file}" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/ccache-doc.css" ) - set(html_files "${html_files}" "${html_file}" PARENT_SCOPE) + set(doc_files "${doc_files}" "${output_file}" PARENT_SCOPE) endfunction() - generate_html(LICENSE.adoc) - generate_html(doc/AUTHORS.adoc) - generate_html(doc/MANUAL.adoc) - generate_html(doc/NEWS.adoc) - - add_custom_target(doc-html DEPENDS "${html_files}") - set(doc_files "${html_files}") + # + # HTML documentation + # + generate_doc(html LICENSE.adoc LICENSE.html) + generate_doc(html doc/AUTHORS.adoc AUTHORS.html) + generate_doc(html doc/MANUAL.adoc MANUAL.html) + generate_doc(html doc/NEWS.adoc NEWS.html) + add_custom_target(doc-html DEPENDS "${doc_files}") # # Man page # - find_program(A2X_EXE a2x) - mark_as_advanced(A2X_EXE) # Don't show in CMake UIs - if(NOT A2X_EXE) - message(WARNING "Could not find a2x; man page will not be generated") - else() - # MANUAL.adoc -> MANUAL.xml -> man page - add_custom_command( - OUTPUT MANUAL.xml - COMMAND - ${ASCIIDOC_EXE} - -o - - -a revnumber=${CCACHE_VERSION} - -d manpage - -b docbook "${CMAKE_SOURCE_DIR}/doc/MANUAL.adoc" - | perl -pe 's!\(.*?\)!\\1!g' - >MANUAL.xml - MAIN_DEPENDENCY "${CMAKE_SOURCE_DIR}/doc/MANUAL.adoc" - ) - add_custom_command( - OUTPUT ccache.1 - COMMAND ${A2X_EXE} --doctype manpage --format manpage MANUAL.xml - MAIN_DEPENDENCY MANUAL.xml - ) - add_custom_target(doc-man-page DEPENDS ccache.1) - install( - FILES "${CMAKE_CURRENT_BINARY_DIR}/ccache.1" - DESTINATION "${CMAKE_INSTALL_MANDIR}/man1" - ) - set(doc_files "${doc_files}" ccache.1) - endif() + generate_doc(manpage doc/MANUAL.adoc ccache.1.tmp) + add_custom_command( + OUTPUT ccache.1 + # Convert monospace to bold since that's typically rendered better when + # viewing the man page. + COMMAND perl -pe "'s!\\\\f\\(CR(.*?)\\\\fP!\\\\fB\\1\\\\fP!g'" ccache.1.tmp >ccache.1 + MAIN_DEPENDENCY ccache.1.tmp + ) + install( + FILES "${CMAKE_CURRENT_BINARY_DIR}/ccache.1" + DESTINATION "${CMAKE_INSTALL_MANDIR}/man1" + ) + add_custom_target(doc-man-page DEPENDS ccache.1) - add_custom_target(doc ALL DEPENDS "${doc_files}") + add_custom_target(doc ALL DEPENDS doc-html doc-man-page) endif() diff --git a/doc/INSTALL.md b/doc/INSTALL.md index a90a24f..e076387 100644 --- a/doc/INSTALL.md +++ b/doc/INSTALL.md @@ -6,70 +6,86 @@ Prerequisites To build ccache you need: -- CMake 3.4.3 or newer. -- A C++11 compiler. See [Supported platforms, compilers and +- CMake 3.15 or newer. +- A C++17 compiler. See [Supported platforms, compilers and languages](https://ccache.dev/platform-compiler-language-support.html) for details. - A C99 compiler. -- [libzstd](http://www.zstd.net). If you don't have libzstd installed and - can't or don't want to install it in a standard system location, there are - two options: - 1. Install zstd in a custom path and set `CMAKE_PREFIX_PATH` to it, e.g. - by passing `-DCMAKE_PREFIX_PATH=/some/custom/path` to `cmake`, or - 2. Pass `-DZSTD_FROM_INTERNET=ON` to `cmake` to make it download libzstd - from the Internet and unpack it in the local binary tree. Ccache will - then be linked statically to the locally built libzstd. +- [libzstd](http://www.zstd.net). If you don't have libzstd installed and can't + or don't want to install it in a standard system location, it will be + automatically downloaded, built and linked statically as part of the build + process. To disable this, pass `-DZSTD_FROM_INTERNET=OFF` to `cmake`. You can + also install zstd in a custom path and pass + `-DCMAKE_PREFIX_PATH=/some/custom/path` to `cmake`. - To link libzstd statically you can use `-DZSTD_LIBRARY=/path/to/libzstd.a`. + To link libzstd statically (and you have a static libzstd available), pass + `-DSTATIC_LINK=ON` to `cmake`. This is the default on Windows. Alternatively, + use `-DZSTD_LIBRARY=/path/to/libzstd.a`. Optional: +- [hiredis](https://github.com/redis/hiredis) for the Redis storage backend. If + you don't have libhiredis installed and can't or don't want to install it in a + standard system location, it will be automatically downloaded, built and + linked statically as part of the build process. To disable this, pass + `-DHIREDIS_FROM_INTERNET=OFF` to cmake. You can also install hiredis in a + custom path and pass `-DCMAKE_PREFIX_PATH=/some/custom/path` to `cmake`. + + To link libhiredis statically (and you have a static libhiredis available), + pass `-DSTATIC_LINK=ON` to `cmake`. This is the default on Windows. + Alternatively, use `-DHIREDIS_LIBRARY=/path/to/libhiredis.a`. - GNU Bourne Again SHell (bash) for tests. -- [AsciiDoc](https://www.methods.co.nz/asciidoc/) to build the HTML - documentation. - - Tip: On Debian-based systems (e.g. Ubuntu), install the `docbook-xml` and - `docbook-xsl` packages in addition to `asciidoc`. Without the former the - man page generation will be very slow. -- [xsltproc](http://xmlsoft.org/XSLT/xsltproc2.html) to build the man page. +- [Asciidoctor](https://asciidoctor.org) to build the HTML documentation. - [Python](https://www.python.org) to debug and run the performance test suite. +Reference configurations: + +- See [CI configurations](../.github/workflows/build.yaml) for a selection of + regularly tested build setups, including cross-compiling and explicit + dependencies required in Debian/Ubuntu environment. Installation ------------ Here is the typical way to build and install ccache: - mkdir build - cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - make - make install +```bash +mkdir build +cd build +cmake -DCMAKE_BUILD_TYPE=Release .. +make +make install +``` You can set the installation directory to e.g. `/usr` by adding `-DCMAKE_INSTALL_PREFIX=/usr` to the `cmake` command. You can set the directory -where the secondary configuration file should be located to e.g. `/etc` by -adding `-DCMAKE_INSTALL_SYSCONFDIR=/etc`. - -There are two ways to use ccache. You can either prefix your compilation -commands with `ccache` or you can create a symbolic link (named as your -compiler) to ccache. The first method is most convenient if you just want to -try out ccache or wish to use it for some specific projects. The second method -is most useful for when you wish to use ccache for all your compilations. - -To install for usage by the first method just copy ccache to somewhere in your -path. - -To install for the second method, do something like this: - - cp ccache /usr/local/bin/ - ln -s ccache /usr/local/bin/gcc - ln -s ccache /usr/local/bin/g++ - ln -s ccache /usr/local/bin/cc - ln -s ccache /usr/local/bin/c++ - -And so forth. This will work as long as `/usr/local/bin` comes before the path -to the compiler (which is usually in `/usr/bin`). After installing you may wish -to run `which gcc` to make sure that the correct link is being used. - -NOTE: Do not use a hard link, use a symbolic link. A hard link will cause -"interesting" problems. +where the system configuration file should be located to e.g. `/etc` by adding +`-DCMAKE_INSTALL_SYSCONFDIR=/etc`. + +There are two different ways to use ccache to cache a compilation: + +1. Prefix your compilation command with `ccache`. This method is most convenient + if you just want to try out ccache or wish to use it for some specific + projects. +2. Let ccache masquerade as the compiler. This method is most useful when you + wish to use ccache for all your compilations. To do this, create a symbolic + link to ccache named as the compiler. For example, here is how to set up + ccache to masquerade as `gcc` and `g++`: + + ```bash + cp ccache /usr/local/bin/ + ln -s ccache /usr/local/bin/gcc + ln -s ccache /usr/local/bin/g++ + ``` + + On platforms that don't support symbolic links you can simply copy ccache to the + compiler name instead for a similar effect: + + ```bash + cp ccache /usr/local/bin/gcc + cp ccache /usr/local/bin/g++ + ``` + + And so forth. This will work as long as the directory with symbolic links or + ccache copies comes before the directory with the compiler (typically + `/usr/bin`) in `PATH`. diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index 25c34c0..48328fa 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -1,27 +1,20 @@ -CCACHE(1) -========= -:man source: ccache -:man version: {revnumber} -:man manual: ccache Manual += ccache(1) +:mansource: Ccache {revnumber} - -Name ----- +== Name ccache - a fast C/C++ compiler cache -Synopsis --------- +== Synopsis [verse] *ccache* [_options_] *ccache* _compiler_ [_compiler options_] -_compiler_ [_compiler options_] (via symbolic link) +_compiler_ [_compiler options_] (ccache masquerading as the compiler) -Description ------------ +== Description Ccache is a compiler cache. It speeds up recompilation by caching the result of previous compilations and detecting when the same compilation is being done @@ -30,57 +23,61 @@ again. Ccache has been carefully written to always produce exactly the same compiler output that you would get without the cache. The only way you should be able to tell that you are using ccache is the speed. Currently known exceptions to this -goal are listed under <<_caveats,CAVEATS>>. If you discover an undocumented case -where ccache changes the output of your compiler, please let us know. - +goal are listed under _<>_. If you discover an undocumented case where +ccache changes the output of your compiler, please let us know. -Run modes ---------- -There are two ways to use ccache. You can either prefix your compilation -commands with *ccache* or you can let ccache masquerade as the compiler by -creating a symbolic link (named as the compiler) to ccache. The first method is -most convenient if you just want to try out ccache or wish to use it for some -specific projects. The second method is most useful for when you wish to use -ccache for all your compilations. +== Run modes -To use the first method, just make sure that *ccache* is in your *PATH*. - -To use the symlinks method, do something like this: +There are two different ways to use ccache to cache a compilation: +1. Prefix your compilation command with `ccache`. This method is most convenient + if you just want to try out ccache or wish to use it for some specific + projects. Example: ++ +------------------------------------------------------------------------------- +ccache gcc -c example.c +------------------------------------------------------------------------------- ++ +2. Let ccache masquerade as the compiler. This method is most useful when you + wish to use ccache for all your compilations. To do this, create a symbolic + link to ccache named as the compiler. For example, here is set up ccache to + masquerade as `gcc` and `g++`: ++ ------------------------------------------------------------------------------- cp ccache /usr/local/bin/ ln -s ccache /usr/local/bin/gcc ln -s ccache /usr/local/bin/g++ -ln -s ccache /usr/local/bin/cc -ln -s ccache /usr/local/bin/c++ ------------------------------------------------------------------------------- - -And so forth. This will work as long as the directory with symlinks comes -before the path to the compiler (which is usually in `/usr/bin`). After -installing you may wish to run ``which gcc'' to make sure that the correct link -is being used. - ++ +On platforms that don't support symbolic links you can simply copy ccache to the +compiler name instead for a similar effect: ++ +------------------------------------------------------------------------------- +cp ccache /usr/local/bin/gcc +cp ccache /usr/local/bin/g++ +------------------------------------------------------------------------------- ++ +And so forth. This will work as long as the directory with symbolic links or +ccache copies comes before the directory with the compiler (typically +`/usr/bin`) in `PATH`. ++ WARNING: The technique of letting ccache masquerade as the compiler works well, -but currently doesn't interact well with other tools that do the same thing. -See _<<_using_ccache_with_other_compiler_wrappers,Using ccache with other -compiler wrappers>>_. +but currently doesn't interact well with other tools that do the same thing. See +_<>_. -WARNING: Use a symbolic links for masquerading, not hard links. -Command line options --------------------- +== Command line options -These command line options only apply when you invoke ccache as ``ccache''. -When invoked as a compiler (via a symlink as described in the previous -section), the normal compiler options apply and you should refer to the -compiler's documentation. +These command line options only apply when you invoke ccache as "`ccache`". When +ccache masquerades as a compiler (as described in the previous section), the +normal compiler options apply and you should refer to the compiler's +documentation. -Common options -~~~~~~~~~~~~~~ +=== Common options -*`-c`*, *`--cleanup`*:: +*-c*, *--cleanup*:: Clean up the cache by removing old cached files until the specified file number and cache size limits are not exceeded. This also recalculates the @@ -90,41 +87,47 @@ Common options cleanup is mostly useful if you manually modify the cache contents or believe that the cache size statistics may be inaccurate. -*`-C`*, *`--clear`*:: +*-C*, *--clear*:: Clear the entire cache, removing all cached files, but keeping the configuration file. -*`--config-path`* _PATH_:: +*--config-path* _PATH_:: - Let the subsequent command line options operate on configuration file - _PATH_ instead of the default. Using this option has the same effect as - setting the environment variable `CCACHE_CONFIGPATH` temporarily. + Let the command line options operate on configuration file _PATH_ instead of + the default. Using this option has the same effect as setting (overriding) + the environment variable `CCACHE_CONFIGPATH` temporarily. -*`-d`*, *`--directory`* _PATH_:: +*-d*, *--dir* _PATH_:: - Let the subsequent command line options operate on cache directory _PATH_ - instead of the default. For example, to show statistics for a cache - directory at `/shared/ccache` you can run `ccache -d /shared/ccache -s`. - Using this option has the same effect as setting the environment variable - `CCACHE_DIR` temporarily. + Let the command line options operate on cache directory _PATH_ instead of + the default. For example, to show statistics for a cache directory at + `/shared/ccache` you can run `ccache -d /shared/ccache -s`. Using this option + has the same effect as setting the environment variable `CCACHE_DIR` + temporarily. -*`--evict-older-than`* _AGE_:: +*--evict-namespace* _NAMESPACE_:: + + Remove files created in the given <> from the + cache. + +*--evict-older-than* _AGE_:: Remove files older than _AGE_ from the cache. _AGE_ should be an unsigned - integer with a `d` (days) or `s` (seconds) suffix. + integer with a `d` (days) or `s` (seconds) suffix. If combined with + `--evict-namespace`, only remove old files within that namespace. -*`-h`*, *`--help`*:: +*-h*, *--help*:: Print a summary of command line options. -*`-F`* _NUM_, *`--max-files`* _NUM_:: +*-F* _NUM_, *--max-files* _NUM_:: Set the maximum number of files allowed in the cache to _NUM_. Use 0 for no limit. The value is stored in a configuration file in the cache directory and applies to all future compilations. -*`-M`* _SIZE_, *`--max-size`* _SIZE_:: +*-M* _SIZE_, *--max-size* _SIZE_:: Set the maximum size of the files stored in the cache. _SIZE_ should be a number followed by an optional suffix: k, M, G, T (decimal), Ki, Mi, Gi or @@ -132,97 +135,134 @@ Common options stored in a configuration file in the cache directory and applies to all future compilations. -*`-X`* _LEVEL_, *`--recompress`* _LEVEL_:: +*-X* _LEVEL_, *--recompress* _LEVEL_:: Recompress the cache to level _LEVEL_ using the Zstandard algorithm. The level can be an integer, with the same semantics as the - <> configuration option), or + <> configuration option, or the special value *uncompressed* for no compression. See - _<<_cache_compression,Cache compression>>_ for more information. This can - potentionally take a long time since all files in the cache need to be - visited. Only files that are currently compressed with a different level - than _LEVEL_ will be recompressed. + _<>_ for more information. This can potentionally take a + long time since all files in the cache need to be visited. Only files that + are currently compressed with a different level than _LEVEL_ will be + recompressed. -*`-o`* _KEY=VALUE_, *`--set-config`* _KEY_=_VALUE_:: +*-o* _KEY=VALUE_, *--set-config* _KEY_=_VALUE_:: - Set configuration option _KEY_ to _VALUE_. See - _<<_configuration,Configuration>>_ for more information. + Set configuration option _KEY_ to _VALUE_. See _<>_ for more + information. -*`-x`*, *`--show-compression`*:: +*-x*, *--show-compression*:: - Print cache compression statistics. See _<<_cache_compression,Cache - compression>>_ for more information. This can potentionally take a long - time since all files in the cache need to be visited. + Print cache compression statistics. See _<>_ for more + information. This can potentionally take a long time since all files in the + cache need to be visited. -*`-p`*, *`--show-config`*:: +*-p*, *--show-config*:: Print current configuration options and from where they originate (environment variable, configuration file or compile-time default) in human-readable format. -*`-s`*, *`--show-stats`*:: +*--show-log-stats*:: + + Print statistics counters from the stats log in human-readable format. See + <>. Use `-v`/`--verbose` once or twice for + more details. + +*-s*, *--show-stats*:: Print a summary of configuration and statistics counters in human-readable - format. + format. Use `-v`/`--verbose` once or twice for more details. + +*-v*, *--verbose*:: -*`-V`*, *`--version`*:: + Increase verbosity. The option can be given multiple times. + +*-V*, *--version*:: Print version and copyright information. -*`-z`*, *`--zero-stats`*:: +*-z*, *--zero-stats*:: Zero the cache statistics (but not the configuration options). -Options for scripting or debugging -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Options for remote file-based storage + +*--trim-dir* _PATH_:: + + Remove old files from directory _PATH_ until it is at most the size + specified by `--trim-max-size`. ++ +WARNING: Don't use this option to trim the local cache. To trim the local cache +directory to a certain size, use `CCACHE_MAXSIZE=_SIZE_ ccache -c`. + +*--trim-max-size* _SIZE_:: + + Specify the maximum size for `--trim-dir`. _SIZE_ should be a number + followed by an optional suffix: k, M, G, T (decimal), Ki, Mi, Gi or Ti + (binary). The default suffix is G. -*`--checksum-file`* _PATH_:: +*--trim-method* _METHOD_:: + + Specify the method to trim a directory with `--trim-dir`. Possible values + are: ++ +-- +*atime*:: + LRU (least recently used) using the file access timestamp. This is the + default. +*mtime*:: + LRU (least recently used) using the file modification timestamp. +-- - Print the checksum (64 bit XXH3) of the file at _PATH_. +=== Options for scripting or debugging -*`--dump-manifest`* _PATH_:: +*--checksum-file* _PATH_:: - Dump manifest file at _PATH_ in text format to standard output. This is - only useful when debugging ccache and its behavior. + Print the checksum (128 bit XXH3) of the file at _PATH_ (`-` for standard + input). -*`--dump-result`* _PATH_:: +*--extract-result* _PATH_:: - Dump result file at _PATH_ in text format to standard output. This is only - useful when debugging ccache and its behavior. + Extract data stored in the result file at _PATH_ (`-` for standard input). + The data will be written to `ccache-result.*` files in to the current + working directory. This option is only useful when debugging ccache and its + behavior. -*`--extract-result`* _PATH_:: +*-k* _KEY_, *--get-config* _KEY_:: - Extract data stored in the result file at _PATH_. The data will be written - to *ccache-result.** files in to the current working directory. This is - only useful when debugging ccache and its behavior. + Print the value of configuration option _KEY_. See _<>_ for + more information. -*`-k`* _KEY_, *`--get-config`* _KEY_:: +*--hash-file* _PATH_:: - Print the value of configuration option _KEY_. See - _<<_configuration,Configuration>>_ for more information. + Print the hash (160 bit BLAKE3) of the file at _PATH_ (`-` for standard + input). This is only useful when debugging ccache and its behavior. -*`--hash-file`* _PATH_:: +*--inspect* _PATH_:: - Print the hash (160 bit BLAKE3) of the file at _PATH_. This is only useful - when debugging ccache and its behavior. + Print the content of a result or manifest file at _PATH_ (`-` for standard + input) to standard output in human-readable format. File content embedded in + a result file will however not be printed; use `--extract-result` to extract + the file content. This option is only useful when debugging ccache and its + behavior. -*`--print-stats`*:: +*--print-stats*:: Print statistics counter IDs and corresponding values in machine-parsable (tab-separated) format. -Extra options -~~~~~~~~~~~~~ +=== Extra options When run as a compiler, ccache usually just takes the same command line options as the compiler you are using. The only exception to this is the option -*--ccache-skip*. That option can be used to tell ccache to avoid interpreting +`--ccache-skip`. That option can be used to tell ccache to avoid interpreting the next option in any way and to pass it along to the compiler as-is. -NOTE: *--ccache-skip* currently only tells ccache not to interpret the next +NOTE: `--ccache-skip` currently only tells ccache not to interpret the next option as a special compiler option -- the option will still be included in the direct mode hash. @@ -231,60 +271,74 @@ line and determine what is an input filename and what is a compiler option, as it needs the input filename to determine the name of the resulting object file (among other things). The heuristic ccache uses when parsing the command line is that any argument that exists as a file is treated as an input file name. By -using *--ccache-skip* you can force an option to not be treated as an input +using `--ccache-skip` you can force an option to not be treated as an input file name and instead be passed along to the compiler as a command line option. -Another case where *--ccache-skip* can be useful is if ccache interprets an +Another case where `--ccache-skip` can be useful is if ccache interprets an option specially but shouldn't, since the option has another meaning for your compiler than what ccache thinks. -Configuration -------------- +== Configuration -ccache's default behavior can be overridden by options in configuration files, +Ccache's default behavior can be overridden by options in configuration files, which in turn can be overridden by environment variables with names starting -with *CCACHE_*. Ccache normally reads configuration from two files: first a +with `CCACHE_`. Ccache normally reads configuration from two files: first a system-level configuration file and secondly a cache-specific configuration file. The priorities of configuration options are as follows (where 1 is highest): 1. Environment variables. -2. The primary (cache-specific) configuration file (see below). -3. The secondary (system-wide read-only) configuration file - *__/ccache.conf* (typically */etc/ccache.conf* or - */usr/local/etc/ccache.conf*). +2. The cache-specific configuration file (see below). +3. The system (read-only) configuration file `/ccache.conf` + (typically `/etc/ccache.conf` or `/usr/local/etc/ccache.conf`). 4. Compile-time defaults. -As a special case, if the the environment variable *CCACHE_CONFIGPATH* is set -it specifies the primary configuration file and the secondary (system-wide) -configuration file won't be read. +As a special case, if the environment variable `CCACHE_CONFIGPATH` is set it +specifies the configuration file, and the system configuration file won't be +read. -Location of the primary configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Location of the configuration file -The location of the primary (cache-specific) configuration is determined like -this: +The location of the cache-specific configuration file is determined like this on +non-Windows systems: -1. If *CCACHE_CONFIGPATH* is set, use that path. -2. Otherwise, if the environment variable *CCACHE_DIR* is set then use - *$CCACHE_DIR/ccache.conf*. -3. Otherwise, if <> is set in the secondary - (system-wide) configuration file then use */ccache.conf*. -4. Otherwise, if there is a legacy *$HOME/.ccache* directory then use - *$HOME/.ccache/ccache.conf*. -5. Otherwise, if *XDG_CONFIG_HOME* is set then use - *$XDG_CONFIG_HOME/ccache/ccache.conf*. -6. Otherwise, use *%APPDATA%/ccache/ccache.conf* (Windows), - *$HOME/Library/Preferences/ccache/ccache.conf* (macOS) or - *$HOME/.config/ccache/ccache.conf* (other systems). +1. If `CCACHE_CONFIGPATH` is set, use that path. +2. Otherwise, if the environment variable `CCACHE_DIR` is set then use + `$CCACHE_DIR/ccache.conf`. +3. Otherwise, if <> is set in the system + configuration file then use `/ccache.conf`. +4. Otherwise, if there is a legacy `$HOME/.ccache` directory then use + `$HOME/.ccache/ccache.conf`. +5. Otherwise, if `XDG_CONFIG_HOME` is set then use + `$XDG_CONFIG_HOME/ccache/ccache.conf`. +6. Otherwise, use + `$HOME/Library/Preferences/ccache/ccache.conf` (macOS) or + `$HOME/.config/ccache/ccache.conf` (other systems). +On Windows, this is the method used to find the configuration file: -Configuration file syntax -~~~~~~~~~~~~~~~~~~~~~~~~~ +1. If `CCACHE_CONFIGPATH` is set, use that path. +2. Otherwise, if the environment variable `CCACHE_DIR` is set then use + `%CCACHE_DIR%/ccache.conf`. +3. Otherwise, if <> is set in the system + configuration file then use `\ccache.conf`. The + system-wide configuration on Windows is + `%ALLUSERSPROFILE%\ccache\ccache.conf` by default. The `ALLUSERSPROFILE` + environment variable is usually `C:\ProgramData`. +4. Otherwise, if there is a legacy `%USERPROFILE%\.ccache` directory then use + `%USERPROFILE%\.ccache\ccache.conf`. +5. Otherwise, use `%LOCALAPPDATA%\ccache\ccache.conf` if it exists. +6. Otherwise, use `%APPDATA%\ccache\ccache.conf`. -Configuration files are in a simple ``key = value'' format, one option per +See also the <> configuration option for how the +cache directory location is determined. + + +=== Configuration file syntax + +Configuration files are in a simple "`key = value`" format, one option per line. Lines starting with a hash sign are comments. Blank lines are ignored, as is whitespace surrounding keys and values. Example: @@ -293,31 +347,31 @@ is whitespace surrounding keys and values. Example: max_size = 10G ------------------------------------------------------------------------------- -Boolean values -~~~~~~~~~~~~~~ + +=== Boolean values Some configuration options are boolean values (i.e. truth values). In a configuration file, such values must be set to the string *true* or *false*. For the corresponding environment variables, the semantics are a bit different: -* A set environment variable means ``true'' (even if set to the empty string). +* A set environment variable means "`true`" (even if set to the empty string). * The following case-insensitive negative values are considered an error (instead of surprising the user): *0*, *false*, *disable* and *no*. -* An unset environment variable means ``false''. +* An unset environment variable means "`false`". Each boolean environment variable also has a negated form starting with -*CCACHE_NO*. For example, *CCACHE_COMPRESS* can be set to force compression and -*CCACHE_NOCOMPRESS* can be set to force no compression. +`CCACHE_NO`. For example, `CCACHE_COMPRESS` can be set to force compression and +`CCACHE_NOCOMPRESS` can be set to force no compression. -Configuration options -~~~~~~~~~~~~~~~~~~~~~~ +=== Configuration options Below is a list of available configuration options. The corresponding environment variable name is indicated in parentheses after each configuration option key. -[[config_absolute_paths_in_stderr]] *absolute_paths_in_stderr* (*CCACHE_ABSSTDERR*):: +[#config_absolute_paths_in_stderr] +*absolute_paths_in_stderr* (*CCACHE_ABSSTDERR*):: This option specifies whether ccache should rewrite relative paths in the compiler's standard error output to absolute paths. This can be useful if @@ -326,16 +380,16 @@ option key. working directory, which makes relative paths in compiler errors or warnings incorrect. The default is false. -[[config_base_dir]] *base_dir* (*CCACHE_BASEDIR*):: +[#config_base_dir] +*base_dir* (*CCACHE_BASEDIR*):: This option should be an absolute path to a directory. If set, ccache will - rewrite absolute paths into paths relative to the current working - directory, but only absolute paths that begin with *base_dir*. Cache - results can then be shared for compilations in different directories even - if the project uses absolute paths in the compiler command line. See also - the discussion under _<<_compiling_in_different_directories,Compiling in - different directories>>_. If set to the empty string (which is the - default), no rewriting is done. + rewrite absolute paths into paths relative to the current working directory, + but only absolute paths that begin with *base_dir*. Cache results can then + be shared for compilations in different directories even if the project uses + absolute paths in the compiler command line. See also the discussion under + _<>_. If set to the empty string (which + is the default), no rewriting is done. + A typical path to use as *base_dir* is your home directory or another directory that is a parent of your project directories. Don't use `/` as the base @@ -381,32 +435,48 @@ differ. With *base_dir* set to `/` there will be a cache miss since the relative path to `/usr/include/example` will be different. With *base_dir* set to `/home/bob/stuff/project1` there will a cache miss since the path to project2 will be a different absolute path. ++ +WARNING: Rewriting absolute paths to relative is kind of a brittle hack. It +works OK in many cases, but there might be cases where things break. One known +issue is that absolute paths are not reproduced in dependency files, which can +mess up dependency detection in tools like Make and Ninja. If possible, use +relative paths in the first place instead instead of using *base_dir*. -[[config_cache_dir]] *cache_dir* (*CCACHE_DIR*):: +[#config_cache_dir] +*cache_dir* (*CCACHE_DIR*):: This option specifies where ccache will keep its cached compiler outputs. - The default is *$XDG_CACHE_HOME/ccache* if *XDG_CACHE_HOME* is set, - otherwise *$HOME/.cache/ccache*. Exception: If the legacy directory - *$HOME/.ccache* exists then that directory is the default. + -See also _<<_location_of_the_primary_configuration_file,Location of the primary -configuration file>>_. +On non-Windows systems, the default is `$HOME/.ccache` if such a directory +exists, otherwise `$XDG_CACHE_HOME/ccache` if `XDG_CACHE_HOME` is set, otherwise +`$HOME/Library/Caches/ccache` (macOS) or `$HOME/.config/ccache` (other systems). + -If you want to use another *CCACHE_DIR* value temporarily for one ccache -invocation you can use the `-d/--directory` command line option instead. +On Windows, the default is `%USERPROFILE%\.ccache` if such a directory exists, +otherwise `%LOCALAPPDATA%\ccache`. ++ +WARNING: Previous ccache versions defaulted to storing the cache in +`%APPDATA%\ccache` on Windows. This can result in large network file transfers +of the cache in domain environments and similar problems. Please check this +directory for cache directories and either delete them or the whole directory, +or move them to the `%LOCALAPPDATA%\ccache` directory. ++ +See also _<>_. -[[config_compiler]] *compiler* (*CCACHE_COMPILER* or (deprecated) *CCACHE_CC*):: +[#config_compiler] +*compiler* (*CCACHE_COMPILER* or (deprecated) *CCACHE_CC*):: This option can be used to force the name of the compiler to use. If set to the empty string (which is the default), ccache works it out from the command line. -[[config_compiler_check]] *compiler_check* (*CCACHE_COMPILERCHECK*):: +[#config_compiler_check] +*compiler_check* (*CCACHE_COMPILERCHECK*):: - By default, ccache includes the modification time (``mtime'') and size of + By default, ccache includes the modification time ("`mtime`") and size of the compiler in the hash to ensure that results retrieved from the cache - are accurate. This option can be used to select another strategy. Possible - values are: + are accurate. If compiler plugins are used, these plugins will also be + added to the hash. This option can be used to select another strategy. + Possible values are: + -- *content*:: @@ -433,35 +503,32 @@ _a command string_:: path to the compiler. Several commands can be specified with semicolon as separator. Examples: + --- - ---- %compiler% -v ---- - ++ ---- %compiler% -dumpmachine; %compiler% -dumpversion ---- - ++ You should make sure that the specified command is as fast as possible since it will be run once for each ccache invocation. - ++ Identifying the compiler using a command is useful if you want to avoid cache misses when the compiler has been rebuilt but not changed. - ++ Another case is when the compiler (as seen by ccache) actually isn't the real compiler but another compiler wrapper -- in that case, the default *mtime* method will hash the mtime and size of the other compiler wrapper, which means -that ccache won't be able to detect a compiler upgrade. Using a suitable -command to identify the compiler is thus safer, but it's also slower, so you -should consider continue using the *mtime* method in combination with the +that ccache won't be able to detect a compiler upgrade. Using a suitable command +to identify the compiler is thus safer, but it's also slower, so you should +consider continue using the *mtime* method in combination with the *prefix_command* option if possible. See -_<<_using_ccache_with_other_compiler_wrappers,Using ccache with other compiler -wrappers>>_. --- +_<>_. -- -[[config_compiler_type]] *compiler_type* (*CCACHE_COMPILERTYPE*):: +[#config_compiler_type] +*compiler_type* (*CCACHE_COMPILERTYPE*):: Ccache normally guesses the compiler type based on the compiler name. The *compiler_type* option lets you force a compiler type. This can be useful @@ -474,17 +541,22 @@ wrappers>>_. symlinks). This is the default. *clang*:: Clang-based compiler. +*clang-cl*:: + clang-cl. *gcc*:: GCC-based compiler. +*icl*:: + Intel compiler on Windows. +*msvc*:: + Microsoft Visual C++ (MSVC). *nvcc*:: NVCC (CUDA) compiler. *other*:: Any compiler other than the known types. -*pump*:: - distcc's "pump" script. -- -[[config_compression]] *compression* (*CCACHE_COMPRESS* or *CCACHE_NOCOMPRESS*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_compression] +*compression* (*CCACHE_COMPRESS* or *CCACHE_NOCOMPRESS*, see _<>_ above):: If true, ccache will compress data it puts in the cache. However, this option has no effect on how files are retrieved from the cache; compressed @@ -500,7 +572,8 @@ Compression will be disabled if file cloning (the <> option) or hard linking (the <> option) is enabled. -[[config_compression_level]] *compression_level* (*CCACHE_COMPRESSLEVEL*):: +[#config_compression_level] +*compression_level* (*CCACHE_COMPRESSLEVEL*):: This option determines the level at which ccache will compress object files using the real-time compression algorithm Zstandard. It only has effect if @@ -520,11 +593,11 @@ Semantics of *compression_level*: essentially the same for all levels. As a rule of thumb, use level 5 or lower since higher levels may slow down compilations noticeably. Higher levels are however useful when recompressing the cache with command line - option *-X/--recompress*. + option `-X`/`--recompress`. *< 0*:: - A negative value corresponds to Zstandard's ``ultra-fast'' compression + A negative value corresponds to Zstandard's "`ultra-fast`" compression levels, which are even faster than level 1 but with less good compression - ratios. For instance, level *-3* corresponds to ``--fast=3'' for the *zstd* + ratios. For instance, level *-3* corresponds to `--fast=3` for the `zstd` command line tool. In practice, there is little use for levels lower than *-5* or so. *0* (default):: @@ -534,26 +607,28 @@ Semantics of *compression_level*: + See the http://zstd.net[Zstandard documentation] for more information. -[[config_cpp_extension]] *cpp_extension* (*CCACHE_EXTENSION*):: +[#config_cpp_extension] +*cpp_extension* (*CCACHE_EXTENSION*):: This option can be used to force a certain extension for the intermediate preprocessed file. The default is to automatically determine the extension to use for intermediate preprocessor files based on the type of file being compiled, but that sometimes doesn't work. For example, when using the - ``aCC'' compiler on HP-UX, set the cpp extension to *i*. + "`aCC`" compiler on HP-UX, set the cpp extension to *i*. -[[config_debug]] *debug* (*CCACHE_DEBUG* or *CCACHE_NODEBUG*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_debug] +*debug* (*CCACHE_DEBUG* or *CCACHE_NODEBUG*, see _<>_ above):: If true, enable the debug mode. The debug mode creates per-object debug files that are helpful when debugging unexpected cache misses. Note however - that ccache performance will be reduced slightly. See - _<<_cache_debugging,Cache debugging>>_ for more information. The default is - false. + that ccache performance will be reduced slightly. See _<>_ + for more information. The default is false. -[[config_debug_dir]] *debug_dir* (*CCACHE_DEBUGDIR*):: +[#config_debug_dir] +*debug_dir* (*CCACHE_DEBUGDIR*):: - Specifies where to write per-object debug files if the _<>_ is enabled. If set to the empty string, the files will be written + Specifies where to write per-object debug files if the <> is enabled. If set to the empty string, the files will be written next to the object file. If set to a directory, the debug files will be written with full absolute paths in that directory, creating it if needed. The default is the empty string. @@ -561,35 +636,40 @@ See the http://zstd.net[Zstandard documentation] for more information. For example, if *debug_dir* is set to `/example`, the current working directory is `/home/user` and the object file is `build/output.o` then the debug log will be written to `/example/home/user/build/output.o.ccache-log`. See also -_<<_cache_debugging,Cache debugging>>_. +_<>_. -[[config_depend_mode]] *depend_mode* (*CCACHE_DEPEND* or *CCACHE_NODEPEND*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_depend_mode] +*depend_mode* (*CCACHE_DEPEND* or *CCACHE_NODEPEND*, see _<>_ above):: If true, the depend mode will be used. The default is false. See - _<<_the_depend_mode,The depend mode>>_. + _<>_. -[[config_direct_mode]] *direct_mode* (*CCACHE_DIRECT* or *CCACHE_NODIRECT*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_direct_mode] +*direct_mode* (*CCACHE_DIRECT* or *CCACHE_NODIRECT*, see _<>_ above):: If true, the direct mode will be used. The default is true. See - _<<_the_direct_mode,The direct mode>>_. + _<>_. -[[config_disable]] *disable* (*CCACHE_DISABLE* or *CCACHE_NODISABLE*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_disable] +*disable* (*CCACHE_DISABLE* or *CCACHE_NODISABLE*, see _<>_ above):: When true, ccache will just call the real compiler, bypassing the cache completely. The default is false. -[[config_extra_files_to_hash]] *extra_files_to_hash* (*CCACHE_EXTRAFILES*):: +[#config_extra_files_to_hash] +*extra_files_to_hash* (*CCACHE_EXTRAFILES*):: This option is a list of paths to files that ccache will include in the the hash sum that identifies the build. The list separator is semicolon on Windows systems and colon on other systems. -[[config_file_clone]] *file_clone* (*CCACHE_FILECLONE* or *CCACHE_NOFILECLONE*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_file_clone] +*file_clone* (*CCACHE_FILECLONE* or *CCACHE_NOFILECLONE*, see _<>_ above):: - If true, ccache will attempt to use file cloning (also known as ``copy on - write'', ``CoW'' or ``reflinks'') to store and fetch cached compiler results. - *file_clone* has priority over <>. The - default is false. + If true, ccache will attempt to use file cloning (also known as "`copy on + write`", "`CoW`" or "`reflinks`") to store and fetch cached compiler + results. *file_clone* has priority over <>. + The default is false. + Files stored by cloning cannot be compressed, so the cache size will likely be significantly larger if this option is enabled. However, performance may be @@ -600,7 +680,8 @@ safe to use, but not all file systems support the feature. For such file systems, ccache will fall back to use plain copying (or hard links if <> is enabled). -[[config_hard_link]] *hard_link* (*CCACHE_HARDLINK* or *CCACHE_NOHARDLINK*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_hard_link] +*hard_link* (*CCACHE_HARDLINK* or *CCACHE_NOHARDLINK*, see _<>_ above):: If true, ccache will attempt to use hard links to store and fetch cached object files. The default is false. @@ -614,32 +695,32 @@ WARNING: Do not enable this option unless you are aware of these caveats: * If the resulting file is modified, the file in the cache will also be modified since they share content, which corrupts the cache entry. As of version 4.0, ccache makes stored and fetched object files read-only as a - safety measure guard. Furthermore, a simple integrity check is made for - cached object files by verifying that their sizes are correct. This means - that mistakes like `strip file.o` or `echo >file.o` will be detected even if - the object file is made writeable, but a modification that doesn't change the - file size will not. + safety measure. Furthermore, a simple integrity check is made for cached + object files by verifying that their sizes are correct. This means that + mistakes like `strip file.o` or `echo >file.o` will be detected even if the + object file is made writeable, but a modification that doesn't change the file + size will not. * Programs that don't expect that files from two different identical compilations are hard links to each other can fail. -* Programs that rely on modification times (like ``make'') can be confused if +* Programs that rely on modification times (like `make`) can be confused if several users (or one user with several build trees) use the same cache directory. The reason for this is that the object files share i-nodes and - therefore modification times. If *file.o* is in build tree A (hard-linked - from the cache) and *file.o* then is produced by ccache in build tree B by + therefore modification times. If `file.o` is in build tree *A* (hard-linked + from the cache) and `file.o` then is produced by ccache in build tree *B* by hard-linking from the cache, the modification timestamp will be updated for - *file.o* in build tree A as well. This can retrigger relinking in build tree - A even though nothing really has changed. + `file.o` in build tree *A* as well. This can retrigger relinking in build tree + *A* even though nothing really has changed. -[[config_hash_dir]] *hash_dir* (*CCACHE_HASHDIR* or *CCACHE_NOHASHDIR*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_hash_dir] +*hash_dir* (*CCACHE_HASHDIR* or *CCACHE_NOHASHDIR*, see _<>_ above):: If true (which is the default), ccache will include the current working directory (CWD) in the hash that is used to distinguish two compilations - when generating debug info (compiler option *-g* with variations). + when generating debug info (compiler option `-g` with variations). Exception: The CWD will not be included in the hash if <> is set (and matches the CWD) and the - compiler option *-fdebug-prefix-map* is used. See also the discussion under - _<<_compiling_in_different_directories,Compiling in different - directories>>_. + compiler option `-fdebug-prefix-map` is used. See also the discussion under + _<>_. + The reason for including the CWD in the hash by default is to prevent a problem with the storage of the current working directory in the debug info of an @@ -650,7 +731,8 @@ You can disable this option to get cache hits when compiling the same source code in different directories if you don't mind that CWD in the debug info might be incorrect. -[[config_ignore_headers_in_manifest]] *ignore_headers_in_manifest* (*CCACHE_IGNOREHEADERS*):: +[#config_ignore_headers_in_manifest] +*ignore_headers_in_manifest* (*CCACHE_IGNOREHEADERS*):: This option is a list of paths to files (or directories with headers) that ccache will *not* include in the manifest list that makes up the direct @@ -658,39 +740,44 @@ might be incorrect. change. The list separator is semicolon on Windows systems and colon on other systems. -[[config_ignore_options]] *ignore_options* (*CCACHE_IGNOREOPTIONS*):: +[#config_ignore_options] +*ignore_options* (*CCACHE_IGNOREOPTIONS*):: This option is a space-delimited list of compiler options that ccache will exclude from the hash. Excluding a compiler option from the hash can be useful when you know it doesn't affect the result (but ccache doesn't know that), or when it does and you don't care. If a compiler option in the list is suffixed with an asterisk (`*`) it will be matched as a prefix. For - example, `-fmessage-length=*` will match both `-fmessage-length=20` and + example, `+-fmessage-length=*+` will match both `-fmessage-length=20` and `-fmessage-length=70`. -[[config_inode_cache]] *inode_cache* (*CCACHE_INODECACHE* or *CCACHE_NOINODECACHE*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_inode_cache] +*inode_cache* (*CCACHE_INODECACHE* or *CCACHE_NOINODECACHE*, see _<>_ above):: - If true, enables caching of source file hashes based on device, inode and - timestamps. This will reduce the time spent on hashing included files as - the result can be resused between compilations. -+ -The feature is still experimental and thus off by default. It is currently not -available on Windows. + If true, ccache will cache source file hashes based on device, inode and + timestamps. This reduces the time spent on hashing include files since the + result can be resused between compilations. The default is true. The feature + requires <> to be located on a local + filesystem of a supported type. + -The feature requires *temporary_dir* to be located on a local filesystem. +NOTE: The inode cache feature is currently not available on Windows. -[[config_keep_comments_cpp]] *keep_comments_cpp* (*CCACHE_COMMENTS* or *CCACHE_NOCOMMENTS*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_keep_comments_cpp] +*keep_comments_cpp* (*CCACHE_COMMENTS* or *CCACHE_NOCOMMENTS*, see _<>_ above):: If true, ccache will not discard the comments before hashing preprocessor - output. This can be used to check documentation with *-Wdocumentation*. + output. The default is false. This can be used to check documentation with + `-Wdocumentation`. -[[config_limit_multiple]] *limit_multiple* (*CCACHE_LIMIT_MULTIPLE*):: +[#config_limit_multiple] +*limit_multiple* (*CCACHE_LIMIT_MULTIPLE*):: Sets the limit when cleaning up. Files are deleted (in LRU order) until the levels are below the limit. The default is 0.8 (= 80%). See - _<<_automatic_cleanup,Automatic cleanup>>_ for more information. + _<>_ for more information. -[[config_log_file]] *log_file* (*CCACHE_LOGFILE*):: +[#config_log_file] +*log_file* (*CCACHE_LOGFILE*):: If set to a file path, ccache will write information on what it is doing to the specified file. This is useful for tracking down problems. @@ -706,76 +793,139 @@ file in `/etc/rsyslog.d`: & ~ ------------------------------------------------------------------------------- -[[config_max_files]] *max_files* (*CCACHE_MAXFILES*):: +[#config_max_files] +*max_files* (*CCACHE_MAXFILES*):: This option specifies the maximum number of files to keep in the cache. Use - 0 for no limit (which is the default). See also - _<<_cache_size_management,Cache size management>>_. + 0 for no limit (which is the default). See also _<>_. + +[#config_max_size] +*max_size* (*CCACHE_MAXSIZE*):: + + This option specifies the maximum size of the cache. Use 0 for no limit. The + default value is 5G. Available suffixes: k, M, G, T (decimal) and Ki, Mi, + Gi, Ti (binary). The default suffix is G. See also + _<>_. -[[config_max_size]] *max_size* (*CCACHE_MAXSIZE*):: +[#config_msvc_dep_prefix] +*msvc_dep_prefix* (*CCACHE_MSVC_DEP_PREFIX*):: - This option specifies the maximum size of the cache. Use 0 for no limit. - The default value is 5G. Available suffixes: k, M, G, T (decimal) and Ki, - Mi, Gi, Ti (binary). The default suffix is G. See also - _<<_cache_size_management,Cache size management>>_. + This option specifies the prefix of included files output for MSVC compiler. + The default prefix is "`Note: including file:`". If you use a localized + compiler, this should be set accordingly. -[[config_path]] *path* (*CCACHE_PATH*):: +[#config_namespace] +*namespace* (*CCACHE_NAMESPACE*):: + + If set, the namespace string will be added to the hashed data for each + compilation. This will make the associated cache entries logically separate + from cache entries with other namespaces, but they will still share the same + storage space. Cache entries can also be selectively removed from the local + cache with the command line option `--evict-namespace`, potentially in + combination with `--evict-older-than`. ++ +For instance, if you use the same local cache for several disparate projects, +you can use a unique namespace string for each one. This allows you to remove +cache entries that belong to a certain project if you stop working with that +project. + +[#config_path] +*path* (*CCACHE_PATH*):: If set, ccache will search directories in this list when looking for the real compiler. The list separator is semicolon on Windows systems and colon on other systems. If not set, ccache will look for the first executable - matching the compiler name in the normal *PATH* that isn't a symbolic link + matching the compiler name in the normal `PATH` that isn't a symbolic link to ccache itself. -[[config_pch_external_checksum]] *pch_external_checksum* (*CCACHE_PCH_EXTSUM* or *CCACHE_NOPCH_EXTSUM*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_pch_external_checksum] +*pch_external_checksum* (*CCACHE_PCH_EXTSUM* or *CCACHE_NOPCH_EXTSUM*, see _<>_ above):: When this option is set, and ccache finds a precompiled header file, - ccache will look for a file with the extension ``.sum'' added - (e.g. ``pre.h.gch.sum''), and if found, it will hash this file instead + ccache will look for a file with the extension "`.sum`" added + (e.g. "`pre.h.gch.sum`"), and if found, it will hash this file instead of the precompiled header itself to work around the performance penalty of hashing very large files. -[[config_prefix_command]] *prefix_command* (*CCACHE_PREFIX*):: +[#config_prefix_command] +*prefix_command* (*CCACHE_PREFIX*):: - This option adds a list of prefixes (separated by space) to the command - line that ccache uses when invoking the compiler. See also - _<<_using_ccache_with_other_compiler_wrappers,Using ccache with other - compiler wrappers>>_. + This option adds a list of prefixes (separated by space) to the command line + that ccache uses when invoking the compiler. See also + _<>_. -[[config_prefix_command_cpp]] *prefix_command_cpp* (*CCACHE_PREFIX_CPP*):: +[#config_prefix_command_cpp] +*prefix_command_cpp* (*CCACHE_PREFIX_CPP*):: This option adds a list of prefixes (separated by space) to the command line that ccache uses when invoking the preprocessor. -[[config_read_only]] *read_only* (*CCACHE_READONLY* or *CCACHE_NOREADONLY*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_read_only] +*read_only* (*CCACHE_READONLY* or *CCACHE_NOREADONLY*, see _<>_ above):: If true, ccache will attempt to use existing cached results, but it will not - add new results to the cache. Statistics counters will still be updated, - though, unless the <> option is set to *false*. + add new results to any cache backend. Statistics counters will still be + updated, though, unless the <> option is set to + *false*. + If you are using this because your ccache directory is read-only, you need to set <> since ccache will fail to create temporary files otherwise. You may also want to set <> to *false* make ccache not even try to update stats files. -[[config_read_only_direct]] *read_only_direct* (*CCACHE_READONLY_DIRECT* or *CCACHE_NOREADONLY_DIRECT*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_read_only_direct] +*read_only_direct* (*CCACHE_READONLY_DIRECT* or *CCACHE_NOREADONLY_DIRECT*, see _<>_ above):: Just like <> except that ccache will only try to retrieve results from the cache using the direct mode, not the preprocessor mode. See documentation for <> regarding using a read-only ccache directory. -[[config_recache]] *recache* (*CCACHE_RECACHE* or *CCACHE_NORECACHE*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_recache] +*recache* (*CCACHE_RECACHE* or *CCACHE_NORECACHE*, see _<>_ above):: If true, ccache will not use any previously stored result. New results will still be cached, possibly overwriting any pre-existing results. -[[config_run_second_cpp]] *run_second_cpp* (*CCACHE_CPP2* or *CCACHE_NOCPP2*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_remote_only] +*remote_only* (*CCACHE_REMOTE_ONLY* or *CCACHE_NOREMOTE_ONLY*, see _<>_ above):: + + If true, ccache will only use <>. The + default is false. Note that cache statistics counters will still be kept in + the local cache directory unless <> is false. See also + _<>_. + +[#config_remote_storage] +*remote_storage* (*CCACHE_REMOTE_STORAGE*):: + + This option specifies one or several storage backends (separated by space) + to query after checking the local cache (unless + <> is true). See _<>_ for documentation of syntax and available backends. ++ +Examples: ++ +* `+file:/shared/nfs/directory+` +* `+file:///shared/nfs/one|read-only file:///shared/nfs/two+` +* `+http://example.com/cache+` +* `+redis://example.com+` ++ +NOTE: In previous ccache versions this option was called *secondary_storage* +(*CCACHE_SECONDARY_STORAGE*), which can still be used as an alias. + +[#config_reshare] +*reshare* (*CCACHE_RESHARE* or *CCACHE_NORESHARE*, see _<>_ above):: + + If true, ccache will write results to remote storage even for local storage + cache hits. The default is false. + +[#config_run_second_cpp] +*run_second_cpp* (*CCACHE_CPP2* or *CCACHE_NOCPP2*, see _<>_ above):: If true, ccache will first run the preprocessor to preprocess the source - code (see _<<_the_preprocessor_mode,The preprocessor mode>>_) and then on a - cache miss run the compiler on the source code to get hold of the object - file. This is the default. + code (see _<>_) and then on a cache miss run the + compiler on the source code to get hold of the object file. This is the + default. + If false, ccache will first run preprocessor to preprocess the source code and then on a cache miss run the compiler on the _preprocessed source code_ instead @@ -785,13 +935,17 @@ compilers won't produce the same result (for instance diagnostics warnings) when compiling preprocessed source code. + A solution to the above mentioned downside is to set *run_second_cpp* to false -and pass *-fdirectives-only* (for GCC) or *-frewrite-includes* (for Clang) to +and pass `-fdirectives-only` (for GCC) or `-frewrite-includes` (for Clang) to the compiler. This will cause the compiler to leave the macros and other preprocessor information, and only process the *#include* directives. When run in this way, the preprocessor arguments will be passed to the compiler since it still has to do _some_ preprocessing (like macros). ++ +This option is ignored with MSVC, as there is no way to make it compile without +preprocessing first. -[[config_sloppiness]] *sloppiness* (*CCACHE_SLOPPINESS*):: +[#config_sloppiness] +*sloppiness* (*CCACHE_SLOPPINESS*):: By default, ccache tries to give as few false cache hits as possible. However, in certain situations it's possible that you know things that @@ -802,7 +956,7 @@ still has to do _some_ preprocessing (like macros). + -- *clang_index_store*:: - Ignore the Clang compiler option *-index-store-path* and its argument when + Ignore the Clang compiler option `-index-store-path` and its argument when computing the manifest hash. This is useful if you use Xcode, which uses an index store path derived from the local project path. Note that the index store won't be updated correctly on cache hits if you enable this @@ -814,80 +968,297 @@ still has to do _some_ preprocessing (like macros). *file_stat_matches_ctime*:: Ignore ctimes when *file_stat_matches* is enabled. This can be useful when backdating files' mtimes in a controlled way. +*gcno_cwd*:: + By default, ccache will include the current working directory in the hash + when producing a `.gcno` file (when compiling with `-ftest-coverage` or + `--coverage`). This is because GCC 9+ includes the current working directory + in the `.gcno` file. The *gcno_cwd* sloppiness makes ccache not hash the + current working directory so that you can get cache hits when compiling in + different directories, with the tradeoff of potentially getting an incorrect + directory in the `.gcno` file. *gcno_cwd* also disables hashing of the + current working directory if `-fprofile-abs-path` is used. *include_file_ctime*:: - By default, ccache will not cache a file if it includes a header whose ctime - is too new. This sloppiness disables that check. See also - _<<_handling_of_newly_created_header_files,Handling of newly created header - files>>_. + By default, ccache will disable the direct mode if an include file has too + new ctime. This sloppiness disables that check. See also _<>_. *include_file_mtime*:: - By default, ccache will not cache a file if it includes a header whose - mtime is too new. This sloppiness disables that check. See also - _<<_handling_of_newly_created_header_files,Handling of newly created header - files>>_. + By default, ccache will disable the direct mode if an include file has too + new mtime. This sloppiness disables that check. See also _<>_. *ivfsoverlay*:: - Ignore the Clang compiler option *-ivfsoverlay* and its argument. This is + Ignore the Clang compiler option `-ivfsoverlay` and its argument. This is useful if you use Xcode, which uses a virtual file system (VFS) for things like combining Objective-C and Swift code. *locale*:: - Ccache includes the environment variables *LANG*, *LC_ALL*, *LC_CTYPE* and - *LC_MESSAGES* in the hash by default since they may affect localization of + Ccache includes the environment variables `LANG`, `LC_ALL`, `LC_CTYPE` and + `LC_MESSAGES` in the hash by default since they may affect localization of compiler warning messages. Set this sloppiness to tell ccache not to do that. -*pch_defines*:: - Be sloppy about **#define**s when precompiling a header file. See - _<<_precompiled_headers,Precompiled headers>>_ for more information. *modules*:: - By default, ccache will not cache compilations if *-fmodules* is used since + By default, ccache will not cache compilations if `-fmodules` is used since it cannot hash the state of compiler's internal representation of relevant modules. This sloppiness allows caching in such a case. See - _<<_c_modules,C++ modules>>_ for more information. + _<>_ for more information. +*pch_defines*:: + Be sloppy about `#define` directives when precompiling a header file. See + _<>_ for more information. +*random_seed*:: + Ignore the `-frandom-seed` option and its arguments when computing the input + hash. This is useful if your build system generates different seeds between + builds and you are OK with reusing cached results. *system_headers*:: By default, ccache will also include all system headers in the manifest. With this sloppiness set, ccache will only include system headers in the hash but not add the system header files to the list of include files. *time_macros*:: - Ignore `__DATE__`, `__TIME__` and `__TIMESTAMP__` being present in the + Ignore `+__DATE__+`, `+__TIME__+` and `+__TIMESTAMP__+` being present in the source code. -- + -See the discussion under _<<_troubleshooting,Troubleshooting>>_ for more -information. +See the discussion under _<>_ for more information. -[[config_stats]] *stats* (*CCACHE_STATS* or *CCACHE_NOSTATS*, see _<<_boolean_values,Boolean values>>_ above):: +[#config_stats] +*stats* (*CCACHE_STATS* or *CCACHE_NOSTATS*, see _<>_ above):: If true, ccache will update the statistics counters on each compilation. The default is true. -[[config_temporary_dir]] *temporary_dir* (*CCACHE_TEMPDIR*):: +[#config_stats_log] +*stats_log* (*CCACHE_STATSLOG*):: + + If set to a file path, ccache will write statistics counter updates to the + specified file. This is useful for getting statistics for individual builds. + To show a summary of the current stats log, use `ccache --show-log-stats`. ++ +NOTE: Lines in the stats log starting with a hash sign (`#`) are comments. + +[#config_temporary_dir] +*temporary_dir* (*CCACHE_TEMPDIR*):: This option specifies where ccache will put temporary files. The default is - */run/user//ccache-tmp* if */run/user/* exists, otherwise - */tmp*. + `$XDG_RUNTIME_DIR/ccache-tmp` (typically `/run/user//ccache-tmp`) if + `XDG_RUNTIME_DIR` is set and the directory exists, otherwise + `/tmp`. + NOTE: In previous versions of ccache, *CCACHE_TEMPDIR* had to be on the same -filesystem as the *CCACHE_DIR* path, but this requirement has been relaxed.) +filesystem as the `CCACHE_DIR` path, but this requirement has been relaxed. + +[#config_umask] +*umask* (*CCACHE_UMASK*):: + + This option (an octal integer) specifies the umask for files and directories + in the cache directory. This is mostly useful when you wish to share your + cache with other users. + + +== Remote storage backends + +The <> option lets you configure ccache +to use one or several remote storage backends. By default, the local cache +directory located in <> will be queried first and +remote storage second, but <> can be set to +true to disable local storage. Note that cache statistics counters will still be +kept in the local cache directory -- remote storage backends only store +compilation results and manifests. + +A remote storage backend is specified with a URL, optionally followed by a pipe +(`|`) and a pipe-separated list of attributes. An attribute is _key_=_value_ or +just _key_ as a short form of _key_=*true*. Attribute values must be +https://en.wikipedia.org/wiki/Percent-encoding[percent-encoded] if they contain +percent, pipe or space characters. + +=== Attributes for all backends + +These optional attributes are available for all remote storage backends: + +* *read-only*: If *true*, only read from this backend, don't write. The default + is *false*. +* *shards*: A comma-separated list of names for sharding (partitioning) the + cache entries using + https://en.wikipedia.org/wiki/Rendezvous_hashing[Rendezvous hashing], + typically to spread the cache over a server cluster. When set, the storage URL + must contain an asterisk (`+*+`), which will be replaced by one of the shard + names to form a real URL. A shard name can optionally have an appended weight + within parentheses to indicate how much of the key space should be associated + with that shard. A shard with weight *w* will contain *w*/*S* of the cache, + where *S* is the sum of all shard weights. A weight could for instance be set + to represent the available memory for a memory cache on a specific server. The + default weight is *1*. ++ +Examples: ++ +-- +* `+redis://cache-*.example.com|shards=a(3),b(1),c(1.5)+` will put 55% (3/5.5) + of the cache on `+redis://cache-a.example.com+`, 18% (1/5.5) on + `+redis://cache-b.example.com+` and 27% (1.5/5.5) on + `+redis://cache-c.example.com+`. +* `+http://example.com/*|shards=alpha,beta+` will put 50% of the cache on + `+http://example.com/alpha+` and 50% on `+http://example.com/beta+`. +-- + + +=== Storage interaction + +The table below describes the interaction between local and remote storage on +cache hits and misses if <> is false (which is +the default): -[[config_umask]] *umask* (*CCACHE_UMASK*):: +[options="header",cols="20%,20%,60%"] +|============================================================================== +| *Local storage* | *Remote storage* | *What happens* + +| miss | miss | Compile, write to local, write to remote^[1]^ +| miss | hit | Read from remote, write to local +| hit | - | Read from local, don't write to remote^[2]^ - This option specifies the umask for files and directories in the cache - directory. This is mostly useful when you wish to share your cache with - other users. +|============================================================================== +^[1]^ Unless remote storage has attribute `read-only=true`. + +^[2]^ Unless local storage is set to share its cache hits with the +<> option. -Cache size management ---------------------- +If <> is true: + +[options="header",cols="20%,20%,60%"] +|============================================================================== +| *Local storage* | *Remote storage* | *What happens* + +| - | miss | Compile, write to remote, don't write to local +| - | hit | Read from remote, don't write to local + +|============================================================================== + +=== File storage backend + +URL format: `+file:DIRECTORY+` or `+file://[HOST]DIRECTORY+` + +This backend stores data as separate files in a directory structure below +*DIRECTORY*, similar (but not identical) to the local cache storage. A typical +use case for this backend would be sharing a cache on an NFS directory. +*DIRECTORY* must start with a slash. *HOST* can be the empty string or +localhost. On Windows, *HOST* can also be the name of a server hosting a shared +folder. + +IMPORTANT: ccache will not perform any cleanup of the storage -- that has to be +done by other means, for instance by running `ccache --trim-dir` periodically. + +Examples: + +* `+file:/shared/nfs/directory+` +* `+file:///shared/nfs/directory|umask=002|update-mtime=true+` +* `+file://example.com/shared/folder+` + +Optional attributes: + +* *layout*: How to store file under the cache directory. Available values: ++ +-- +* *flat*: Store all files directly under the cache directory. +* *subdirs*: Store files in 256 subdirectories of the cache directory. +-- ++ +The default is *subdirs*. +* *umask*: This attribute (an octal integer) overrides the umask to use for + files and directories in the cache directory. +* *update-mtime*: If *true*, update the modification time (mtime) of cache + entries that are read. The default is *false*. + + +=== HTTP storage backend + +URL format: `+http://HOST[:PORT][/PATH]+` + +This backend stores data in an HTTP-compatible server. The required HTTP methods +are `GET`, `PUT` and `DELETE`. + +IMPORTANT: ccache will not perform any cleanup of the storage -- that has to be +done by other means, for instance by running `ccache --trim-dir` periodically. + +NOTE: HTTPS is not supported. + +TIP: See https://ccache.dev/howto/http-storage.html[How to set up HTTP storage] +for hints on how to set up an HTTP server for use with ccache. + +Examples: + +* `+http://localhost+` +* `+http://someusername:p4ssw0rd@example.com/cache/+` +* `+http://localhost:8080|layout=bazel|connect-timeout=50+` + +Optional attributes: + +* *bearer-token*: Bearer token used to authorize the HTTP requests. +* *connect-timeout*: Timeout (in ms) for network connection. The default is 100. +* *keep-alive*: If *true*, keep the HTTP connection to the storage server open + to avoid reconnects. The default is *true*. +* *layout*: How to map key names to the path part of the URL. Available values: ++ +-- +* *bazel*: Store values in a format compatible with the Bazel HTTP caching + protocol. More specifically, the entries will be stored as 64 hex digits + under the `/ac/` part of the cache. ++ +NOTE: You may have to disable verification of action cache values in the server +for this to work since ccache entries are not valid action result metadata +values. +* *flat*: Append the key directly to the path part of the URL (with a leading + slash if needed). +* *subdirs*: Append the first two characters of the key to the URL (with a + leading slash if needed), followed by a slash and the rest of the key. This + divides the entries into 256 buckets. +-- ++ +The default is *subdirs*. +* *operation-timeout*: Timeout (in ms) for HTTP requests. The default is 10000. + + +=== Redis storage backend + +URL formats: + +`+redis://[[USERNAME:]PASSWORD@]HOST[:PORT][/DBNUMBER]+` + +`+redis+unix:SOCKET_PATH[?db=DBNUMBER]+` + +`+redis+unix://[[USERNAME:]PASSWORD@localhost]SOCKET_PATH[?db=DBNUMBER]+` + +This backend stores data in a https://redis.io[Redis] (or Redis-compatible) +server. There are implementations for both memory-based and disk-based storage. +*PORT* defaults to *6379* and *DBNUMBER* defaults to *0*. + +NOTE: ccache will not perform any cleanup of the Redis storage, but you can +https://redis.io/topics/lru-cache[configure LRU eviction]. + +TIP: See https://ccache.dev/howto/redis-storage.html[How to set up Redis +storage] for hints on setting up a Redis server for use with ccache. + +TIP: You can set up a cluster of Redis servers using the `shards` attribute +described in _<>_. + +Examples: + +* `+redis://localhost+` +* `+redis://p4ssw0rd@cache.example.com:6379/0|connect-timeout=50+` +* `+redis+unix:/run/redis.sock+` +* `+redis+unix:///run/redis.sock+` +* `+redis+unix://p4ssw0rd@localhost/run/redis.sock?db=0+` + +Optional attributes: + +* *connect-timeout*: Timeout (in ms) for network connection. The default is 100. +* *operation-timeout*: Timeout (in ms) for Redis commands. The default is 10000. + + +== Cache size management By default, ccache has a 5 GB limit on the total size of files in the cache and no limit on the number of files. You can set different limits using the command -line options *-M*/*--max-size* and *-F*/*--max-files*. Use *ccache --s/--show-stats* to see the cache size and the currently configured limits (in -addition to other various statistics). +line options `-M`/`--max-size` and `-F`/`--max-files`. Use the +`-s`/`--show-stats` option to see the cache size and the currently configured +limits (in addition to other various statistics). Cleanup can be triggered in two different ways: automatic and manual. -Automatic cleanup -~~~~~~~~~~~~~~~~~ +=== Automatic cleanup Ccache maintains counters for various statistics about the cache, including the size and number of all cached files. In order to improve performance and reduce @@ -915,11 +1286,14 @@ The reason for removing more files than just those needed to not exceed the max limits is that a cleanup is a fairly slow operation, so it would not be a good idea to trigger it often, like after each cache miss. +The LRU cleanup makes use of the file modification time (mtime) of cache +entries; ccache updates mtime of the cache entries read on a cache hit to mark +them as "recently used". -Manual cleanup -~~~~~~~~~~~~~~ -You can run *ccache -c/--cleanup* to force cleanup of the whole cache, i.e. all +=== Manual cleanup + +You can run `ccache -c/--cleanup` to force cleanup of the whole cache, i.e. all of the sixteen subdirectories. This will recalculate the statistics counters and make sure that the configuration options *max_size* and <> are not exceeded. Note that @@ -927,8 +1301,7 @@ and make sure that the configuration options *max_size* and cleanup. -Cache compression ------------------ +== Cache compression Ccache will by default compress all data it puts into the cache using the compression algorithm http://zstd.net[Zstandard] (zstd) using compression level @@ -939,24 +1312,24 @@ course is redundant. See the documentation for the configuration options <> and <> for more information. -You can use the command line option *-x/--show-compression* to print +You can use the command line option `-x`/`--show-compression` to print information related to compression. Example: ------------------------------------------------------------------------------- -Total data: 14.8 GB (16.0 GB disk blocks) -Compressed data: 11.3 GB (30.6% of original size) - - Original data: 36.9 GB - - Compression ratio: 3.267 x (69.4% space savings) -Incompressible data: 3.5 GB +Total data: 14.8 GB (16.0 GB disk blocks) +Compressed data: 11.3 GB (30.6% of original size) + Original size: 36.9 GB + Compression ratio: 3.267 x (69.4% space savings) +Incompressible data: 3.5 GB ------------------------------------------------------------------------------- Notes: -* The ``disk blocks'' size is the cache size when taking disk block size into - account. This value should match the ``cache size'' value from ``ccache - --show-stats''. The other size numbers refer to actual content sizes. -* ``Compressed data'' refers to result and manifest files stored in the cache. -* ``Incompressible data'' refers to files that are always stored uncompressed +* The "`disk blocks`" size is the cache size when taking disk block size into + account. This value should match the "`Cache size`" value from "`ccache + --show-stats`". The other size numbers refer to actual content sizes. +* "`Compressed data`" refers to result and manifest files stored in the cache. +* "`Incompressible data`" refers to files that are always stored uncompressed (triggered by enabling <> or <>) or unknown files (for instance files created by older ccache versions). @@ -964,7 +1337,7 @@ Notes: <>. The cache data can also be recompressed to another compression level (or made -uncompressed) with the command line option *-X/--recompress*. If you choose to +uncompressed) with the command line option `-X`/`--recompress`. If you choose to disable compression by default or to use a low compression level, you can (re)compress newly cached data with a higher compression level after the build or at another time when there are more CPU cycles available, for instance every @@ -973,130 +1346,112 @@ are currently compressed with a different level than the target level will be recompressed. -Cache statistics ----------------- +== Cache statistics + +`ccache --show-stats` shows a summary of statistics, including cache size, +cleanups (number of performed cleanups, either implicitly due to a cache size +limit being reached or due to explicit `ccache -c` calls), overall hit rate, hit +rate for <>/<> modes +and hit rate for local and <>. -*ccache -s/--show-stats* can show the following statistics: +The summary also includes counters called "`Errors`" and "`Uncacheable`", which +are sums of more detailed counters. To see those detailed counters, use the +`-v`/`--verbose` flag. The verbose mode can show the following counters: [options="header",cols="30%,70%"] |============================================================================== -|Name | Description -| autoconf compile/link | -Uncachable compilation or linking by an autoconf test. +| *Counter* | *Description* + +| Autoconf compile/link | +Uncacheable compilation or linking by an Autoconf test. -| bad compiler arguments | +| Bad compiler arguments | Malformed compiler argument, e.g. missing a value for a compiler option that requires an argument or failure to read a file specified by a compiler option argument. -| cache file missing | -A file was unexpectedly missing from the cache. This only happens in rare -situations, e.g. if one ccache instance is about to get a file from the cache -while another instance removed the file as part of cache cleanup. - -| cache hit (direct) | -A result was successfully found using <<_the_direct_mode,the direct mode>>. - -| cache hit (preprocessed) | -A result was successfully found using <<_the_preprocessor_mode,the preprocessor -mode>>. - -| cache miss | -No result was found. - -| cache size | -Current size of the cache. - -| called for link | +| Called for linking | The compiler was called for linking, not compiling. Ccache only supports -compilation of a single file, i.e. calling the compiler with the *-c* option to +compilation of a single file, i.e. calling the compiler with the `-c` option to produce a single object file from a single source file. -| called for preprocessing | +| Called for preprocessing | The compiler was called for preprocessing, not compiling. -| can't use precompiled header | -Preconditions for using <<_precompiled_headers,precompiled headers>> were not -fulfilled. - -| can't use modules | -Preconditions for using <<_c_modules,C++ modules>> were not fulfilled. +| Could not use modules | +Preconditions for using <> were not fulfilled. -| ccache internal error | -Unexpected failure, e.g. due to problems reading/writing the cache. +| Could not use precompiled header | +Preconditions for using <> were not +fulfilled. -| cleanups performed | -Number of cleanups performed, either implicitly due to the cache size limit -being reached or due to explicit *ccache -c/--cleanup* calls. +| Could not write to output file | +The output path specified with `-o` could not be written to. -| compile failed | +| Compilation failed | The compilation failed. No result stored in the cache. -| compiler check failed | +| Compiler check failed | A compiler check program specified by <> (*CCACHE_COMPILERCHECK*) failed. -| compiler produced empty output | -The compiler's output file (typically an object file) was empty after +| Compiler output file missing | +One of the files expected to be produced by the compiler was missing after compilation. -| compiler produced no output | -The compiler's output file (typically an object file) was missing after +| Compiler produced empty output | +The compiler's output file (typically an object file) was empty after compilation. -| compiler produced stdout | -The compiler wrote data to standard output. This is something that compilers -normally never do, so ccache is not designed to store such output in the cache. - -| couldn't find the compiler | +| Could not find the compiler | The compiler to execute could not be found. -| error hashing extra file | +| Error hashing extra file | Failure reading a file specified by <> (*CCACHE_EXTRAFILES*). -| files in cache | -Current number of files in the cache. +| Forced recache | +<> was used to overwrite an existing result. -| multiple source files | +| Internal error | +Unexpected failure, e.g. due to problems reading/writing the cache. + +| Missing cache file | +A file was unexpectedly missing from the cache. This only happens in rare +situations, e.g. if one ccache instance is about to get a file from the cache +while another instance removed the file as part of cache cleanup. + +| Multiple source files | The compiler was called to compile multiple source files in one go. This is not supported by ccache. -| no input file | +| No input file | No input file was specified to the compiler. -| output to a non-regular file | -The output path specified with *-o* is not a file (e.g. a directory or a device -node). +| Output to stdout | +The compiler was instructed to write its output to standard output using `-o -`. +This is not supported by ccache. -| output to stdout | -The compiler was instructed to write its output to standard output using *-o --*. This is not supported by ccache. +| Preprocessing failed | +Preprocessing the source code using the compiler's `-E` option failed. -| preprocessor error | -Preprocessing the source code using the compiler's *-E* option failed. - -| stats updated | -When statistics were updated the last time. - -| stats zeroed | -When *ccache -z* was called the last time. - -| unsupported code directive | -Code like the assembler *.incbin* directive was found. This is not supported +| Unsupported code directive | +Code like the assembler `.incbin` directive was found. This is not supported by ccache. -| unsupported compiler option | +| Unsupported compiler option | A compiler option not supported by ccache was found. -| unsupported source language | -A source language e.g. specified with *-x* was unsupported by ccache. +| Unsupported environment variable | +An environment variable not supported by ccache was set. + +| Unsupported source language | +A source language e.g. specified with `-x` was unsupported by ccache. |============================================================================== -How ccache works ----------------- +== How ccache works The basic idea is to detect when you are compiling exactly the same code a second time and reuse the previously produced output. The detection is done by @@ -1119,20 +1474,18 @@ cache: The direct mode is generally faster since running the preprocessor has some overhead. -If no previous result is detected (i.e., there is a cache miss) using the -direct mode, ccache will fall back to the preprocessor mode unless the *depend -mode* is enabled. In the depend mode, ccache never runs the preprocessor, not -even on cache misses. Read more in _<<_the_depend_mode,The depend mode>>_ -below. +If no previous result is detected (i.e., there is a cache miss) using the direct +mode, ccache will fall back to the preprocessor mode unless the *depend mode* is +enabled. In the depend mode, ccache never runs the preprocessor, not even on +cache misses. Read more in _<>_ below. -Common hashed information -~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Common hashed information The following information is always included in the hash: * the extension used by the compiler for a file with preprocessor output - (normally *.i* for C code and *.ii* for C++ code) + (normally `.i` for C code and `.ii` for C++ code) * the compiler's size and modification time (or other compiler-specific information specified by <>) * the name of the compiler @@ -1141,14 +1494,13 @@ The following information is always included in the hash: <> (if any) -The preprocessor mode -~~~~~~~~~~~~~~~~~~~~~ +=== The preprocessor mode In the preprocessor mode, the hash is formed of the common information and: -* the preprocessor output from running the compiler with *-E* -* the command line options except those that affect include files (*-I*, - *-include*, *-D*, etc; the theory is that these command line options will +* the preprocessor output from running the compiler with `-E` +* the command line options except those that affect include files (`-I`, + `-include`, `-D`, etc; the theory is that these command line options will change the preprocessor output if they have any effect at all) * any standard error output generated by the preprocessor @@ -1156,15 +1508,14 @@ Based on the hash, the cached compilation result can be looked up directly in the cache. -The direct mode -~~~~~~~~~~~~~~~ +=== The direct mode In the direct mode, the hash is formed of the common information and: * the input source file * the compiler options -Based on the hash, a data structure called ``manifest'' is looked up in the +Based on the hash, a data structure called "`manifest`" is looked up in the cache. The manifest contains: * references to cached compilation results (object file, dependency file, etc) @@ -1195,19 +1546,19 @@ The direct mode will be disabled if any of the following holds: * a modification time of one of the include files is too new (needed to avoid a race condition) * a compiler option not supported by the direct mode is used: -** a *-Wp,_X_* compiler option other than *-Wp,-MD,_path_*, - *-Wp,-MMD,_path_* and *-Wp,-D_define_* -** *-Xpreprocessor* -* the string `__TIME__` is present in the source code +** a `-Wp,++*++` compiler option other than `-Wp,-MD,`, `-Wp,-MMD,` + and `-Wp,-D` +** `-Xpreprocessor` +* the string `+__TIME__+` is present in the source code -The depend mode -~~~~~~~~~~~~~~~ +=== The depend mode If the depend mode is enabled, ccache will not use the preprocessor at all. The hash used to identify results in the cache will be based on the direct mode hash described above plus information about include files read from the -dependency file generated by the compiler with *-MD* or *-MMD*. +dependency list generated by MSVC with `/showIncludes`, or the dependency file +generated by other compilers with `-MD` or `-MMD`. Advantages: @@ -1223,24 +1574,24 @@ Disadvantages: setup where ccache will fall back to the preprocessor mode, which is tolerant to some types of changes of compiler options and source code changes. * If -MD is used, the manifest entries will include system header files as - well, thus slowing down cache hits slightly, just as using -MD slows down - make. -* If -MMD is used, the manifest entries will not include system header files, + well, thus slowing down cache hits slightly, just as using `-MD` slows down + make. This is also the case for MSVC with `/showIncludes`. +* If `-MMD` is used, the manifest entries will not include system header files, which means ccache will ignore changes in them. The depend mode will be disabled if any of the following holds: * <> is false. * <> is false. -* The compiler is not generating dependencies using *-MD* or *-MMD*. +* The compiler is not generating dependencies using `-MD` or `-MMD` (for MSVC, + `/showIncludes` is added automatically if not specified by the user). -Handling of newly created header files --------------------------------------- +== Handling of newly created header files If modification time (mtime) or status change time (ctime) of one of the include -files is the same second as the time compilation is being done, ccache disables -the direct mode (or, in the case of a <<_precompiled_headers,precompiled +files is equal to (or newer than) the time compilation is being done, ccache +disables the direct mode (or, in the case of a <>, disables caching completely). This done as a safety measure to avoid a race condition (see below). @@ -1263,12 +1614,11 @@ For reference, the race condition mentioned above consists of these events: 5. The wrong object file is stored in the cache. -Cache debugging ---------------- +== Cache debugging To find out what information ccache actually is hashing, you can enable the debug mode via the configuration option <> or by setting -*CCACHE_DEBUG* in the environment. This can be useful if you are investigating +`CCACHE_DEBUG` in the environment. This can be useful if you are investigating why you don't get cache hits. Note that performance will be reduced slightly. When the debug mode is enabled, ccache will create up to five additional files @@ -1276,61 +1626,65 @@ next to the object file: [options="header",cols="30%,70%"] |============================================================================== -|Filename | Description -| *.ccache-input-c* | +| *Filename* | *Description* + +| `..ccache-input-c` | Binary input hashed by both the direct mode and the preprocessor mode. -| *.ccache-input-d* | +| `..ccache-input-d` | Binary input only hashed by the direct mode. -| *.ccache-input-p* | +| `..ccache-input-p` | Binary input only hashed by the preprocessor mode. -| *.ccache-input-text* | +| `..ccache-input-text` | Human-readable combined diffable text version of the three files above. -| *.ccache-log* | +| `..ccache-log` | Log for this object file. |============================================================================== -If <> (environment variable *CCACHE_DEBUGDIR*) is + The timestamp format is +`__`. + +If <> (environment variable `CCACHE_DEBUGDIR`) is set, the files above will be written to that directory with full absolute paths instead of next to the object file. In the direct mode, ccache uses the 160 bit BLAKE3 hash of the -*ccache-input-c* + *ccache-input-d* data (where *+* means concatenation), while -the *ccache-input-c* + *ccache-input-p* data is used in the preprocessor mode. +"`ccache-input-c`" + "`ccache-input-d`" data (where *+* means concatenation), +while the "`ccache-input-c`" + "`ccache-input-p`" data is used in the +preprocessor mode. -The *ccache-input-text* file is a combined text version of the three -binary input files. It has three sections (``COMMON'', ``DIRECT MODE'' and -``PREPROCESSOR MODE''), which is turn contain annotations that say what kind of +The "`ccache-input-text`" file is a combined text version of the three binary +input files. It has three sections ("`COMMON`", "`DIRECT MODE`" and +"`PREPROCESSOR MODE`"), which is turn contain annotations that say what kind of data comes next. To debug why you don't get an expected cache hit for an object file, you can do something like this: -1. Build with debug mode enabled. -2. Save the *.ccache-** files. -3. Build again with debug mode enabled. -4. Compare *.ccache-input-text* for the two builds. This together - with the *.ccache-log* files should give you some clues about - what is happening. +1. Enable `debug` (`CCACHE_DEBUG`). +2. Build. +3. Clean and build again. +4. Compare the `..ccache-input-text` files for the two + builds. This together with the `..ccache-log` files + should give you some clues about what is happening. -Compiling in different directories ----------------------------------- +== Compiling in different directories Some information included in the hash that identifies a unique compilation can contain absolute paths: * The preprocessed source code may contain absolute paths to include files if - the compiler option *-g* is used or if absolute paths are given to *-I* and + the compiler option `-g` is used or if absolute paths are given to `-I` and similar compiler options. -* Paths specified by compiler options (such as *-I*, *-MF*, etc) on the command +* Paths specified by compiler options (such as `-I`, `-MF`, etc) on the command line may be absolute. * The source code file path may be absolute, and that path may substituted for - `__FILE__` macros in the source code or included in warnings emitted to + `+__FILE__+` macros in the source code or included in warnings emitted to standard error by the preprocessor. This means that if you compile the same code in different locations, you can't @@ -1341,83 +1695,78 @@ hash. Here's what can be done to enable cache hits between different build directories: -* If you build with *-g* (or similar) to add debug information to the object +* If you build with `-g` (or similar) to add debug information to the object file, you must either: -+ --- -** use the compiler option *-fdebug-prefix-map=_old_=_new_* for relocating - debug info to a common prefix (e.g. *-fdebug-prefix-map=$PWD=.*); or +** use the compiler option `-fdebug-prefix-map==` for relocating + debug info to a common prefix (e.g. `-fdebug-prefix-map=$PWD=.`); or ** set *hash_dir = false*. --- * If you use absolute paths anywhere on the command line (e.g. the source code - file path or an argument to compiler options like *-I* and *-MF*), you must - set <> to an absolute path to a ``base - directory''. Ccache will then rewrite absolute paths under that directory to + file path or an argument to compiler options like `-I` and `-MF`), you must + set <> to an absolute path to a "`base + directory`". Ccache will then rewrite absolute paths under that directory to relative before computing the hash. -Precompiled headers -------------------- +== Precompiled headers -Ccache has support for GCC's precompiled headers. However, you have to do some -things to make it work properly: +Ccache has support for precompiled headers with GCC and Clang. However, you have +to do some things to make it work properly: * You must set <> to *pch_defines,time_macros*. - The reason is that ccache can't tell whether `__TIME__`, `__DATE__` or - `__TIMESTAMP__` is used when using a precompiled header. Further, it can't - detect changes in **#define**s in the source code because of how - preprocessing works in combination with precompiled headers. + The reason is that ccache can't tell whether `+__TIME__+`, `+__DATE__+` or + `+__TIMESTAMP__+` is used when using a precompiled header. Further, it can't + detect changes in ``#define``s in the source code because of how preprocessing + works in combination with precompiled headers. * You may also want to include *include_file_mtime,include_file_ctime* in <>. See - _<<_handling_of_newly_created_header_files,Handling of newly created header - files>>_. + _<>_. * You must either: + -- -** use the compiler option *-include* to include the precompiled header (i.e., - don't use *#include* in the source code to include the header; the filename - itself must be sufficient to find the header, i.e. *-I* paths are not +* use the compiler option `-include` to include the precompiled header (i.e., + don't use `#include` in the source code to include the header; the filename + itself must be sufficient to find the header, i.e. `-I` paths are not searched); or -** (for the Clang compiler) use the compiler option *-include-pch* to include +* (for the Clang compiler) use the compiler option `-include-pch` to include the PCH file generated from the precompiled header; or -** (for the GCC compiler) add the compiler option *-fpch-preprocess* when +* (for the GCC compiler) add the compiler option `-fpch-preprocess` when compiling. - -If you don't do this, either the non-precompiled version of the header file -will be used (if available) or ccache will fall back to running the real -compiler and increase the statistics counter ``preprocessor error'' (if the -non-precompiled header file is not available). -- ++ +* If you use Clang, you must compile with `-fno-pch-timestamp`. + +If you don't do this, either the non-precompiled version of the header file will +be used (if available) or ccache will fall back to running the real compiler and +increase the statistics counter "`Preprocessing failed`" (if the non-precompiled +header file is not available). -C++ modules ------------ +== C++ modules -Ccache has support for Clang's *-fmodules* option. In practice ccache only -additionally hashes *module.modulemap* files; it does not know how Clang +Ccache has support for Clang's `-fmodules` option. In practice ccache only +additionally hashes `module.modulemap` files; it does not know how Clang handles its cached binary form of modules so those are ignored. This should not -matter in practice: as long as everything else (including *module.modulemap* +matter in practice: as long as everything else (including `module.modulemap` files) is the same the cached result should work. Still, you must set <> to *modules* to allow caching. -You must use both <<_the_direct_mode,*direct mode*>> and -<<_the_depend_mode,*depend mode*>>. When using <<_the_preprocessor_mode,the -preprocessor mode>> Clang does not provide enough information to allow hashing -of *module.modulemap* files. +You must use both <> and +<>. When using +<> Clang does not provide enough +information to allow hashing of `module.modulemap` files. -Sharing a cache ---------------- +== Sharing a local cache -A group of developers can increase the cache hit rate by sharing a cache -directory. To share a cache without unpleasant side effects, the following +A group of developers can increase the cache hit rate by sharing a local cache +directory. To share a local cache without unpleasant side effects, the following conditions should to be met: * Use the same cache directory. * Make sure that the configuration option <> is false (which is the default). * Make sure that all users are in the same group. -* Set the configuration option <> to 002. This ensures +* Set the configuration option <> to *002*. This ensures that cached files are accessible to everyone in the group. * Make sure that all users have write permission in the entire cache directory (and that you trust all users of the shared cache). @@ -1425,11 +1774,9 @@ conditions should to be met: tells the filesystem to inherit group ownership for new directories. The following command might be useful for this: + --- ---- find $CCACHE_DIR -type d | xargs chmod g+s ---- --- The reason to avoid the hard link mode is that the hard links cause unwanted side effects, as all links to a cached file share the file's modification @@ -1442,8 +1789,7 @@ You may also want to make sure that a base directory is set appropriately, as discussed in a previous section. -Sharing a cache on NFS ----------------------- +== Sharing a cache on NFS It is possible to put the cache directory on an NFS filesystem (or similar filesystems), but keep in mind that: @@ -1462,26 +1808,27 @@ systems. One way of improving cache hit rate in that case is to set <> to *system_headers* to ignore system headers. +An alternative to putting the main cache directory on NFS is to set up a +<> file cache. + -Using ccache with other compiler wrappers ------------------------------------------ +== Using ccache with other compiler wrappers The recommended way of combining ccache with another compiler wrapper (such as -``distcc'') is by letting ccache execute the compiler wrapper. This is +"`distcc`") is by letting ccache execute the compiler wrapper. This is accomplished by defining <>, for -example by setting the environment variable *CCACHE_PREFIX* to the name of the -wrapper (e.g. *distcc*). Ccache will then prefix the command line with the +example by setting the environment variable `CCACHE_PREFIX` to the name of the +wrapper (e.g. `distcc`). Ccache will then prefix the command line with the specified command when running the compiler. To specify several prefix commands, set <> to a colon-separated list of commands. Unless you set <> to a suitable command -(see the description of that configuration option), it is not recommended to -use the form *ccache anotherwrapper compiler args* as the compilation command. -It's also not recommended to use the masquerading technique for the other -compiler wrapper. The reason is that by default, ccache will in both cases hash -the mtime and size of the other wrapper instead of the real compiler, which -means that: +(see the description of that configuration option), it is not recommended to use +the form `ccache anotherwrapper compiler args` as the compilation command. It's +also not recommended to use the masquerading technique for the other compiler +wrapper. The reason is that by default, ccache will in both cases hash the mtime +and size of the other wrapper instead of the real compiler, which means that: * Compiler upgrades will not be detected properly. * The cached results will not be shared between compilations with and without @@ -1491,31 +1838,27 @@ Another minor thing is that if <> is used, ccache will not invoke the other wrapper when running the preprocessor, which increases performance. You can use <> if you also want to invoke -the other wrapper when doing preprocessing (normally by adding *-E*). +the other wrapper when doing preprocessing (normally by adding `-E`). -Caveats -------- +== Caveats * The direct mode fails to pick up new header files in some rare scenarios. See - _<<_the_direct_mode,The direct mode>>_ above. + _<>_ above. -Troubleshooting ---------------- +== Troubleshooting -General -~~~~~~~ +=== General A general tip for getting information about what ccache is doing is to enable debug logging by setting the configuration option <> (or -the environment variable *CCACHE_DEBUG*); see _<<_cache_debugging,Cache -debugging>>_ for more information. Another way of keeping track of what is +the environment variable *CCACHE_DEBUG*); see _<>_ +for more information. Another way of keeping track of what is happening is to check the output of *ccache -s*. -Performance -~~~~~~~~~~~ +=== Performance Ccache has been written to perform well out of the box, but sometimes you may have to do some adjustments of how you use the compiler and ccache in order to @@ -1525,77 +1868,74 @@ Since ccache works best when I/O is fast, put the cache directory on a fast storage device if possible. Having lots of free memory so that files in the cache directory stay in the disk cache is also preferable. -A good way of monitoring how well ccache works is to run *ccache -s* before and +A good way of monitoring how well ccache works is to run `ccache -s` before and after your build and then compare the statistics counters. Here are some common problems and what may be done to increase the hit rate: -* If ``cache hit (preprocessed)'' has been incremented instead of ``cache hit - (direct)'', ccache has fallen back to preprocessor mode, which is generally - slower. Some possible reasons are: +* If the counter for preprocessed cache hits has been incremented instead of the + one for direct cache hits, ccache has fallen back to preprocessor mode, which + is generally slower. Some possible reasons are: ** The source code has been modified in such a way that the preprocessor output is not affected. ** Compiler arguments that are hashed in the direct mode but not in the - preprocessor mode have changed (*-I*, *-include*, *-D*, etc) and they didn't + preprocessor mode have changed (`-I`, `-include`, `-D`, etc) and they didn't affect the preprocessor output. -** The compiler option *-Xpreprocessor* or *-Wp,_X_* (except *-Wp,-MD,_path_*, - *-Wp,-MMD,_path_*, and *-Wp,-D_define_*) is used. +** The compiler option `-Xpreprocessor` or `-Wp,++*++` (except `-Wp,-MD,`, + `-Wp,-MMD,`, and `-Wp,-D`) is used. ** This was the first compilation with a new value of the <>. ** A modification or status change time of one of the include files is too new (created the same second as the compilation is being done). See - _<<_handling_of_newly_created_header_files,Handling of newly created header - files>>_. -** The `__TIME__` preprocessor macro is (potentially) being used. Ccache turns - off direct mode if `__TIME__` is present in the source code. This is done as - a safety measure since the string indicates that a `__TIME__` macro _may_ - affect the output. (To be sure, ccache would have to run the preprocessor, - but the sole point of the direct mode is to avoid that.) If you know that - `__TIME__` isn't used in practise, or don't care if ccache produces objects - where `__TIME__` is expanded to something in the past, you can set - <> to *time_macros*. -** The `__DATE__` preprocessor macro is (potentially) being used and the date - has changed. This is similar to how `__TIME__` is handled. If `__DATE__` is - present in the source code, ccache hashes the current date in order to be - able to produce the correct object file if the `__DATE__` macro affects the - output. If you know that `__DATE__` isn't used in practise, or don't care if - ccache produces objects where `__DATE__` is expanded to something in the + _<>_. +** The `+__TIME__+` preprocessor macro is (potentially) being used. Ccache turns + off direct mode if `+__TIME__+` is present in the source code. This is done + as a safety measure since the string indicates that a `+__TIME__+` macro + _may_ affect the output. (To be sure, ccache would have to run the + preprocessor, but the sole point of the direct mode is to avoid that.) If you + know that `+__TIME__+` isn't used in practise, or don't care if ccache + produces objects where `+__TIME__+` is expanded to something in the past, you + can set <> to *time_macros*. +** The `+__DATE__+` preprocessor macro is (potentially) being used and the date + has changed. This is similar to how `+__TIME__+` is handled. If `+__DATE__+` + is present in the source code, ccache hashes the current date in order to be + able to produce the correct object file if the `+__DATE__+` macro affects the + output. If you know that `+__DATE__+` isn't used in practise, or don't care + if ccache produces objects where `+__DATE__+` is expanded to something in the past, you can set <> to *time_macros*. -** The `__TIMESTAMP__` preprocessor macro is (potentially) being used and the +** The `+__TIMESTAMP__+` preprocessor macro is (potentially) being used and the source file's modification time has changed. This is similar to how - `__TIME__` is handled. If `__TIMESTAMP__` is present in the source code, + `+__TIME__+` is handled. If `+__TIMESTAMP__+` is present in the source code, ccache hashes the string representation of the source file's modification time in order to be able to produce the correct object file if the - `__TIMESTAMP__` macro affects the output. If you know that `__TIMESTAMP__` - isn't used in practise, or don't care if ccache produces objects where - `__TIMESTAMP__` is expanded to something in the past, you can set - <> to *time_macros*. + `+__TIMESTAMP__+` macro affects the output. If you know that + `+__TIMESTAMP__+` isn't used in practise, or don't care if ccache produces + objects where `+__TIMESTAMP__+` is expanded to something in the past, you can + set <> to *time_macros*. ** The input file path has changed. Ccache includes the input file path in the direct mode hash to be able to take relative include files into account and - to produce a correct object file if the source code includes a `__FILE__` + to produce a correct object file if the source code includes a `+__FILE__+` macro. -* If ``cache miss'' has been incremented even though the same code has been +* If a cache hit counter was not incremented even though the same code has been compiled and cached before, ccache has either detected that something has changed anyway or a cleanup has been performed (either explicitly or - implicitly when a cache limit has been reached). Some perhaps unobvious - things that may result in a cache miss are usage of `__TIME__`, `__DATE__` or - `__TIMESTAMP__` macros, or use of automatically generated code that contains + implicitly when a cache limit has been reached). Some perhaps unobvious things + that may result in a cache miss are usage of `+__TIME__+`, `+__DATE__+` or + `+__TIMESTAMP__+` macros, or use of automatically generated code that contains a timestamp, build counter or other volatile information. -* If ``multiple source files'' has been incremented, it's an indication that - the compiler has been invoked on several source code files at once. Ccache - doesn't support that. Compile the source code files separately if possible. -* If ``unsupported compiler option'' has been incremented, enable debug logging +* If "`Multiple source files`" has been incremented, it's an indication that the + compiler has been invoked on several source code files at once. Ccache doesn't + support that. Compile the source code files separately if possible. +* If "`Unsupported compiler option`" has been incremented, enable debug logging and check which compiler option was rejected. -* If ``preprocessor error'' has been incremented, one possible reason is that - precompiled headers are being used. See _<<_precompiled_headers,Precompiled - headers>>_ for how to remedy this. -* If ``can't use precompiled header'' has been incremented, see - _<<_precompiled_headers,Precompiled headers>>_. -* If ``can't use modules'' has been incremented, see _<<_c_modules,C++ - modules>>_. +* If "`Preprocessing failed`" has been incremented, one possible reason is that + precompiled headers are being used. See _<>_ for how to + remedy this. +* If "`Could not use precompiled header`" has been incremented, see + _<>_. +* If "`Could not use modules`" has been incremented, see _<>_. -Corrupt object files -~~~~~~~~~~~~~~~~~~~~ +=== Corrupt object files It should be noted that ccache is susceptible to general storage problems. If a bad object file sneaks into the cache for some reason, it will of course stay @@ -1606,9 +1946,9 @@ happens, the easiest way of fixing it is this: 1. Build so that the bad object file ends up in the build tree. 2. Remove the bad object file from the build tree. -3. Rebuild with *CCACHE_RECACHE* set. +3. Rebuild with `CCACHE_RECACHE` set. -An alternative is to clear the whole cache with *ccache -C* if you don't mind +An alternative is to clear the whole cache with `ccache -C` if you don't mind losing other cached results. There are no reported issues about ccache producing broken object files @@ -1616,15 +1956,13 @@ reproducibly. That doesn't mean it can't happen, so if you find a repeatable case, please report it. -More information ----------------- +== More information Credits, mailing list information, bug reporting instructions, source code, etc, can be found on ccache's web site: . -Author ------- +== Author Ccache was originally written by Andrew Tridgell and is currently developed and maintained by Joel Rosdahl. See AUTHORS.txt or AUTHORS.html and diff --git a/doc/NEWS.adoc b/doc/NEWS.adoc index 7d69bd4..7249904 100644 --- a/doc/NEWS.adoc +++ b/doc/NEWS.adoc @@ -1,29 +1,1140 @@ -Ccache news -=========== += Ccache news + +== Ccache 4.7.4 + +Release date: 2022-11-21 + + +=== Bug fixes + +- Fixed an inode cache race condition. + + [small]#_[contributed by Joel Rosdahl]_# + +- The default temporary directory is now `$XDG_RUNTIME_DIR/ccache-tmp` instead + of a hardcoded `/run/user//ccache-tmp`. If `XDG_RUNTIME_DIR` is not set, + `/tmp` is used. This avoids creating `/run/user/` on systems + that don't have it if compiling as root. + + [small]#_[contributed by Joel Rosdahl and Oleg Sidorkin]_# + +- Added a fallback in case `posix_fallocate` returns `EINVAL` when creating the + inode cache file. + + [small]#_[contributed by Oleg Sidorkin]_# + +- Connection timeout for an HTTP connection is now reported as a timeout instead + of an error. + + [small]#_[contributed by Joel Rosdahl]_# + +- Temporary files found in the cache are no longer counted in + `--show-compression`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Removed duplicate magic header in output from `--inspect`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Ccache now properly waits for all recompression jobs to finish when there is + no `f` subdirectory in the cache. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Other minor improvements + +- Improved inode cache logging. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Documentation improvements + +- Removed stray parenthesis. + + [small]#_[contributed by Joel Rosdahl]_# + +- Improved description of how header files are handled. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added a hint about using `-fno-pch-timestamp` for precompiled headers with + Clang. + + [small]#_[contributed by Joel Rosdahl]_# + +- Removed obsolete description of compiler type "`pump`". + + [small]#_[contributed by Joel Rosdahl]_# + + + +== Ccache 4.7.3 + +Release date: 2022-11-05 + + +=== New features and improvements + +- Re-added support for handling a `-Wp,-MD` or `-Wp,-MMD` option with `-o` but + without `-MMD`, `-MQ` or `-MT` for GCC and Clang. (This combination of options + is used by the Linux kernel build system and became unsupported as a + side-effect of a feature added in ccache 4.7.) + + [small]#_[contributed by Joel Rosdahl]_# + +- Variables that affect Clang version on macOS are now added to the input + hash. + + [small]#_[contributed by Joel Rosdahl]_# + +- Variables that affect the underlying compiler used by ICC (the Intel compiler) + are now added to the input hash. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Bug fixes + +- Fixed parsing of sloppiness with a trailing delimiter. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Other minor improvements + +- Removed a redundant slash in URLs when querying remote HTTP storage. + + [small]#_[contributed by Joel Rosdahl]_# + +- Improved logging related to the inode cache. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Documentation improvements + +- Added documentation of the default value of *keep_comments_cpp*. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Test improvements + +- Silenced various benign test warnings. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.7.2 + +Release date: 2022-10-29 + + +=== Bug fixes + +- Fixed a problem when using 32-bit and 64-bit ccache binaries with the same + inode cache file. + + [small]#_[contributed by Joel Rosdahl]_# + +- Ccache now processes the argument following a `-Xarch` option. + + [small]#_[contributed by an anonymous user]_# + +- Made sure to use the configured umask for command line operations like + `--zero-stats` so that newly created cache directories will have correct + permissions. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Build improvements + +- Include `limits.h` for `PATH_MAX`. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Test improvements + +- Disabled flaky Windows profiling tests. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.7.1 + +Release date: 2022-10-22 + + +=== Bug fixes + +- Fixed a regression in 4.7 related to using the `-MD` or `-MMD` option when + compiling an assembler file. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Build improvements + +- Added support for compiling ccache with GCC 8. + + [small]#_[contributed by Orgad Shaneh]_# + +- Removed unneeded C++11 flag in libatomic test. + + [small]#_[contributed by Orgad Shaneh]_# + + +== Ccache 4.7 + +Release date: 2022-10-17 + + +=== Compatibility notes + +- The cache entry format has changed, so ccache 4.7 will not share cache entries + with earlier versions. Different ccache versions can however still use the + same cache storage without any issues. + +- The https://ccache.dev/manual/4.7.html#config_cache_dir[default location of + the cache directory] on Windows has been changed from `%APPDATA%\ccache` to + `%LOCALAPPDATA%\ccache`. Please remove any existing `%APPDATA%\ccache` + directory or move it to `%LOCALAPPDATA%\ccache` to keep it. + + +=== Changed tooling + +- A C++17 compiler and CMake 3.15 or newer are now required to + build ccache. + + +=== New and improved features + +- The https://ccache.dev/manual/4.7.html#config_inode_cache[inode cache] is now + enabled by default, but only if the filesystem where ccache's temporary + directory is located is known to work with the inode cache. + + [small]#_[contributed by Joel Rosdahl]_# + +- Improved performance of cache entry reading and writing. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added a https://ccache.dev/manual/4.7.html#config_remote_only[*remote_only*] + configuration option, which tells ccache to disable local storage. This way, + ccache can use a shared + link:pass:[https://ccache.dev/manual/4.7.html#_remote_storage_backends][network cache] + without writing cache entries locally, which can be useful if the local + storage is ephemeral. + + [small]#_[contributed by Joel Rosdahl]_# + +- Renamed configuration option *secondary_storage* to + https://ccache.dev/manual/4.7.html#config_remote_storage[*remote_storage*]. + The old spelling will still work as a deprecated alias. + + [small]#_[contributed by Joel Rosdahl]_# + +- Improved handling of manifests when using both local and remote storage. ++ +On a cache miss, ccache used to update the local manifest file and write it to +remote storage, thereby overwriting any preexisting manifest entries on remote +storage. Similarly, the local manifest could be replaced by the remote manifest +on a remote cache hit, thus discarding local entries. This has now been improved +so that ccache will merge local and remote manifest entries transparently. + +[small]#_[contributed by Joel Rosdahl and an anonymous user]_# + +- Added support for caching assembler listing files with compiler option + `-Wa,-a=file`. This also fixes a bug in ccache 4.6.2+ where usage of + `-Wa,-march=...` makes ccache fall back to running the compiler. + + [small]#_[contributed by Joel Rosdahl]_# + +- Improved statistics shown by `ccache --show-stats`. Most values are now shown + in relation to a total count, and some less useful statistics counters have + been changed to be displayed only with `--verbose`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Improved statistics for remote hits and misses. + + [small]#_[contributed by Joel Rosdahl]_# + +- Renamed "`Primary config`" to "`Config file`" and "`Secondary config`" to + "`System config file`" in the output of `ccache --show-stats --verbose`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added support for + link:pass:[https://ccache.dev/manual/4.7.html#_run_modes][masquerading as a + compiler] via a copy or hard link of the ccache executable. Previously, ccache + only supported masquerading as a compiler by using a symbolic link. + + [small]#_[contributed by Joel Rosdahl]_# + +- A timestamp is now included in + link:pass:[https://ccache.dev/manual/4.7.html#_cache_debugging][per-object + debug filenames]. This makes it easier to compare two builds without having to + save previous debug files before the second build. It also makes sure debug + files won't be overwritten if an object file is compiled several times during + one build. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added support for Clang's `--` option. + + [small]#_[contributed by Joel Rosdahl]_# + +- Cache entries are now shared for different `-MT`/`-MQ` options. + + [small]#_[contributed by Joel Rosdahl]_# + +- The https://ccache.dev/manual/4.7.html#config_cache_dir[default location of + the cache directory] on Windows has been changed to `%LOCALAPPDATA%\ccache`, + both in Windows Bash and Windows native environments such as CMD or + Powershell. ++ +Previous ccache versions defaulted to storing the cache in `%APPDATA%\ccache` on +Windows in native environment (i.e., when the `%USER%` variable was not set), +which could result in large network file transfers of the cache in domain +environments and similar problems. + +[small]#_[contributed by Rafael Kitover]_# + +- Ccache now uses subsecond resolution timestamps when checking for "`too new + include files`". In practice, this means that the direct mode will no longer + be disabled for newly generated include files. + + [small]#_[contributed by Joel Rosdahl]_# + +- Started using subsecond resolution timestamps in manifest files. This improves + accuracy with *file_stat_matches* + https://ccache.dev/manual/4.7.html#config_sloppiness[sloppiness]. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added support for subsecond timestamps on macOS. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added link:pass:[https://ccache.dev/manual/4.7.html#_the_depend_mode][depend + mode] support for MSVC. + + [small]#_[contributed by Orgad Shaneh and Luboš Luňák]_# + +- Added support for the Intel compiler on Windows. + + [small]#_[contributed by Daniel Richtmann]_# + +- Ccache now sets `CCACHE_DISABLE` when running the compiler. This avoids + running ccache twice (and potentially storing two different results in the + cache) if, for instance, the compiler happens to be a wrapper script that in + turn runs `ccache $compiler ...`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added support for Redis over Unix sockets. + + [small]#_[contributed by Anders F Björklund]_# + +- Added support for GCC's `-fprofile-abs-path` option by including the current + working directory in the input hash. To opt out of this, set *gcno_cwd* + https://ccache.dev/manual/4.7.html#config_sloppiness[sloppiness]. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made it possible to exclude `-frandom-seed=_seed_` options from the input hash + with a new *random_seed* + https://ccache.dev/manual/4.7.html#config_sloppiness[sloppiness]. + + [small]#_[contributed by Raihaan Shouhell]_# + +- Added support for server name in + link:pass:[https://ccache.dev/manual/4.7.html#_file_storage_backend][remote + storage file URLs] on Windows. + + [small]#_[contributed by Joel Rosdahl]_# + +- Configuration file locations are now included in the + link:pass:[https://ccache.dev/manual/4.7.html#_cache_debugging][debug log]. + + [small]#_[contributed by Joel Rosdahl]_# + +- The inode cache file is no longer removed with `ccache --clear`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Result format version and number of files are now printed when inspecting a + result cache entry file. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Removed features + +- Removed the *share-hits* attribute for remote storage. It has been superseded + by the https://ccache.dev/manual/4.7.html#config_remote_only[*remote_only*] + configuration option. + + [small]#_[contributed by Joel Rosdahl]_# + +- Buggy support for GCC-specific environment variables `DEPENDENCIES_OUTPUT` and + `SUNPRO_DEPENDENCIES` has been removed. If any of those variables are set, + ccache now just falls back to running the real compiler. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Bug fixes + +- The correct dependency target will now be produced even when a + https://ccache.dev/manual/4.7.html#config_base_dir[base directory] is used -- + the dependency target will still be an absolute path. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed a bug that could lead to bad modification timestamp (mtime) for object + files on systems that lack `utimensat` and `utimes` system calls (such as + Windows). + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed ordering of MSVC include directory options when using + `/external:I`. + + [small]#_[contributed by Raihaan Shouhell]_# + +- Fixed capturing of MSVC stdout/stderr when running from Visual Studio. + + [small]#_[contributed by Orgad Shaneh]_# + +- Carriage return characters are now retained in the compiler output on + Windows. + + [small]#_[contributed by Orgad Shaneh]_# + +- Made sure not to increment the "`preprocessed_cache_miss`" counter in recache + mode. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed an issue with the inode cache in combination with + `+__DATE__+`/`+__TIME__+`/`+__TIMESTAMP__+` macros in the source code. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made sure to enable the inode cache only if subsecond timestamps are + available. + + [small]#_[contributed by Joel Rosdahl]_# + +- Cache entries created with enabled hard linking or file cloning are no longer + written to remote storage since they won't be possible to retrieve correctly + anyway. + + [small]#_[contributed by Joel Rosdahl]_# + +- Ccache now accepts spaces between target and colon when parsing dependency + files in the depend mode. + + [small]#_[contributed by Louis Caron]_# + +- Fixed a crash when failing to write an error message to stderr after failing + to write to the log file. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made ccache able to retrieve an object file from the cache even if the + destination object file exists and is read-only. + + [small]#_[contributed by Joel Rosdahl]_# + +- The `.exe` extension is now stripped from the ccache executable name in + `ccache --version` on Windows. + + [small]#_[contributed by Orgad Shaneh]_# + +- Fixed naming of temporary files written to the cache directory. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Build improvements + +- Enabled static runtime linking for MSVC. + + [small]#_[contributed by Cristian Adam and Rafael Kitover]_# + +- The Zstandard and Hiredis dependencies are now by default downloaded from the + Internet when unavailable. + + [small]#_[contributed by Rafael Kitover]_# + +- Added support for pkgconfig to find a Zstandard installation. + + [small]#_[contributed by Rosen Penev]_# + +- Removed usage of the deprecated `codecvt` header. + + [small]#_[contributed by Orgad Shaneh]_# + +- Added headers to CMake project files. + + [small]#_[contributed by Orgad Shaneh]_# + + +=== Test improvements + +- Made the integration tests work on Windows. + + [small]#_[contributed by Orgad Shaneh and R. Voggenauer]_# + +- Improved diagnostics from the "`Version output readable`" test. + + [small]#_[contributed by Orgad Shaneh]_# + +- Made setting the `KEEP_TESTDIR` variable actually work. + + [small]#_[contributed by Louis Caron]_# + +- Fixed a typo in depend mode tests. + + [small]#_[contributed by Louis Caron]_# + +- Added more depend mode tests. + + [small]#_[contributed by Louis Caron]_# + + +=== Documentation improvements + +- Improved the + https://github.com/ccache/ccache/blob/v4.7/doc/INSTALL.md[installation + guide]. + + [small]#_[contributed by Rafael Kitover]_# + +- Fixed a typo in the help text. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.6.3 + +Release date: 2022-08-27 + + +=== Bug fixes + +- Fixed MSVC support (regression in ccache 4.6.2). + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed detection of PCH header for concatenated `-include` option (e.g., + `-includepch.h` instead of `-include pch.h`). + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Build improvements + +- Fixed build with musl when using GCC 12. + + [small]#_[contributed by Khem Raj]_# + + +=== Test improvements + +- Disabled the "`Failure to write output file`" test when running on a + filesystem that doesn't support read-only directories. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.6.2 + +Release date: 2022-08-22 + + +=== Bug fixes + +- Fixed a race condition that could lead to a crash if a file in the cache is + removed with unlucky timing, e.g. by another ccache process doing cache + cleanup. + + [small]#_[contributed by Joel Rosdahl]_# + +- Dependency file rewriting will now always be performed if + `base_dir`/`CCACHE_BASEDIR` is active. This fixes a problem with the + dependency file content when Clang is used with `-fsanitize=address`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed handling of error condition for `--hash-file`/`--checksum-file`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made sure to enable the inode cache only if subsecond `stat` timestamps are + available. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added a work-around for a Clang bug when writing to a full NFS file system. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made failure writing to the output file increment the "`bad output file`" + counter instead of "`cache miss`". + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed false positive cache hits for code constructions similar to + `__asm__(".incbin" " \"file\"")`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed false success for `-fcolor-diagnostics` probe with GCC. A side effect of + this is that a compiler type that ccache can't identify from the compiler name + (such as `/usr/bin/cc` where `cc` is not a symlink) from now on won't produce + color diagnostics when used via ccache even if the compiler actually is GCC or + Clang. + + [small]#_[contributed by Joel Rosdahl]_# + +- More cases of invalid secondary storage URLs are now handled gracefully. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed the display of maximum cache size in `ccache -s` if it's 0 (= + unlimited). + + [small]#_[contributed by Joel Rosdahl]_# + +- Removed AsciiDoc markup from help text of `--trim-dir`. + + [small]#_[contributed by Joel Rosdahl]_# + +- The temporary directory is now cleaned up properly even if it's left + unconfigured. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made cleanup of the temporary directory not rely a directory timestamp. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made sure to retain mtime/atime when recompressing cache files with + `-X`/`--recompress`. + + [small]#_[contributed by Joel Rosdahl]_# + +- The correct umask is now used when populating the primary cache from a + secondary cache. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed creation of temporary files on file systems that don't support hard + links (such as FAT32). + + [small]#_[contributed by Joel Rosdahl]_# + +- Added knowledge about `-Wa,...=file` so that ccache then falls back to running + the real compiler. + + [small]#_[contributed by Joel Rosdahl]_# + +- Corrected handling of space in paths when using response file on Windows. + + [small]#_[contributed by Sergey Semushin]_# + +- Fixed crash due to empty include filename in preprocessor output generated by + `f2c`. + + [small]#_[contributed by Oleg Sidorkin]_# + + +=== Build improvements + +- Fixed build problems with a development version of GCC 13. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed build problems with MSVC. + + [small]#_[contributed by Florin Trofin]_# + + +=== Test improvements + +- Clang warnings from the "`-fdebug-prefix-map`" test are now suppressed. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made sure to only run the "`-ftest-coverage + -fprofile-dir`" test with GCC. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed printing of error messages with embedded newlines. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed warning when running the "`inode_cache`" test in isolation. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed test failure when the compiler used for testing is an old ccache version + masquerading as the compiler. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Documentation improvements + +- Mentioned that mtime is used for LRU cleanup. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.6.1 + +Release date: 2022-05-15 + + +=== New features + +- Added support for passing a directory to the MSVC `/Fo` option. + + [small]#_[contributed by Orgad Shaneh]_# + +- Added knowledge about the `-imsvc` compiler option. + + [small]#_[contributed by Jacob Young]_# + +- Added knowledge about the `-z` linker option. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Bug fixes + +- Improved handling of `.gcno` files in combination with absolute input file + paths. + + [small]#_[contributed by Joel Rosdahl]_# + +- Adapted to changes in GCC 9+ `.gcno` files, which contain the current working + directory, by including said directory in the hash. You can opt out of this by + enabling "`gcno_cwd sloppiness`". + + [small]#_[contributed by Joel Rosdahl]_# + +- A preexisting object file is no longer considered when using + `-fsyntax-only`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Authenticate with Redis before database selection. + + [small]#_[contributed by an anonymous user]_# + +- Don't exit with an error on failure reading a cached file. + + [small]#_[contributed by an anonymous user]_# + +- Bail out on too hard MSVC environment variables `CL` and `+_CL_+`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Only use `/run/user//ccache-tmp` as the temporary directory if it's + writable. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed handling of the final newline in cached standard output from the + compiler. + + [small]#_[contributed by Orgad Shaneh]_# + +- Fixed a bug related to distcc markers in standard error output. + + [small]#_[contributed by Joel Rosdahl]_# + +- Paths to `base_dir` are now properly normalized on Windows. + + [small]#_[contributed by Vili Väinölä and Joel Rosdahl]_# + +- Fixed handling of MSVC `/Fp` and `/Yu` options with concatenated path. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed "`Multiple precompiled headers used`" error if MSVC `/Yu` option is used + after `/Fp`. + + [small]#_[contributed by Alexey Telishev]_# + +- Check for short reads when reading strings in result/manifest files. + + [small]#_[contributed by Gregor Jasny]_# + +- Log expanded secondary storage URL in put/remove. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed logging of statistics counters with value higher than one in debug log + and stats log. + + [small]#_[contributed by Joel Rosdahl]_# + +- Avoid incorrect error log message for Redis write operations in `reshare` + mode. + + [small]#_[contributed by Joel Rosdahl]_# + +- Support Redis URL without host (meaning localhost). + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Build improvements + +- Prefer CMake find module for hiredis and zstd packages. + + [small]#_[contributed by Cristian Adam and Joel Rosdahl]_# + +- Fixed building and linking BLAKE3 with MSVC. + + [small]#_[contributed by Rafael Kitover]_# + +- Fixed static linkage with hiredis on Windows. + + [small]#_[contributed by Orgad Shaneh]_# + +- Fixed miscompile of nonstd::expected on MSVC v19.22. + + [small]#_[contributed by Jacob Young]_# + +- Fixed build arguments to clang-cl. + + [small]#_[contributed by Jacob Young]_# + +- Fixed parsing of MSVC response files. + + [small]#_[contributed by Jacob Young]_# + +- Support Git 1.x when determining ccache version. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Test improvements + +- Worked around an endianness problem which affected builds and tests on + big-endian systems. + + [small]#_[contributed by Joel Rosdahl]_# + +- A C++-capable compiler is no longer required for the test suite. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed an issue with inode cache tests, leading to sporadic failures in the + inode test suite when running many parallel tests. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed sporadic failures in the profiling test suite. + + [small]#_[contributed by Jacob Young]_# + + +=== Documentation improvements + +- Added reference to example build configs in installation instructions. + + [small]#_[contributed by an anonymous user]_# + +- Default cache locations are now mentioned for Windows and macOS as well. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added a warning about usage of `base_dir`. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.6 + +Release date: 2022-02-27 + + +=== New features + +- Added support for caching calls to Microsoft Visual C++ (MSVC) and clang-cl + (MSVC compatibility for Clang). + + [small]#_[contributed by Cristian Adam, Luboš Luňák, Orgad Shaneh and Joel + Rosdahl]_# + +- Added an option to use a bearer token with the HTTP backend. This makes it + possible to use e.g. Google Cloud Storage as a secondary storage backend. + + [small]#_[contributed by an anonymous user]_# + +- Added support for caching standard output from the compiler. + + [small]#_[contributed by Luboš Luňák and Joel Rosdahl]_# + +- Added a new `--inspect` option for debugging cache entries, replacing the + previous `--dump-manifest` and `--dump-result` options. + + [small]#_[contributed by Joel Rosdahl]_# + +- Enabled HTTP keep-alive by default. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Bug fixes + +- Fixed copying of binary files on Windows. + + [small]#_[contributed by R. Voggenauer]_# + +- Improved detection of the `.incbin` assembler directive to reduce false + positives. + + [small]#_[contributed by Alexey Sheplyakov]_# + +- Ccache now verifies that `/run/user//ccache-tmp` is writable before using + it for temporary files. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed statistics output for secondary storage. + + [small]#_[contributed by Orgad Shaneh]_# + +- Fixed a problem when copying a cache entry from secondary storage into an + empty primary storage. + + [small]#_[contributed by Joel Rosdahl]_# + +- Visual Studio .rsp files with UTF-16LE encoding are now handled correctly. + + [small]#_[contributed by Vili Väinölä]_# + +- Made conversion to relative paths more reliable on Windows. + + [small]#_[contributed by Marius Zwicker]_# + +- The process umask is now respected when making hard linked files read only. + + [small]#_[contributed by Joel Rosdahl]_# + +- A forced recache is no longer considered a "`direct cache miss`". + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Documentation improvements + +- Corrected reference to the `debug_dir` option. + + [small]#_[contributed by Joel Rosdahl]_# + +- Improved documentation of `--config-path`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added documentation that compiler plugins are hashed too. + + [small]#_[contributed by Philipp Gortan]_# + + +=== Test improvements + +- The "`trim_dir`" test suite is now only run when cleanup tests are enabled. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.5.1 + +Release date: 2021-11-17 + + +=== Bug fixes + +- Fixed entry_size field for result entries. This bug affected the recompression + feature (`-X`/`--recompress`) in ccache 4.5. + + [small]#_[contributed by Joel Rosdahl]_# + +- The actual compression level is now once again stored in the cache entry + header. + + [small]#_[contributed by Joel Rosdahl]_# + +- Corrected error handling for non-constructible secondary storage backends. For + instance, this avoids a crash when a Redis server can't be reached. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.5 + +Release date: 2021-11-13 + + +=== New features + +- Made various improvements to the cache entry format. Among other things, the + header now contains a creation timestamp and a note of the ccache version used + to create the entry. The cache entry checksum has also been upgraded to use + 128-bit XXH3 instead 64-bit XXH3. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added support for cache namespaces. If a namespace configured, e.g. using + `CCACHE_NAMESPACE=some_string`, the namespace string will be added to the + hashed data for each compilation. This will make the associated cache entries + logically separate from cache entries in other namespaces, but they will still + share the same storage space. Cache entries can also be selectively removed + from the primary cache with the new command line option `--evict-namespace`, + potentially in combination with `--evict-older-than`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made HTTP keep-alive configurable, defaulting to off for now. + + [small]#_[contributed by Gregor Jasny]_# + +- Added support for rewriting absolute path to Clang option `--gcc-toolchain`. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Compatibility notes + +- A consequence of the changed cache entry format is that ccache 4.5 will not + share cache entries with earlier versions. Different ccache versions can + however still use the same cache storage without any issues. + + +=== Bug fixes + +- Fixed a problem with special characters in the user info part of URLs for HTTP + storage. + + [small]#_[contributed by Russell McClellan]_# + +- Fixed win32 log messages about file locks. + + [small]#_[contributed by Luboš Luňák]_# + +- Fixed debug directory handling on Windows. + + [small]#_[contributed by Luboš Luňák]_# + +- The hard link and file clone features are now disabled when secondary storage + is used since they only work for the local primary cache. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.4.2 + +Release date: 2021-09-28 + + +=== Bug fixes + +- Fixed a bug introduced in 4.4 where ccache could produce false direct cache + hits in some situations if it decides to disable the direct mode temporarily + (e.g. due to "`too new header`" file). + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Test improvements + +- Use shell builtin pwd command for basedir test. + + [small]#_[contributed by Kira Bruneau]_# + +- Cope with CC being a wrapper script that uses ccache. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.4.1 + +Release date: 2021-09-11 + + +=== New features + +- The secondary storage statistics section of `-s/--show-stats` is now shown + only if it's non-empty or with two verbose options. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added display of statistics counters for misses. Previously they were only + implicit in the "`hits + misses`" sums. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Bug fixes + +- Fixed spurious crashes when using the HTTP or Redis backend and the remote + connection hung up. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made sure to always store configuration origin value. + + [small]#_[contributed by Gregor Jasny]_# + + +=== Build improvements + +- The matching version of lld is now used for Clang. + + [small]#_[contributed by Gregor Jasny]_# + +- The standard linker is now used if IPO (LTO) is enabled. + + [small]#_[contributed by Gregor Jasny]_# + +- Disabled IPO (LTO) for MinGW toolchains since they seem to be generally + broken. + + [small]#_[contributed by Gregor Jasny]_# + +- Fixed build errors with Clang on Windows. + + [small]#_[contributed by Orgad Shaneh]_# + + +=== Test improvements + +- Fixed .incbin test with newer binutil versions. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed basedir test suite failure when using a symlinked CWD. + + [small]#_[contributed by Joel Rosdahl]_# + +- Improved output of differing text files on failure. + + [small]#_[contributed by Joel Rosdahl]_# + + +== Ccache 4.4 + +Release date: 2021-08-19 + + +=== New features + +- Made it possible to share a cache over network or on a local filesystem. The + configuration option `secondary_storage`/`CCACHE_SECONDARY_STORAGE` specifies + one or several storage backends to query after the primary local cache + storage. It is also possible to configure sharding (partitioning) of the cache + to spread it over a server cluster using + https://en.wikipedia.org/wiki/Rendezvous_hashing[Rendezvous hashing]. See the + _https://ccache.dev/manual/4.4.html#_secondary_storage_backends[Secondary + storage backends]_ chapter in the manual for details. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added an HTTP backend for secondary storage on any HTTP server that supports + GET/PUT/DELETE methods. See https://ccache.dev/howto/http-storage.html[How to + set up HTTP storage] for hints on how to set up an HTTP server for use with + ccache. + + [small]#_[contributed by Gregor Jasny]_# + +- Added a Redis backend for secondary storage on any server that supports the + Redis protocol. See https://ccache.dev/howto/redis-storage.html[How to set up + Redis storage] for hints on how to set up a Redis server for use with + ccache. + + [small]#_[contributed by Anders F Björklund]_# + +- Added a filesystem backend for secondary storage. It can for instance be used + for a shared cache over networked filesystems such as NFS, or for mounting a + secondary read-only cache layer into a build container. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added `--trim-dir`, `--trim-max-size` and `--trim-method` options that can be + used to trim a secondary storage directory to a certain size, e.g. via + cron. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added a configuration option `reshare`/`CCACHE_RESHARE` which makes ccache + send results to secondary storage even for primary storage cache hits. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added new statistics counters for direct/preprocessed cache misses, primary + storage hits/misses, secondary storage hits/misses/errors/timeouts and forced + recaches. + + [small]#_[contributed by Joel Rosdahl]_# + +- Improved statistics summary. The `-s`/`--show-stats` option now prints a more + condensed overview where the counters representing "`uncacheable calls`" are + summed as uncacheable and errors counters. The summary shows hit rate for + direct/preprocessed hits/misses, as well as primary/secondary storage + hits/misses. More details are shown with `-v`/`--verbose`. Note: Scripts + should use `--print-stats` (available since ccache 3.7) instead of trying to + parse the output of `--show-stats`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added a "`stats log`" feature (configuration option + `stats_log`/`CCACHE_STATSLOG`), which tells ccache to store statistics in a + separate log file specified by the user. It can for instance be used to + collect statistics for a single build without interference from other + concurrent builds. Statistics from the log file can then be viewed with + `ccache --show-log-stats`. + + [small]#_[contributed by Anders F Björklund]_# + +- Added support for clang's `--config` option. + + [small]#_[contributed by Tom Stellard]_# + +- Added support for one `-Xarch_*` option that matches a corresponding `-arch` + option. + + [small]#_[contributed by Joel Rosdahl]_# + +- Renamed the `--directory` option to `--dir` for consistency with other + options. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made the `--config-path` and `--dir` options affect the whole command line so + that they don't have to be put before `-s`/`--show-stats`. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made `--dump-manifest` and `--dump-result` accept filename `-` for reading + from standard input. + + [small]#_[contributed by Anders F Björklund]_# + +- Made the output of `--print-stats` sorted. + + [small]#_[contributed by Joel Rosdahl]_# + +- Added more internal trace points. + + [small]#_[contributed by Joel Rosdahl]_# + + +=== Bug fixes + +- Fixed a crash if using `base_dir` and `$PWD` is set to a relative path. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed a bug with `-fprofile-generate` where ccache could give false positive + cache hits when compiling with relative paths in another directory. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed a bug in `debug_dir`/`CCACHE_DEBUGDIR`. The absolute path to the object + file was not created correctly if the object file didn't already exist. + + [small]#_[contributed by Joel Rosdahl]_# + +- Disabled preprocessor hits for pre-compiled headers with Clang again. + + [small]#_[contributed by Arne Hasselbring]_# + +- Fixed a problem when using the Gold linker on MIPS by only probing for a + faster linker in dev build mode and on x86_64. + + [small]#_[contributed by Joel Rosdahl]_# + +- Made the `-DENABLE_TRACING=1` mode work again. + + [small]#_[contributed by Anders F Björklund]_# + + +=== Changed tooling + +- A C++14 compiler or newer is now required to build ccache. For GCC, this means + version 6 or newer in practice. + +- CMake 3.10 or newer is now required to build ccache. + +- https://asciidoctor.org[Asciidoctor] is now required to build ccache + documentation. + + +=== Build/test/documentation improvements + +- Fixed an issue in the modules test suite that showed up when running the + ccache test suite with the clang wrapper provided by Nixpkgs. + + [small]#_[contributed by Ryan Burns]_# + +- Made the nvcc_ldir test suite require a working NVCC. + + [small]#_[contributed by Michael Kruse]_# + +- Made the ivfsoverlay test suite more robust. + + [small]#_[contributed by Michael Kruse]_# + +- Fixed issue with usage of `/FI` when building ccache with MSVC. + + [small]#_[contributed by Michael Kruse]_# + +- Fixed Apple Clang detection in the integration test suite. + + [small]#_[contributed by Gregor Jasny]_# + +- Made clang the default compiler when running the test suite on macOS. + + [small]#_[contributed by Gregor Jasny]_# + +- Silenced stray printout from "-P -c" test case. + + [small]#_[contributed by Joel Rosdahl]_# + +- Fixed selection of the ccache binary to use when running the test suite with + multi-config generators like Xcode. + + [small]#_[contributed by Gregor Jasny]_# + +- Fixed CMake feature detection for `ctim`/`mtim` fields in `struct stat`. + + [small]#_[contributed by Gregor Jasny]_# + +- Fixed issue with not linking to .lib correctly on Windows. + + [small]#_[contributed by R. Voggenauer]_# + +- Made it possible to override `CCACHE_DEV_MODE` on the command line. + + [small]#_[contributed by Joel Rosdahl]_# + +- Improved HTML documentation style. + + [small]#_[contributed by Joel Rosdahl with minor fixes by Orgad Shaneh]_# + + +== Ccache 4.3 -Ccache 4.3 ----------- Release date: 2021-05-09 -New features -~~~~~~~~~~~~ + +=== New features - Ccache now ignores the Clang compiler option `-ivfsoverlay` and its argument - if you opt in to ``ivfsoverlay sloppiness''. This is useful if you use Xcode, + if you opt in to "`ivfsoverlay sloppiness`". This is useful if you use Xcode, which uses a virtual file system (VFS) for things like combining Objective-C and Swift code. -- When using `-P` in combination with `-E`, ccache now reports this as ``called - for preprocessing'' instead of ``unsupported compiler option''. +- When using `-P` in combination with `-E`, ccache now reports this as "`called + for preprocessing`" instead of "`unsupported compiler option`". - Added support for `-specs file.specs` and `--specs file.specs` without an equal sign between the arguments. -Bug fixes -~~~~~~~~~ +=== Bug fixes -- "Split dwarf" code paths are now disabled when outputting to `/dev/null`. This +- "`Split dwarf`" code paths are now disabled when outputting to `/dev/null`. This avoids an attempt to delete `/dev/null.dwo`. - Made the `stat`/`lstat` wrapper function for Windows treat pending deletes as @@ -32,13 +1143,12 @@ Bug fixes - Fixed a bug that made ccache process header files redundantly for some relative headers when using Clang. -- The object path in now included in the input hash when using `-fprofile-arcs` +- The object path is now included in the input hash when using `-fprofile-arcs` (or `--coverage`) since the object file embeds a `.gcda` path based on the object file path. -Build improvements -~~~~~~~~~~~~~~~~~~ +=== Build improvements - Added an `ENABLE_DOCUMENTATION` build option (default: true) that can be used to disable the build of documentation. @@ -50,12 +1160,12 @@ Build improvements expands to the empty string. -Ccache 4.2.1 ------------- +== Ccache 4.2.1 + Release date: 2021-03-27 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Ccache now only duplicates the stderr file descriptor into `$UNCACHED_ERR_FD` for calls to the preprocessor/compiler. This works around a complex bug in @@ -87,8 +1197,7 @@ Bug fixes - Fixed handling of long command lines on Windows. -Portability and build improvements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Portability and build improvements - Build configuration scripts now probe for atomic increment as well. This fixes a linking error on Sparc. @@ -98,29 +1207,28 @@ Portability and build improvements - Added support for building ccache with xlclang++ on AIX 7.2. -- Fixed assertion in the "Debug option" test. +- Fixed assertion in the "`Debug option`" test. - Made build configuration skip using ccache when building with MSVC. - Upgraded to doctest 2.4.6. This fixes a build error with glibc >= 2.34. -Documentation -~~~~~~~~~~~~~ +=== Documentation -- Fixed markup of `compiler_type` value "other". +- Fixed markup of `compiler_type` value `other`. - Fixed markup of `debug_dir` documentation. - Fixed references to the `extra_files_to_hash` configuration option. -Ccache 4.2 ----------- +== Ccache 4.2 + Release date: 2021-02-02 -New features -~~~~~~~~~~~~ + +=== New features - Improved calculation of relative paths when using `base_dir` to also consider canonical paths (i.e. paths with dereferenced symlinks) as candidates. @@ -133,12 +1241,11 @@ New features - The value of the `SOURCE_DATE_EPOCH` variable is now only hashed if it potentially affects the output from ccache. This means that ccache now (like before version 4.0) will be able to produce cache hits for source code that - doesn't contain `__DATE__` or `__TIME__` macros regardless of the value of + doesn't contain `+__DATE__+` or `+__TIME__+` macros regardless of the value of `SOURCE_DATE_EPOCH`. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed a bug where a non-Clang compiler would silently accept the Clang-specific `-f(no-)color-diagnostics` option when run via ccache. This @@ -165,8 +1272,7 @@ Bug fixes - Fixed retrieval of the object file the destination is `/dev/null`. -Portability and build improvements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Portability and build improvements - Additional compiler flags like `-Wextra -Werror` are now only added when building ccache in developer mode. @@ -207,8 +1313,7 @@ Portability and build improvements - Took steps towards being able to run the test suite on Windows. -Documentation -~~~~~~~~~~~~~ +=== Documentation - Improved wording of `compiler_check` string values. @@ -225,16 +1330,16 @@ Documentation - Mention that ccache requires the `-c` compiler option. -Ccache 4.1 ----------- +== Ccache 4.1 + Release date: 2020-11-22 -New features -~~~~~~~~~~~~ + +=== New features - Symlinks are now followed when guessing the compiler. This makes ccache able - to guess compiler type “GCC” for a common symlink chain like this: - `/usr/bin/cc` → `/etc/alternatives/cc` → `/usr/bin/gcc` → `gcc-9` → + to guess compiler type "`GCC`" for a common symlink chain like this: + `/usr/bin/cc` -> `/etc/alternatives/cc` -> `/usr/bin/gcc` -> `gcc-9` -> `x86_64-linux-gnu-gcc-9`. - Added a new `compiler_type` (`CCACHE_COMPILERTYPE`) configuration option that @@ -247,20 +1352,19 @@ New features `CCACHE_CONFIGPATH` temporarily. -Bug fixes -~~~~~~~~~ +=== Bug fixes - The original color diagnostics options are now retained when forcing colored output. This fixes a bug where feature detection of the `-fcolor-diagnostics` option would succeed when run via ccache even though the actual compiler - doesn’t support it (e.g. GCC <4.9). + doesn't support it (e.g. GCC <4.9). - Fixed a bug related to umask when using the `umask` (`CCACHE_UMASK`) configuration option. - Allow `ccache ccache compiler ...` (repeated `ccache`) again. -- Fixed parsing of dependency file in the “depend mode” so that filenames with +- Fixed parsing of dependency file in the "`depend mode`" so that filenames with space or other special characters are handled correctly. - Fixed rewriting of the dependency file content when the object filename @@ -273,16 +1377,15 @@ Bug fixes found out at runtime that file cloning is unsupported by the OS. -Portability and build fixes -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Portability and build fixes - The ccache binary is now linked with `libatomic` if needed. This fixes build problems with GCC on ARM and PowerPC. - Fixed build of BLAKE3 code with Clang 3.4 and 3.5. -- Fixed “use of undeclared identifier 'CLONE_NOOWNERCOPY'” build error on macOS - 10.12. +- Fixed "`use of undeclared identifier 'CLONE_NOOWNERCOPY'`" build error on + macOS 10.12. - Fixed build problems related to missing AVX2 and AVX512 support on older macOS versions. @@ -290,7 +1393,7 @@ Portability and build fixes - Fixed static linkage with libgcc and libstdc++ for MinGW and made it optional. -- Fixed conditional compilation of “robust mutex” code for the inode cache +- Fixed conditional compilation of "`robust mutex`" code for the inode cache routines. - Fixed badly named man page filename (`Ccache.1` instead of `ccache.1`). @@ -298,8 +1401,7 @@ Portability and build fixes - Disabled some tests on ancient Clang versions. -Other improvements and fixes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Other improvements and fixes - The man page is now built by default if the required tools are available. @@ -307,20 +1409,19 @@ Other improvements and fixes - Improved build errors when building ccache with very old compiler versions. -- Fall back to version “unknown” when Git is not installed. +- Fall back to version "`unknown`" when Git is not installed. - Documented the relationship between `CCACHE_DIR` and `-d/--directory`. - Fixed incorrect reference and bad markup in the manual. -Ccache 4.0 ----------- +== Ccache 4.0 + Release date: 2020-10-18 -Summary of major changes -~~~~~~~~~~~~~~~~~~~~~~~~ +=== Summary of major changes - Changed the default cache directory location to follow the XDG base directory specification. @@ -337,31 +1438,29 @@ Summary of major changes - Improved cache directory structure. -- Added support for using file cloning (AKA “reflinks”). +- Added support for using file cloning (AKA "`reflinks`"). -- Added an experimental “inode cache” for file hashes. +- Added an experimental "`inode cache`" for file hashes. -Compatibility notes -~~~~~~~~~~~~~~~~~~~ +=== Compatibility notes - The default location of the cache directory has changed to follow the XDG - base directory specification (<<_detailed_functional_changes,more details + base directory specification (<>). This means that scripts can no longer assume that the cache directory is `~/.ccache` by default. The `CCACHE_DIR` environment variable still overrides the default location just like before. - The cache directory structure has changed compared to previous versions - (<<_detailed_functional_changes,more details below>>). This means that ccache + (<>). This means that ccache 4.0 will not share cache results with earlier versions. It is however safe to run ccache 4.0 and earlier versions against the same cache directory: cache bookkeeping, statistics and cleanup are backward compatible, with the minor - exception that some statistics counters incremented by ccache 4.0 won’t be + exception that some statistics counters incremented by ccache 4.0 won't be visible when running `ccache -s` with an older version. -Changed tooling -~~~~~~~~~~~~~~~ +=== Changed tooling - CMake is now used instead of Autoconf for configuration and building. @@ -371,10 +1470,9 @@ Changed tooling - Ccache can now be built using Microsoft Visual C++. -Detailed functional changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== Detailed functional changes -- All data of a cached result is now stored in a single file called “result” +- All data of a cached result is now stored in a single file called "`result`" instead of up to seven files. This reduces inode usage and improves data locality. @@ -410,12 +1508,12 @@ Detailed functional changes The `cache_dir_levels` (`CCACHE_NLEVELS`) configuration option has therefore been removed. -- Added an experimental “inode cache” for file hashes, allowing computed hash +- Added an experimental "`inode cache`" for file hashes, allowing computed hash values to be reused both within and between builds. The inode cache is off by default but can be enabled by setting `inode_cache` (`CCACHE_INODECACHE`) to `true`. -- Added support for using file cloning (AKA “reflinks”) on Btrfs, XFS and APFS +- Added support for using file cloning (AKA "`reflinks`") on Btrfs, XFS and APFS to copy data to and from the cache very efficiently. - Two measures have been implemented to make the hard link mode safer: @@ -445,9 +1543,9 @@ Detailed functional changes `-x/--show-compression` and `-X/--recompress`. - When supported by the CPU, a SIMD-friendly (using AVX2) algorithm is now used - to scan input source code for `__DATE__`, `__TIME__` and `__TIMESTAMP__` - macros. This can decrease the number of CPU cycles for a direct cache hit - with up to 15% in some cases. + to scan input source code for `+__DATE__+`, `+__TIME__+` and `+__TIMESTAMP__+` + macros. This can decrease the number of CPU cycles for a direct cache hit with + up to 15% in some cases. - Some unnecessary `stat(2)` system calls are now avoided when verifying header files. @@ -465,8 +1563,8 @@ Detailed functional changes - Added optional logging to syslog if `log_file` (`CCACHE_LOGFILE`) is set to `syslog`. -- The compiler option `-fmodules` is now handled in the “depend mode”. If - “depend mode” is disabled the option is still considered too hard and ccache +- The compiler option `-fmodules` is now handled in the "`depend mode`". If + "`depend mode`" is disabled the option is still considered too hard and ccache will fall back to running the compiler. - Ccache can now cache compilations with coverage notes (`.gcno` files) @@ -491,19 +1589,19 @@ Detailed functional changes - Ccache is now able to share cache entries for different object file names when using `-MD` or `-MMD`. -- Clang’s `-Xclang` (used by CMake for precompiled headers), +- Clang's `-Xclang` (used by CMake for precompiled headers), `-fno-pch-timestamp`, `-emit-pch`, `-emit-pth` and `-include-pth` options are now understood. -- Added support for the HIP (“C++ Heterogeneous-Compute Interface for - Portability”) language. +- Added support for the HIP ("`C++ Heterogeneous-Compute Interface for + Portability`") language. - The manifest format now allows for header files larger than 4 GiB. -- Made it possible to once again cache compilations with `__DATE__` in the +- Made it possible to once again cache compilations with `+__DATE__+` in the source code. -- Added handling of the `__TIMESTAMP__` macro. +- Added handling of the `+__TIMESTAMP__+` macro. - An absolute input source path is now rewritten to a relative path when using `base_dir`. @@ -513,8 +1611,8 @@ Detailed functional changes - Made handling of `.dwo` files and interaction between `-gsplit-dwarf` and other `-g*` options more robust. -- The “couldn't find compiler” statistics counter is no longer incremented when - ccache exits with a fatal error. +- The "`couldn't find compiler`" statistics counter is no longer incremented + when ccache exits with a fatal error. - Failure to run a `compiler_check` command is no longer a fatal error. @@ -537,8 +1635,7 @@ Detailed functional changes older than 3.1 (released 2010). -Other improvements -~~~~~~~~~~~~~~~~~~ +=== Other improvements - Improved help text and documentation of command line options. @@ -549,7 +1646,7 @@ Other improvements - Added HTML anchors to configuration options in the manual so that it is possible link to a specific option. -- Tweaked placement of “(readonly)” in output of `ccache -s`. +- Tweaked placement of "`(readonly)`" in output of `ccache -s`. - Improved visibility of color output from the test suite. @@ -563,13 +1660,12 @@ Other improvements - Disabled read-only tests on file systems that lack such support. -ccache 3.7.12 -------------- +== Ccache 3.7.12 Release date: 2020-10-01 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Coverage files (`.gcno`) produced by GCC 9+ when using `-fprofile-dir=dir` are now handled gracefully by falling back to running the compiler. @@ -578,8 +1674,7 @@ Bug fixes 32-bit mode. -Other -~~~~~ +=== Other - Improved documentation about sharing a cache on NFS. @@ -588,35 +1683,33 @@ Other - Fixed test case failures with GCC 4.4. -ccache 3.7.11 -------------- +== Ccache 3.7.11 Release date: 2020-07-21 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Added knowledge about `-fprofile-{correction,reorder-functions,values}`. - ccache now handles the Intel compiler option `-xCODE` (where `CODE` is a processor feature code) correctly. -- Added support for NVCC’s `-Werror` and `--Werror` options. +- Added support for NVCC's `-Werror` and `--Werror` options. -Other -~~~~~ +=== Other -- ccache’s “Directory is not hashed if using -gz[=zlib]” tests are now skipped +- ccache's "`Directory is not hashed if using -gz[=zlib]`" tests are now skipped for GCC 6. -ccache 3.7.10 ------------- +== Ccache 3.7.10 + Release date: 2020-06-22 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Improved handling of profiling options. ccache should now work correctly for profiling options like `-fprofile-{generate,use}[=path]` for GCC ≥ 9 and @@ -626,11 +1719,11 @@ Bug fixes - ccache now copies files directly from the cache to the destination file instead of via a temporary file. This avoids problems when using filenames - long enough to be near the file system’s filename max limit. + long enough to be near the file system's filename max limit. - When the hard-link mode is enabled, ccache now only uses hard links for object files, not other files like dependency files. This is because - compilers unlink object files before writing to them but they don’t do that + compilers unlink object files before writing to them but they don't do that for dependency files, so the latter can become overwritten and therefore corrupted in the cache. @@ -641,38 +1734,37 @@ Bug fixes - Temporary files are now deleted immediately on signals like SIGTERM and SIGINT instead of some time later in a cleanup phase. -- Fixed a bug that affected ccache’s `-o/--set-config` option for the +- Fixed a bug that affected ccache's `-o/--set-config` option for the `base_dir` and `cache_dir_levels` keys. -ccache 3.7.9 ------------- +== Ccache 3.7.9 + Release date: 2020-03-29 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed replacing of /dev/null when building as root with hard link mode enabled and using `-o /dev/null`. -- Removed incorrect assertion resulting in “ccache: error: Internal error in - format” when using `-fdebug-prefix-map=X=` with X equal to `$PWD`. +- Removed incorrect assertion resulting in "`ccache: error: Internal error in + format`" when using `-fdebug-prefix-map=X=` with X equal to `$PWD`. -Other -~~~~~ +=== Other - Improved CUDA/NVCC support: Recognize `-dc` and `-x cu` options. - Improved name of temporary file used in NFS-safe unlink. -ccache 3.7.8 ------------- +== Ccache 3.7.8 + Release date: 2020-03-16 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Use `$PWD` instead of the real CWD (current working directory) when checking for CWD in preprocessed output. This fixes a problem when `$PWD` includes a @@ -684,8 +1776,7 @@ Bug fixes - If `localtime_r` fails the epoch time is now logged instead of garbage. -Other -~~~~~ +=== Other - Improved error message when a boolean environment variable has an invalid value. @@ -693,28 +1784,28 @@ Other - Improved the regression fix in ccache 3.7.5 related to not passing compilation-only options to the preprocessor. -- ccache’s PCH test suite now skips running the tests if it detects broken PCH +- ccache's PCH test suite now skips running the tests if it detects broken PCH compiler support. - Fixed unit test failure on Windows. -- Fixed “stringop-truncation” build warning on Windows. +- Fixed "`stringop-truncation`" build warning on Windows. -- Improved “x_rename” implementation on Windows. +- Improved "`x_rename`" implementation on Windows. - Improved removal of temporary file when rewriting absolute paths to relative in the dependency file. -- Clarified “include_file_ctime sloppiness” in the Performance section in the +- Clarified "`include_file_ctime sloppiness`" in the Performance section in the manual. -ccache 3.7.7 ------------- +== Ccache 3.7.7 + Release date: 2020-01-05 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed a bug related to object file location in the dependency file (if using `-MD` or `-MMD` but not `-MF` and the build directory is not the same as the @@ -724,32 +1815,32 @@ Bug fixes compilers again. (A better fix for this is planned for ccache 4.0.) - Removed the unify mode since it has bugs and shortcomings that are non-trivial - or impossible to fix: it doesn’t work with the direct mode, it doesn’t handle + or impossible to fix: it doesn't work with the direct mode, it doesn't handle C++ raw strings correctly, it can give false cache hits for `.incbin` - directives, it’s turned off when using `-g` and it can make line numbers in + directives, it's turned off when using `-g` and it can make line numbers in warning messages and `__LINE__` macros incorrect. - mtime and ctime values are now stored in the manifest files only when sloppy_file_stat is set. This avoids adding superfluous manifest file entries on direct mode cache misses. -- A “Result:” line is now always printed to the log. +- A "`Result:`" line is now always printed to the log. -- The “cache miss” statistics counter will now be updated for read-only cache +- The "`cache miss`" statistics counter will now be updated for read-only cache misses, making it consistent with the cache hit case. -ccache 3.7.6 ------------- +== Ccache 3.7.6 + Release date: 2019-11-17 -Bug fixes -~~~~~~~~~ -- The opt-in “file_macro sloppiness” mode has been removed so that the input +=== Bug fixes + +- The opt-in "`file_macro sloppiness`" mode has been removed so that the input file path now is always included in the direct mode hash. This fixes a bug - that could result in false cache hits in an edge case when “file_macro - sloppiness” is enabled and several identical source files include a relative + that could result in false cache hits in an edge case when "`file_macro + sloppiness`" is enabled and several identical source files include a relative header file with the same name but in different directories. - Statistics files are no longer lost when the filesystem of the cache is full. @@ -760,26 +1851,25 @@ Bug fixes - Properly handle color diagnostics in the depend mode as well. -ccache 3.7.5 ------------- +== Ccache 3.7.5 + Release date: 2019-10-22 -New features -~~~~~~~~~~~~ + +=== New features - Added support for `-MF=arg` (with an extra equal sign) as understood by EDG-based compilers. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed a regression in 3.7.2 that could result in a warning message instead of - an error in an edge case related to usage of “-Werror”. + an error in an edge case related to usage of "`-Werror`". - An implicit `-MQ` is now passed to the preprocessor only if the object file extension is non-standard. This will make it easier to use EDG-based - compilers (e.g. GHS) which don’t understand `-MQ`. (This is a bug fix of the + compilers (e.g. GHS) which don't understand `-MQ`. (This is a bug fix of the corresponding improvement implemented in ccache 3.4.) - ccache now falls back to running the real compiler instead of failing fataly @@ -792,45 +1882,45 @@ Bug fixes - Fixed warning during configure in out-of-tree build in developer mode. -ccache 3.7.4 ------------- +== Ccache 3.7.4 + Release date: 2019-09-12 -Improvements -~~~~~~~~~~~~ + +=== Improvements - Added support for the `-gz[=type]` compiler option (previously ccache would - think that “-gz” alone would enable debug information, thus potentially + think that "`-gz`" alone would enable debug information, thus potentially including the current directory in the hash). -- Added support for converting paths like “/c/users/...” into relative paths on - Windows. +- Added support for converting paths like "`/c/users/...`" into relative paths + on Windows. -ccache 3.7.3 ------------- +== Ccache 3.7.3 + Release date: 2019-08-17 -Bug fixes -~~~~~~~~~ -- The cache size (which is counted in “used disk blocks”) is now correct on +=== Bug fixes + +- The cache size (which is counted in "`used disk blocks`") is now correct on filesystems that use more or less disk blocks than conventional filesystems, e.g. ecryptfs or btrfs/zfs with transparent compression. This also fixes a - related problem with ccache’s own test suite when run on such file systems. + related problem with ccache's own test suite when run on such file systems. -- Fixed a regression in 3.7.2 when using the compiler option “-Werror” and then - “-Wno-error” later on the command line. +- Fixed a regression in 3.7.2 when using the compiler option "`-Werror`" and + then "`-Wno-error`" later on the command line. -ccache 3.7.2 ------------- +== Ccache 3.7.2 + Release date: 2019-07-19 -Bug fixes -~~~~~~~~~ -- The compiler option `-gdwarf*` no longer forces “run_second_cpp = true”. +=== Bug fixes + +- The compiler option `-gdwarf*` no longer forces "`run_second_cpp = true`". - Added verification that the value passed to the `-o/--set-config` option is valid. @@ -847,11 +1937,10 @@ Bug fixes - Unknown manifest versions are now handled gracefully in `--dump-manifest`. -- Fixed `make check` with “funny” locales. +- Fixed `make check` with "`funny`" locales. -Documentation -~~~~~~~~~~~~~ +=== Documentation - Added a hint about not running `autogen.sh` when building from a release archive. @@ -859,28 +1948,28 @@ Documentation - Mention that `xsltproc` is needed when building from the source repository. -ccache 3.7.1 ------------- +== Ccache 3.7.1 + Release date: 2019-05-01 -Changes -~~~~~~~ + +=== Changes - Fixed a problem when using the compiler option `-MF /dev/null`. - Long commandlines are now handled gracefully on Windows by using the `@file` syntax to avoid hitting the commandline size limit. -- Fixed complaint from GCC 9’s `-Werror=format-overflow` when compiling ccache +- Fixed complaint from GCC 9's `-Werror=format-overflow` when compiling ccache itself. -ccache 3.7 ----------- +== Ccache 3.7 + Release date: 2019-04-23 -Changes -~~~~~~~ + +=== Changes - Fixed crash when the debug mode is enabled and the output file is in a non-writable directory, e.g. when the output file is `/dev/null`. @@ -924,12 +2013,12 @@ Changes machine-parsable (tab-separated) format. - ccache no longer creates a missing output directory, thus mimicking the - compiler behavior for `-o out/obj.o` when “out” doesn’t exist. + compiler behavior for `-o out/obj.o` when `out` doesn't exist. -- `-fdebug-prefix-map=ARG`, `-ffile-prefix-map=ARG` and - `-fmacro-prefix-map=ARG` are now included in the hash, but only the part - before “ARG”. This fixes a bug where compiler feature detection of said flags - would not work correctly with ccache. +- `-fdebug-prefix-map=ARG`, `-ffile-prefix-map=ARG` and `-fmacro-prefix-map=ARG` + are now included in the hash, but only the part before "`ARG`". This fixes a + bug where compiler feature detection of said flags would not work correctly + with ccache. - Bail out on too hard compiler option `-gtoggle`. @@ -952,19 +2041,19 @@ Changes involved. -ccache 3.6 ----------- +== Ccache 3.6 + Release date: 2019-01-14 -Changes -~~~~~~~ -- ccache now has an opt-in “depend mode”. When enabled, ccache never executes +=== Changes + +- ccache now has an opt-in "`depend mode`". When enabled, ccache never executes the preprocessor, which results in much lower cache miss overhead at the expense of a lower potential cache hit rate. The depend mode is only possible to use when the compiler option `-MD` or `-MMD` is used. -- Added support for GCC’s `-ffile-prefix-map` option. The `-fmacro-prefix-map` +- Added support for GCC's `-ffile-prefix-map` option. The `-fmacro-prefix-map` option is now also skipped from the hash. - Added support for multiple `-fsanitize-blacklist` arguments. @@ -976,7 +2065,7 @@ Changes - Fixed a problem due to Clang overwriting the output file when compiling an assembler file. -- Clarified the manual to explain the reasoning behind the “file_macro” +- Clarified the manual to explain the reasoning behind the "`file_macro`" sloppiness setting in a better way. - ccache now handles several levels of nonexistent directories when rewriting @@ -985,27 +2074,27 @@ Changes - A new sloppiness setting `clang_index_store` makes ccache skip the Clang compiler option `-index-store-path` and its argument when computing the manifest hash. This is useful if you use Xcode, which uses an index store - path derived from the local project path. Note that the index store won’t be + path derived from the local project path. Note that the index store won't be updated correctly on cache hits if you enable this option. - Rename sloppiness `no_system_headers` to `system_headers` for consistency with other options. `no_system_headers` can still be used as an (undocumented) alias. -- The GCC variables “DEPENDENCIES_OUTPUT” and “SUNPRO_DEPENDENCIES” are now +- The GCC variables "`DEPENDENCIES_OUTPUT`" and "`SUNPRO_DEPENDENCIES`" are now supported correctly. -- The algorithm that scans for `__DATE_` and `__TIME__` tokens in the hashed - source code now doesn’t produce false positives for tokens where `__DATE__` - or `__TIME__` is a substring. +- The algorithm that scans for `__DATE_` and `+__TIME__+` tokens in the hashed + source code now doesn't produce false positives for tokens where `+__DATE__+` + or `+__TIME__+` is a substring. + +== Ccache 3.5.1 -ccache 3.5.1 ------------- Release date: 2019-01-02 -Changes -~~~~~~~ + +=== Changes - Added missing getopt_long.c source file to release archive. @@ -1016,17 +2105,17 @@ Changes - Improved development mode build flags. -ccache 3.5 ----------- +== Ccache 3.5 + Release date: 2018-10-15 -Changes -~~~~~~~ + +=== Changes - Added a boolean `debug` (`CCACHE_DEBUG`) configuration option. When enabled, - ccache will create per-object debug files that are helpful e.g. when - debugging unexpected cache misses. See also the new “Cache debugging” section - in the manual. + ccache will create per-object debug files that are helpful e.g. when debugging + unexpected cache misses. See also the new "`Cache debugging`" section in the + manual. - Renamed `CCACHE_CC` to `CCACHE_COMPILER` (keeping the former as a deprecated alias). @@ -1042,36 +2131,36 @@ Changes - Improved performance substantially when using `hash_dir = false` on platforms like macOS where `getcwd()` is slow. -- Added “stats updated” timestamp in `ccache -s` output. This can be useful if +- Added "`stats updated`" timestamp in `ccache -s` output. This can be useful if you wonder whether ccache actually was used for your last build. -- Renamed “stats zero time” to “stats zeroed” and documented it. The counter is - also now only present in `ccache -s` output when `ccache -z` actually has +- Renamed "`stats zero time`" to "`stats zeroed`" and documented it. The counter + is also now only present in `ccache -s` output when `ccache -z` actually has been called. - The content of the `-fsanitize-blacklist` file is now included in the hash, so updates to the file will now correctly result in separate cache entries. -- It’s now possible to opt out of building and installing man pages when +- It's now possible to opt out of building and installing man pages when running `make install` in the source repository. -- If the compiler type can’t be detected (e.g. if it is named `cc`), use safer - defaults that won’t trip up Clang. +- If the compiler type can't be detected (e.g. if it is named `cc`), use safer + defaults that won't trip up Clang. - Made the ccache test suite work on FreeBSD. - Added `file_stat_matches_ctime` option to disable ctime check if `file_stat_matches` is enabled. -- Made “./configure --without-bundled-zlib” do what’s intended. +- Made "`./configure --without-bundled-zlib`" do what's intended. + +== Ccache 3.4.3 -ccache 3.4.3 ------------ Release date: 2018-09-02 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed a race condition when creating the initial config file in the cache directory. @@ -1085,12 +2174,12 @@ Bug fixes - Upgraded bundled zlib to version 1.2.11. -ccache 3.4.2 ------------- +== Ccache 3.4.2 + Release date: 2018-03-25 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - The cleanup algorithm has been fixed to not misbehave when files are removed by another process while the cleanup process is running. Previously, too many @@ -1098,42 +2187,42 @@ Bug fixes triggered at the same time, in extreme cases trimming the cache to a much smaller size than the configured limits. -- Correctly hash preprocessed headers located in a “.gch directory”. +- Correctly hash preprocessed headers located in a "`.gch directory`". Previously, ccache would not pick up changes to such precompiled headers, risking false positive cache hits. - Fixed build failure when using the bundled zlib sources. - ccache 3.3.5 added a workaround for not triggering Clang errors when a - precompiled header’s dependency has an updated timestamp (but identical + precompiled header's dependency has an updated timestamp (but identical content). That workaround is now only applied when the compiler is Clang. - Made it possible to perform out-of-source builds in dev mode again. -ccache 3.4.1 ------------- +== Ccache 3.4.1 + Release date: 2018-02-11 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed printing of version number in `ccache --version`. -ccache 3.4 ----------- +== Ccache 3.4 + Release date: 2018-02-11 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +=== New features and enhancements - The compiler option form `--sysroot arg` is now handled like the documented `--sysroot=arg` form. - Added support for caching `.su` files generated by GCC flag `-fstack-usage`. -- ccache should now work with distcc’s “pump” wrapper. +- ccache should now work with distcc's "`pump`" wrapper. - The optional unifier is no longer disabled when the direct mode is enabled. @@ -1142,7 +2231,7 @@ New features and enhancements - Boolean environment variable settings no longer accept the following (case-insensitive) values: `0`, `false`, `disable` and `no`. All other values - are accepted and taken to mean “true”. This is to stop users from setting + are accepted and taken to mean "`true`". This is to stop users from setting e.g. `CCACHE_DISABLE=0` and then expect the cache to be used. - Improved support for `run_second_cpp = false`: If combined with passing @@ -1151,7 +2240,7 @@ New features and enhancements - An implicit `-MQ` is now passed to the preprocessor only if the object file extension is non-standard. This should make it easier to use EDG-based - compilers (e.g. GHS) which don’t understand `-MQ`. + compilers (e.g. GHS) which don't understand `-MQ`. - ccache now treats an unreadable configuration file just like a missing configuration file. @@ -1161,8 +2250,7 @@ New features and enhancements - Documented caveats related to colored warnings from compilers. -Bug fixes -~~~~~~~~~ +=== Bug fixes - File size and number counters are now updated correctly when files are overwritten in the cache, e.g. when using `CCACHE_RECACHE`. @@ -1172,39 +2260,35 @@ Bug fixes - Fixed how the NVCC options `-optf` and `-odir` are handled. -ccache 3.3.6 ------------- +== Ccache 3.3.6 + Release date: 2018-01-28 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Improved instructions on how to get cache hits between different working directories. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed regression in ccache 3.3.5 related to the `UNCACHED_ERR_FD` feature. -ccache 3.3.5 ------------- +== Ccache 3.3.5 + Release date: 2018-01-13 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Documented how automatic cache cleanup works. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed a regression where the original order of debug options could be lost. - This reverts the “Improved parsing of `-g*` options” feature in ccache 3.3. + This reverts the "`Improved parsing of `-g*` options`" feature in ccache 3.3. - Multiple `-fdebug-prefix-map` options should now be handled correctly. @@ -1222,9 +2306,9 @@ Bug fixes - `ccache -c/--cleanup` now works like documented: it just recalculates size counters and trims the cache to not exceed the max size and file number - limits. Previously, the forced cleanup took “limit_multiple” into account, so - that `ccache -c/--cleanup` by default would trim the cache to 80% of the max - limit. + limits. Previously, the forced cleanup took "`limit_multiple`" into account, + so that `ccache -c/--cleanup` by default would trim the cache to 80% of the + max limit. - ccache no longer ignores linker arguments for Clang since Clang warns about them. @@ -1236,46 +2320,44 @@ Bug fixes `-remap` or `-trigraphs` option in preprocessor mode. -ccache 3.3.4 ------------- +== Ccache 3.3.4 + Release date: 2017-02-17 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Documented the different cache statistics counters. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed a regression in ccache 3.3 related to potentially bad content of dependency files when compiling identical source code but with different source paths. This was only partially fixed in 3.3.2 and reverts the new - “Names of included files are no longer included in the hash of the compiler’s - preprocessed output” feature in 3.3. + "`Names of included files are no longer included in the hash of the compiler's + preprocessed output`" feature in 3.3. - Corrected statistics counter for `-optf`/`--options-file` failure. - Fixed undefined behavior warnings in ccache found by `-fsanitize=undefined`. -ccache 3.3.3 ------------- +== Ccache 3.3.3 + Release date: 2016-10-26 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - ccache now detects usage of `.incbin` assembler directives in the source code and avoids caching such compilations. -ccache 3.3.2 ------------- +== Ccache 3.3.2 + Release date: 2016-09-28 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed a regression in ccache 3.3 related to potentially bad content of dependency files when compiling identical source code but with different @@ -1286,14 +2368,14 @@ Bug fixes resulting in missing dependency files from direct mode cache hits. -ccache 3.3.1 ------------- +== Ccache 3.3.1 + Release date: 2016-09-07 -Bug fixes -~~~~~~~~~ -- Fixed a problem in the “multiple `-arch` options” support introduced in 3.3. +=== Bug fixes + +- Fixed a problem in the "`multiple `-arch` options`" support introduced in 3.3. When using the direct mode (the default), different combinations of `-arch` options were not detected properly. @@ -1302,22 +2384,20 @@ Bug fixes (`CCACHE_CPP2`) is enabled. -ccache 3.3 ----------- +== Ccache 3.3 + Release date: 2016-08-27 -Notes -~~~~~ +=== Notes - A C99-compatible compiler is now required to build ccache. -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - The configuration option `run_second_cpp` (`CCACHE_CPP2`) now defaults to - true. This improves ccache’s out-of-the-box experience for compilers that - can’t compile their own preprocessed output with the same outcome as if they + true. This improves ccache's out-of-the-box experience for compilers that + can't compile their own preprocessed output with the same outcome as if they compiled the real source code directly, e.g. newer versions of GCC and Clang. - The configuration option `hash_dir` (`CCACHE_HASHDIR`) now defaults to true. @@ -1341,22 +2421,22 @@ New features and enhancements - Added a new statistics counter that tracks the number of performed cleanups due to the cache size being over the limit. The value is shown in the output - of “ccache -s”. + of "`ccache -s`". - Added support for relocating debug info directory using `-fdebug-prefix-map`. This allows for cache hits even when `hash_dir` is used in combination with `base_dir`. -- Added a new “cache hit rate” field to the output of “ccache -s”. +- Added a new "`cache hit rate`" field to the output of "`ccache -s`". -- Added support for caching compilation of assembler code produced by e.g. “gcc - -S file.c”. +- Added support for caching compilation of assembler code produced by e.g. "`gcc + -S file.c`". - Added support for cuda including the -optf/--options-file option. - Added support for Fortran 77. -- Added support for multiple `-arch` options to produce “fat binaries”. +- Added support for multiple `-arch` options to produce "`fat binaries`". - Multiple identical `-arch` arguments are now handled without bailing. @@ -1378,7 +2458,7 @@ New features and enhancements - ccache now understands the undocumented `-coverage` (only one dash) GCC option. -- Names of included files are no longer included in the hash of the compiler’s +- Names of included files are no longer included in the hash of the compiler's preprocessed output. This leads to more potential cache hits when not using the direct mode. @@ -1386,8 +2466,7 @@ New features and enhancements slightly. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Bail out on too hard compiler option `-P`. @@ -1396,24 +2475,24 @@ Bug fixes - Fixed build and test for MinGW32 and Windows. -ccache 3.2.9 ------------- +== Ccache 3.2.9 + Release date: 2016-09-28 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed a regression in ccache 3.2.8: ccache could get confused when using the compiler option `-Wp,` to pass multiple options to the preprocessor, resulting in missing dependency files from direct mode cache hits. -ccache 3.2.8 ------------- +== Ccache 3.2.8 + Release date: 2016-09-07 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed an issue when compiler option `-Wp,-MT,path` is used instead of `-MT path` (and similar for `-MF`, `-MP` and `-MQ`) and `run_second_cpp` @@ -1423,86 +2502,83 @@ Bug fixes option. -ccache 3.2.7 ------------- +== Ccache 3.2.7 + Release date: 2016-07-20 -Bug fixes -~~~~~~~~~ + +=== Bug fixes - Fixed a bug which could lead to false cache hits for compiler command lines with a missing argument to an option that takes an argument. -- ccache now knows how to work around a glitch in the output of GCC 6’s +- ccache now knows how to work around a glitch in the output of GCC 6's preprocessor. -ccache 3.2.6 ------------- +== Ccache 3.2.6 + Release date: 2016-07-12 -Bug fixes -~~~~~~~~~ -- Fixed build problem on QNX, which lacks “SA_RESTART”. +=== Bug fixes + +- Fixed build problem on QNX, which lacks "`SA_RESTART`". - Bail out on compiler option `-fstack-usage` since it creates a `.su` file - which ccache currently doesn’t handle. + which ccache currently doesn't handle. - Fixed a bug where (due to ccache rewriting paths) the compiler could choose incorrect include files if `CCACHE_BASEDIR` is used and the source file path is absolute and is a symlink. -ccache 3.2.5 ------------- +== Ccache 3.2.5 + Release date: 2016-04-17 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Only pass Clang-specific `-stdlib=` to the preprocessor. - Improved handling of stale NFS handles. -- Made it harder to misinterpret documentation of boolean environment settings’ +- Made it harder to misinterpret documentation of boolean environment settings' semantics. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Include m4 files used by configure.ac in the source dist archives. -- Corrected “Performance” section in the manual regarding `__DATE_`, `__TIME__` - and `__FILE__` macros. +- Corrected "`Performance`" section in the manual regarding `__DATE_`, + `+__TIME__+` and `+__FILE__+` macros. - Fixed build on Solaris 10+ and AIX 7. - Fixed failure to create directories on QNX. -- Don’t (try to) update manifest file in “read-only” and “read-only direct” +- Don't (try to) update manifest file in "`read-only`" and "`read-only direct`" modes. -- Fixed a bug in caching of `stat` system calls in “file_stat_matches - sloppiness mode”. +- Fixed a bug in caching of `stat` system calls in "`file_stat_matches + sloppiness mode`". - Fixed bug in hashing of Clang plugins, leading to unnecessary cache misses. -- Fixed --print-config to show “pch_defines sloppiness”. +- Fixed --print-config to show "`pch_defines sloppiness`". -- The man page is now built when running “make install” from Git repository +- The man page is now built when running "`make install`" from Git repository sources. -ccache 3.2.4 ------------- +== Ccache 3.2.4 + Release date: 2015-10-08 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed build error related to zlib on systems with older make versions (regression in ccache 3.2.3). @@ -1521,19 +2597,17 @@ Bug fixes 64 GiB on 32-bit systems. -ccache 3.2.3 ------------- +== Ccache 3.2.3 + Release date: 2015-08-16 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Added support for compiler option `-gsplit-dwarf`. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Support external zlib in nonstandard directory. @@ -1543,34 +2617,32 @@ Bug fixes - Bail out on compiler option `--save-temps` in addition to `-save-temps`. -- Only log “Disabling direct mode” once when failing to read potential include +- Only log "`Disabling direct mode`" once when failing to read potential include files. -ccache 3.2.2 ------------- +== Ccache 3.2.2 + Release date: 2015-05-10 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Added support for `CCACHE_COMPILERCHECK=string:`. This is a faster - alternative to `CCACHE_COMPILERCHECK=` if the command’s output can + alternative to `CCACHE_COMPILERCHECK=` if the command's output can be precalculated by the build system. - Add support for caching code coverage results (compiling for gcov). -Bug fixes -~~~~~~~~~ +=== Bug fixes - Made hash of cached result created with and without `CCACHE_CPP2` different. This makes it possible to rebuild with `CCACHE_CPP2` set without having to clear the cache to get new results. -- Don’t try to reset a nonexistent stats file. This avoids “No such file or - directory” messages in the ccache log when the cache directory doesn’t exist. +- Don't try to reset a nonexistent stats file. This avoids "`No such file or + directory`" messages in the ccache log when the cache directory doesn't exist. - Fixed a bug where ccache deleted Clang diagnostics after compiler failures. @@ -1592,13 +2664,12 @@ Bug fixes - Fixed build error when compiling ccache with recent Clang versions. -ccache 3.2.1 ------------- +== Ccache 3.2.1 + Release date: 2014-12-10 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed regression in temporary file handling, which lead to incorrect permissions for stats, manifest and ccache.conf files in the cache. @@ -1616,13 +2687,12 @@ Bug fixes options. -ccache 3.2 ----------- +== Ccache 3.2 + Release date: 2014-11-17 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Added support for configuring ccache via one or several configuration files instead of via environment variables. Environment variables still have @@ -1634,13 +2704,13 @@ New features and enhancements - Made creation of temporary directories and cache directories smarter to avoid unnecessary `stat` calls. -- Improved efficiency of the algorithm that scans for `__DATE_` and `__TIME__` +- Improved efficiency of the algorithm that scans for `__DATE_` and `+__TIME__+` tokens in the hashed source code. - Added support for several binaries (separated by space) in `CCACHE_PREFIX`. - The `-c` option is no longer passed to the preprocessor. This fixes problems - with Clang and Solaris’s C++ compiler. + with Clang and Solaris's C++ compiler. - ccache no longer passes preprocessor options like `-D` and `-I` to the compiler when compiling preprocessed output. This fixes warnings emitted by @@ -1649,7 +2719,7 @@ New features and enhancements - Compiler options `-fprofile-generate`, `-fprofile-arcs`, `-fprofile-use` and `-fbranch-probabilities` are now handled without bailing. -- Added support for Clang’s `--serialize-diagnostic` option, storing the +- Added support for Clang's `--serialize-diagnostic` option, storing the diagnostic file (`.dia`) in the cache. - Added support for precompiled headers when using Clang. @@ -1664,19 +2734,19 @@ New features and enhancements the other way around. This is needed to support compiler options like `-fprofile-arcs` and `--serialize-diagnostics`. -- ccache now checks that included files’ ctimes aren’t too new. This check can - be turned off by adding `include_file_ctime` to the “ccache sloppiness” +- ccache now checks that included files' ctimes aren't too new. This check can + be turned off by adding `include_file_ctime` to the "`ccache sloppiness`" setting. - Added possibility to get cache hits based on filename, size, mtime and ctime only. On other words, source code files are not even read, only stat-ed. This - operation mode is opt-in by adding `file_stat_matches` to the “ccache - sloppiness” setting. + operation mode is opt-in by adding `file_stat_matches` to the "`ccache + sloppiness`" setting. - The filename part of options like `-Wp,-MDfilename` is no longer included in - the hash since the filename doesn’t have any bearing on the result. + the hash since the filename doesn't have any bearing on the result. -- Added a “read-only direct” configuration setting, which is like the ordinary +- Added a "`read-only direct`" configuration setting, which is like the ordinary read-only setting except that ccache will only try to retrieve results from the cache using the direct mode, not the preprocessor mode. @@ -1691,7 +2761,7 @@ New features and enhancements - Added support for `@file` and `-@file` arguments (reading options from a file). -- `-Wl,` options are no longer included in the hash since they don’t affect +- `-Wl,` options are no longer included in the hash since they don't affect compilation. - Bail out on too hard compiler option `-Wp,-P`. @@ -1715,8 +2785,7 @@ New features and enhancements - Various other improvements of the test suite. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Any previous `.stderr` is now removed from the cache when recaching. @@ -1727,26 +2796,24 @@ Bug fixes - Fixed test suite failures when `CC` is a ccache-wrapped compiler. -ccache 3.1.12 -------------- +== Ccache 3.1.12 + Release date: 2016-07-12 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed a bug where (due to ccache rewriting paths) the compiler could choose incorrect include files if `CCACHE_BASEDIR` is used and the source file path is absolute and is a symlink. -ccache 3.1.11 -------------- +== Ccache 3.1.11 + Release date: 2015-03-07 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed bug which could result in false cache hits when source code contains `'"'` followed by `" /*"` or `" //"` (with variations). @@ -1755,17 +2822,16 @@ Bug fixes This makes it possible to rebuild with `CCACHE_CPP2` set without having to clear the cache to get new results. -- Don’t try to reset a nonexistent stats file. This avoids “No such file or - directory” messages in the ccache log when the cache directory doesn’t exist. +- Don't try to reset a nonexistent stats file. This avoids "`No such file or + directory`" messages in the ccache log when the cache directory doesn't exist. -ccache 3.1.10 -------------- +== Ccache 3.1.10 + Release date: 2014-10-19 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Added support for the `-Xclang` compiler option. @@ -1780,21 +2846,20 @@ New features and enhancements `CCACHE_BASEDIR` to reuse results across different directories.) - Added note in documentation that `--ccache-skip` currently does not mean - “don’t hash the following option”. + "`don't hash the following option`". - To enable support for precompiled headers (PCH), `CCACHE_SLOPPINESS` now also needs to include the new `pch_defines` sloppiness. This is because ccache - can’t detect changes in the source code when only defined macros have been + can't detect changes in the source code when only defined macros have been changed. - Stale files in the internal temporary directory (`/tmp`) are now cleaned up if they are older than one hour. -Bug fixes -~~~~~~~~~ +=== Bug fixes -- Fixed path canonicalization in `make_relative_path()` when path doesn’t +- Fixed path canonicalization in `make_relative_path()` when path doesn't exist. - Fixed bug in `common_dir_prefix_length()`. This corrects the `CCACHE_BASEDIR` @@ -1809,13 +2874,12 @@ Bug fixes - Fixed problem with logging of current working directory. -ccache 3.1.9 ------------- +== Ccache 3.1.9 + Release date: 2013-01-06 -Bug fixes -~~~~~~~~~ +=== Bug fixes - The EAGAIN signal is now handled correctly when emitting cached stderr output. This fixes a problem triggered by large error outputs from the @@ -1823,7 +2887,7 @@ Bug fixes - Subdirectories in the cache are no longer created in read-only mode. -- Fixed so that ccache’s log file descriptor is not made available to the +- Fixed so that ccache's log file descriptor is not made available to the compiler. - Improved error reporting when failing to create temporary stdout/stderr files @@ -1832,19 +2896,17 @@ Bug fixes - Disappearing temporary stdout/stderr files are now handled gracefully. -Other -~~~~~ +=== Other - Fixed test suite to work on ecryptfs. -ccache 3.1.8 ------------- +== Ccache 3.1.8 + Release date: 2012-08-11 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Made paths to dependency files relative in order to increase cache hits. @@ -1854,8 +2916,7 @@ New features and enhancements - Clang plugins are now hashed to catch plugin upgrades. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Fixed crash when the current working directory has been removed. @@ -1868,28 +2929,26 @@ Bug fixes base directory. -Other -~~~~~ +=== Other - Made git version macro work when compiling outside of the source directory. - Fixed `static_assert` macro definition clash with GCC 4.7. -ccache 3.1.7 ------------- +== Ccache 3.1.7 + Release date: 2012-01-08 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Non-writable `CCACHE_DIR` is now handled gracefully when `CCACHE_READONLY` is set. - Made failure to create files (typically due to bad directory permissions) in the cache directory fatal. Previously, such failures were silently and - erroneously flagged as “compiler produced stdout”. + erroneously flagged as "`compiler produced stdout`". - Both the `-specs=file` and `--specs=file` forms are now recognized. @@ -1908,8 +2967,7 @@ Bug fixes versions.) -Other -~~~~~ +=== Other - Corrected license header for `mdfour.c`. @@ -1917,34 +2975,31 @@ Other -ccache 3.1.6 ------------- +== Ccache 3.1.6 + Release date: 2011-08-21 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Rewrite argument to `--sysroot` if `CCACHE_BASEDIR` is used. -Bug fixes -~~~~~~~~~ +=== Bug fixes + +- Don't crash if `getcwd()` fails. -- Don’t crash if `getcwd()` fails. +- Fixed alignment of "`called for preprocessing`" counter. -- Fixed alignment of “called for preprocessing” counter. +== Ccache 3.1.5 -ccache 3.1.5 ------------- Release date: 2011-05-29 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements -- Added a new statistics counter named “called for preprocessing”. +- Added a new statistics counter named "`called for preprocessing`". - The original command line is now logged to the file specified with `CCACHE_LOGFILE`. @@ -1957,8 +3012,7 @@ New features and enhancements - Improved order of statistics counters in `ccache -s` output. -Bug fixes -~~~~~~~~~ +=== Bug fixes - The `-MF`/`-MT`/`-MQ` options with concatenated argument are now handled correctly when they are last on the command line. @@ -1968,53 +3022,49 @@ Bug fixes - Fixed a minor memory leak. -- Systems that lack (and don’t need to be linked with) libm are now supported. +- Systems that lack (and don't need to be linked with) libm are now supported. + +== Ccache 3.1.4 -ccache 3.1.4 ------------- Release date: 2011-01-09 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Made a work-around for a bug in `gzputc()` in zlib 1.2.5. -- Corrupt manifest files are now removed so that they won’t block direct mode +- Corrupt manifest files are now removed so that they won't block direct mode hits. -- ccache now copes with file systems that don’t know about symbolic links. +- ccache now copes with file systems that don't know about symbolic links. -- The file handle in now correctly closed on write error when trying to create +- The file handle is now correctly closed on write error when trying to create a cache dir tag. -ccache 3.1.3 ------------- +== Ccache 3.1.3 + Release date: 2010-11-28 -Bug fixes -~~~~~~~~~ +=== Bug fixes - The -MFarg, -MTarg and -MQarg compiler options (i.e, without space between option and argument) are now handled correctly. -Other -~~~~~ +=== Other + +- Portability fixes for HP-UX 11.00 and other less common systems. -- Portability fixes for HP-UX 11.00 and other esoteric systems. +== Ccache 3.1.2 -ccache 3.1.2 ------------- Release date: 2010-11-21 -Bug fixes -~~~~~~~~~ +=== Bug fixes - Bail out on too hard compiler options `-fdump-*`. @@ -2024,24 +3074,22 @@ Bug fixes - Fixed issue when parsing precompiler output on AIX. -Other -~~~~~ +=== Other - Improved documentation on which information is included in the hash sum. -- Made the “too new header file” test case work on file systems with +- Made the "`too new header file`" test case work on file systems with unsynchronized clocks. - The test suite now also works on systems that lack a /dev/zero. -ccache 3.1.1 ------------- +== Ccache 3.1.1 + Release date: 2010-11-07 -Bug fixes -~~~~~~~~~ +=== Bug fixes - ccache now falls back to preprocessor mode when a non-regular include file (device, socket, etc) has been detected so that potential hanging due to @@ -2056,27 +3104,25 @@ Bug fixes - Fixed configure detection of ar. - ccache development version (set by dev.mk) now works with gits whose - `describe` command doesn’t understand `--dirty`. + `describe` command doesn't understand `--dirty`. -Other -~~~~~ +=== Other - Minor debug log message improvements. -ccache 3.1 ----------- +== Ccache 3.1 + Release date: 2010-09-16 -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements - Added support for hashing the output of a custom command (e.g. `%compiler% --version`) to identify the compiler instead of stat-ing or hashing the compiler binary. This can improve robustness when the compiler (as seen by - ccache) actually isn’t the real compiler but another compiler wrapper. + ccache) actually isn't the real compiler but another compiler wrapper. - Added support for caching compilations that use precompiled headers. (See the manual for important instructions regarding this.) @@ -2095,9 +3141,9 @@ New features and enhancements - Reading and writing of statistics counters has been made forward-compatible (unknown counters are retained). -- Files are now read without using `mmap()`. This has two benefits: it’s more +- Files are now read without using `mmap()`. This has two benefits: it's more robust against file changes during reading and it improves performance on - poor systems where `mmap()` doesn’t use the disk cache. + poor systems where `mmap()` doesn't use the disk cache. - Added `.cp` and `.CP` as known C++ suffixes. @@ -2107,8 +3153,7 @@ New features and enhancements statistics when using the Darwin linker.) -Bug fixes -~~~~~~~~~ +=== Bug fixes - Non-fatal error messages are now never printed to stderr but logged instead. @@ -2119,8 +3164,7 @@ Bug fixes - EINTR is now handled correctly. -Other -~~~~~ +=== Other - Work on porting ccache to win32 (native), mostly done by Ramiro Polla. The port is not yet finished, but will hopefully be complete in some subsequent @@ -2146,22 +3190,21 @@ Other - New `HACKING.txt` file with some notes about ccache code conventions. -ccache 3.0.1 ------------- +== Ccache 3.0.1 + Release date: 2010-07-15 -Bug fixes -~~~~~~~~~ +=== Bug fixes -- The statistics counter “called for link” is now correctly updated when +- The statistics counter "`called for link`" is now correctly updated when linking with a single object file. - Fixed a problem with out-of-source builds. -ccache 3.0 ----------- +== Ccache 3.0 + Release date: 2010-06-20 @@ -2172,19 +3215,17 @@ General or later. -Upgrade notes -~~~~~~~~~~~~~ +=== Upgrade notes -- The way the hashes are calculated has changed, so you won’t get cache hits +- The way the hashes are calculated has changed, so you won't get cache hits for compilation results stored by older ccache versions. Because of this, you might as well clear the old cache directory with `ccache --clear` if you want, unless you plan to keep using an older ccache version. -New features and enhancements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=== New features and enhancements -- ccache now has a “direct mode” where it computes a hash of the source code +- ccache now has a "`direct mode`" where it computes a hash of the source code (including all included files) and compiler options without running the preprocessor. By not running the preprocessor, CPU usage is reduced; the speed is somewhere between 1 and 5 times that of ccache running in @@ -2202,19 +3243,19 @@ New features and enhancements - Object files are now optionally stored compressed in the cache. The runtime cost is negligible, and more files will fit in the ccache directory and in the disk cache. Set `CCACHE_COMPRESS` to enable object file compression. Note - that you can’t use compression in combination with the hard link feature. + that you can't use compression in combination with the hard link feature. - A `CCACHE_COMPILERCHECK` option has been added. This option tells ccache what compiler-identifying information to hash to ensure that results retrieved - from the cache are accurate. Possible values are: none (don’t hash anything), - mtime (hash the compiler’s mtime and size) and content (hash the content of + from the cache are accurate. Possible values are: none (don't hash anything), + mtime (hash the compiler's mtime and size) and content (hash the content of the compiler binary). The default is mtime. - It is now possible to specify extra files whose contents should be included in the hash sum by setting the `CCACHE_EXTRAFILES` option. - Added support for Objective-C and Objective-C\+\+. The statistics counter - “not a C/C++ file” has been renamed to “unsupported source language”. + "`not a C/C++ file`" has been renamed to "`unsupported source language`". - Added support for the `-x` compiler option. @@ -2237,13 +3278,13 @@ New features and enhancements - Temporary files that later will be moved into the cache are now created in the cache directory they will end up in. This makes ccache more friendly to - Linux’s directory layout. + Linux's directory layout. - Improved the test suite and added tests for most of the new functionality. - It’s now also possible to specify a subset of tests to run. + It's now also possible to specify a subset of tests to run. - Standard error output from the compiler is now only stored in the cache if - it’s non-empty. + it's non-empty. - If the compiler produces no object file or an empty object file, but gives a zero exit status (could be due to a file system problem, a buggy program @@ -2251,7 +3292,7 @@ New features and enhancements - Added `installcheck` and `distcheck` make targets. -- Clarified cache size limit options’ and cleanup semantics. +- Clarified cache size limit options' and cleanup semantics. - Improved display of cache max size values. @@ -2260,8 +3301,7 @@ New features and enhancements `-iwithprefixbefore`, `-nostdinc`, `-nostdinc++` and `-U`. -Bug fixes -~~~~~~~~~ +=== Bug fixes - Various portability improvements. @@ -2280,7 +3320,7 @@ Bug fixes `-save-temps`. Also bail out on `@file` style options. - Errors when using multiple `-arch` compiler options are now noted as - “unsupported compiler option”. + "`unsupported compiler option`". - `-MD`/`-MMD` options without `-MT`/`-MF` are now handled correctly. diff --git a/doc/ccache-doc.css b/doc/ccache-doc.css new file mode 100644 index 0000000..86a0ed9 --- /dev/null +++ b/doc/ccache-doc.css @@ -0,0 +1,50 @@ +@import url(https://fonts.googleapis.com/css?family=Montserrat|Open+Sans); +@import url(https://cdn.jsdelivr.net/gh/asciidoctor/asciidoctor@2.0/data/stylesheets/asciidoctor-default.css); /* Default asciidoc style framework - important */ + +:root{ +--maincolor:#ffffff; +--primarycolor:#294172; +--secondarycolor:#5f646c; +--tertiarycolor:#cccccc; +--highlightcolor:#f5f5f5; +--sidebarbackground:#f5f5f5; +--linkcolor:#02a; +--linkcoloralternate:#db3279; +--white:#ffffff; +--black:#000000; +} + +/* Text styles */ + +body{font-family: "Open Sans", sans-serif;background-color: var(--maincolor);color:var(--black);} + +h1{color:var(--primarycolor) !important;font-family:"Montserrat",sans-serif;} +h2,h3,h4,h5,h6{color:var(--secondarycolor) !important;font-family:"Montserrat",sans-serif;} +.title{color:var(--black) !important;font-family:"Open Sans",sans-serif;font-style: normal; font-weight: normal;} +a{text-decoration: none;} +a:hover{color: #3958da; text-decoration: underline;} +p{font-family: "Open Sans",sans-serif ! important} +#toc.toc2 a:link{color:var(--linkcolor);} +blockquote{color:var(--linkcoloralternate) !important} +.quoteblock blockquote:before{color:var(--linkcoloralternate)} +code{color:var(--black);background-color: var(--highlightcolor) !important} +mark{background-color: var(--highlightcolor)} /* Text highlighting color */ + +/* Table styles */ +th{background-color: var(--maincolor);color:var(--black) !important;} +td{background-color: var(--maincolor);color: var(--black) !important} + + +#toc.toc2{background-color:var(--sidebarbackground);} +#toctitle{color:var(--black);} + +/* Responsiveness fixes */ +video { + max-width: 100%; +} + +@media all and (max-width: 600px) { +table { + width: 55vw!important; + font-size: 3vw; +} diff --git a/dockerfiles/README b/dockerfiles/README index c792398..d1de685 100644 --- a/dockerfiles/README +++ b/dockerfiles/README @@ -3,7 +3,7 @@ different build environments. For instance, run something like this to build ccache in Ubuntu 20.04: - misc/build-in-docker ubuntu-20-focal + misc/build-in-docker ubuntu-20.04 The above command will first build the Ubuntu 20.04 Docker image if needed and finally build ccache and run the ccache test suite. diff --git a/dockerfiles/alpine-3.11/Dockerfile b/dockerfiles/alpine-3.11/Dockerfile new file mode 100644 index 0000000..7bc1d29 --- /dev/null +++ b/dockerfiles/alpine-3.11/Dockerfile @@ -0,0 +1,20 @@ +FROM alpine:3.11 + +RUN apk add --no-cache \ + bash \ + ccache \ + clang \ + cmake \ + elfutils \ + g++ \ + gcc \ + hiredis-dev \ + libc-dev \ + make \ + perl \ + python3 \ + redis \ + zstd-dev + +# Redirect all compilers to ccache. +RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff --git a/dockerfiles/alpine-3.12/Dockerfile b/dockerfiles/alpine-3.12/Dockerfile deleted file mode 100644 index 324921a..0000000 --- a/dockerfiles/alpine-3.12/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM alpine:3.12 - -RUN apk add --no-cache \ - bash \ - ccache \ - clang \ - cmake \ - elfutils \ - g++ \ - gcc \ - libc-dev \ - make \ - perl \ - zstd-dev - -# Redirect all compilers to ccache. -RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff --git a/dockerfiles/alpine-3.15/Dockerfile b/dockerfiles/alpine-3.15/Dockerfile new file mode 100644 index 0000000..b97b38c --- /dev/null +++ b/dockerfiles/alpine-3.15/Dockerfile @@ -0,0 +1,20 @@ +FROM alpine:3.15 + +RUN apk add --no-cache \ + bash \ + ccache \ + clang \ + cmake \ + elfutils \ + g++ \ + gcc \ + hiredis-dev \ + libc-dev \ + make \ + perl \ + python3 \ + redis \ + zstd-dev + +# Redirect all compilers to ccache. +RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff --git a/dockerfiles/alpine-3.4/Dockerfile b/dockerfiles/alpine-3.4/Dockerfile deleted file mode 100644 index ad42a00..0000000 --- a/dockerfiles/alpine-3.4/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# Released 2016, this is the first release to contain cmake >= 3.4.3 - -FROM alpine:3.4 - -RUN apk add --no-cache \ - bash \ - ccache \ - cmake \ - g++ \ - gcc \ - libc-dev \ - make \ - perl - -# Redirect all compilers to ccache. -RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff --git a/dockerfiles/centos-7/Dockerfile b/dockerfiles/centos-7/Dockerfile index 89a122f..29c3cd8 100644 --- a/dockerfiles/centos-7/Dockerfile +++ b/dockerfiles/centos-7/Dockerfile @@ -1,21 +1,23 @@ FROM centos:7 RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \ + && yum install -y centos-release-scl \ && yum install -y \ - asciidoc \ autoconf \ bash \ ccache \ clang \ cmake3 \ + devtoolset-11 \ elfutils \ gcc \ gcc-c++ \ libzstd-devel \ make \ -# Remove superfluous dependencies brought in by asciidoc: - && rpm -e --nodeps graphviz \ + python3 \ && yum autoremove -y \ && yum clean all \ && cp /usr/bin/cmake3 /usr/bin/cmake \ && cp /usr/bin/ctest3 /usr/bin/ctest + +ENTRYPOINT ["scl", "enable", "devtoolset-11", "--"] diff --git a/dockerfiles/centos-8/Dockerfile b/dockerfiles/centos-8/Dockerfile deleted file mode 100644 index b7ab38f..0000000 --- a/dockerfiles/centos-8/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM centos:8 - -RUN dnf install -y epel-release \ - && dnf install -y \ - asciidoc \ - autoconf \ - bash \ - ccache \ - clang \ - cmake \ - diffutils \ - elfutils \ - gcc \ - gcc-c++ \ - libzstd-devel \ - make \ - && dnf clean all diff --git a/dockerfiles/debian-10/Dockerfile b/dockerfiles/debian-10/Dockerfile deleted file mode 100644 index 703bdc9..0000000 --- a/dockerfiles/debian-10/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM debian:10 - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - bash \ - build-essential \ - ccache \ - clang \ - cmake \ - elfutils \ - gcc-multilib \ - libzstd-dev \ - xsltproc \ - && rm -rf /var/lib/apt/lists/* - -# Redirect all compilers to ccache. -RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff --git a/dockerfiles/debian-11/Dockerfile b/dockerfiles/debian-11/Dockerfile new file mode 100644 index 0000000..ce7db7b --- /dev/null +++ b/dockerfiles/debian-11/Dockerfile @@ -0,0 +1,20 @@ +FROM debian:11 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + build-essential \ + ccache \ + clang \ + cmake \ + elfutils \ + gcc-multilib \ + libhiredis-dev \ + libzstd-dev \ + python3 \ + redis-server \ + redis-tools \ + && rm -rf /var/lib/apt/lists/* + +# Redirect all compilers to ccache. +RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff --git a/dockerfiles/debian-9/Dockerfile b/dockerfiles/debian-9/Dockerfile deleted file mode 100644 index b4c6585..0000000 --- a/dockerfiles/debian-9/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM debian:9 - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - bash \ - build-essential \ - ccache \ - clang \ - cmake \ - elfutils \ - gcc-multilib \ - libzstd-dev \ - xsltproc \ - && rm -rf /var/lib/apt/lists/* - -# Redirect all compilers to ccache. -RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff --git a/dockerfiles/fedora-32/Dockerfile b/dockerfiles/fedora-32/Dockerfile deleted file mode 100644 index 65c883d..0000000 --- a/dockerfiles/fedora-32/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM fedora:32 - -RUN dnf install -y \ - autoconf \ - bash \ - ccache \ - clang \ - cmake \ - diffutils \ - elfutils \ - findutils \ - gcc \ - gcc-c++ \ - libzstd-devel \ - make \ - && dnf clean all diff --git a/dockerfiles/fedora-36/Dockerfile b/dockerfiles/fedora-36/Dockerfile new file mode 100644 index 0000000..5817fca --- /dev/null +++ b/dockerfiles/fedora-36/Dockerfile @@ -0,0 +1,19 @@ +FROM fedora:36 + +RUN dnf install -y \ + autoconf \ + bash \ + ccache \ + clang \ + cmake \ + diffutils \ + elfutils \ + findutils \ + gcc \ + gcc-c++ \ + hiredis-devel \ + libzstd-devel \ + make \ + python3 \ + redis \ + && dnf clean all diff --git a/dockerfiles/ubuntu-14.04/Dockerfile b/dockerfiles/ubuntu-14.04/Dockerfile deleted file mode 100644 index 5b3d1db..0000000 --- a/dockerfiles/ubuntu-14.04/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM ubuntu:14.04 - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - asciidoc \ - bash \ - build-essential \ - ccache \ - clang-3.4 \ - curl \ - docbook-xml \ - docbook-xsl \ - elfutils \ - g++-multilib \ - ninja-build \ - wget \ - xsltproc \ - && rm -rf /var/lib/apt/lists/* - -# Redirect all compilers to ccache. -RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done - -# The distribution's CMake it too old (2.8.12.2). -RUN curl -sSL https://cmake.org/files/v3.5/cmake-3.5.2-Linux-x86_64.tar.gz | sudo tar -xzC /opt \ - && cp -a /opt/cmake-3.5.2-Linux-x86_64/bin /usr/local \ - && cp -a /opt/cmake-3.5.2-Linux-x86_64/share /usr/local diff --git a/dockerfiles/ubuntu-16.04/Dockerfile b/dockerfiles/ubuntu-16.04/Dockerfile deleted file mode 100644 index 7c9ddbc..0000000 --- a/dockerfiles/ubuntu-16.04/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM ubuntu:16.04 - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - asciidoc \ - bash \ - build-essential \ - ccache \ - clang \ - cmake \ - docbook-xml \ - docbook-xsl \ - elfutils \ - gcc-multilib \ - libzstd1-dev \ - xsltproc \ - && rm -rf /var/lib/apt/lists/* - -# Redirect all compilers to ccache. -RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff --git a/dockerfiles/ubuntu-18.04/Dockerfile b/dockerfiles/ubuntu-18.04/Dockerfile index 4ab985b..49ae973 100644 --- a/dockerfiles/ubuntu-18.04/Dockerfile +++ b/dockerfiles/ubuntu-18.04/Dockerfile @@ -2,18 +2,29 @@ FROM ubuntu:18.04 RUN apt-get update \ && apt-get install -y --no-install-recommends \ - asciidoc \ + asciidoctor \ bash \ build-essential \ ccache \ clang \ - cmake \ docbook-xml \ docbook-xsl \ elfutils \ gcc-multilib \ + gpg \ + libhiredis-dev \ libzstd-dev \ - xsltproc \ + python3 \ + redis-server \ + redis-tools \ + software-properties-common \ + wget \ + && add-apt-repository ppa:ubuntu-toolchain-r/test \ + && apt install -y --no-install-recommends g++-9 \ + && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg --dearmor - >/usr/share/keyrings/kitware-archive-keyring.gpg \ + && echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ bionic main' >/etc/apt/sources.list.d/kitware.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends cmake \ && rm -rf /var/lib/apt/lists/* # Redirect all compilers to ccache. diff --git a/dockerfiles/ubuntu-20.04/Dockerfile b/dockerfiles/ubuntu-20.04/Dockerfile index f2be356..46f4135 100644 --- a/dockerfiles/ubuntu-20.04/Dockerfile +++ b/dockerfiles/ubuntu-20.04/Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu:20.04 # Non-interactive: do not set up timezone settings. RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ - asciidoc \ + asciidoctor \ bash \ build-essential \ ccache \ @@ -13,8 +13,11 @@ RUN apt-get update \ docbook-xsl \ elfutils \ gcc-multilib \ + libhiredis-dev \ libzstd-dev \ - xsltproc \ + python3 \ + redis-server \ + redis-tools \ && rm -rf /var/lib/apt/lists/* # Redirect all compilers to ccache. diff --git a/dockerfiles/ubuntu-22.04/Dockerfile b/dockerfiles/ubuntu-22.04/Dockerfile new file mode 100644 index 0000000..1c6d561 --- /dev/null +++ b/dockerfiles/ubuntu-22.04/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:22.04 + +# Non-interactive: do not set up timezone settings. +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ + asciidoctor \ + bash \ + build-essential \ + ccache \ + clang \ + cmake \ + docbook-xml \ + docbook-xsl \ + elfutils \ + gcc-multilib \ + gcc-12 \ + g++-12 \ + libhiredis-dev \ + libzstd-dev \ + python3 \ + redis-server \ + redis-tools \ + && rm -rf /var/lib/apt/lists/* + +# Redirect all compilers to ccache. +RUN for t in gcc g++ cc c++ clang clang++; do ln -vs /usr/bin/ccache /usr/local/bin/$t; done diff --git a/misc/ccache.magic b/misc/ccache.magic new file mode 100644 index 0000000..ad08262 --- /dev/null +++ b/misc/ccache.magic @@ -0,0 +1,4 @@ +0 beshort 0xCCAC ccache entry +>2 byte x \b, format version %d +>3 byte 0 \b, result +>3 byte 1 \b, manifest diff --git a/misc/codespell-allowlist.txt b/misc/codespell-allowlist.txt index 60f3e5c..876d528 100644 --- a/misc/codespell-allowlist.txt +++ b/misc/codespell-allowlist.txt @@ -1,6 +1,8 @@ copyable creat files' +fo +hda pase seve stoll diff --git a/misc/download-redis b/misc/download-redis new file mode 100755 index 0000000..0352b47 --- /dev/null +++ b/misc/download-redis @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +# This script downloads the contents of the local cache from a Redis remote +# storage. + +import redis +import os + +import progress.bar +import progress.spinner +import humanize + +config = os.getenv("REDIS_CONF", "localhost") +if ":" in config: + host, port = config.rsplit(":", 1) + sock = None +elif config.startswith("/"): + host, port, sock = None, None, config +else: + host, port, sock = config, 6379, None + +username = os.getenv("REDIS_USERNAME") +password = os.getenv("REDIS_PASSWORD") +context = redis.Redis(host=host, port=port, unix_socket_path=sock, password=password) +pipe = context.pipeline(transaction=False) + +ccache = os.getenv("CCACHE_DIR", os.path.expanduser("~/.cache/ccache")) + +try: + count = context.info()["db0"]["keys"] +except Exception: + count = None # only for showing progress +files = result = manifest = objects = 0 +size = 0 +columns = os.get_terminal_size()[0] +width = min(columns - 24, 100) +bar = progress.bar.Bar( + "Downloading...", max=count, fill="=", suffix="%(percent).1f%%", width=width +) +if not count: + bar = progress.spinner.Spinner("Downloading... ") + +# Note: doesn't work with the SSDB SCAN command +# syntax: keys key_start, key_end, limit +for key in context.scan_iter(): + if not key.startswith(b"ccache:"): + bar.next() + continue + base = key.decode().replace("ccache:", "") + pipe.get(key) + val = pipe.execute()[-1] + if val is None: + continue + if val[0:2] == b"\xcc\xac": # magic + objects += 1 + if val[2] == 0 and val[3] == 0: + ext = "R" + result += 1 + elif val[2] == 0 and val[3] == 1: + ext = "M" + manifest += 1 + else: + bar.next() + continue + filename = os.path.join(ccache, base[0], base[1], base[2:] + ext) + if not os.path.exists(filename): + dirname = os.path.join(ccache, base[0], base[1]) + os.makedirs(dirname, mode=0o755, exist_ok=True) + with open(filename, "wb") as out: + out.write(val) + files += 1 + size += len(val) + bar.next() +bar.finish() + +print( + "%d files, %d result (%d manifest) = %d objects (%s)" + % (files, result, manifest, objects, humanize.naturalsize(size, binary=True)) +) diff --git a/misc/format-files b/misc/format-files index 44340b9..6671f8e 100755 --- a/misc/format-files +++ b/misc/format-files @@ -44,7 +44,9 @@ for file in "$@"; do if [ -n "$check" ]; then status=1 echo "Error: $file not formatted with Clang-Format" - echo 'Run "make format" or apply this diff:' + echo + echo 'Please run "make format" (or "ninja format"), or apply this diff:' + echo git diff $cf_color --no-index "$file" "$tmp_file" \ | sed -E -e "s!^---.*!--- a/$file!" \ -e "s!^\+\+\+.*!+++ b/$file!" \ diff --git a/misc/performance b/misc/performance index d33ad64..93cb2f4 100755 --- a/misc/performance +++ b/misc/performance @@ -263,9 +263,7 @@ def main(argv): op.add_option( "--no-compression", help="disable compression", action="store_true" ) - op.add_option( - "--compression-level", help="set compression level", type=int - ) + op.add_option("--compression-level", help="set compression level", type=int) op.add_option( "-d", "--directory", diff --git a/misc/test-all-systems b/misc/test-all-systems index a95c392..1383f95 100755 --- a/misc/test-all-systems +++ b/misc/test-all-systems @@ -14,41 +14,39 @@ build() { local cxx=$3 local test_cc=$4 shift 4 + local cmake_params="$*" + if command -v >/dev/null ccache; then + cmake_params="${cmake_params} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + fi echo "Build in Docker: $name CC=$cc CXX=$cxx TEST_CC=$test_cc CMAKE_PARAMS=\"$*\"" - ASM=$cc CC=$cc CXX=$cxx TEST_CC=$test_cc CMAKE_PARAMS="$*" $build_in_docker $name + ASM=$cc CC=$cc CXX=$cxx TEST_CC=$test_cc CMAKE_PARAMS="${cmake_params}" $build_in_docker $name } -# NAME CC CXX TEST_CC CMAKE_PARAMS +# NAME CC CXX TEST_CC CMAKE_PARAMS -build debian-9 gcc g++ gcc -build debian-9 clang clang++ clang +build debian-11 gcc g++ gcc +build debian-11 clang clang++ clang -build debian-10 gcc g++ gcc -build debian-10 clang clang++ clang +build ubuntu-18.04 gcc-9 g++-9 gcc +build ubuntu-18.04 clang clang++ clang -build ubuntu-14.04 gcc g++ gcc -DZSTD_FROM_INTERNET=ON -build ubuntu-14.04 gcc g++ clang -DZSTD_FROM_INTERNET=ON +build ubuntu-20.04 gcc g++ gcc +build ubuntu-20.04 clang clang++ clang -build ubuntu-16.04 gcc g++ gcc -build ubuntu-16.04 clang clang++ clang +build ubuntu-22.04 gcc g++ gcc +build ubuntu-22.04 clang clang++ clang -build ubuntu-18.04 gcc g++ gcc -build ubuntu-18.04 clang clang++ clang +build ubuntu-22.04 gcc-12 g++-12 gcc +build ubuntu-22.04 clang clang++ clang -build ubuntu-20.04 gcc g++ gcc -build ubuntu-20.04 clang clang++ clang +build centos-7 gcc g++ gcc -DHIREDIS_FROM_INTERNET=ON +build centos-7 gcc g++ clang -DHIREDIS_FROM_INTERNET=ON -build centos-7 gcc g++ gcc -DWARNINGS_AS_ERRORS=false -build centos-7 gcc g++ clang -DWARNINGS_AS_ERRORS=false +build fedora-36 gcc g++ gcc +build fedora-36 clang clang++ clang -build centos-8 gcc g++ gcc -build centos-8 clang clang++ clang +build alpine-3.11 gcc g++ gcc +build alpine-3.11 gcc g++ clang -build fedora-32 gcc g++ gcc -build fedora-32 clang clang++ clang - -build alpine-3.4 gcc g++ gcc -DZSTD_FROM_INTERNET=ON -build alpine-3.4 gcc g++ clang -DZSTD_FROM_INTERNET=ON - -build alpine-3.12 gcc g++ gcc -build alpine-3.12 clang clang++ clang +build alpine-3.15 gcc g++ gcc +build alpine-3.15 clang clang++ clang diff --git a/misc/update-authors b/misc/update-authors index 0755026..80a58b6 100755 --- a/misc/update-authors +++ b/misc/update-authors @@ -1,5 +1,7 @@ #!/bin/sh +anonymous="^(6d5CfLQ3dYAb|bengtj|Delgan|luzpaz|rblx-kbuck|RW|vsplesk)$" + if [ -d .git ]; then # Fetch full Git history if needed, e.g. when run via CI. git fetch --unshallow 2>/dev/null @@ -8,6 +10,7 @@ if [ -d .git ]; then # a "Co-authored-by:" in the commit message. (git log | grep -Po "(?<=Co-authored-by: )(.*)(?= <)"; \ git log --format="%aN") \ + | grep -Ev "$anonymous" \ | sed 's/^/* /' \ | LANG=en_US.utf8 sort -uf \ | perl -00 -p -i -e 's/^\*.*/ . "\n"/es' doc/AUTHORS.adoc diff --git a/misc/upload-redis b/misc/upload-redis new file mode 100755 index 0000000..b3595c5 --- /dev/null +++ b/misc/upload-redis @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +# This script uploads the contents of the local cache to a Redis remote storage. + +import redis +import os + +import progress.bar +import humanize + +config = os.getenv("REDIS_CONF", "localhost") +if ":" in config: + host, port = config.rsplit(":", 1) + sock = None +elif config.startswith("/"): + host, port, sock = None, None, config +else: + host, port, sock = config, 6379, None +username = os.getenv("REDIS_USERNAME") +password = os.getenv("REDIS_PASSWORD") +context = redis.Redis(host=host, port=port, unix_socket_path=sock, password=password) +pipe = context.pipeline(transaction=False) + +ccache = os.getenv("CCACHE_DIR", os.path.expanduser("~/.cache/ccache")) +filelist = [] +for dirpath, dirnames, filenames in os.walk(ccache): + # sort by modification time, most recently used last + for filename in filenames: + if filename.endswith(".lock"): + continue + stat = os.stat(os.path.join(dirpath, filename)) + filelist.append((stat.st_mtime, dirpath, filename, stat.st_size)) +filelist.sort() + +files = result = manifest = objects = 0 +size = 0 +batchsize = 0 +columns = os.get_terminal_size()[0] +width = min(columns - 22, 100) +bar = progress.bar.Bar( + "Uploading...", max=len(filelist), fill="=", suffix="%(percent).1f%%", width=width +) +for mtime, dirpath, filename, filesize in filelist: + dirname = dirpath.replace(ccache + os.path.sep, "") + if dirname == "tmp": + continue + elif filename == "CACHEDIR.TAG" or filename == "stats": + # ignore these + files += 1 + else: + (base, ext) = filename[:-1], filename[-1:] + if ext == "R" or ext == "M": + if ext == "R": + result += 1 + if ext == "M": + manifest += 1 + key = "ccache:" + "".join(list(os.path.split(dirname)) + [base]) + val = open(os.path.join(dirpath, filename), "rb").read() or None + if val: + # print("%s: %s %d" % (key, ext, len(val))) + pipe.setnx(key, val) + objects += 1 + files += 1 + size += filesize + batchsize += filesize + if batchsize > 64 * 1024 * 1024: + pipe.execute() + batchsize = 0 + bar.next() +pipe.execute() +bar.finish() + +print( + "%d files, %d result (%d manifest) = %d objects (%s)" + % (files, result, manifest, objects, humanize.naturalsize(size, binary=True)) +) diff --git a/src/.clang-tidy b/src/.clang-tidy index f30529d..48c5e0a 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -13,6 +13,7 @@ Checks: '-*, -readability-implicit-bool-conversion, -readability-magic-numbers, -readability-else-after-return, + -readability-function-cognitive-complexity, -readability-named-parameter, -readability-qualified-auto, -readability-redundant-declaration, @@ -24,6 +25,7 @@ Checks: '-*, -modernize-return-braced-init-list, -modernize-use-auto, -modernize-use-default-member-init, + -modernize-use-nodiscard, -modernize-use-trailing-return-type, cppcoreguidelines-*, -cppcoreguidelines-pro-bounds-array-to-pointer-decay, @@ -69,13 +71,13 @@ CheckOptions: # If you hit a limit, please consider changing the code instead of the limit. - key: readability-function-size.LineThreshold - value: 700 + value: 999999 - key: readability-function-size.StatementThreshold value: 999999 - key: readability-function-size.BranchThreshold - value: 170 + value: 999999 - key: readability-function-size.ParameterThreshold - value: 6 + value: 7 - key: readability-function-size.NestingThreshold value: 999999 - key: readability-function-size.VariableThreshold diff --git a/src/Args.cpp b/src/Args.cpp index 5270989..50cfa67 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -20,9 +20,10 @@ #include "Util.hpp" -using nonstd::nullopt; -using nonstd::optional; -using nonstd::string_view; +#include +#include +#include +#include Args::Args(Args&& other) noexcept : m_args(std::move(other.m_args)) { @@ -37,7 +38,7 @@ Args::from_argv(int argc, const char* const* argv) } Args -Args::from_string(const std::string& command) +Args::from_string(std::string_view command) { Args args; for (const std::string& word : Util::split_into_strings(command, " \t\r\n")) { @@ -46,20 +47,19 @@ Args::from_string(const std::string& command) return args; } -optional -Args::from_gcc_atfile(const std::string& filename) +std::optional +Args::from_atfile(const std::string& filename, AtFileFormat format) { - std::string argtext; - try { - argtext = Util::read_file(filename); - } catch (Error&) { - return nullopt; + const auto argtext = util::read_file(filename); + if (!argtext) { + LOG("Failed to read atfile {}: {}", filename, argtext.error()); + return std::nullopt; } Args args; - auto pos = argtext.c_str(); + auto pos = argtext->c_str(); std::string argbuf; - argbuf.resize(argtext.length() + 1); + argbuf.resize(argtext->length() + 1); auto argpos = argbuf.begin(); // Used to track quoting state; if \0 we are not inside quotes. Otherwise @@ -70,13 +70,27 @@ Args::from_gcc_atfile(const std::string& filename) switch (*pos) { case '\\': pos++; - if (*pos == '\0') { - continue; + switch (format) { + case AtFileFormat::gcc: + if (*pos == '\0') { + continue; + } + break; + case AtFileFormat::msvc: + if (*pos != '"' && *pos != '\\') { + pos--; + } + break; } break; - case '"': case '\'': + if (format == AtFileFormat::msvc) { + break; + } + [[fallthrough]]; + + case '"': if (quoting != '\0') { if (quoting == *pos) { quoting = '\0'; @@ -98,7 +112,7 @@ Args::from_gcc_atfile(const std::string& filename) if (quoting) { break; } - // Fall through. + [[fallthrough]]; case '\0': // End of token @@ -156,7 +170,7 @@ Args::to_string() const } void -Args::erase_last(string_view arg) +Args::erase_last(std::string_view arg) { const auto it = std::find(m_args.rbegin(), m_args.rend(), arg); if (it != m_args.rend()) { @@ -165,12 +179,12 @@ Args::erase_last(string_view arg) } void -Args::erase_with_prefix(string_view prefix) +Args::erase_with_prefix(std::string_view prefix) { m_args.erase(std::remove_if(m_args.begin(), m_args.end(), - [&prefix](const std::string& s) { - return Util::starts_with(s, prefix); + [&prefix](const auto& s) { + return util::starts_with(s, prefix); }), m_args.end()); } diff --git a/src/Args.hpp b/src/Args.hpp index 48fb77d..16bc7fe 100644 --- a/src/Args.hpp +++ b/src/Args.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020-2021 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,27 +18,32 @@ #pragma once -#include "system.hpp" - #include "NonCopyable.hpp" -#include "Util.hpp" - -#include "third_party/nonstd/optional.hpp" -#include "third_party/nonstd/string_view.hpp" #include +#include #include +#include +#include class Args { public: + enum class AtFileFormat { + gcc, // '\'' and '"' quote, '\\' escapes any character + msvc, // '"' quotes, '\\' escapes only '"' and '\\' + }; + Args() = default; Args(const Args& other) = default; Args(Args&& other) noexcept; static Args from_argv(int argc, const char* const* argv); - static Args from_string(const std::string& command); - static nonstd::optional from_gcc_atfile(const std::string& filename); + static Args from_string(std::string_view command); + + static std::optional + from_atfile(const std::string& filename, + AtFileFormat format = AtFileFormat::gcc); Args& operator=(const Args& other) = default; Args& operator=(Args&& other) noexcept; @@ -61,10 +66,10 @@ public: std::string to_string() const; // Remove last argument equal to `arg`, if any. - void erase_last(nonstd::string_view arg); + void erase_last(std::string_view arg); // Remove all arguments with prefix `prefix`. - void erase_with_prefix(nonstd::string_view prefix); + void erase_with_prefix(std::string_view prefix); // Insert arguments in `args` at position `index`. void insert(size_t index, const Args& args); diff --git a/src/ArgsInfo.hpp b/src/ArgsInfo.hpp index 4d9eaf3..b131f42 100644 --- a/src/ArgsInfo.hpp +++ b/src/ArgsInfo.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,28 +18,37 @@ #pragma once -#include "system.hpp" - #include "Args.hpp" +#include #include #include // This class holds meta-information derived from the compiler arguments. struct ArgsInfo { - // The source file. + // The source file path. + std::string orig_input_file; + + // The source file path, potentially rewritten into relative. std::string input_file; + // The source file path run through Util::normalize_concrete_absolute_path. + std::string normalized_input_file; + // In normal compiler operation an output file is created if there is no // compiler error. However certain flags like -fsyntax-only change this // behavior. bool expect_output_obj = true; // The output file being compiled to. + std::string orig_output_obj; + + // The output file being compiled to, potentially rewritten into relative. std::string output_obj; - // The path to the dependency file (implicit or specified with -MF). + // The path to the dependency file (implicit or specified with -MFdepfile, + // -Wp,-MD,depfile or -Wp,-MMD,depfile). std::string output_dep; // The path to the stack usage (implicit when using -fstack-usage). @@ -51,6 +60,12 @@ struct ArgsInfo // Split dwarf information (GCC 4.8 and up). Contains pathname if not empty. std::string output_dwo; + // Assembler listing file. + std::string output_al; + + // The .gch/.pch/.pth file used for compilation. + std::string included_pch_file; + // Language to use for the compilation target (see language.c). std::string actual_language; @@ -60,11 +75,12 @@ struct ArgsInfo // Is the compiler being asked to output dependencies? bool generating_dependencies = false; - // Seen -MD or -MMD? - bool seen_MD_MMD = false; + // Is the compiler being asked to output includes (MSVC /showIncludes)? + bool generating_includes = false; - // Is the dependency makefile target name specified with -MT or -MQ? - bool dependency_target_specified = false; + // The dependency target in the dependency file (the object file unless + // overridden via e.g. -MT or -MQ). + std::optional dependency_target; // Is the compiler being asked to output coverage? bool generating_coverage = false; @@ -79,6 +95,9 @@ struct ArgsInfo // Whether to strip color codes from diagnostic messages on output. bool strip_diagnostics_colors = false; + // Have we seen --? + bool seen_double_dash = false; + // Have we seen -gsplit-dwarf? bool seen_split_dwarf = false; diff --git a/src/AtomicFile.cpp b/src/AtomicFile.cpp index d02679b..c9d82c3 100644 --- a/src/AtomicFile.cpp +++ b/src/AtomicFile.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -21,11 +21,13 @@ #include "TemporaryFile.hpp" #include "Util.hpp" #include "assertions.hpp" -#include "exceptions.hpp" + +#include +#include AtomicFile::AtomicFile(const std::string& path, Mode mode) : m_path(path) { - TemporaryFile tmp_file(path + ".tmp"); + TemporaryFile tmp_file(path); m_stream = fdopen(tmp_file.fd.release(), mode == Mode::binary ? "w+b" : "w+"); m_tmp_path = std::move(tmp_file.path); } @@ -40,18 +42,20 @@ AtomicFile::~AtomicFile() } void -AtomicFile::write(const std::string& data) +AtomicFile::write(std::string_view data) { if (fwrite(data.data(), data.size(), 1, m_stream) != 1) { - throw Error("failed to write data to {}: {}", m_path, strerror(errno)); + throw core::Error( + FMT("failed to write data to {}: {}", m_path, strerror(errno))); } } void -AtomicFile::write(const std::vector& data) +AtomicFile::write(nonstd::span data) { if (fwrite(data.data(), data.size(), 1, m_stream) != 1) { - throw Error("failed to write data to {}: {}", m_path, strerror(errno)); + throw core::Error( + FMT("failed to write data to {}: {}", m_path, strerror(errno))); } } @@ -63,7 +67,8 @@ AtomicFile::commit() m_stream = nullptr; if (result == EOF) { Util::unlink_tmp(m_tmp_path); - throw Error("failed to write data to {}: {}", m_path, strerror(errno)); + throw core::Error( + FMT("failed to write data to {}: {}", m_path, strerror(errno))); } Util::rename(m_tmp_path, m_path); } diff --git a/src/AtomicFile.hpp b/src/AtomicFile.hpp index 118c310..37ac372 100644 --- a/src/AtomicFile.hpp +++ b/src/AtomicFile.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,10 +18,11 @@ #pragma once -#include "system.hpp" +#include +#include +#include #include -#include // This class represents a file whose data will be atomically written to a path // by renaming a temporary file in place. @@ -35,8 +36,8 @@ public: FILE* stream(); - void write(const std::string& data); - void write(const std::vector& data); + void write(std::string_view data); + void write(nonstd::span data); // Close the temporary file and rename it to the destination file. Note: The // destructor will not do this automatically to avoid half-written data in the diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index beefd81..79f0fd1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,81 +2,77 @@ set( source_files Args.cpp AtomicFile.cpp - CacheEntryReader.cpp - CacheEntryWriter.cpp - CacheFile.cpp - Compression.cpp - Compressor.cpp Config.cpp Context.cpp - Counters.cpp - Decompressor.cpp Depfile.cpp + Fd.cpp Hash.cpp - Lockfile.cpp Logging.cpp - Manifest.cpp - MiniTrace.cpp - NullCompressor.cpp - NullDecompressor.cpp ProgressBar.cpp - Result.cpp - ResultDumper.cpp - ResultExtractor.cpp - ResultRetriever.cpp SignalHandler.cpp Stat.cpp - Statistics.cpp TemporaryFile.cpp ThreadPool.cpp Util.cpp - ZstdCompressor.cpp - ZstdDecompressor.cpp argprocessing.cpp assertions.cpp ccache.cpp - cleanup.cpp compopt.cpp - compress.cpp execute.cpp hashutil.cpp language.cpp - version.cpp) + version.cpp +) if(INODE_CACHE_SUPPORTED) list(APPEND source_files InodeCache.cpp) endif() +if(MTR_ENABLED) + list(APPEND source_files MiniTrace.cpp) +endif() + if(WIN32) list(APPEND source_files Win32Util.cpp) endif() -add_library(ccache_lib STATIC ${source_files}) -target_compile_definitions( - ccache_lib PUBLIC -Dnssv_CONFIG_SELECT_STRING_VIEW=nssv_STRING_VIEW_NONSTD -) +file(GLOB headers *.hpp) +list(APPEND source_files ${headers}) + +add_library(ccache_framework STATIC ${source_files}) if(WIN32) - target_link_libraries(ccache_lib PRIVATE ws2_32 "psapi") + list(APPEND CCACHE_EXTRA_LIBS psapi) +endif() - if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - if(STATIC_LINK) - target_link_libraries(ccache_lib PRIVATE -static-libgcc -static-libstdc++ -static winpthread -dynamic) - else() - target_link_libraries(ccache_lib PRIVATE winpthread) - endif() - elseif(STATIC_LINK AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - target_link_libraries(ccache_lib PRIVATE -static c++ -dynamic) - endif() +if(CCACHE_EXTRA_LIBS) + target_link_libraries( + ccache_framework + PRIVATE "${CCACHE_EXTRA_LIBS}" + ) endif() set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) target_link_libraries( - ccache_lib - PRIVATE standard_settings standard_warnings ZSTD::ZSTD - Threads::Threads third_party_lib) + ccache_framework + PRIVATE standard_settings standard_warnings ZSTD::ZSTD Threads::Threads third_party +) + +target_include_directories(ccache_framework PUBLIC ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +if(REDIS_STORAGE_BACKEND) + target_compile_definitions(ccache_framework PUBLIC -DHAVE_REDIS_STORAGE_BACKEND) + target_link_libraries( + ccache_framework + PUBLIC standard_settings standard_warnings HIREDIS::HIREDIS third_party + ) +endif() -target_include_directories(ccache_lib PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) +add_executable(test-lockfile test_lockfile.cpp) +target_link_libraries(test-lockfile PRIVATE standard_settings standard_warnings ccache_framework) +add_subdirectory(core) +add_subdirectory(storage) add_subdirectory(third_party) +add_subdirectory(util) diff --git a/src/CacheEntryReader.cpp b/src/CacheEntryReader.cpp deleted file mode 100644 index 36dd393..0000000 --- a/src/CacheEntryReader.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "CacheEntryReader.hpp" - -#include "Compressor.hpp" -#include "exceptions.hpp" -#include "fmtmacros.hpp" - -#include "third_party/fmt/core.h" - -CacheEntryReader::CacheEntryReader(FILE* stream, - const uint8_t* expected_magic, - uint8_t expected_version) -{ - uint8_t header_bytes[15]; - if (fread(header_bytes, sizeof(header_bytes), 1, stream) != 1) { - throw Error("Error reading header"); - } - - memcpy(m_magic, header_bytes, sizeof(m_magic)); - m_version = header_bytes[4]; - m_compression_type = Compression::type_from_int(header_bytes[5]); - m_compression_level = header_bytes[6]; - Util::big_endian_to_int(header_bytes + 7, m_content_size); - - if (memcmp(m_magic, expected_magic, sizeof(m_magic)) != 0) { - throw Error("Bad magic value 0x{:02x}{:02x}{:02x}{:02x}", - m_magic[0], - m_magic[1], - m_magic[2], - m_magic[3]); - } - if (m_version != expected_version) { - throw Error( - "Unknown version (actual {}, expected {})", m_version, expected_version); - } - - m_checksum.update(header_bytes, sizeof(header_bytes)); - m_decompressor = Decompressor::create_from_type(m_compression_type, stream); -} - -void -CacheEntryReader::dump_header(FILE* dump_stream) -{ - PRINT(dump_stream, "Magic: {:.4}\n", m_magic); - PRINT(dump_stream, "Version: {}\n", m_version); - PRINT(dump_stream, - "Compression type: {}\n", - Compression::type_to_string(m_compression_type)); - PRINT(dump_stream, "Compression level: {}\n", m_compression_level); - PRINT(dump_stream, "Content size: {}\n", m_content_size); -} - -void -CacheEntryReader::read(void* data, size_t count) -{ - m_decompressor->read(data, count); - m_checksum.update(data, count); -} - -void -CacheEntryReader::finalize() -{ - uint64_t actual_digest = m_checksum.digest(); - - uint8_t buffer[8]; - read(buffer, sizeof(buffer)); - uint64_t expected_digest; - Util::big_endian_to_int(buffer, expected_digest); - - if (actual_digest != expected_digest) { - throw Error("Incorrect checksum (actual 0x{:016x}, expected 0x{:016x})", - actual_digest, - expected_digest); - } - - m_decompressor->finalize(); -} diff --git a/src/CacheEntryReader.hpp b/src/CacheEntryReader.hpp deleted file mode 100644 index 37db80f..0000000 --- a/src/CacheEntryReader.hpp +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Checksum.hpp" -#include "Decompressor.hpp" -#include "Util.hpp" - -#include - -// This class knows how to read a cache entry with a common header and a -// payload part that is different depending on the cache entry type (result or -// manifest). -class CacheEntryReader -{ -public: - // Constructor. - // - // Parameters: - // - stream: Stream to read header and payload from. - // - expected_magic: Expected file format magic (first four bytes of the - // file). - // - expected_version: Expected file format version. - CacheEntryReader(FILE* stream, - const uint8_t* expected_magic, - uint8_t expected_version); - - // Dump header information in text format. - // - // Parameters: - // - dump_stream: Stream to write to. - void dump_header(FILE* dump_stream); - - // Read data into a buffer from the payload. - // - // Parameters: - // - data: Buffer to write data to. - // - count: How many bytes to write. - // - // Throws Error on failure. - void read(void* data, size_t count); - - // Read an unsigned integer from the payload. - // - // Parameters: - // - value: Variable to write to. - // - // Throws Error on failure. - template void read(T& value); - - // Close for reading. - // - // This method potentially verifies the end state after reading the cache - // entry and throws Error if any integrity issues are found. - void finalize(); - - // Get size of the payload, - uint64_t payload_size() const; - - // Get content magic. - const uint8_t* magic() const; - - // Get content version. - uint8_t version() const; - - // Get compression type. - Compression::Type compression_type() const; - - // Get compression level. - int8_t compression_level() const; - - // Get size of the content (header + payload + checksum). - uint64_t content_size() const; - -private: - std::unique_ptr m_decompressor; - Checksum m_checksum; - uint8_t m_magic[4]; - uint8_t m_version; - Compression::Type m_compression_type; - int8_t m_compression_level; - uint64_t m_content_size; -}; - -template -inline void -CacheEntryReader::read(T& value) -{ - uint8_t buffer[sizeof(T)]; - read(buffer, sizeof(T)); - Util::big_endian_to_int(buffer, value); -} - -inline const uint8_t* -CacheEntryReader::magic() const -{ - return m_magic; -} - -inline uint8_t -CacheEntryReader::version() const -{ - return m_version; -} - -inline Compression::Type -CacheEntryReader::compression_type() const -{ - return m_compression_type; -} - -inline int8_t -CacheEntryReader::compression_level() const -{ - return m_compression_level; -} - -inline uint64_t -CacheEntryReader::payload_size() const -{ - return m_content_size - 15 - 8; -} - -inline uint64_t -CacheEntryReader::content_size() const -{ - return m_content_size; -} diff --git a/src/CacheEntryWriter.cpp b/src/CacheEntryWriter.cpp deleted file mode 100644 index b936947..0000000 --- a/src/CacheEntryWriter.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "CacheEntryWriter.hpp" - -CacheEntryWriter::CacheEntryWriter(FILE* stream, - const uint8_t* magic, - uint8_t version, - Compression::Type compression_type, - int8_t compression_level, - uint64_t payload_size) - // clang-format off - : m_compressor( - Compressor::create_from_type(compression_type, stream, compression_level)) -// clang-format on -{ - uint8_t header_bytes[15]; - memcpy(header_bytes, magic, 4); - header_bytes[4] = version; - header_bytes[5] = static_cast(compression_type); - header_bytes[6] = m_compressor->actual_compression_level(); - uint64_t content_size = 15 + payload_size + 8; - Util::int_to_big_endian(content_size, header_bytes + 7); - if (fwrite(header_bytes, sizeof(header_bytes), 1, stream) != 1) { - throw Error("Failed to write cache entry header"); - } - m_checksum.update(header_bytes, sizeof(header_bytes)); -} - -void -CacheEntryWriter::write(const void* data, size_t count) -{ - m_compressor->write(data, count); - m_checksum.update(data, count); -} - -void -CacheEntryWriter::finalize() -{ - uint8_t buffer[8]; - Util::int_to_big_endian(m_checksum.digest(), buffer); - m_compressor->write(buffer, sizeof(buffer)); - m_compressor->finalize(); -} diff --git a/src/CacheEntryWriter.hpp b/src/CacheEntryWriter.hpp deleted file mode 100644 index dc9226f..0000000 --- a/src/CacheEntryWriter.hpp +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Checksum.hpp" -#include "Compressor.hpp" -#include "Util.hpp" - -#include - -// This class knows how to write a cache entry with a common header and a -// payload part that is different depending on the cache entry type (result or -// manifest). -class CacheEntryWriter -{ -public: - // Constructor. - // - // Parameters: - // - stream: Stream to write header + payload to. - // - magic: File format magic (first four bytes of the file). - // - version: File format version. - // - compression_type: Compression type to use. - // - compression_level: Compression level to use. - // - payload_size: Payload size. - CacheEntryWriter(FILE* stream, - const uint8_t* magic, - uint8_t version, - Compression::Type compression_type, - int8_t compression_level, - uint64_t payload_size); - - // Write data to the payload from a buffer. - // - // Parameters: - // - data: Data to write. - // - count: Size of data to write. - // - // Throws Error on failure. - void write(const void* data, size_t count); - - // Write an unsigned integer to the payload. - // - // Parameters: - // - value: Value to write. - // - // Throws Error on failure. - template void write(T value); - - // Close for writing. - // - // This method potentially verifies the end state after writing the cache - // entry and throws Error if any integrity issues are found. - void finalize(); - -private: - std::unique_ptr m_compressor; - Checksum m_checksum; -}; - -template -inline void -CacheEntryWriter::write(T value) -{ - uint8_t buffer[sizeof(T)]; - Util::int_to_big_endian(value, buffer); - write(buffer, sizeof(T)); -} diff --git a/src/CacheFile.cpp b/src/CacheFile.cpp deleted file mode 100644 index 68072df..0000000 --- a/src/CacheFile.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "CacheFile.hpp" - -#include "Manifest.hpp" -#include "Result.hpp" -#include "Util.hpp" - -const Stat& -CacheFile::lstat() const -{ - if (!m_stat) { - m_stat = Stat::lstat(m_path); - } - - return *m_stat; -} - -CacheFile::Type -CacheFile::type() const -{ - if (Util::ends_with(m_path, Manifest::k_file_suffix)) { - return Type::manifest; - } else if (Util::ends_with(m_path, Result::k_file_suffix)) { - return Type::result; - } else { - return Type::unknown; - } -} diff --git a/src/CacheFile.hpp b/src/CacheFile.hpp deleted file mode 100644 index 0541068..0000000 --- a/src/CacheFile.hpp +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Stat.hpp" - -#include "third_party/nonstd/optional.hpp" - -#include - -class CacheFile -{ -public: - enum class Type { result, manifest, unknown }; - - explicit CacheFile(const std::string& path); - - const Stat& lstat() const; - const std::string& path() const; - Type type() const; - -private: - std::string m_path; - mutable nonstd::optional m_stat; -}; - -inline CacheFile::CacheFile(const std::string& path) : m_path(path) -{ -} - -inline const std::string& -CacheFile::path() const -{ - return m_path; -} diff --git a/src/Checksum.hpp b/src/Checksum.hpp deleted file mode 100644 index e2c6274..0000000 --- a/src/Checksum.hpp +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#ifdef USE_XXH_DISPATCH -# include "third_party/xxh_x86dispatch.h" -#else -# include "third_party/xxhash.h" -#endif - -class Checksum -{ -public: - Checksum(); - ~Checksum(); - - void reset(); - void update(const void* data, size_t length); - uint64_t digest() const; - -private: - XXH3_state_t* m_state; -}; - -inline Checksum::Checksum() : m_state(XXH3_createState()) -{ - reset(); -} - -inline Checksum::~Checksum() -{ - XXH3_freeState(m_state); -} - -inline void -Checksum::reset() -{ - XXH3_64bits_reset(m_state); -} - -inline void -Checksum::update(const void* data, size_t length) -{ - XXH3_64bits_update(m_state, data, length); -} - -inline uint64_t -Checksum::digest() const -{ - return XXH3_64bits_digest(m_state); -} diff --git a/src/Compression.cpp b/src/Compression.cpp deleted file mode 100644 index aa2a182..0000000 --- a/src/Compression.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Compression.hpp" - -#include "Config.hpp" -#include "Context.hpp" -#include "assertions.hpp" -#include "exceptions.hpp" - -namespace Compression { - -int8_t -level_from_config(const Config& config) -{ - return config.compression() ? config.compression_level() : 0; -} - -Type -type_from_config(const Config& config) -{ - return config.compression() ? Type::zstd : Type::none; -} - -Type -type_from_int(uint8_t type) -{ - switch (type) { - case static_cast(Type::none): - return Type::none; - - case static_cast(Type::zstd): - return Type::zstd; - } - - throw Error("Unknown type: {}", type); -} - -std::string -type_to_string(Type type) -{ - switch (type) { - case Type::none: - return "none"; - - case Type::zstd: - return "zstd"; - } - - ASSERT(false); -} - -} // namespace Compression diff --git a/src/Compression.hpp b/src/Compression.hpp deleted file mode 100644 index 24e1746..0000000 --- a/src/Compression.hpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include - -class Config; - -namespace Compression { - -enum class Type : uint8_t { - none = 0, - zstd = 1, -}; - -int8_t level_from_config(const Config& config); - -Type type_from_config(const Config& config); - -Type type_from_int(uint8_t type); - -std::string type_to_string(Compression::Type type); - -} // namespace Compression diff --git a/src/Compressor.cpp b/src/Compressor.cpp deleted file mode 100644 index efb1257..0000000 --- a/src/Compressor.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Compressor.hpp" - -#include "NullCompressor.hpp" -#include "StdMakeUnique.hpp" -#include "ZstdCompressor.hpp" -#include "assertions.hpp" - -std::unique_ptr -Compressor::create_from_type(Compression::Type type, - FILE* stream, - int8_t compression_level) -{ - switch (type) { - case Compression::Type::none: - return std::make_unique(stream); - - case Compression::Type::zstd: - return std::make_unique(stream, compression_level); - } - - ASSERT(false); -} diff --git a/src/Compressor.hpp b/src/Compressor.hpp deleted file mode 100644 index 3295874..0000000 --- a/src/Compressor.hpp +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Compression.hpp" - -#include - -class Compressor -{ -public: - virtual ~Compressor() = default; - - // Create a compressor for the specified type. - // - // Parameters: - // - type: The type. - // - stream: The stream to write to. - // - compression_level: Desired compression level. - static std::unique_ptr create_from_type(Compression::Type type, - FILE* stream, - int8_t compression_level); - - // Get the actual compression level used for the compressed stream. - virtual int8_t actual_compression_level() const = 0; - - // Write data from a buffer to the compressed stream. - // - // Parameters: - // - data: Data to write. - // - count: Size of data to write. - // - // Throws Error on failure. - virtual void write(const void* data, size_t count) = 0; - - // Write an unsigned integer to the compressed stream. - // - // Parameters: - // - value: Value to write. - // - // Throws Error on failure. - template void write(T value); - - // Finalize compression. - // - // This method checks that the end state of the compressed stream is correct - // and throws Error if not. - virtual void finalize() = 0; -}; diff --git a/src/Config.cpp b/src/Config.cpp index 9e70923..75a1358 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -19,15 +19,27 @@ #include "Config.hpp" #include "AtomicFile.hpp" -#include "Compression.hpp" -#include "Sloppiness.hpp" +#include "MiniTrace.hpp" #include "Util.hpp" #include "assertions.hpp" -#include "exceptions.hpp" -#include "fmtmacros.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include "third_party/fmt/core.h" +#ifdef HAVE_UNISTD_H +# include +#endif + #include #include #include @@ -36,8 +48,12 @@ #include #include -using nonstd::nullopt; -using nonstd::optional; +#ifndef environ +DLLIMPORT extern char** environ; +#endif + +// Make room for binary patching at install time. +const char k_sysconfdir[4096 + 1] = SYSCONFDIR; namespace { @@ -68,6 +84,8 @@ enum class ConfigItem { log_file, max_files, max_size, + msvc_dep_prefix, + namespace_, path, pch_external_checksum, prefix_command, @@ -75,52 +93,72 @@ enum class ConfigItem { read_only, read_only_direct, recache, + remote_only, + remote_storage, + reshare, run_second_cpp, sloppiness, stats, + stats_log, temporary_dir, umask, }; -const std::unordered_map k_config_key_table = { - {"absolute_paths_in_stderr", ConfigItem::absolute_paths_in_stderr}, - {"base_dir", ConfigItem::base_dir}, - {"cache_dir", ConfigItem::cache_dir}, - {"compiler", ConfigItem::compiler}, - {"compiler_check", ConfigItem::compiler_check}, - {"compiler_type", ConfigItem::compiler_type}, - {"compression", ConfigItem::compression}, - {"compression_level", ConfigItem::compression_level}, - {"cpp_extension", ConfigItem::cpp_extension}, - {"debug", ConfigItem::debug}, - {"debug_dir", ConfigItem::debug_dir}, - {"depend_mode", ConfigItem::depend_mode}, - {"direct_mode", ConfigItem::direct_mode}, - {"disable", ConfigItem::disable}, - {"extra_files_to_hash", ConfigItem::extra_files_to_hash}, - {"file_clone", ConfigItem::file_clone}, - {"hard_link", ConfigItem::hard_link}, - {"hash_dir", ConfigItem::hash_dir}, - {"ignore_headers_in_manifest", ConfigItem::ignore_headers_in_manifest}, - {"ignore_options", ConfigItem::ignore_options}, - {"inode_cache", ConfigItem::inode_cache}, - {"keep_comments_cpp", ConfigItem::keep_comments_cpp}, - {"limit_multiple", ConfigItem::limit_multiple}, - {"log_file", ConfigItem::log_file}, - {"max_files", ConfigItem::max_files}, - {"max_size", ConfigItem::max_size}, - {"path", ConfigItem::path}, - {"pch_external_checksum", ConfigItem::pch_external_checksum}, - {"prefix_command", ConfigItem::prefix_command}, - {"prefix_command_cpp", ConfigItem::prefix_command_cpp}, - {"read_only", ConfigItem::read_only}, - {"read_only_direct", ConfigItem::read_only_direct}, - {"recache", ConfigItem::recache}, - {"run_second_cpp", ConfigItem::run_second_cpp}, - {"sloppiness", ConfigItem::sloppiness}, - {"stats", ConfigItem::stats}, - {"temporary_dir", ConfigItem::temporary_dir}, - {"umask", ConfigItem::umask}, +enum class ConfigKeyType { normal, alias }; + +struct ConfigKeyTableEntry +{ + ConfigItem item; + std::optional alias = std::nullopt; +}; + +const std::unordered_map k_config_key_table = + { + {"absolute_paths_in_stderr", {ConfigItem::absolute_paths_in_stderr}}, + {"base_dir", {ConfigItem::base_dir}}, + {"cache_dir", {ConfigItem::cache_dir}}, + {"compiler", {ConfigItem::compiler}}, + {"compiler_check", {ConfigItem::compiler_check}}, + {"compiler_type", {ConfigItem::compiler_type}}, + {"compression", {ConfigItem::compression}}, + {"compression_level", {ConfigItem::compression_level}}, + {"cpp_extension", {ConfigItem::cpp_extension}}, + {"debug", {ConfigItem::debug}}, + {"debug_dir", {ConfigItem::debug_dir}}, + {"depend_mode", {ConfigItem::depend_mode}}, + {"direct_mode", {ConfigItem::direct_mode}}, + {"disable", {ConfigItem::disable}}, + {"extra_files_to_hash", {ConfigItem::extra_files_to_hash}}, + {"file_clone", {ConfigItem::file_clone}}, + {"hard_link", {ConfigItem::hard_link}}, + {"hash_dir", {ConfigItem::hash_dir}}, + {"ignore_headers_in_manifest", {ConfigItem::ignore_headers_in_manifest}}, + {"ignore_options", {ConfigItem::ignore_options}}, + {"inode_cache", {ConfigItem::inode_cache}}, + {"keep_comments_cpp", {ConfigItem::keep_comments_cpp}}, + {"limit_multiple", {ConfigItem::limit_multiple}}, + {"log_file", {ConfigItem::log_file}}, + {"max_files", {ConfigItem::max_files}}, + {"max_size", {ConfigItem::max_size}}, + {"msvc_dep_prefix", {ConfigItem::msvc_dep_prefix}}, + {"namespace", {ConfigItem::namespace_}}, + {"path", {ConfigItem::path}}, + {"pch_external_checksum", {ConfigItem::pch_external_checksum}}, + {"prefix_command", {ConfigItem::prefix_command}}, + {"prefix_command_cpp", {ConfigItem::prefix_command_cpp}}, + {"read_only", {ConfigItem::read_only}}, + {"read_only_direct", {ConfigItem::read_only_direct}}, + {"recache", {ConfigItem::recache}}, + {"remote_only", {ConfigItem::remote_only}}, + {"remote_storage", {ConfigItem::remote_storage}}, + {"reshare", {ConfigItem::reshare}}, + {"run_second_cpp", {ConfigItem::run_second_cpp}}, + {"secondary_storage", {ConfigItem::remote_storage, "remote_storage"}}, + {"sloppiness", {ConfigItem::sloppiness}}, + {"stats", {ConfigItem::stats}}, + {"stats_log", {ConfigItem::stats_log}}, + {"temporary_dir", {ConfigItem::temporary_dir}}, + {"umask", {ConfigItem::umask}}, }; const std::unordered_map k_env_variable_table = { @@ -152,6 +190,8 @@ const std::unordered_map k_env_variable_table = { {"LOGFILE", "log_file"}, {"MAXFILES", "max_files"}, {"MAXSIZE", "max_size"}, + {"MSVC_DEP_PREFIX", "msvc_dep_prefix"}, + {"NAMESPACE", "namespace"}, {"PATH", "path"}, {"PCH_EXTSUM", "pch_external_checksum"}, {"PREFIX", "prefix_command"}, @@ -159,15 +199,20 @@ const std::unordered_map k_env_variable_table = { {"READONLY", "read_only"}, {"READONLY_DIRECT", "read_only_direct"}, {"RECACHE", "recache"}, + {"REMOTE_ONLY", "remote_only"}, + {"REMOTE_STORAGE", "remote_storage"}, + {"RESHARE", "reshare"}, + {"SECONDARY_STORAGE", "remote_storage"}, // Alias for CCACHE_REMOTE_STORAGE {"SLOPPINESS", "sloppiness"}, {"STATS", "stats"}, + {"STATSLOG", "stats_log"}, {"TEMPDIR", "temporary_dir"}, {"UMASK", "umask"}, }; bool parse_bool(const std::string& value, - const optional env_var_key, + const std::optional env_var_key, bool negate) { if (env_var_key) { @@ -181,12 +226,12 @@ parse_bool(const std::string& value, std::string lower_value = Util::to_lowercase(value); if (value == "0" || lower_value == "false" || lower_value == "disable" || lower_value == "no") { - throw Error( - "invalid boolean environment variable value \"{}\" (did you mean to" - " set \"CCACHE_{}{}=true\"?)", - value, - negate ? "" : "NO", - *env_var_key); + throw core::Error( + FMT("invalid boolean environment variable value \"{}\" (did you mean to" + " set \"CCACHE_{}{}=true\"?)", + value, + negate ? "" : "NO", + *env_var_key)); } return !negate; } else if (value == "true") { @@ -194,7 +239,7 @@ parse_bool(const std::string& value, } else if (value == "false") { return false; } else { - throw Error("not a boolean value: \"{}\"", value); + throw core::Error(FMT("not a boolean value: \"{}\"", value)); } } @@ -204,22 +249,6 @@ format_bool(bool value) return value ? "true" : "false"; } -double -parse_double(const std::string& value) -{ - size_t end; - double result; - try { - result = std::stod(value, &end); - } catch (std::exception& e) { - throw Error(e.what()); - } - if (end != value.size()) { - throw Error("invalid floating point: \"{}\"", value); - } - return result; -} - std::string format_cache_size(uint64_t value) { @@ -231,94 +260,104 @@ parse_compiler_type(const std::string& value) { if (value == "clang") { return CompilerType::clang; + } else if (value == "clang-cl") { + return CompilerType::clang_cl; } else if (value == "gcc") { return CompilerType::gcc; + } else if (value == "icl") { + return CompilerType::icl; + } else if (value == "msvc") { + return CompilerType::msvc; } else if (value == "nvcc") { return CompilerType::nvcc; } else if (value == "other") { return CompilerType::other; - } else if (value == "pump") { - return CompilerType::pump; } else { // Allow any unknown value for forward compatibility. return CompilerType::auto_guess; } } -uint32_t +core::Sloppiness parse_sloppiness(const std::string& value) { - size_t start = 0; - size_t end = 0; - uint32_t result = 0; - while (end != std::string::npos) { - end = value.find_first_of(", ", start); - std::string token = - Util::strip_whitespace(value.substr(start, end - start)); - if (token == "file_stat_matches") { - result |= SLOPPY_FILE_STAT_MATCHES; + core::Sloppiness result; + + for (const auto token : util::Tokenizer(value, ", ")) { + if (token == "clang_index_store") { + result.enable(core::Sloppy::clang_index_store); + } else if (token == "file_stat_matches") { + result.enable(core::Sloppy::file_stat_matches); } else if (token == "file_stat_matches_ctime") { - result |= SLOPPY_FILE_STAT_MATCHES_CTIME; + result.enable(core::Sloppy::file_stat_matches_ctime); + } else if (token == "gcno_cwd") { + result.enable(core::Sloppy::gcno_cwd); } else if (token == "include_file_ctime") { - result |= SLOPPY_INCLUDE_FILE_CTIME; + result.enable(core::Sloppy::include_file_ctime); } else if (token == "include_file_mtime") { - result |= SLOPPY_INCLUDE_FILE_MTIME; - } else if (token == "system_headers" || token == "no_system_headers") { - result |= SLOPPY_SYSTEM_HEADERS; - } else if (token == "pch_defines") { - result |= SLOPPY_PCH_DEFINES; - } else if (token == "time_macros") { - result |= SLOPPY_TIME_MACROS; - } else if (token == "clang_index_store") { - result |= SLOPPY_CLANG_INDEX_STORE; + result.enable(core::Sloppy::include_file_mtime); + } else if (token == "ivfsoverlay") { + result.enable(core::Sloppy::ivfsoverlay); } else if (token == "locale") { - result |= SLOPPY_LOCALE; + result.enable(core::Sloppy::locale); } else if (token == "modules") { - result |= SLOPPY_MODULES; - } else if (token == "ivfsoverlay") { - result |= SLOPPY_IVFSOVERLAY; + result.enable(core::Sloppy::modules); + } else if (token == "pch_defines") { + result.enable(core::Sloppy::pch_defines); + } else if (token == "random_seed") { + result.enable(core::Sloppy::random_seed); + } else if (token == "system_headers" || token == "no_system_headers") { + result.enable(core::Sloppy::system_headers); + } else if (token == "time_macros") { + result.enable(core::Sloppy::time_macros); } // else: ignore unknown value for forward compatibility - start = value.find_first_not_of(", ", end); } + return result; } std::string -format_sloppiness(uint32_t sloppiness) +format_sloppiness(core::Sloppiness sloppiness) { std::string result; - if (sloppiness & SLOPPY_INCLUDE_FILE_MTIME) { - result += "include_file_mtime, "; - } - if (sloppiness & SLOPPY_INCLUDE_FILE_CTIME) { - result += "include_file_ctime, "; - } - if (sloppiness & SLOPPY_TIME_MACROS) { - result += "time_macros, "; - } - if (sloppiness & SLOPPY_PCH_DEFINES) { - result += "pch_defines, "; + if (sloppiness.is_enabled(core::Sloppy::clang_index_store)) { + result += "clang_index_store, "; } - if (sloppiness & SLOPPY_FILE_STAT_MATCHES) { + if (sloppiness.is_enabled(core::Sloppy::file_stat_matches)) { result += "file_stat_matches, "; } - if (sloppiness & SLOPPY_FILE_STAT_MATCHES_CTIME) { + if (sloppiness.is_enabled(core::Sloppy::file_stat_matches_ctime)) { result += "file_stat_matches_ctime, "; } - if (sloppiness & SLOPPY_SYSTEM_HEADERS) { - result += "system_headers, "; + if (sloppiness.is_enabled(core::Sloppy::gcno_cwd)) { + result += "gcno_cwd, "; } - if (sloppiness & SLOPPY_CLANG_INDEX_STORE) { - result += "clang_index_store, "; + if (sloppiness.is_enabled(core::Sloppy::include_file_ctime)) { + result += "include_file_ctime, "; } - if (sloppiness & SLOPPY_LOCALE) { + if (sloppiness.is_enabled(core::Sloppy::include_file_mtime)) { + result += "include_file_mtime, "; + } + if (sloppiness.is_enabled(core::Sloppy::ivfsoverlay)) { + result += "ivfsoverlay, "; + } + if (sloppiness.is_enabled(core::Sloppy::locale)) { result += "locale, "; } - if (sloppiness & SLOPPY_MODULES) { + if (sloppiness.is_enabled(core::Sloppy::modules)) { result += "modules, "; } - if (sloppiness & SLOPPY_IVFSOVERLAY) { - result += "ivfsoverlay, "; + if (sloppiness.is_enabled(core::Sloppy::pch_defines)) { + result += "pch_defines, "; + } + if (sloppiness.is_enabled(core::Sloppy::random_seed)) { + result += "random_seed, "; + } + if (sloppiness.is_enabled(core::Sloppy::system_headers)) { + result += "system_headers, "; + } + if (sloppiness.is_enabled(core::Sloppy::time_macros)) { + result += "time_macros, "; } if (!result.empty()) { // Strip last ", ". @@ -327,36 +366,21 @@ format_sloppiness(uint32_t sloppiness) return result; } -uint32_t -parse_umask(const std::string& value) -{ - if (value.empty()) { - return std::numeric_limits::max(); - } - - size_t end; - uint32_t result = std::stoul(value, &end, 8); - if (end != value.size()) { - throw Error("not an octal integer: \"{}\"", value); - } - return result; -} - std::string -format_umask(uint32_t umask) +format_umask(std::optional umask) { - if (umask == std::numeric_limits::max()) { - return {}; + if (umask) { + return FMT("{:03o}", *umask); } else { - return FMT("{:03o}", umask); + return {}; } } void verify_absolute_path(const std::string& value) { - if (!Util::is_absolute_path(value)) { - throw Error("not an absolute path: \"{}\"", value); + if (!util::is_absolute_path(value)) { + throw core::Error(FMT("not an absolute path: \"{}\"", value)); } } @@ -366,7 +390,7 @@ parse_line(const std::string& line, std::string* value, std::string* error_message) { - std::string stripped_line = Util::strip_whitespace(line); + std::string stripped_line = util::strip_whitespace(line); if (stripped_line.empty() || stripped_line[0] == '#') { return true; } @@ -377,8 +401,8 @@ parse_line(const std::string& line, } *key = stripped_line.substr(0, equal_pos); *value = stripped_line.substr(equal_pos + 1); - *key = Util::strip_whitespace(*key); - *value = Util::strip_whitespace(*value); + *key = util::strip_whitespace(*key); + *value = util::strip_whitespace(*value); return true; } @@ -408,11 +432,11 @@ parse_config_file(const std::string& path, std::string value; std::string error_message; if (!parse_line(line, &key, &value, &error_message)) { - throw Error(error_message); + throw core::Error(error_message); } config_line_handler(line, key, value); - } catch (const Error& e) { - throw Error("{}:{}: {}", path, line_number, e.what()); + } catch (const core::Error& e) { + throw core::Error(FMT("{}:{}: {}", path, line_number, e.what())); } } return true; @@ -420,6 +444,28 @@ parse_config_file(const std::string& path, } // namespace +#ifndef _WIN32 +static std::string +default_cache_dir(const std::string& home_dir) +{ +# ifdef __APPLE__ + return home_dir + "/Library/Caches/ccache"; +# else + return home_dir + "/.cache/ccache"; +# endif +} + +static std::string +default_config_dir(const std::string& home_dir) +{ +# ifdef __APPLE__ + return home_dir + "/Library/Preferences/ccache"; +# else + return home_dir + "/.config/ccache"; +# endif +} +#endif + std::string compiler_type_to_string(CompilerType compiler_type) { @@ -430,53 +476,163 @@ compiler_type_to_string(CompilerType compiler_type) switch (compiler_type) { case CompilerType::auto_guess: return "auto"; + case CompilerType::clang_cl: + return "clang-cl"; CASE(clang); CASE(gcc); + CASE(icl); + CASE(msvc); CASE(nvcc); CASE(other); - CASE(pump); } #undef CASE ASSERT(false); } +void +Config::read() +{ + const std::string home_dir = Util::get_home_directory(); + const std::string legacy_ccache_dir = Util::make_path(home_dir, ".ccache"); + const bool legacy_ccache_dir_exists = + Stat::stat(legacy_ccache_dir).is_directory(); +#ifdef _WIN32 + const char* const env_appdata = getenv("APPDATA"); + const char* const env_local_appdata = getenv("LOCALAPPDATA"); +#else + const char* const env_xdg_cache_home = getenv("XDG_CACHE_HOME"); + const char* const env_xdg_config_home = getenv("XDG_CONFIG_HOME"); +#endif + + const char* env_ccache_configpath = getenv("CCACHE_CONFIGPATH"); + if (env_ccache_configpath) { + set_config_path(env_ccache_configpath); + } else { + // Only used for ccache tests: + const char* const env_ccache_configpath2 = getenv("CCACHE_CONFIGPATH2"); + + std::string sysconfdir = Util::make_path(k_sysconfdir); +#ifdef _WIN32 + if (const char* program_data = getenv("ALLUSERSPROFILE")) + sysconfdir = Util::make_path(program_data, "ccache"); +#endif + + set_system_config_path(env_ccache_configpath2 + ? env_ccache_configpath2 + : Util::make_path(sysconfdir, "ccache.conf")); + MTR_BEGIN("config", "conf_read_system"); + // A missing config file in SYSCONFDIR is OK so don't check return value. + update_from_file(system_config_path()); + MTR_END("config", "conf_read_system"); + + const char* const env_ccache_dir = getenv("CCACHE_DIR"); + std::string config_dir; + if (env_ccache_dir && *env_ccache_dir) { + config_dir = env_ccache_dir; + } else if (!cache_dir().empty() && !env_ccache_dir) { + config_dir = cache_dir(); + } else if (legacy_ccache_dir_exists) { + config_dir = legacy_ccache_dir; +#ifdef _WIN32 + } else if (env_local_appdata + && Stat::stat( + Util::make_path(env_local_appdata, "ccache", "ccache.conf"))) { + config_dir = Util::make_path(env_local_appdata, "ccache"); + } else if (env_appdata + && Stat::stat( + Util::make_path(env_appdata, "ccache", "ccache.conf"))) { + config_dir = Util::make_path(env_appdata, "ccache"); + } else if (env_local_appdata) { + config_dir = Util::make_path(env_local_appdata, "ccache"); + } else { + throw core::Fatal( + "could not find configuration file and the LOCALAPPDATA environment" + " variable is not set"); + } +#else + } else if (env_xdg_config_home) { + config_dir = Util::make_path(env_xdg_config_home, "ccache"); + } else { + config_dir = default_config_dir(home_dir); + } +#endif + set_config_path(Util::make_path(config_dir, "ccache.conf")); + } + + const std::string& cache_dir_before_config_file_was_read = cache_dir(); + + MTR_BEGIN("config", "conf_read"); + update_from_file(config_path()); + MTR_END("config", "conf_read"); + + // Ignore cache_dir set in configuration file + set_cache_dir(cache_dir_before_config_file_was_read); + + MTR_BEGIN("config", "conf_update_from_environment"); + update_from_environment(); + // (cache_dir is set above if CCACHE_DIR is set.) + MTR_END("config", "conf_update_from_environment"); + + if (cache_dir().empty()) { + if (legacy_ccache_dir_exists) { + set_cache_dir(legacy_ccache_dir); +#ifdef _WIN32 + } else if (env_local_appdata) { + set_cache_dir(Util::make_path(env_local_appdata, "ccache")); + } else { + throw core::Fatal( + "could not find cache directory and the LOCALAPPDATA environment" + " variable is not set"); + } +#else + } else if (env_xdg_cache_home) { + set_cache_dir(Util::make_path(env_xdg_cache_home, "ccache")); + } else { + set_cache_dir(default_cache_dir(home_dir)); + } +#endif + } + // else: cache_dir was set explicitly via environment or via system config. + + // We have now determined config.cache_dir and populated the rest of config in + // prio order (1. environment, 2. cache-specific config, 3. system config). +} + const std::string& -Config::primary_config_path() const +Config::config_path() const { - return m_primary_config_path; + return m_config_path; } const std::string& -Config::secondary_config_path() const +Config::system_config_path() const { - return m_secondary_config_path; + return m_system_config_path; } void -Config::set_primary_config_path(std::string path) +Config::set_config_path(std::string path) { - m_primary_config_path = std::move(path); + m_config_path = std::move(path); } void -Config::set_secondary_config_path(std::string path) +Config::set_system_config_path(std::string path) { - m_secondary_config_path = std::move(path); + m_system_config_path = std::move(path); } bool Config::update_from_file(const std::string& path) { - return parse_config_file(path, - [&](const std::string& /*line*/, - const std::string& key, - const std::string& value) { - if (!key.empty()) { - set_item(key, value, nullopt, false, path); - } - }); + return parse_config_file( + path, [&](const auto& /*line*/, const auto& key, const auto& value) { + if (!key.empty()) { + this->set_item(key, value, std::nullopt, false, path); + } + }); } void @@ -485,7 +641,7 @@ Config::update_from_environment() for (char** env = environ; *env; ++env) { std::string setting = *env; const std::string prefix = "CCACHE_"; - if (!Util::starts_with(setting, prefix)) { + if (!util::starts_with(setting, prefix)) { continue; } size_t equal_pos = setting.find('='); @@ -495,7 +651,7 @@ Config::update_from_environment() std::string key = setting.substr(prefix.size(), equal_pos - prefix.size()); std::string value = setting.substr(equal_pos + 1); - bool negate = Util::starts_with(key, "NO"); + bool negate = util::starts_with(key, "NO"); if (negate) { key = key.substr(2); } @@ -509,8 +665,9 @@ Config::update_from_environment() try { set_item(config_key, value, key, negate, "environment"); - } catch (const Error& e) { - throw Error("CCACHE_{}{}: {}", negate ? "NO" : "", key, e.what()); + } catch (const core::Error& e) { + throw core::Error( + FMT("CCACHE_{}{}: {}", negate ? "NO" : "", key, e.what())); } } } @@ -520,10 +677,10 @@ Config::get_string_value(const std::string& key) const { auto it = k_config_key_table.find(key); if (it == k_config_key_table.end()) { - throw Error("unknown configuration option \"{}\"", key); + throw core::Error(FMT("unknown configuration option \"{}\"", key)); } - switch (it->second) { + switch (it->second.item) { case ConfigItem::absolute_paths_in_stderr: return format_bool(m_absolute_paths_in_stderr); @@ -602,6 +759,12 @@ Config::get_string_value(const std::string& key) const case ConfigItem::max_size: return format_cache_size(m_max_size); + case ConfigItem::msvc_dep_prefix: + return m_msvc_dep_prefix; + + case ConfigItem::namespace_: + return m_namespace; + case ConfigItem::path: return m_path; @@ -623,6 +786,15 @@ Config::get_string_value(const std::string& key) const case ConfigItem::recache: return format_bool(m_recache); + case ConfigItem::remote_only: + return format_bool(m_remote_only); + + case ConfigItem::remote_storage: + return m_remote_storage; + + case ConfigItem::reshare: + return format_bool(m_reshare); + case ConfigItem::run_second_cpp: return format_bool(m_run_second_cpp); @@ -632,6 +804,9 @@ Config::get_string_value(const std::string& key) const case ConfigItem::stats: return format_bool(m_stats); + case ConfigItem::stats_log: + return m_stats_log; + case ConfigItem::temporary_dir: return m_temporary_dir; @@ -645,42 +820,43 @@ Config::get_string_value(const std::string& key) const void Config::set_value_in_file(const std::string& path, const std::string& key, - const std::string& value) + const std::string& value) const { + UmaskScope umask_scope(m_umask); + if (k_config_key_table.find(key) == k_config_key_table.end()) { - throw Error("unknown configuration option \"{}\"", key); + throw core::Error(FMT("unknown configuration option \"{}\"", key)); } // Verify that the value is valid; set_item will throw if not. Config dummy_config; - dummy_config.set_item(key, value, nullopt, false, ""); + dummy_config.set_item(key, value, std::nullopt, false, ""); const auto resolved_path = Util::real_path(path); const auto st = Stat::stat(resolved_path); if (!st) { Util::ensure_dir_exists(Util::dir_name(resolved_path)); - try { - Util::write_file(resolved_path, ""); - } catch (const Error& e) { - throw Error("failed to write to {}: {}", resolved_path, e.what()); + const auto result = util::write_file(resolved_path, ""); + if (!result) { + throw core::Error( + FMT("failed to write to {}: {}", resolved_path, result.error())); } } AtomicFile output(resolved_path, AtomicFile::Mode::text); bool found = false; - if (!parse_config_file(path, - [&](const std::string& c_line, - const std::string& c_key, - const std::string& /*c_value*/) { - if (c_key == key) { - output.write(FMT("{} = {}\n", key, value)); - found = true; - } else { - output.write(FMT("{}\n", c_line)); - } - })) { - throw Error("failed to open {}: {}", path, strerror(errno)); + if (!parse_config_file( + path, + [&](const auto& c_line, const auto& c_key, const auto& /*c_value*/) { + if (c_key == key) { + output.write(FMT("{} = {}\n", key, value)); + found = true; + } else { + output.write(FMT("{}\n", c_line)); + } + })) { + throw core::Error(FMT("failed to open {}: {}", path, strerror(errno))); } if (!found) { @@ -696,8 +872,10 @@ Config::visit_items(const ItemVisitor& item_visitor) const std::vector keys; keys.reserve(k_config_key_table.size()); - for (const auto& item : k_config_key_table) { - keys.emplace_back(item.first); + for (const auto& [key, entry] : k_config_key_table) { + if (!entry.alias) { + keys.emplace_back(key); + } } std::sort(keys.begin(), keys.end()); for (const auto& key : keys) { @@ -710,7 +888,7 @@ Config::visit_items(const ItemVisitor& item_visitor) const void Config::set_item(const std::string& key, const std::string& value, - const optional& env_var_key, + const std::optional& env_var_key, bool negate, const std::string& origin) { @@ -720,7 +898,7 @@ Config::set_item(const std::string& key, return; } - switch (it->second) { + switch (it->second.item) { case ConfigItem::absolute_paths_in_stderr: m_absolute_paths_in_stderr = parse_bool(value, env_var_key, negate); break; @@ -729,7 +907,7 @@ Config::set_item(const std::string& key, m_base_dir = Util::expand_environment_variables(value); if (!m_base_dir.empty()) { // The empty string means "disable" verify_absolute_path(m_base_dir); - m_base_dir = Util::normalize_absolute_path(m_base_dir); + m_base_dir = Util::normalize_abstract_absolute_path(m_base_dir); } break; @@ -753,11 +931,10 @@ Config::set_item(const std::string& key, m_compression = parse_bool(value, env_var_key, negate); break; - case ConfigItem::compression_level: { - m_compression_level = - Util::parse_signed(value, INT8_MIN, INT8_MAX, "compression_level"); + case ConfigItem::compression_level: + m_compression_level = util::value_or_throw( + util::parse_signed(value, INT8_MIN, INT8_MAX, "compression_level")); break; - } case ConfigItem::cpp_extension: m_cpp_extension = value; @@ -816,7 +993,8 @@ Config::set_item(const std::string& key, break; case ConfigItem::limit_multiple: - m_limit_multiple = Util::clamp(parse_double(value), 0.0, 1.0); + m_limit_multiple = std::clamp( + util::value_or_throw(util::parse_double(value)), 0.0, 1.0); break; case ConfigItem::log_file: @@ -824,13 +1002,22 @@ Config::set_item(const std::string& key, break; case ConfigItem::max_files: - m_max_files = Util::parse_unsigned(value, nullopt, nullopt, "max_files"); + m_max_files = util::value_or_throw( + util::parse_unsigned(value, std::nullopt, std::nullopt, "max_files")); break; case ConfigItem::max_size: m_max_size = Util::parse_size(value); break; + case ConfigItem::msvc_dep_prefix: + m_msvc_dep_prefix = Util::expand_environment_variables(value); + break; + + case ConfigItem::namespace_: + m_namespace = Util::expand_environment_variables(value); + break; + case ConfigItem::path: m_path = Util::expand_environment_variables(value); break; @@ -859,6 +1046,18 @@ Config::set_item(const std::string& key, m_recache = parse_bool(value, env_var_key, negate); break; + case ConfigItem::remote_only: + m_remote_only = parse_bool(value, env_var_key, negate); + break; + + case ConfigItem::remote_storage: + m_remote_storage = Util::expand_environment_variables(value); + break; + + case ConfigItem::reshare: + m_reshare = parse_bool(value, env_var_key, negate); + break; + case ConfigItem::run_second_cpp: m_run_second_cpp = parse_bool(value, env_var_key, negate); break; @@ -871,40 +1070,60 @@ Config::set_item(const std::string& key, m_stats = parse_bool(value, env_var_key, negate); break; + case ConfigItem::stats_log: + m_stats_log = Util::expand_environment_variables(value); + break; + case ConfigItem::temporary_dir: m_temporary_dir = Util::expand_environment_variables(value); m_temporary_dir_configured_explicitly = true; break; case ConfigItem::umask: - m_umask = parse_umask(value); + if (!value.empty()) { + const auto umask = util::parse_umask(value); + if (!umask) { + throw core::Error(umask.error()); + } + m_umask = *umask; + } break; } - m_origins.emplace(key, origin); + const std::string canonical_key = it->second.alias ? *it->second.alias : key; + const auto& [element, inserted] = m_origins.emplace(canonical_key, origin); + if (!inserted) { + element->second = origin; + } } void Config::check_key_tables_consistency() { - for (const auto& item : k_env_variable_table) { - if (k_config_key_table.find(item.second) == k_config_key_table.end()) { - throw Error( - "env var {} mapped to {} which is missing from k_config_key_table", - item.first, - item.second); + for (const auto& [key, value] : k_env_variable_table) { + if (k_config_key_table.find(value) == k_config_key_table.end()) { + throw core::Error( + FMT("env var {} mapped to {} which is missing from k_config_key_table", + key, + value)); } } } std::string -Config::default_temporary_dir(const std::string& cache_dir) +Config::default_temporary_dir() const { -#ifdef HAVE_GETEUID - std::string user_tmp_dir = FMT("/run/user/{}", geteuid()); - if (Stat::stat(user_tmp_dir).is_directory()) { - return user_tmp_dir + "/ccache-tmp"; - } + static const std::string run_user_tmp_dir = [] { +#ifndef _WIN32 + const char* const xdg_runtime_dir = getenv("XDG_RUNTIME_DIR"); + if (xdg_runtime_dir && Stat::stat(xdg_runtime_dir).is_directory()) { + auto dir = FMT("{}/ccache-tmp", xdg_runtime_dir); + if (Util::create_dir(dir) && access(dir.c_str(), W_OK) == 0) { + return dir; + } + } #endif - return cache_dir + "/tmp"; + return std::string(); + }(); + return !run_user_tmp_dir.empty() ? run_user_tmp_dir : m_cache_dir + "/tmp"; } diff --git a/src/Config.hpp b/src/Config.hpp index eb574dc..b75ff43 100644 --- a/src/Config.hpp +++ b/src/Config.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,28 +18,38 @@ #pragma once -#include "system.hpp" - #include "NonCopyable.hpp" -#include "Util.hpp" -#include "third_party/nonstd/optional.hpp" +#include + +#include +#include #include #include +#include #include #include -enum class CompilerType { auto_guess, clang, gcc, nvcc, other, pump }; +enum class CompilerType { + auto_guess, + clang, + clang_cl, + gcc, + icl, + msvc, + nvcc, + other +}; std::string compiler_type_to_string(CompilerType compiler_type); -class Config +class Config : NonCopyable { public: Config() = default; - Config(Config&) = default; - Config& operator=(const Config&) = default; + + void read(); bool absolute_paths_in_stderr() const; const std::string& base_dir() const; @@ -67,6 +77,7 @@ public: const std::string& log_file() const; uint64_t max_files() const; uint64_t max_size() const; + const std::string& msvc_dep_prefix() const; const std::string& path() const; bool pch_external_checksum() const; const std::string& prefix_command() const; @@ -74,33 +85,50 @@ public: bool read_only() const; bool read_only_direct() const; bool recache() const; + bool remote_only() const; + const std::string& remote_storage() const; + bool reshare() const; bool run_second_cpp() const; - uint32_t sloppiness() const; + core::Sloppiness sloppiness() const; bool stats() const; + const std::string& stats_log() const; + const std::string& namespace_() const; const std::string& temporary_dir() const; - uint32_t umask() const; + std::optional umask() const; + + // Return true for Clang and clang-cl. + bool is_compiler_group_clang() const; + + // Return true for MSVC (cl.exe), clang-cl, and icl. + bool is_compiler_group_msvc() const; + + std::string default_temporary_dir() const; void set_base_dir(const std::string& value); void set_cache_dir(const std::string& value); - void set_cpp_extension(const std::string& value); void set_compiler(const std::string& value); void set_compiler_type(CompilerType value); - void set_depend_mode(bool value); + void set_cpp_extension(const std::string& value); void set_debug(bool value); + void set_depend_mode(bool value); void set_direct_mode(bool value); + void set_file_clone(bool value); + void set_hard_link(bool value); void set_ignore_options(const std::string& value); void set_inode_cache(bool value); void set_max_files(uint64_t value); void set_max_size(uint64_t value); + void set_msvc_dep_prefix(const std::string& value); void set_run_second_cpp(bool value); + void set_temporary_dir(const std::string& value); // Where to write configuration changes. - const std::string& primary_config_path() const; - // Secondary, read-only configuration file (if any). - const std::string& secondary_config_path() const; + const std::string& config_path() const; + // System (read-only) configuration file (if any). + const std::string& system_config_path() const; - void set_primary_config_path(std::string path); - void set_secondary_config_path(std::string path); + void set_config_path(std::string path); + void set_system_config_path(std::string path); using ItemVisitor = std::function::max(); // Don't set umask + std::optional m_umask; bool m_temporary_dir_configured_explicitly = false; @@ -178,11 +212,9 @@ private: void set_item(const std::string& key, const std::string& value, - const nonstd::optional& env_var_key, + const std::optional& env_var_key, bool negate, const std::string& origin); - - static std::string default_temporary_dir(const std::string& cache_dir); }; inline bool @@ -221,6 +253,21 @@ Config::compiler_type() const return m_compiler_type; } +inline bool +Config::is_compiler_group_clang() const +{ + return m_compiler_type == CompilerType::clang + || m_compiler_type == CompilerType::clang_cl; +} + +inline bool +Config::is_compiler_group_msvc() const +{ + return m_compiler_type == CompilerType::msvc + || m_compiler_type == CompilerType::clang_cl + || m_compiler_type == CompilerType::icl; +} + inline bool Config::compression() const { @@ -341,6 +388,12 @@ Config::max_size() const return m_max_size; } +inline const std::string& +Config::msvc_dep_prefix() const +{ + return m_msvc_dep_prefix; +} + inline const std::string& Config::path() const { @@ -383,13 +436,31 @@ Config::recache() const return m_recache; } +inline bool +Config::reshare() const +{ + return m_reshare; +} + inline bool Config::run_second_cpp() const { return m_run_second_cpp; } -inline uint32_t +inline bool +Config::remote_only() const +{ + return m_remote_only; +} + +inline const std::string& +Config::remote_storage() const +{ + return m_remote_storage; +} + +inline core::Sloppiness Config::sloppiness() const { return m_sloppiness; @@ -401,13 +472,25 @@ Config::stats() const return m_stats; } +inline const std::string& +Config::stats_log() const +{ + return m_stats_log; +} + +inline const std::string& +Config::namespace_() const +{ + return m_namespace; +} + inline const std::string& Config::temporary_dir() const { return m_temporary_dir; } -inline uint32_t +inline std::optional Config::umask() const { return m_umask; @@ -424,7 +507,7 @@ Config::set_cache_dir(const std::string& value) { m_cache_dir = value; if (!m_temporary_dir_configured_explicitly) { - m_temporary_dir = default_temporary_dir(m_cache_dir); + m_temporary_dir = default_temporary_dir(); } } @@ -464,6 +547,18 @@ Config::set_direct_mode(bool value) m_direct_mode = value; } +inline void +Config::set_file_clone(const bool value) +{ + m_file_clone = value; +} + +inline void +Config::set_hard_link(const bool value) +{ + m_hard_link = value; +} + inline void Config::set_ignore_options(const std::string& value) { @@ -488,8 +583,20 @@ Config::set_max_size(uint64_t value) m_max_size = value; } +inline void +Config::set_msvc_dep_prefix(const std::string& value) +{ + m_msvc_dep_prefix = value; +} + inline void Config::set_run_second_cpp(bool value) { m_run_second_cpp = value; } + +inline void +Config::set_temporary_dir(const std::string& value) +{ + m_temporary_dir = value; +} diff --git a/src/Context.cpp b/src/Context.cpp index a7ce450..c46d368 100644 --- a/src/Context.cpp +++ b/src/Context.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,26 +18,55 @@ #include "Context.hpp" -#include "Counters.hpp" #include "Logging.hpp" #include "SignalHandler.hpp" #include "Util.hpp" #include "hashutil.hpp" +#include +#include +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + #include #include #include -using nonstd::string_view; - Context::Context() : actual_cwd(Util::get_actual_cwd()), - apparent_cwd(Util::get_apparent_cwd(actual_cwd)) + apparent_cwd(Util::get_apparent_cwd(actual_cwd)), + storage(config) #ifdef INODE_CACHE_SUPPORTED , inode_cache(config) #endif { + time_of_invocation = util::TimePoint::now(); +} + +void +Context::initialize() +{ + config.read(); + Logging::init(config); + + ignore_header_paths = + util::split_path_list(config.ignore_headers_in_manifest()); + set_ignore_options(Util::split_into_strings(config.ignore_options(), " ")); + + // Set default umask for all files created by ccache from now on (if + // configured to). This is intentionally done after calling Logging::init so + // that the log file won't be affected by the umask but before creating the + // initial configuration file. The intention is that all files and directories + // in the cache directory should be affected by the configured umask and that + // no other files and directories should. + if (config.umask()) { + original_umask = Util::set_umask(*config.umask()); + } } Context::~Context() diff --git a/src/Context.hpp b/src/Context.hpp index c021124..883c02f 100644 --- a/src/Context.hpp +++ b/src/Context.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,26 +18,26 @@ #pragma once -#include "system.hpp" - #include "Args.hpp" #include "ArgsInfo.hpp" #include "Config.hpp" -#include "Counters.hpp" #include "Digest.hpp" #include "File.hpp" #include "MiniTrace.hpp" #include "NonCopyable.hpp" -#include "Sloppiness.hpp" #ifdef INODE_CACHE_SUPPORTED # include "InodeCache.hpp" #endif -#include "third_party/nonstd/optional.hpp" -#include "third_party/nonstd/string_view.hpp" +#include +#include +#include +#include +#include #include +#include #include #include @@ -49,6 +49,10 @@ public: Context(); ~Context(); + // Read configuration, initialize logging, etc. Typically not called from unit + // tests. + void initialize(); + ArgsInfo args_info; Config config; @@ -61,58 +65,39 @@ public: // The original argument list. Args orig_args; - // Name (represented as a hash) of the file containing the manifest for the - // cached result. - const nonstd::optional& manifest_name() const; - - // Full path to the file containing the manifest (cachedir/a/b/cdef[...]M), if - // any. - const nonstd::optional& manifest_path() const; - - // Name (represented as a hash) of the file containing the cached result. - const nonstd::optional& result_name() const; - - // Full path to the file containing the result (cachedir/a/b/cdef[...]R). - const nonstd::optional& result_path() const; + // Time of ccache invocation. + util::TimePoint time_of_invocation; // Time of compilation. Used to see if include files have changed after // compilation. - time_t time_of_compilation = 0; + util::TimePoint time_of_compilation; // Files included by the preprocessor and their hashes. std::unordered_map included_files; - // Uses absolute path for some include files. - bool has_absolute_include_headers = false; - // Have we tried and failed to get colored diagnostics? bool diagnostics_color_failed = false; // The name of the temporary preprocessed file. std::string i_tmpfile; - // The name of the cpp stderr file. - std::string cpp_stderr; - - // The .gch/.pch/.pth file used for compilation. - std::string included_pch_file; + // The preprocessor's stderr output. + util::Bytes cpp_stderr_data; // Headers (or directories with headers) to ignore in manifest mode. std::vector ignore_header_paths; + // Storage (fronting local and remote storage backends). + storage::Storage storage; + + // Direct mode manifest. + core::Manifest manifest; + #ifdef INODE_CACHE_SUPPORTED // InodeCache that caches source file hashes when enabled. mutable InodeCache inode_cache; #endif - // Statistics updates which get written into the statistics file belonging to - // the result. - Counters counter_updates; - - // Statistics updates which get written into the statistics file belonging to - // the manifest. - Counters manifest_counter_updates; - // PID of currently executing compiler that we have started, if any. 0 means // no ongoing compilation. pid_t compiler_pid = 0; @@ -126,28 +111,21 @@ public: // Original umask before applying the `umask`/`CCACHE_UMASK` configuration, or // `nullopt` if there is no such configuration. - nonstd::optional original_umask; + std::optional original_umask; #ifdef MTR_ENABLED // Internal tracing. std::unique_ptr mini_trace; #endif - void set_manifest_name(const Digest& name); - void set_manifest_path(const std::string& path); - void set_result_name(const Digest& name); - void set_result_path(const std::string& path); + // Whether we have added "/showIncludes" ourselves since it's missing and + // depend mode is enabled. + bool auto_depend_mode = false; // Register a temporary file to remove at program exit. void register_pending_tmp_file(const std::string& path); private: - nonstd::optional m_manifest_name; - nonstd::optional m_manifest_path; - - nonstd::optional m_result_name; - nonstd::optional m_result_path; - // Options to ignore for the hash. std::vector m_ignore_options; @@ -163,56 +141,8 @@ private: void unlink_pending_tmp_files_signal_safe(); // called from signal handler }; -inline const nonstd::optional& -Context::manifest_name() const -{ - return m_manifest_name; -} - -inline const nonstd::optional& -Context::manifest_path() const -{ - return m_manifest_path; -} - -inline const nonstd::optional& -Context::result_name() const -{ - return m_result_name; -} - -inline const nonstd::optional& -Context::result_path() const -{ - return m_result_path; -} - inline const std::vector& Context::ignore_options() const { return m_ignore_options; } - -inline void -Context::set_manifest_name(const Digest& name) -{ - m_manifest_name = name; -} - -inline void -Context::set_manifest_path(const std::string& path) -{ - m_manifest_path = path; -} - -inline void -Context::set_result_name(const Digest& name) -{ - m_result_name = name; -} - -inline void -Context::set_result_path(const std::string& path) -{ - m_result_path = path; -} diff --git a/src/Counters.cpp b/src/Counters.cpp deleted file mode 100644 index 1263d9d..0000000 --- a/src/Counters.cpp +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (C) 2010-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Counters.hpp" - -#include "Statistic.hpp" -#include "assertions.hpp" - -#include - -Counters::Counters() : m_counters(static_cast(Statistic::END)) -{ -} - -uint64_t -Counters::get(Statistic statistic) const -{ - const auto index = static_cast(statistic); - ASSERT(index < static_cast(Statistic::END)); - return index < m_counters.size() ? m_counters[index] : 0; -} - -void -Counters::set(Statistic statistic, uint64_t value) -{ - const auto index = static_cast(statistic); - ASSERT(index < static_cast(Statistic::END)); - m_counters[index] = value; -} - -uint64_t -Counters::get_raw(size_t index) const -{ - ASSERT(index < size()); - return m_counters[index]; -} - -void -Counters::set_raw(size_t index, uint64_t value) -{ - if (index >= m_counters.size()) { - m_counters.resize(index + 1); - } - m_counters[index] = value; -} - -void -Counters::increment(Statistic statistic, int64_t value) -{ - const auto i = static_cast(statistic); - if (i >= m_counters.size()) { - m_counters.resize(i + 1); - } - auto& counter = m_counters[i]; - counter = - std::max(static_cast(0), static_cast(counter + value)); -} - -void -Counters::increment(const Counters& other) -{ - m_counters.resize(std::max(size(), other.size())); - for (size_t i = 0; i < other.size(); ++i) { - auto& counter = m_counters[i]; - counter = std::max(static_cast(0), - static_cast(counter + other.m_counters[i])); - } -} - -size_t -Counters::size() const -{ - return m_counters.size(); -} - -bool -Counters::all_zero() const -{ - return !std::any_of( - m_counters.begin(), m_counters.end(), [](unsigned v) { return v != 0; }); -} diff --git a/src/Counters.hpp b/src/Counters.hpp deleted file mode 100644 index d19227b..0000000 --- a/src/Counters.hpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (C) 2010-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include - -enum class Statistic; - -// A simple wrapper around a vector of integers used for the statistics -// counters. -class Counters -{ -public: - Counters(); - - uint64_t get(Statistic statistic) const; - void set(Statistic statistic, uint64_t value); - - uint64_t get_raw(size_t index) const; - void set_raw(size_t index, uint64_t value); - - void increment(Statistic statistic, int64_t value = 1); - void increment(const Counters& other); - - size_t size() const; - - // Return true if all counters are zero, false otherwise. - bool all_zero() const; - -private: - std::vector m_counters; -}; diff --git a/src/Decompressor.cpp b/src/Decompressor.cpp deleted file mode 100644 index b53cd95..0000000 --- a/src/Decompressor.cpp +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Decompressor.hpp" - -#include "NullDecompressor.hpp" -#include "StdMakeUnique.hpp" -#include "ZstdDecompressor.hpp" -#include "assertions.hpp" - -std::unique_ptr -Decompressor::create_from_type(Compression::Type type, FILE* stream) -{ - switch (type) { - case Compression::Type::none: - return std::make_unique(stream); - - case Compression::Type::zstd: - return std::make_unique(stream); - } - - ASSERT(false); -} diff --git a/src/Decompressor.hpp b/src/Decompressor.hpp deleted file mode 100644 index 5955889..0000000 --- a/src/Decompressor.hpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Compression.hpp" - -#include - -class Decompressor -{ -public: - virtual ~Decompressor() = default; - - // Create a decompressor for the specified type. - // - // Parameters: - // - type: The type. - // - stream: The stream to read from. - static std::unique_ptr create_from_type(Compression::Type type, - FILE* stream); - - // Read data into a buffer from the compressed stream. - // - // Parameters: - // - data: Buffer to write decompressed data to. - // - count: How many bytes to write. - // - // Throws Error on failure. - virtual void read(void* data, size_t count) = 0; - - // Finalize decompression. - // - // This method checks that the end state of the compressed stream is correct - // and throws Error if not. - virtual void finalize() = 0; -}; diff --git a/src/Depfile.cpp b/src/Depfile.cpp index 13106ce..f1ef39a 100644 --- a/src/Depfile.cpp +++ b/src/Depfile.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020-2021 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -23,6 +23,12 @@ #include "Logging.hpp" #include "assertions.hpp" +#include +#include +#include + +#include + static inline bool is_blank(const std::string& s) { @@ -32,7 +38,7 @@ is_blank(const std::string& s) namespace Depfile { std::string -escape_filename(nonstd::string_view filename) +escape_filename(std::string_view filename) { std::string result; result.reserve(filename.size()); @@ -54,32 +60,39 @@ escape_filename(nonstd::string_view filename) return result; } -nonstd::optional -rewrite_paths(const Context& ctx, const std::string& file_content) +std::optional +rewrite_source_paths(const Context& ctx, std::string_view file_content) { ASSERT(!ctx.config.base_dir().empty()); - ASSERT(ctx.has_absolute_include_headers); // Fast path for the common case: if (file_content.find(ctx.config.base_dir()) == std::string::npos) { - return nonstd::nullopt; + return std::nullopt; } std::string adjusted_file_content; adjusted_file_content.reserve(file_content.size()); bool content_rewritten = false; - for (const auto& line : Util::split_into_views(file_content, "\n")) { + bool seen_target_token = false; + + using util::Tokenizer; + for (const auto line : Tokenizer(file_content, + "\n", + Tokenizer::Mode::include_empty, + Tokenizer::IncludeDelimiter::yes)) { const auto tokens = Util::split_into_views(line, " \t"); for (size_t i = 0; i < tokens.size(); ++i) { - DEBUG_ASSERT(line.length() > 0); // line.empty() -> no tokens + DEBUG_ASSERT(!line.empty()); // line.empty() -> no tokens + DEBUG_ASSERT(!tokens[i].empty()); + if (i > 0 || line[0] == ' ' || line[0] == '\t') { adjusted_file_content.push_back(' '); } const auto& token = tokens[i]; bool token_rewritten = false; - if (Util::is_absolute_path(token)) { + if (seen_target_token && util::is_absolute_path(token)) { const auto new_path = Util::make_relative_path(ctx, token); if (new_path != token) { adjusted_file_content.append(new_path); @@ -91,14 +104,17 @@ rewrite_paths(const Context& ctx, const std::string& file_content) } else { adjusted_file_content.append(token.begin(), token.end()); } + + if (tokens[i].back() == ':') { + seen_target_token = true; + } } - adjusted_file_content.push_back('\n'); } if (content_rewritten) { return adjusted_file_content; } else { - return nonstd::nullopt; + return std::nullopt; } } @@ -110,33 +126,52 @@ make_paths_relative_in_output_dep(const Context& ctx) LOG_RAW("Base dir not set, skip using relative paths"); return; // nothing to do } - if (!ctx.has_absolute_include_headers) { - LOG_RAW( - "No absolute path for included files found, skip using relative paths"); - return; // nothing to do - } const std::string& output_dep = ctx.args_info.output_dep; - std::string file_content; - try { - file_content = Util::read_file(output_dep); - } catch (const Error& e) { - LOG("Cannot open dependency file {}: {}", output_dep, e.what()); + const auto file_content = util::read_file(output_dep); + if (!file_content) { + LOG("Failed to read dependency file {}: {}", + output_dep, + file_content.error()); return; } - const auto new_content = rewrite_paths(ctx, file_content); + const auto new_content = rewrite_source_paths(ctx, *file_content); if (new_content) { - Util::write_file(output_dep, *new_content); + util::write_file(output_dep, *new_content); } else { LOG("No paths in dependency file {} made relative", output_dep); } } std::vector -tokenize(nonstd::string_view file_content) +tokenize(std::string_view file_content) { // A dependency file uses Makefile syntax. This is not perfect parser but // should be enough for parsing a regular dependency file. + // + // Note that this is pretty complex because of Windows paths that can be + // identical to a target-colon-prerequisite without spaces (e.g. cat:/meow vs. + // c:/meow). + // + // Here are tests on Windows on how GNU Make 4.3 handles different scenarios: + // + // cat:/meow -> sees "cat" and "/meow" + // cat:\meow -> sees "cat" and "\meow" + // cat:\ meow -> sees "cat" and " meow" + // cat:c:/meow -> sees "cat" and "c:/meow" + // cat:c:\meow -> sees "cat" and "c:\meow" + // cat:c: -> target pattern contains no '%'. Stop. + // cat:c:\ -> target pattern contains no '%'. Stop. + // cat:c:/ -> sees "cat" and "c:/" + // cat:c:meow -> target pattern contains no '%'. Stop. + // c:c:/meow -> sees "c" and "c:/meow" + // c:c:\meow -> sees "c" and "c:\meow" + // c:z:\meow -> sees "c" and "z:\meow" + // c:cd:\meow -> target pattern contains no '%'. Stop. + // + // Thus, if there is a colon and the previous token is one character long and + // the following character is a slash (forward or backward), then it is + // interpreted as a Windows path. std::vector result; const size_t length = file_content.size(); @@ -144,19 +179,45 @@ tokenize(nonstd::string_view file_content) size_t p = 0; while (p < length) { - // Each token is separated by whitespace. - if (isspace(file_content[p])) { + char c = file_content[p]; + + if (c == ':' && p + 1 < length && !is_blank(token) && token.length() == 1) { + const char next = file_content[p + 1]; + if (next == '/' || next == '\\') { + // It's a Windows path, so the colon is not a separator and instead + // added to the token. + token.push_back(c); + ++p; + continue; + } + } + + // Each token is separated by whitespace or a colon. + if (isspace(c) || c == ':') { + // Chomp all spaces before next character. while (p < length && isspace(file_content[p])) { ++p; } if (!is_blank(token)) { + // If there were spaces between a token and the colon, add the colon the + // token to make sure it is seen as a target and not as a dependency. + if (p < length) { + const char next = file_content[p]; + if (next == ':') { + token.push_back(next); + ++p; + // Chomp all spaces before next character. + while (p < length && isspace(file_content[p])) { + ++p; + } + } + } result.push_back(token); } token.clear(); continue; } - char c = file_content[p]; switch (c) { case '\\': if (p + 1 < length) { @@ -173,7 +234,7 @@ tokenize(nonstd::string_view file_content) ++p; break; // Backslash followed by newline is interpreted like a space, so simply - // the backslash. + // discard the backslash. case '\n': ++p; continue; diff --git a/src/Depfile.hpp b/src/Depfile.hpp index 7250a4c..af73ff8 100644 --- a/src/Depfile.hpp +++ b/src/Depfile.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -23,18 +23,22 @@ class Hash; #include "Digest.hpp" -#include "third_party/nonstd/optional.hpp" -#include "third_party/nonstd/string_view.hpp" - +#include #include +#include #include namespace Depfile { -std::string escape_filename(nonstd::string_view filename); -nonstd::optional rewrite_paths(const Context& ctx, - const std::string& file_content); +std::string escape_filename(std::string_view filename); + +std::optional rewrite_source_paths(const Context& ctx, + std::string_view file_content); + void make_paths_relative_in_output_dep(const Context& ctx); -std::vector tokenize(nonstd::string_view file_content); + +// Tokenize `file_content` into a list of files, where the first token is the +// target and ends with a colon. +std::vector tokenize(std::string_view file_content); } // namespace Depfile diff --git a/src/Digest.hpp b/src/Digest.hpp index 6a7b53e..189b537 100644 --- a/src/Digest.hpp +++ b/src/Digest.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,12 +18,11 @@ #pragma once -#include "system.hpp" - #include "Util.hpp" #include "third_party/fmt/core.h" +#include #include // Digest represents the binary form of the final digest (AKA hash or checksum) diff --git a/src/Fd.cpp b/src/Fd.cpp new file mode 100644 index 0000000..0828258 --- /dev/null +++ b/src/Fd.cpp @@ -0,0 +1,31 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "Fd.hpp" + +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + +bool +Fd::close() +{ + return m_fd != -1 && ::close(release()) == 0; +} diff --git a/src/Fd.hpp b/src/Fd.hpp index 6316e45..3a47278 100644 --- a/src/Fd.hpp +++ b/src/Fd.hpp @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include "NonCopyable.hpp" #include "assertions.hpp" @@ -87,12 +85,6 @@ Fd::operator=(Fd&& other_fd) noexcept return *this; } -inline bool -Fd::close() -{ - return m_fd != -1 && ::close(release()) == 0; -} - inline int Fd::release() { diff --git a/src/File.hpp b/src/File.hpp index f8f6c0b..eea2c09 100644 --- a/src/File.hpp +++ b/src/File.hpp @@ -18,16 +18,16 @@ #pragma once -#include "system.hpp" - #include "NonCopyable.hpp" +#include #include class File : public NonCopyable { public: File() = default; + File(FILE* file); File(const std::string& path, const char* mode); File(File&& other) noexcept; ~File(); @@ -43,16 +43,26 @@ public: private: FILE* m_file = nullptr; + bool m_owned = false; }; +inline File::File(FILE* const file) +{ + m_file = file; + m_owned = false; +} + inline File::File(const std::string& path, const char* mode) { open(path, mode); } -inline File::File(File&& other) noexcept : m_file(other.m_file) +inline File::File(File&& other) noexcept + : m_file(other.m_file), + m_owned(other.m_owned) { other.m_file = nullptr; + other.m_owned = false; } inline File::~File() @@ -64,7 +74,9 @@ inline File& File::operator=(File&& other) noexcept { m_file = other.m_file; + m_owned = other.m_owned; other.m_file = nullptr; + other.m_owned = false; return *this; } @@ -73,15 +85,17 @@ File::open(const std::string& path, const char* mode) { close(); m_file = fopen(path.c_str(), mode); + m_owned = true; } inline void File::close() { - if (m_file) { + if (m_file && m_owned) { fclose(m_file); m_file = nullptr; } + m_owned = false; } inline File::operator bool() const diff --git a/src/Finalizer.hpp b/src/Finalizer.hpp index 74f6b78..c5d2033 100644 --- a/src/Finalizer.hpp +++ b/src/Finalizer.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include class Finalizer diff --git a/src/FormatNonstdStringView.hpp b/src/FormatNonstdStringView.hpp deleted file mode 100644 index 0770a90..0000000 --- a/src/FormatNonstdStringView.hpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "third_party/fmt/core.h" -#include "third_party/nonstd/string_view.hpp" - -// Specialization of fmt::formatter for nonstd::string_view. -namespace fmt { - -template<> struct formatter -{ - template - constexpr auto - parse(ParseContext& ctx) const -> decltype(ctx.begin()) - { - return ctx.begin(); - } - - template - auto - format(const nonstd::string_view& sv, FormatContext& ctx) - -> decltype(ctx.out()) - { - return format_to(ctx.out(), "{}", fmt::string_view(sv.data(), sv.size())); - } -}; - -} // namespace fmt diff --git a/src/Hash.cpp b/src/Hash.cpp index 61cc5a3..442369e 100644 --- a/src/Hash.cpp +++ b/src/Hash.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020-2021 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -22,9 +22,18 @@ #include "Logging.hpp" #include "fmtmacros.hpp" -using nonstd::string_view; +#include +#include -const string_view HASH_DELIMITER("\000cCaChE\000", 8); +#include +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + +const std::string_view HASH_DELIMITER("\000cCaChE\000", 8); Hash::Hash() { @@ -32,7 +41,7 @@ Hash::Hash() } void -Hash::enable_debug(string_view section_name, +Hash::enable_debug(std::string_view section_name, FILE* debug_binary, FILE* debug_text) { @@ -55,11 +64,11 @@ Hash::digest() const } Hash& -Hash::hash_delimiter(string_view type) +Hash::hash_delimiter(std::string_view type) { hash_buffer(HASH_DELIMITER); hash_buffer(type); - hash_buffer(string_view("", 1)); // NUL + hash_buffer(std::string_view("\x00", 1)); add_debug_text("### "); add_debug_text(type); add_debug_text("\n"); @@ -69,7 +78,7 @@ Hash::hash_delimiter(string_view type) Hash& Hash::hash(const void* data, size_t size, HashType hash_type) { - string_view buffer(static_cast(data), size); + std::string_view buffer(static_cast(data), size); hash_buffer(buffer); switch (hash_type) { @@ -88,7 +97,7 @@ Hash::hash(const void* data, size_t size, HashType hash_type) } Hash& -Hash::hash(string_view data) +Hash::hash(std::string_view data) { hash(data.data(), data.length()); return *this; @@ -97,33 +106,32 @@ Hash::hash(string_view data) Hash& Hash::hash(int64_t x) { - hash_buffer(string_view(reinterpret_cast(&x), sizeof(x))); + hash_buffer(std::string_view(reinterpret_cast(&x), sizeof(x))); add_debug_text(FMT("{}\n", x)); return *this; } -bool +nonstd::expected Hash::hash_fd(int fd) { - return Util::read_fd( + return util::read_fd( fd, [this](const void* data, size_t size) { hash(data, size); }); } -bool +nonstd::expected Hash::hash_file(const std::string& path) { Fd fd(open(path.c_str(), O_RDONLY | O_BINARY)); if (!fd) { LOG("Failed to open {}: {}", path, strerror(errno)); - return false; + return nonstd::make_unexpected(strerror(errno)); } - bool ret = hash_fd(*fd); - return ret; + return hash_fd(*fd); } void -Hash::hash_buffer(string_view buffer) +Hash::hash_buffer(std::string_view buffer) { blake3_hasher_update(&m_hasher, buffer.data(), buffer.size()); if (!buffer.empty() && m_debug_binary) { @@ -132,7 +140,7 @@ Hash::hash_buffer(string_view buffer) } void -Hash::add_debug_text(string_view text) +Hash::add_debug_text(std::string_view text) { if (!text.empty() && m_debug_text) { (void)fwrite(text.data(), 1, text.length(), m_debug_text); diff --git a/src/Hash.hpp b/src/Hash.hpp index d2686af..3e8d557 100644 --- a/src/Hash.hpp +++ b/src/Hash.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,12 +18,14 @@ #pragma once -#include "system.hpp" - #include "Digest.hpp" #include "third_party/blake3/blake3.h" -#include "third_party/nonstd/string_view.hpp" +#include + +#include +#include +#include // This class represents a hash state. class Hash @@ -37,7 +39,7 @@ public: Hash& operator=(const Hash& other) = default; // Enable debug logging of the hashed input to a binary and a text file. - void enable_debug(nonstd::string_view section_name, + void enable_debug(std::string_view section_name, FILE* debug_binary, FILE* debug_text); @@ -52,7 +54,7 @@ public: // conditional hashing of information in a safe way (e.g., if we want to // hash information X if CCACHE_A is set and information Y if CCACHE_B is // set, there should never be a hash collision risk). - Hash& hash_delimiter(nonstd::string_view type); + Hash& hash_delimiter(std::string_view type); // Add bytes to the hash. // @@ -71,7 +73,7 @@ public: // // If hash debugging is enabled, the string is written to the text input file // followed by a newline. - Hash& hash(nonstd::string_view data); + Hash& hash(std::string_view data); // Add an integer to the hash. // @@ -79,27 +81,25 @@ public: // text input file followed by a newline. Hash& hash(int64_t x); - // Add contents read from an open file descriptor to the hash. + // Add file contents to the hash. // // If hash debugging is enabled, the data is written verbatim to the text // input file. - // - // Returns true on success, otherwise false. - bool hash_fd(int fd); + nonstd::expected hash_file(const std::string& path); - // Add file contents to the hash. + // Add contents read from an open file descriptor to the hash. // // If hash debugging is enabled, the data is written verbatim to the text // input file. - // - // Returns true on success, otherwise false. - bool hash_file(const std::string& path); + nonstd::expected hash_fd(int fd); + + // Add `text` to the text debug file. + void add_debug_text(std::string_view text); private: blake3_hasher m_hasher; FILE* m_debug_binary = nullptr; FILE* m_debug_text = nullptr; - void hash_buffer(nonstd::string_view buffer); - void add_debug_text(nonstd::string_view text); + void hash_buffer(std::string_view buffer); }; diff --git a/src/InodeCache.cpp b/src/InodeCache.cpp index 5e473ec..9342279 100644 --- a/src/InodeCache.cpp +++ b/src/InodeCache.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020-2021 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -19,6 +19,7 @@ #include "InodeCache.hpp" #include "Config.hpp" +#include "Digest.hpp" #include "Fd.hpp" #include "Finalizer.hpp" #include "Hash.hpp" @@ -28,9 +29,22 @@ #include "Util.hpp" #include "fmtmacros.hpp" -#include +#include + +#include #include #include +#include + +#ifdef HAVE_LINUX_FS_H +# include +# include +#elif defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) +# include +# include +#endif + +#include #include // The inode cache resides on a file that is mapped into shared memory by @@ -62,22 +76,70 @@ const uint32_t k_num_entries = 4; static_assert(Digest::size() == 20, "Increment version number if size of digest is changed."); -static_assert(IS_TRIVIALLY_COPYABLE(Digest), +static_assert(std::is_trivially_copyable::value, "Digest is expected to be trivially copyable."); static_assert( - static_cast(InodeCache::ContentType::binary) == 0, - "Numeric value is part of key, increment version number if changed."); -static_assert( - static_cast(InodeCache::ContentType::code) == 1, + static_cast(InodeCache::ContentType::raw) == 0, "Numeric value is part of key, increment version number if changed."); static_assert( - static_cast(InodeCache::ContentType::code_with_sloppy_time_macros) == 2, - "Numeric value is part of key, increment version number if changed."); -static_assert( - static_cast(InodeCache::ContentType::precompiled_header) == 3, + static_cast(InodeCache::ContentType::checked_for_temporal_macros) == 1, "Numeric value is part of key, increment version number if changed."); +const void* MMAP_FAILED = reinterpret_cast(-1); // NOLINT: Must cast here + +bool +fd_is_on_known_to_work_file_system(int fd) +{ + bool known_to_work = false; + struct statfs buf; + if (fstatfs(fd, &buf) != 0) { + LOG("fstatfs failed: {}", strerror(errno)); + } else { +#ifdef HAVE_LINUX_FS_H + // statfs's f_type field is a signed 32-bit integer on some platforms. Large + // values therefore cause narrowing warnings, so cast the value to a large + // unsigned type. + const auto f_type = static_cast(buf.f_type); + switch (f_type) { + // Is a filesystem you know works with the inode cache missing in this + // list? Please submit an issue or pull request to the ccache project. + case 0x9123683e: // BTRFS_SUPER_MAGIC + case 0xef53: // EXT2_SUPER_MAGIC + case 0x01021994: // TMPFS_MAGIC + case 0x58465342: // XFS_SUPER_MAGIC + known_to_work = true; + break; + default: + LOG("Filesystem type 0x{:x} not known to work for the inode cache", + f_type); + } +#elif defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) // macOS X and some BSDs + static const std::vector known_to_work_filesystems = { + // Is a filesystem you know works with the inode cache missing in this + // list? Please submit an issue or pull request to the ccache project. + "apfs", + "xfs", + }; + if (std::find(known_to_work_filesystems.begin(), + known_to_work_filesystems.end(), + buf.f_fstypename) + != known_to_work_filesystems.end()) { + known_to_work = true; + } else { + LOG("Filesystem type {} not known to work for the inode cache", + buf.f_fstypename); + } +#else +# error Inconsistency: INODE_CACHE_SUPPORTED is set but we should not get here +#endif + } + if (!known_to_work) { + LOG_RAW("Not using the inode cache"); + } + return known_to_work; +} + } // namespace struct InodeCache::Key @@ -86,18 +148,9 @@ struct InodeCache::Key dev_t st_dev; ino_t st_ino; mode_t st_mode; -#ifdef HAVE_STRUCT_STAT_ST_MTIM timespec st_mtim; -#else - time_t st_mtim; -#endif -#ifdef HAVE_STRUCT_STAT_ST_CTIM timespec st_ctim; // Included for sanity checking. -#else - time_t st_ctim; // Included for sanity checking. -#endif - off_t st_size; // Included for sanity checking. - bool sloppy_time_macros; + off_t st_size; // Included for sanity checking. }; struct InodeCache::Entry @@ -134,17 +187,13 @@ InodeCache::mmap_file(const std::string& inode_cache_file) LOG("Failed to open inode cache {}: {}", inode_cache_file, strerror(errno)); return false; } - bool is_nfs; - if (Util::is_nfs_fd(*fd, &is_nfs) == 0 && is_nfs) { - LOG( - "Inode cache not supported because the cache file is located on nfs: {}", - inode_cache_file); + if (!fd_is_on_known_to_work_file_system(*fd)) { return false; } SharedRegion* sr = reinterpret_cast(mmap( nullptr, sizeof(SharedRegion), PROT_READ | PROT_WRITE, MAP_SHARED, *fd, 0)); fd.close(); - if (sr == reinterpret_cast(-1)) { + if (sr == MMAP_FAILED) { LOG("Failed to mmap {}: {}", inode_cache_file, strerror(errno)); return false; } @@ -162,7 +211,7 @@ InodeCache::mmap_file(const std::string& inode_cache_file) } m_sr = sr; if (m_config.debug()) { - LOG("inode cache file loaded: {}", inode_cache_file); + LOG("Inode cache file loaded: {}", inode_cache_file); } return true; } @@ -178,22 +227,21 @@ InodeCache::hash_inode(const std::string& path, return false; } + // See comment for InodeCache::InodeCache why this check is done. + auto now = util::TimePoint::now(); + if (now - stat.ctime() < m_min_age || now - stat.mtime() < m_min_age) { + LOG("Too new ctime or mtime of {}, not considering for inode cache", path); + return false; + } + Key key; memset(&key, 0, sizeof(Key)); key.type = type; key.st_dev = stat.device(); key.st_ino = stat.inode(); key.st_mode = stat.mode(); -#ifdef HAVE_STRUCT_STAT_ST_MTIM - key.st_mtim = stat.mtim(); -#else - key.st_mtim = stat.mtime(); -#endif -#ifdef HAVE_STRUCT_STAT_ST_CTIM - key.st_ctim = stat.ctim(); -#else - key.st_ctim = stat.ctime(); -#endif + key.st_mtim = stat.mtime().to_timespec(); + key.st_ctim = stat.ctime().to_timespec(); key.st_size = stat.size(); Hash hash; @@ -250,20 +298,13 @@ InodeCache::with_bucket(const Digest& key_digest, bool InodeCache::create_new_file(const std::string& filename) { - LOG_RAW("Creating a new inode cache"); - // Create the new file to a temporary name to prevent other processes from // mapping it before it is fully initialized. TemporaryFile tmp_file(filename); Finalizer temp_file_remover([&] { unlink(tmp_file.path.c_str()); }); - bool is_nfs; - if (Util::is_nfs_fd(*tmp_file.fd, &is_nfs) == 0 && is_nfs) { - LOG( - "Inode cache not supported because the cache file would be located on" - " nfs: {}", - filename); + if (!fd_is_on_known_to_work_file_system(*tmp_file.fd)) { return false; } int err = Util::fallocate(*tmp_file.fd, sizeof(SharedRegion)); @@ -278,7 +319,7 @@ InodeCache::create_new_file(const std::string& filename) MAP_SHARED, *tmp_file.fd, 0)); - if (sr == reinterpret_cast(-1)) { + if (sr == MMAP_FAILED) { LOG("Failed to mmap new inode cache: {}", strerror(errno)); return false; } @@ -308,6 +349,7 @@ InodeCache::create_new_file(const std::string& filename) return false; } + LOG("Created a new inode cache {}", filename); return true; } @@ -342,17 +384,32 @@ InodeCache::initialize() return false; } -InodeCache::InodeCache(const Config& config) : m_config(config) +InodeCache::InodeCache(const Config& config, util::Duration min_age) + : m_config(config), + // CCACHE_DISABLE_INODE_CACHE_MIN_AGE is only for testing purposes; see + // test/suites/inode_cache.bash. + m_min_age(getenv("CCACHE_DISABLE_INODE_CACHE_MIN_AGE") ? util::Duration(0) + : min_age) { } InodeCache::~InodeCache() { if (m_sr) { + LOG("Accumulated stats for inode cache: hits={}, misses={}, errors={}", + m_sr->hits.load(), + m_sr->misses.load(), + m_sr->errors.load()); munmap(m_sr, sizeof(SharedRegion)); } } +bool +InodeCache::available(int fd) +{ + return fd_is_on_known_to_work_file_system(fd); +} + bool InodeCache::get(const std::string& path, ContentType type, @@ -369,7 +426,7 @@ InodeCache::get(const std::string& path, } bool found = false; - const bool success = with_bucket(key_digest, [&](Bucket* const bucket) { + const bool success = with_bucket(key_digest, [&](const auto bucket) { for (uint32_t i = 0; i < k_num_entries; ++i) { if (bucket->entries[i].key_digest == key_digest) { if (i > 0) { @@ -391,18 +448,13 @@ InodeCache::get(const std::string& path, return false; } - LOG("inode cache {}: {}", found ? "hit" : "miss", path); - if (m_config.debug()) { + LOG("Inode cache {}: {}", found ? "hit" : "miss", path); if (found) { ++m_sr->hits; } else { ++m_sr->misses; } - LOG("Accumulated stats for inode cache: hits={}, misses={}, errors={}", - m_sr->hits.load(), - m_sr->misses.load(), - m_sr->errors.load()); } return found; } @@ -422,7 +474,7 @@ InodeCache::put(const std::string& path, return false; } - const bool success = with_bucket(key_digest, [&](Bucket* const bucket) { + const bool success = with_bucket(key_digest, [&](const auto bucket) { memmove(&bucket->entries[1], &bucket->entries[0], sizeof(Entry) * (k_num_entries - 1)); @@ -436,8 +488,9 @@ InodeCache::put(const std::string& path, return false; } - LOG("inode cache insert: {}", path); - + if (m_config.debug()) { + LOG("Inode cache insert: {}", path); + } return true; } @@ -448,6 +501,7 @@ InodeCache::drop() if (unlink(file.c_str()) != 0) { return false; } + LOG("Dropped inode cache {}", file); if (m_sr) { munmap(m_sr, sizeof(SharedRegion)); m_sr = nullptr; @@ -458,7 +512,9 @@ InodeCache::drop() std::string InodeCache::get_file() { - return FMT("{}/inode-cache.v{}", m_config.temporary_dir(), k_version); + const uint8_t arch_bits = 8 * sizeof(void*); + return FMT( + "{}/inode-cache-{}.v{}", m_config.temporary_dir(), arch_bits, k_version); } int64_t diff --git a/src/InodeCache.hpp b/src/InodeCache.hpp index f2d049a..e270ef1 100644 --- a/src/InodeCache.hpp +++ b/src/InodeCache.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,10 +18,9 @@ #pragma once -#include "system.hpp" - -#include "config.h" +#include +#include #include #include @@ -32,21 +31,46 @@ class Digest; class InodeCache { public: - // Specifies in which role a file was hashed, since the hash result does not - // only depend on the actual content but also what we used it for. Source code - // files are scanned for macros while binary files are not as one example. + // Specifies in which mode a file was hashed since the hash result does not + // only depend on the actual content but also on operations that were + // performed that affect the return value. For example, source code files are + // normally scanned for macros while binary files are not. enum class ContentType { - binary = 0, - code = 1, - code_with_sloppy_time_macros = 2, - precompiled_header = 3, + // The file was not scanned for temporal macros. + raw = 0, + // The file was checked for temporal macros (see check_for_temporal_macros + // in hashutil). + checked_for_temporal_macros = 1, }; - InodeCache(const Config& config); + // `min_age` specifies how old a file must be to be put in the cache. The + // reason for this is that there is a race condition that consists of these + // events: + // + // 1. A file is written with content C1, size S and timestamp (ctime/mtime) T. + // 2. Ccache hashes the file content and asks the inode cache to store the + // digest with a hash of S and T (and some other data) as the key. + // 3. The file is quickly thereafter written with content C2 without changing + // size S and timestamp T. The timestamp is not updated since the file + // writes are made within a time interval smaller than the granularity of + // the clock used for file system timestamps. At the time of writing, a + // common granularity on a Linux system is 0.004 s (250 Hz). + // 4. The inode cache is asked for the file digest and the inode cache + // delivers a digest of C1 even though the file's content is C2. + // + // To avoid the race condition, the inode cache only caches inodes whose + // timestamp was updated more than `min_age` ago. The default value is a + // conservative 2 seconds since not all file systems have subsecond + // resolution. + InodeCache(const Config& config, util::Duration min_age = util::Duration(2)); ~InodeCache(); + // Return whether it's possible to use the inode cache on the filesystem + // associated with `fd`. + static bool available(int fd); + // Get saved hash digest and return value from a previous call to - // hash_source_code_file(). + // do_hash_file() in hashutil.cpp. // // Returns true if saved values could be retrieved from the cache, false // otherwise. @@ -55,8 +79,8 @@ public: Digest& file_digest, int* return_value = nullptr); - // Put hash digest and return value from a successful call to - // hash_source_code_file(). + // Put hash digest and return value from a successful call to do_hash_file() + // in hashutil.cpp. // // Returns true if values could be stored in the cache, false otherwise. bool put(const std::string& path, @@ -98,14 +122,14 @@ private: using BucketHandler = std::function; bool mmap_file(const std::string& inode_cache_file); - static bool - hash_inode(const std::string& path, ContentType type, Digest& digest); + bool hash_inode(const std::string& path, ContentType type, Digest& digest); bool with_bucket(const Digest& key_digest, const BucketHandler& bucket_handler); static bool create_new_file(const std::string& filename); bool initialize(); const Config& m_config; + util::Duration m_min_age; struct SharedRegion* m_sr = nullptr; bool m_failed = false; }; diff --git a/src/Lockfile.cpp b/src/Lockfile.cpp deleted file mode 100644 index fb9d5e4..0000000 --- a/src/Lockfile.cpp +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Lockfile.hpp" - -#include "Logging.hpp" -#include "Util.hpp" -#include "fmtmacros.hpp" - -#ifdef _WIN32 -# include "Win32Util.hpp" -#endif - -#include "third_party/fmt/core.h" - -#include -#include -#include - -namespace { - -#ifndef _WIN32 - -bool -do_acquire_posix(const std::string& lockfile, uint32_t staleness_limit) -{ - const uint32_t max_to_sleep = 10000; // Microseconds. - uint32_t to_sleep = 1000; // Microseconds. - uint32_t slept = 0; // Microseconds. - std::string initial_content; - - std::stringstream ss; - ss << Util::get_hostname() << ':' << getpid() << ':' - << std::this_thread::get_id(); - const auto content_prefix = ss.str(); - - while (true) { - auto my_content = FMT("{}:{}", content_prefix, time(nullptr)); - - if (symlink(my_content.c_str(), lockfile.c_str()) == 0) { - // We got the lock. - return true; - } - - int saved_errno = errno; - LOG("lockfile_acquire: symlink {}: {}", lockfile, strerror(saved_errno)); - if (saved_errno == ENOENT) { - // Directory doesn't exist? - if (Util::create_dir(Util::dir_name(lockfile))) { - // OK. Retry. - continue; - } - } - - if (saved_errno == EPERM) { - // The file system does not support symbolic links. We have no choice but - // to grant the lock anyway. - return true; - } - - if (saved_errno != EEXIST) { - // Directory doesn't exist or isn't writable? - return false; - } - - std::string content = Util::read_link(lockfile); - if (content.empty()) { - if (errno == ENOENT) { - // The symlink was removed after the symlink() call above, so retry - // acquiring it. - continue; - } else { - LOG("lockfile_acquire: readlink {}: {}", lockfile, strerror(errno)); - return false; - } - } - - if (content == my_content) { - // Lost NFS reply? - LOG("lockfile_acquire: symlink {} failed but we got the lock anyway", - lockfile); - return true; - } - - // A possible improvement here would be to check if the process holding the - // lock is still alive and break the lock early if it isn't. - LOG("lockfile_acquire: lock info for {}: {}", lockfile, content); - - if (initial_content.empty()) { - initial_content = content; - } - - if (slept <= staleness_limit) { - LOG("lockfile_acquire: failed to acquire {}; sleeping {} microseconds", - lockfile, - to_sleep); - usleep(to_sleep); - slept += to_sleep; - to_sleep = std::min(max_to_sleep, 2 * to_sleep); - } else if (content != initial_content) { - LOG("lockfile_acquire: gave up acquiring {}", lockfile); - return false; - } else { - // The lock seems to be stale -- break it and try again. - LOG("lockfile_acquire: breaking {}", lockfile); - if (!Util::unlink_tmp(lockfile)) { - LOG("Failed to unlink {}: {}", lockfile, strerror(errno)); - return false; - } - to_sleep = 1000; - slept = 0; - initial_content.clear(); - } - } -} - -#else // !_WIN32 - -HANDLE -do_acquire_win32(const std::string& lockfile, uint32_t staleness_limit) -{ - unsigned to_sleep = 1000; // Microseconds. - unsigned max_to_sleep = 10000; // Microseconds. - unsigned slept = 0; // Microseconds. - HANDLE handle; - - while (true) { - DWORD flags = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_DELETE_ON_CLOSE; - handle = CreateFile(lockfile.c_str(), - GENERIC_WRITE, // desired access - 0, // shared mode (0 = not shared) - nullptr, // security attributes - CREATE_ALWAYS, // creation disposition - flags, // flags and attributes - nullptr // template file - ); - if (handle != INVALID_HANDLE_VALUE) { - break; - } - - DWORD error = GetLastError(); - LOG("lockfile_acquire: CreateFile {}: {} ({})", - lockfile, - Win32Util::error_message(error), - error); - if (error == ERROR_PATH_NOT_FOUND) { - // Directory doesn't exist? - if (Util::create_dir(Util::dir_name(lockfile))) { - // OK. Retry. - continue; - } - } - - // ERROR_SHARING_VIOLATION: lock already held. - // ERROR_ACCESS_DENIED: maybe pending delete. - if (error != ERROR_SHARING_VIOLATION && error != ERROR_ACCESS_DENIED) { - // Fatal error, give up. - break; - } - - if (slept > staleness_limit) { - LOG("lockfile_acquire: gave up acquiring {}", lockfile); - break; - } - - LOG("lockfile_acquire: failed to acquire {}; sleeping {} microseconds", - lockfile, - to_sleep); - usleep(to_sleep); - slept += to_sleep; - to_sleep = std::min(max_to_sleep, 2 * to_sleep); - } - - return handle; -} - -#endif // !_WIN32 - -} // namespace - -Lockfile::Lockfile(const std::string& path, uint32_t staleness_limit) - : m_lockfile(path + ".lock") -{ -#ifndef _WIN32 - m_acquired = do_acquire_posix(m_lockfile, staleness_limit); -#else - m_handle = do_acquire_win32(m_lockfile, staleness_limit); -#endif - if (acquired()) { - LOG("Acquired lock {}", m_lockfile); - } else { - LOG("Failed to acquire lock {}", m_lockfile); - } -} - -Lockfile::~Lockfile() -{ - if (acquired()) { - LOG("Releasing lock {}", m_lockfile); -#ifndef _WIN32 - if (!Util::unlink_tmp(m_lockfile)) { - LOG("Failed to unlink {}: {}", m_lockfile, strerror(errno)); - } -#else - CloseHandle(m_handle); -#endif - } -} diff --git a/src/Lockfile.hpp b/src/Lockfile.hpp deleted file mode 100644 index 78baed8..0000000 --- a/src/Lockfile.hpp +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include - -class Lockfile -{ -public: - // Acquire a lock on `path`. Break the lock (or give up, depending on - // implementation) after `staleness_limit` Microseconds. - Lockfile(const std::string& path, uint32_t staleness_limit = 2000000); - - // Release the lock if acquired. - ~Lockfile(); - - // Return whether the lockfile was acquired successfully. - bool acquired() const; - -private: - std::string m_lockfile; -#ifndef _WIN32 - bool m_acquired = false; -#else - HANDLE m_handle = nullptr; -#endif -}; - -inline bool -Lockfile::acquired() const -{ -#ifndef _WIN32 - return m_acquired; -#else - return m_handle != INVALID_HANDLE_VALUE; -#endif -} diff --git a/src/Logging.cpp b/src/Logging.cpp index c6590d8..beac470 100644 --- a/src/Logging.cpp +++ b/src/Logging.cpp @@ -1,5 +1,5 @@ // Copyright (C) 2002 Andrew Tridgell -// Copyright (C) 2009-2021 Joel Rosdahl and other contributors +// Copyright (C) 2009-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -22,16 +22,19 @@ #include "Config.hpp" #include "File.hpp" #include "Util.hpp" -#include "exceptions.hpp" +#include "Win32Util.hpp" #include "execute.hpp" #include "fmtmacros.hpp" +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + #ifdef HAVE_SYSLOG_H # include #endif -#ifdef HAVE_SYS_TIME_H -# include -#endif #ifdef __linux__ # ifdef HAVE_SYS_IOCTL_H @@ -39,14 +42,6 @@ # endif #endif -#ifdef _WIN32 -# include -# include -# include -#endif - -using nonstd::string_view; - namespace { // Logfile path and file handle, read from Config::log_file(). @@ -54,7 +49,7 @@ std::string logfile_path; File logfile; // Whether to use syslog() instead. -bool use_syslog; +bool use_syslog = false; // Buffer used for logs in debug mode. std::string debug_log_buffer; @@ -68,36 +63,39 @@ bool debug_log_enabled = false; print_fatal_error_and_exit() { // Note: Can't throw Fatal since that would lead to recursion. - PRINT(stderr, - "ccache: error: Failed to write to {}: {}\n", - logfile_path, - strerror(errno)); + try { + PRINT(stderr, + "ccache: error: Failed to write to {}: {}\n", + logfile_path, + strerror(errno)); + } catch (std::runtime_error&) { + // Ignore since we can't do anything about it. + } exit(EXIT_FAILURE); } void -do_log(string_view message, bool bulk) +do_log(std::string_view message, bool bulk) { static char prefix[200]; if (!bulk) { char timestamp[100]; - struct timeval tv; - gettimeofday(&tv, nullptr); - auto tm = Util::localtime(tv.tv_sec); + auto now = util::TimePoint::now(); + auto tm = Util::localtime(now); if (tm) { strftime(timestamp, sizeof(timestamp), "%Y-%m-%dT%H:%M:%S", &*tm); } else { snprintf(timestamp, sizeof(timestamp), "%llu", - static_cast(tv.tv_sec)); + static_cast(now.sec())); } snprintf(prefix, sizeof(prefix), "[%s.%06d %-5d] ", timestamp, - static_cast(tv.tv_usec), + static_cast(now.nsec_decimal_part() / 1000), static_cast(getpid())); } @@ -159,7 +157,7 @@ enabled() } void -log(string_view message) +log(std::string_view message) { if (!enabled()) { return; @@ -168,7 +166,7 @@ log(string_view message) } void -bulk_log(string_view message) +bulk_log(std::string_view message) { if (!enabled()) { return; diff --git a/src/Logging.hpp b/src/Logging.hpp index 38553f2..669b112 100644 --- a/src/Logging.hpp +++ b/src/Logging.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,34 +18,29 @@ #pragma once -#include "system.hpp" - -#include "FormatNonstdStringView.hpp" - #include "third_party/fmt/core.h" #include "third_party/fmt/format.h" -#include "third_party/nonstd/optional.hpp" -#include "third_party/nonstd/string_view.hpp" +#include #include +#include #include // Log a raw message (plus a newline character). #define LOG_RAW(message_) \ do { \ if (Logging::enabled()) { \ - Logging::log(nonstd::string_view(message_)); \ + Logging::log(std::string_view(message_)); \ } \ } while (false) // Log a message (plus a newline character) described by a format string with at -// least one placeholder. `format` is compile-time checked if CMAKE_CXX_STANDARD -// >= 14. +// least one placeholder. `format` is checked at compile time. #define LOG(format_, ...) LOG_RAW(fmt::format(FMT_STRING(format_), __VA_ARGS__)) // Log a message (plus a newline character) described by a format string with at // least one placeholder without flushing and with a reused timestamp. `format` -// is compile-time checked if CMAKE_CXX_STANDARD >= 14. +// is checked at compile time. #define BULK_LOG(format_, ...) \ do { \ if (Logging::enabled()) { \ @@ -65,11 +60,11 @@ void init(const Config& config); bool enabled(); // Log `message` (plus a newline character). -void log(nonstd::string_view message); +void log(std::string_view message); // Log `message` (plus a newline character) without flushing and with a reused // timestamp. -void bulk_log(nonstd::string_view message); +void bulk_log(std::string_view message); // Write the current log memory buffer `path`. void dump_log(const std::string& path); diff --git a/src/Manifest.cpp b/src/Manifest.cpp deleted file mode 100644 index 38aec0c..0000000 --- a/src/Manifest.cpp +++ /dev/null @@ -1,639 +0,0 @@ -// Copyright (C) 2009-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Manifest.hpp" - -#include "AtomicFile.hpp" -#include "CacheEntryReader.hpp" -#include "CacheEntryWriter.hpp" -#include "Checksum.hpp" -#include "Config.hpp" -#include "Context.hpp" -#include "Digest.hpp" -#include "File.hpp" -#include "Hash.hpp" -#include "Logging.hpp" -#include "Sloppiness.hpp" -#include "StdMakeUnique.hpp" -#include "fmtmacros.hpp" -#include "hashutil.hpp" - -// Manifest data format -// ==================== -// -// Integers are big-endian. -// -// ::=
::= -// -// ::= 4 bytes ("cCrS") -// ::= uint8_t -// ::= | -// ::= 0 (uint8_t) -// ::= 1 (uint8_t) -// ::= int8_t -// ::= uint64_t ; size of file if stored uncompressed -// ::= ; body is potentially -// ; compressed -// ::= * -// ::= uint32_t -// ::= -// ::= uint16_t -// ::= path_len bytes -// ::= * -// ::= uint32_t -// ::= -// ::= uint32_t -// ::= Digest::size() bytes -// ::= uint64_t ; file size -// ::= int64_t ; modification time -// ::= int64_t ; status change time -// ::= * -// ::= uint32_t -// ::= * -// ::= uint32_t -// ::= uint32_t -// ::= Digest::size() bytes -// ::= -// ::= uint64_t ; XXH3 of content bytes -// -// Sketch of concrete layout: - -// 4 bytes -// 1 byte -// 1 byte -// 1 byte -// 8 bytes -// --- [potentially compressed from here] ------------------------------------- -// 4 bytes -// 2 bytes -// path_len bytes -// ... -// ---------------------------------------------------------------------------- -// 4 bytes -// 4 bytes -// Digest::size() bytes -// 8 bytes -// 8 bytes -// 8 bytes -// ... -// ---------------------------------------------------------------------------- -// 4 bytes -// 4 bytes -// 4 bytes -// ... -// Digest::size() bytes -// ... -// checksum 8 bytes -// -// -// Version history -// =============== -// -// 1: Introduced in ccache 3.0. (Files are always compressed with gzip.) -// 2: Introduced in ccache 4.0. - -using nonstd::nullopt; -using nonstd::optional; - -const uint32_t k_max_manifest_entries = 100; -const uint32_t k_max_manifest_file_info_entries = 10000; - -namespace { - -struct FileInfo -{ - // Index to n_files. - uint32_t index; - // Digest of referenced file. - Digest digest; - // Size of referenced file. - uint64_t fsize; - // mtime of referenced file. - int64_t mtime; - // ctime of referenced file. - int64_t ctime; -}; - -bool -operator==(const FileInfo& lhs, const FileInfo& rhs) -{ - return lhs.index == rhs.index && lhs.digest == rhs.digest - && lhs.fsize == rhs.fsize && lhs.mtime == rhs.mtime - && lhs.ctime == rhs.ctime; -} - -} // namespace - -namespace std { - -template<> struct hash -{ - size_t - operator()(const FileInfo& file_info) const - { - static_assert(sizeof(FileInfo) == 48, "unexpected size"); // No padding. - Checksum checksum; - checksum.update(&file_info, sizeof(file_info)); - return checksum.digest(); - } -}; - -} // namespace std - -namespace { - -struct ResultEntry -{ - // Indexes to file_infos. - std::vector file_info_indexes; - - // Name of the result. - Digest name; -}; - -bool -operator==(const ResultEntry& lhs, const ResultEntry& rhs) -{ - return lhs.file_info_indexes == rhs.file_info_indexes && lhs.name == rhs.name; -} - -struct ManifestData -{ - // Referenced include files. - std::vector files; - - // Information about referenced include files. - std::vector file_infos; - - // Result names plus references to include file infos. - std::vector results; - - bool - add_result_entry( - const Digest& result_digest, - const std::unordered_map& included_files, - time_t time_of_compilation, - bool save_timestamp) - { - std::unordered_map mf_files; - for (uint32_t i = 0; i < files.size(); ++i) { - mf_files.emplace(files[i], i); - } - - std::unordered_map mf_file_infos; - for (uint32_t i = 0; i < file_infos.size(); ++i) { - mf_file_infos.emplace(file_infos[i], i); - } - - std::vector file_info_indexes; - file_info_indexes.reserve(included_files.size()); - - for (const auto& item : included_files) { - file_info_indexes.push_back(get_file_info_index(item.first, - item.second, - mf_files, - mf_file_infos, - time_of_compilation, - save_timestamp)); - } - - ResultEntry entry{std::move(file_info_indexes), result_digest}; - if (std::find(results.begin(), results.end(), entry) == results.end()) { - results.push_back(std::move(entry)); - return true; - } else { - return false; - } - } - -private: - uint32_t - get_file_info_index( - const std::string& path, - const Digest& digest, - const std::unordered_map& mf_files, - const std::unordered_map& mf_file_infos, - time_t time_of_compilation, - bool save_timestamp) - { - struct FileInfo fi; - - auto f_it = mf_files.find(path); - if (f_it != mf_files.end()) { - fi.index = f_it->second; - } else { - files.push_back(path); - fi.index = files.size() - 1; - } - - fi.digest = digest; - - // file_stat.{m,c}time() have a resolution of 1 second, so we can cache the - // file's mtime and ctime only if they're at least one second older than - // time_of_compilation. - // - // file_stat.ctime() may be 0, so we have to check time_of_compilation - // against MAX(mtime, ctime). - // - // ccache only reads mtime/ctime if file_stat_match sloppiness is enabled, - // so mtimes/ctimes are stored as a dummy value (-1) if not enabled. This - // reduces the number of file_info entries for the common case. - - auto file_stat = Stat::stat(path, Stat::OnError::log); - if (file_stat) { - if (save_timestamp - && time_of_compilation - > std::max(file_stat.mtime(), file_stat.ctime())) { - fi.mtime = file_stat.mtime(); - fi.ctime = file_stat.ctime(); - } else { - fi.mtime = -1; - fi.ctime = -1; - } - fi.fsize = file_stat.size(); - } else { - fi.mtime = -1; - fi.ctime = -1; - fi.fsize = 0; - } - - auto fi_it = mf_file_infos.find(fi); - if (fi_it != mf_file_infos.end()) { - return fi_it->second; - } else { - file_infos.push_back(fi); - return file_infos.size() - 1; - } - } -}; - -struct FileStats -{ - uint64_t size; - int64_t mtime; - int64_t ctime; -}; - -std::unique_ptr -read_manifest(const std::string& path, FILE* dump_stream = nullptr) -{ - File file(path, "rb"); - if (!file) { - return {}; - } - - CacheEntryReader reader(file.get(), Manifest::k_magic, Manifest::k_version); - - if (dump_stream) { - reader.dump_header(dump_stream); - } - - auto mf = std::make_unique(); - - uint32_t entry_count; - reader.read(entry_count); - for (uint32_t i = 0; i < entry_count; ++i) { - mf->files.emplace_back(); - auto& entry = mf->files.back(); - - uint16_t length; - reader.read(length); - entry.assign(length, 0); - reader.read(&entry[0], length); - } - - reader.read(entry_count); - for (uint32_t i = 0; i < entry_count; ++i) { - mf->file_infos.emplace_back(); - auto& entry = mf->file_infos.back(); - - reader.read(entry.index); - reader.read(entry.digest.bytes(), Digest::size()); - reader.read(entry.fsize); - reader.read(entry.mtime); - reader.read(entry.ctime); - } - - reader.read(entry_count); - for (uint32_t i = 0; i < entry_count; ++i) { - mf->results.emplace_back(); - auto& entry = mf->results.back(); - - uint32_t file_info_count; - reader.read(file_info_count); - for (uint32_t j = 0; j < file_info_count; ++j) { - uint32_t file_info_index; - reader.read(file_info_index); - entry.file_info_indexes.push_back(file_info_index); - } - reader.read(entry.name.bytes(), Digest::size()); - } - - reader.finalize(); - return mf; -} - -bool -write_manifest(const Config& config, - const std::string& path, - const ManifestData& mf) -{ - uint64_t payload_size = 0; - payload_size += 4; // n_files - for (const auto& file : mf.files) { - payload_size += 2 + file.length(); - } - payload_size += 4; // n_file_infos - payload_size += mf.file_infos.size() * (4 + Digest::size() + 8 + 8 + 8); - payload_size += 4; // n_results - for (const auto& result : mf.results) { - payload_size += 4; // n_file_info_indexes - payload_size += result.file_info_indexes.size() * 4; - payload_size += Digest::size(); - } - - AtomicFile atomic_manifest_file(path, AtomicFile::Mode::binary); - CacheEntryWriter writer(atomic_manifest_file.stream(), - Manifest::k_magic, - Manifest::k_version, - Compression::type_from_config(config), - Compression::level_from_config(config), - payload_size); - writer.write(mf.files.size()); - for (const auto& file : mf.files) { - writer.write(file.length()); - writer.write(file.data(), file.length()); - } - - writer.write(mf.file_infos.size()); - for (const auto& file_info : mf.file_infos) { - writer.write(file_info.index); - writer.write(file_info.digest.bytes(), Digest::size()); - writer.write(file_info.fsize); - writer.write(file_info.mtime); - writer.write(file_info.ctime); - } - - writer.write(mf.results.size()); - for (const auto& result : mf.results) { - writer.write(result.file_info_indexes.size()); - for (auto index : result.file_info_indexes) { - writer.write(index); - } - writer.write(result.name.bytes(), Digest::size()); - } - - writer.finalize(); - atomic_manifest_file.commit(); - return true; -} - -bool -verify_result(const Context& ctx, - const ManifestData& mf, - const ResultEntry& result, - std::unordered_map& stated_files, - std::unordered_map& hashed_files) -{ - for (uint32_t file_info_index : result.file_info_indexes) { - const auto& fi = mf.file_infos[file_info_index]; - const auto& path = mf.files[fi.index]; - - auto stated_files_iter = stated_files.find(path); - if (stated_files_iter == stated_files.end()) { - auto file_stat = Stat::stat(path, Stat::OnError::log); - if (!file_stat) { - return false; - } - FileStats st; - st.size = file_stat.size(); - st.mtime = file_stat.mtime(); - st.ctime = file_stat.ctime(); - stated_files_iter = stated_files.emplace(path, st).first; - } - const FileStats& fs = stated_files_iter->second; - - if (fi.fsize != fs.size) { - return false; - } - - // Clang stores the mtime of the included files in the precompiled header, - // and will error out if that header is later used without rebuilding. - if ((ctx.config.compiler_type() == CompilerType::clang - || ctx.config.compiler_type() == CompilerType::other) - && ctx.args_info.output_is_precompiled_header - && !ctx.args_info.fno_pch_timestamp && fi.mtime != fs.mtime) { - LOG("Precompiled header includes {}, which has a new mtime", path); - return false; - } - - if (ctx.config.sloppiness() & SLOPPY_FILE_STAT_MATCHES) { - if (!(ctx.config.sloppiness() & SLOPPY_FILE_STAT_MATCHES_CTIME)) { - if (fi.mtime == fs.mtime && fi.ctime == fs.ctime) { - LOG("mtime/ctime hit for {}", path); - continue; - } else { - LOG("mtime/ctime miss for {}", path); - } - } else { - if (fi.mtime == fs.mtime) { - LOG("mtime hit for {}", path); - continue; - } else { - LOG("mtime miss for {}", path); - } - } - } - - auto hashed_files_iter = hashed_files.find(path); - if (hashed_files_iter == hashed_files.end()) { - Hash hash; - int ret = hash_source_code_file(ctx, hash, path, fs.size); - if (ret & HASH_SOURCE_CODE_ERROR) { - LOG("Failed hashing {}", path); - return false; - } - if (ret & HASH_SOURCE_CODE_FOUND_TIME) { - return false; - } - - Digest actual = hash.digest(); - hashed_files_iter = hashed_files.emplace(path, actual).first; - } - - if (fi.digest != hashed_files_iter->second) { - return false; - } - } - - return true; -} - -} // namespace - -namespace Manifest { - -const std::string k_file_suffix = "M"; -const uint8_t k_magic[4] = {'c', 'C', 'm', 'F'}; -const uint8_t k_version = 2; - -// Try to get the result name from a manifest file. Returns nullopt on failure. -optional -get(const Context& ctx, const std::string& path) -{ - std::unique_ptr mf; - try { - mf = read_manifest(path); - if (mf) { - // Update modification timestamp to save files from LRU cleanup. - Util::update_mtime(path); - } else { - LOG_RAW("No such manifest file"); - return nullopt; - } - } catch (const Error& e) { - LOG("Error: {}", e.what()); - return nullopt; - } - - std::unordered_map stated_files; - std::unordered_map hashed_files; - - // Check newest result first since it's a bit more likely to match. - for (uint32_t i = mf->results.size(); i > 0; i--) { - if (verify_result( - ctx, *mf, mf->results[i - 1], stated_files, hashed_files)) { - return mf->results[i - 1].name; - } - } - - return nullopt; -} - -// Put the result name into a manifest file given a set of included files. -// Returns true on success, otherwise false. -bool -put(const Config& config, - const std::string& path, - const Digest& result_name, - const std::unordered_map& included_files, - - time_t time_of_compilation, - bool save_timestamp) -{ - // We don't bother to acquire a lock when writing the manifest to disk. A - // race between two processes will only result in one lost entry, which is - // not a big deal, and it's also very unlikely. - - std::unique_ptr mf; - try { - mf = read_manifest(path); - if (!mf) { - // Manifest file didn't exist. - mf = std::make_unique(); - } - } catch (const Error& e) { - LOG("Error: {}", e.what()); - // Manifest file was corrupt, ignore. - mf = std::make_unique(); - } - - if (mf->results.size() > k_max_manifest_entries) { - // Normally, there shouldn't be many result entries in the manifest since - // new entries are added only if an include file has changed but not the - // source file, and you typically change source files more often than - // header files. However, it's certainly possible to imagine cases where - // the manifest will grow large (for instance, a generated header file that - // changes for every build), and this must be taken care of since - // processing an ever growing manifest eventually will take too much time. - // A good way of solving this would be to maintain the result entries in - // LRU order and discarding the old ones. An easy way is to throw away all - // entries when there are too many. Let's do that for now. - LOG("More than {} entries in manifest file; discarding", - k_max_manifest_entries); - mf = std::make_unique(); - } else if (mf->file_infos.size() > k_max_manifest_file_info_entries) { - // Rarely, FileInfo entries can grow large in pathological cases where - // many included files change, but the main file does not. This also puts - // an upper bound on the number of FileInfo entries. - LOG("More than {} FileInfo entries in manifest file; discarding", - k_max_manifest_file_info_entries); - mf = std::make_unique(); - } - - bool added = mf->add_result_entry( - result_name, included_files, time_of_compilation, save_timestamp); - - if (added) { - try { - write_manifest(config, path, *mf); - return true; - } catch (const Error& e) { - LOG("Error: {}", e.what()); - } - } else { - LOG_RAW("The entry already exists in the manifest, not adding"); - } - return false; -} - -bool -dump(const std::string& path, FILE* stream) -{ - std::unique_ptr mf; - try { - mf = read_manifest(path, stream); - } catch (const Error& e) { - PRINT(stream, "Error: {}\n", e.what()); - return false; - } - - if (!mf) { - PRINT(stream, "Error: No such file: {}\n", path); - return false; - } - - PRINT(stream, "File paths ({}):\n", mf->files.size()); - for (size_t i = 0; i < mf->files.size(); ++i) { - PRINT(stream, " {}: {}\n", i, mf->files[i]); - } - PRINT(stream, "File infos ({}):\n", mf->file_infos.size()); - for (size_t i = 0; i < mf->file_infos.size(); ++i) { - PRINT(stream, " {}:\n", i); - PRINT(stream, " Path index: {}\n", mf->file_infos[i].index); - PRINT(stream, " Hash: {}\n", mf->file_infos[i].digest.to_string()); - PRINT(stream, " File size: {}\n", mf->file_infos[i].fsize); - PRINT(stream, " Mtime: {}\n", mf->file_infos[i].mtime); - PRINT(stream, " Ctime: {}\n", mf->file_infos[i].ctime); - } - PRINT(stream, "Results ({}):\n", mf->results.size()); - for (size_t i = 0; i < mf->results.size(); ++i) { - PRINT(stream, " {}:\n", i); - PRINT_RAW(stream, " File info indexes:"); - for (uint32_t file_info_index : mf->results[i].file_info_indexes) { - PRINT(stream, " {}", file_info_index); - } - PRINT_RAW(stream, "\n"); - PRINT(stream, " Name: {}\n", mf->results[i].name.to_string()); - } - - return true; -} - -} // namespace Manifest diff --git a/src/Manifest.hpp b/src/Manifest.hpp deleted file mode 100644 index fda7758..0000000 --- a/src/Manifest.hpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (C) 2009-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "third_party/nonstd/optional.hpp" - -#include -#include - -class Config; -class Context; -class Digest; - -namespace Manifest { - -extern const std::string k_file_suffix; -extern const uint8_t k_magic[4]; -extern const uint8_t k_version; - -nonstd::optional get(const Context& ctx, const std::string& path); -bool put(const Config& config, - const std::string& path, - const Digest& result_name, - const std::unordered_map& included_files, - time_t time_of_compilation, - bool save_timestamp); -bool dump(const std::string& path, FILE* stream); - -} // namespace Manifest diff --git a/src/MiniTrace.cpp b/src/MiniTrace.cpp index 0516562..cd9f4f8 100644 --- a/src/MiniTrace.cpp +++ b/src/MiniTrace.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -16,52 +16,42 @@ // this program; if not, write to the Free Software Foundation, Inc., 51 // Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -#include "system.hpp" +#include "MiniTrace.hpp" -#ifdef MTR_ENABLED +#include "ArgsInfo.hpp" +#include "TemporaryFile.hpp" +#include "Util.hpp" +#include "fmtmacros.hpp" -# include "ArgsInfo.hpp" -# include "MiniTrace.hpp" -# include "TemporaryFile.hpp" -# include "Util.hpp" -# include "fmtmacros.hpp" +#include +#include -# ifdef HAVE_SYS_TIME_H -# include -# endif +#include // NOLINT: PATH_MAX is defined in limits.h + +#ifdef HAVE_UNISTD_H +# include +#endif namespace { std::string get_system_tmp_dir() { -# ifndef _WIN32 +#ifndef _WIN32 const char* tmpdir = getenv("TMPDIR"); if (tmpdir) { return tmpdir; } -# else +#else static char dirbuf[PATH_MAX]; DWORD retval = GetTempPath(PATH_MAX, dirbuf); if (retval > 0 && retval < PATH_MAX) { return dirbuf; } -# endif +#endif return "/tmp"; } -double -time_seconds() -{ -# ifdef HAVE_GETTIMEOFDAY - struct timeval tv; - gettimeofday(&tv, nullptr); - return (double)tv.tv_sec + (double)tv.tv_usec / 1000000.0; -# else - return (double)time(nullptr); -# endif -} - } // namespace MiniTrace::MiniTrace(const ArgsInfo& args_info) @@ -72,7 +62,9 @@ MiniTrace::MiniTrace(const ArgsInfo& args_info) m_tmp_trace_file = tmp_file.path; mtr_init(m_tmp_trace_file.c_str()); - MTR_INSTANT_C("", "", "time", FMT("{:f}", time_seconds()).c_str()); + auto now = util::TimePoint::now(); + m_start_time = FMT("{}.{:06}", now.sec(), now.nsec_decimal_part() / 1000); + MTR_INSTANT_C("", "", "time", m_start_time.c_str()); MTR_META_PROCESS_NAME("ccache"); MTR_START("program", "ccache", m_trace_id); } @@ -88,5 +80,3 @@ MiniTrace::~MiniTrace() } Util::unlink_tmp(m_tmp_trace_file); } - -#endif diff --git a/src/MiniTrace.hpp b/src/MiniTrace.hpp index 3b44360..76aae38 100644 --- a/src/MiniTrace.hpp +++ b/src/MiniTrace.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,13 +18,9 @@ #pragma once -#include "system.hpp" - #include "third_party/minitrace.h" -#ifdef MTR_ENABLED - -# include +#include struct ArgsInfo; @@ -38,6 +34,5 @@ private: const ArgsInfo& m_args_info; const void* const m_trace_id; std::string m_tmp_trace_file; + std::string m_start_time; }; - -#endif diff --git a/src/NullCompressor.cpp b/src/NullCompressor.cpp deleted file mode 100644 index 13494fc..0000000 --- a/src/NullCompressor.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "NullCompressor.hpp" - -#include "exceptions.hpp" - -NullCompressor::NullCompressor(FILE* stream) : m_stream(stream) -{ -} - -int8_t -NullCompressor::actual_compression_level() const -{ - return 0; -} - -void -NullCompressor::write(const void* data, size_t count) -{ - if (fwrite(data, 1, count, m_stream) != count) { - throw Error("failed to write to uncompressed stream"); - } -} - -void -NullCompressor::finalize() -{ - if (fflush(m_stream) != 0) { - throw Error("failed to finalize uncompressed stream"); - } -} diff --git a/src/NullCompressor.hpp b/src/NullCompressor.hpp deleted file mode 100644 index 9a4fcd1..0000000 --- a/src/NullCompressor.hpp +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Compressor.hpp" -#include "NonCopyable.hpp" - -// A compressor of an uncompressed stream. -class NullCompressor : public Compressor, NonCopyable -{ -public: - // Parameters: - // - stream: The file to write data to. - explicit NullCompressor(FILE* stream); - - int8_t actual_compression_level() const override; - void write(const void* data, size_t count) override; - void finalize() override; - -private: - FILE* m_stream; -}; diff --git a/src/NullDecompressor.cpp b/src/NullDecompressor.cpp deleted file mode 100644 index 090bdf5..0000000 --- a/src/NullDecompressor.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "NullDecompressor.hpp" - -#include "exceptions.hpp" - -NullDecompressor::NullDecompressor(FILE* stream) : m_stream(stream) -{ -} - -void -NullDecompressor::read(void* data, size_t count) -{ - if (fread(data, count, 1, m_stream) != 1) { - throw Error("failed to read from uncompressed stream"); - } -} - -void -NullDecompressor::finalize() -{ - if (fgetc(m_stream) != EOF) { - throw Error("garbage data at end of uncompressed stream"); - } -} diff --git a/src/NullDecompressor.hpp b/src/NullDecompressor.hpp deleted file mode 100644 index 514ed7d..0000000 --- a/src/NullDecompressor.hpp +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Decompressor.hpp" -#include "NonCopyable.hpp" - -// A decompressor of an uncompressed stream. -class NullDecompressor : public Decompressor, NonCopyable -{ -public: - // Parameters: - // - stream: The file to read data from. - explicit NullDecompressor(FILE* stream); - - void read(void* data, size_t count) override; - void finalize() override; - -private: - FILE* m_stream; -}; diff --git a/src/ProgressBar.cpp b/src/ProgressBar.cpp index c7b219e..99a37a0 100644 --- a/src/ProgressBar.cpp +++ b/src/ProgressBar.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -20,9 +20,12 @@ #include "fmtmacros.hpp" +#include + #include "third_party/fmt/core.h" -#ifndef _WIN32 +#ifdef _WIN32 +#else # include #endif @@ -30,6 +33,10 @@ # include #endif +#ifdef HAVE_UNISTD_H +# include +#endif + #include namespace { diff --git a/src/ProgressBar.hpp b/src/ProgressBar.hpp index 8aefa89..1be3802 100644 --- a/src/ProgressBar.hpp +++ b/src/ProgressBar.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,8 @@ #pragma once -#include "system.hpp" - +#include +#include #include class ProgressBar diff --git a/src/Result.cpp b/src/Result.cpp deleted file mode 100644 index 8408d2a..0000000 --- a/src/Result.cpp +++ /dev/null @@ -1,433 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Result.hpp" - -#include "AtomicFile.hpp" -#include "CacheEntryReader.hpp" -#include "CacheEntryWriter.hpp" -#include "Config.hpp" -#include "Context.hpp" -#include "Fd.hpp" -#include "File.hpp" -#include "Logging.hpp" -#include "Stat.hpp" -#include "Statistic.hpp" -#include "Util.hpp" -#include "exceptions.hpp" -#include "fmtmacros.hpp" - -#include - -// Result data format -// ================== -// -// Integers are big-endian. -// -// ::=
-//
::= -// -// ::= 4 bytes ("cCrS") -// ::= uint8_t -// ::= | -// ::= 0 (uint8_t) -// ::= 1 (uint8_t) -// ::= int8_t -// ::= uint64_t ; size of file if stored uncompressed -// ::= * ; potentially compressed -// ::= uint8_t -// ::= | -// ::= -// -// ::= 0 (uint8_t) -// ::= uint8_t -// ::= uint64_t -// ::= data_len bytes -// ::= -// ::= 1 (uint8_t) -// ::= uint64_t -// ::= -// ::= uint64_t ; XXH3 of content bytes -// -// Sketch of concrete layout: -// -// 4 bytes -// 1 byte -// 1 byte -// 1 byte -// 8 bytes -// --- [potentially compressed from here] ------------------------------------- -// 1 byte -// 1 byte -// 1 byte -// 8 bytes -// data_len bytes -// ... -// 1 byte -// 1 byte -// key_len bytes -// ... -// checksum 8 bytes -// -// -// Version history -// =============== -// -// 1: Introduced in ccache 4.0. - -using nonstd::nullopt; -using nonstd::optional; -using nonstd::string_view; - -namespace { - -// File data stored inside the result file. -const uint8_t k_embedded_file_marker = 0; - -// File stored as-is in the file system. -const uint8_t k_raw_file_marker = 1; - -std::string -get_raw_file_path(string_view result_path, uint32_t entry_number) -{ - const auto prefix = result_path.substr( - 0, result_path.length() - Result::k_file_suffix.length()); - return FMT("{}{}W", prefix, entry_number); -} - -bool -should_store_raw_file(const Config& config, Result::FileType type) -{ - if (!config.file_clone() && !config.hard_link()) { - return false; - } - - // Only store object files as raw files since there are several problems with - // storing other file types: - // - // 1. The compiler unlinks object files before writing to them but it doesn't - // unlink .d files, so just it's possible to corrupt .d files just by - // running the compiler (see ccache issue 599). - // 2. .d files cause trouble for automake if hard-linked (see ccache issue - // 378). - // 3. It's unknown how the compiler treats other file types, so better safe - // than sorry. - // - // It would be possible to store all files in raw form for the file_clone case - // and only hard link object files. However, most likely it's only object - // files that become large enough that it's of interest to clone or hard link - // them, so we keep things simple for now. This will also save i-nodes in the - // cache. - return type == Result::FileType::object; -} - -} // namespace - -namespace Result { - -const std::string k_file_suffix = "R"; -const uint8_t k_magic[4] = {'c', 'C', 'r', 'S'}; -const uint8_t k_version = 1; -const char* const k_unknown_file_type = ""; - -const char* -file_type_to_string(FileType type) -{ - switch (type) { - case FileType::object: - return ".o"; - - case FileType::dependency: - return ".d"; - - case FileType::stderr_output: - return ""; - - case FileType::coverage_unmangled: - return ".gcno-unmangled"; - - case FileType::stackusage: - return ".su"; - - case FileType::diagnostic: - return ".dia"; - - case FileType::dwarf_object: - return ".dwo"; - - case FileType::coverage_mangled: - return ".gcno-mangled"; - } - - return k_unknown_file_type; -} - -std::string -gcno_file_in_mangled_form(const Context& ctx) -{ - const auto& output_obj = ctx.args_info.output_obj; - const std::string abs_output_obj = - Util::is_absolute_path(output_obj) - ? output_obj - : FMT("{}/{}", ctx.apparent_cwd, output_obj); - std::string hashified_obj = abs_output_obj; - std::replace(hashified_obj.begin(), hashified_obj.end(), '/', '#'); - return Util::change_extension(hashified_obj, ".gcno"); -} - -std::string -gcno_file_in_unmangled_form(const Context& ctx) -{ - return Util::change_extension(ctx.args_info.output_obj, ".gcno"); -} - -Result::Reader::Reader(const std::string& result_path) - : m_result_path(result_path) -{ -} - -optional -Result::Reader::read(Consumer& consumer) -{ - LOG("Reading result {}", m_result_path); - - try { - if (read_result(consumer)) { - return nullopt; - } else { - return "No such result file"; - } - } catch (const Error& e) { - return e.what(); - } -} - -bool -Reader::read_result(Consumer& consumer) -{ - File file(m_result_path, "rb"); - if (!file) { - // Cache miss. - return false; - } - - CacheEntryReader cache_entry_reader(file.get(), k_magic, k_version); - - consumer.on_header(cache_entry_reader); - - uint8_t n_entries; - cache_entry_reader.read(n_entries); - - uint32_t i; - for (i = 0; i < n_entries; ++i) { - read_entry(cache_entry_reader, i, consumer); - } - - if (i != n_entries) { - throw Error("Too few entries (read {}, expected {})", i, n_entries); - } - - cache_entry_reader.finalize(); - return true; -} - -void -Reader::read_entry(CacheEntryReader& cache_entry_reader, - uint32_t entry_number, - Reader::Consumer& consumer) -{ - uint8_t marker; - cache_entry_reader.read(marker); - - switch (marker) { - case k_embedded_file_marker: - case k_raw_file_marker: - break; - - default: - throw Error("Unknown entry type: {}", marker); - } - - UnderlyingFileTypeInt type; - cache_entry_reader.read(type); - FileType file_type = FileType(type); - - uint64_t file_len; - cache_entry_reader.read(file_len); - - if (marker == k_embedded_file_marker) { - consumer.on_entry_start(entry_number, file_type, file_len, nullopt); - - uint8_t buf[READ_BUFFER_SIZE]; - size_t remain = file_len; - while (remain > 0) { - size_t n = std::min(remain, sizeof(buf)); - cache_entry_reader.read(buf, n); - consumer.on_entry_data(buf, n); - remain -= n; - } - } else { - ASSERT(marker == k_raw_file_marker); - - auto raw_path = get_raw_file_path(m_result_path, entry_number); - auto st = Stat::stat(raw_path, Stat::OnError::throw_error); - if (st.size() != file_len) { - throw Error("Bad file size of {} (actual {} bytes, expected {} bytes)", - raw_path, - st.size(), - file_len); - } - - consumer.on_entry_start(entry_number, file_type, file_len, raw_path); - } - - consumer.on_entry_end(); -} - -Writer::Writer(Context& ctx, const std::string& result_path) - : m_ctx(ctx), - m_result_path(result_path) -{ -} - -void -Writer::write(FileType file_type, const std::string& file_path) -{ - m_entries_to_write.emplace_back(file_type, file_path); -} - -optional -Writer::finalize() -{ - try { - do_finalize(); - return nullopt; - } catch (const Error& e) { - return e.what(); - } -} - -void -Writer::do_finalize() -{ - uint64_t payload_size = 0; - payload_size += 1; // n_entries - for (const auto& pair : m_entries_to_write) { - const auto& path = pair.second; - auto st = Stat::stat(path, Stat::OnError::throw_error); - - payload_size += 1; // embedded_file_marker - payload_size += 1; // embedded_file_type - payload_size += 8; // data_len - payload_size += st.size(); // data - } - - AtomicFile atomic_result_file(m_result_path, AtomicFile::Mode::binary); - CacheEntryWriter writer(atomic_result_file.stream(), - k_magic, - k_version, - Compression::type_from_config(m_ctx.config), - Compression::level_from_config(m_ctx.config), - payload_size); - - writer.write(m_entries_to_write.size()); - - uint32_t entry_number = 0; - for (const auto& pair : m_entries_to_write) { - const auto file_type = pair.first; - const auto& path = pair.second; - LOG("Storing result {}", path); - - const bool store_raw = should_store_raw_file(m_ctx.config, file_type); - uint64_t file_size = Stat::stat(path, Stat::OnError::throw_error).size(); - - LOG("Storing {} file #{} {} ({} bytes) from {}", - store_raw ? "raw" : "embedded", - entry_number, - file_type_to_string(file_type), - file_size, - path); - - writer.write(store_raw ? k_raw_file_marker - : k_embedded_file_marker); - writer.write(UnderlyingFileTypeInt(file_type)); - writer.write(file_size); - - if (store_raw) { - write_raw_file_entry(path, entry_number); - } else { - write_embedded_file_entry(writer, path, file_size); - } - - ++entry_number; - } - - writer.finalize(); - atomic_result_file.commit(); -} - -void -Result::Writer::write_embedded_file_entry(CacheEntryWriter& writer, - const std::string& path, - uint64_t file_size) -{ - Fd file(open(path.c_str(), O_RDONLY | O_BINARY)); - if (!file) { - throw Error("Failed to open {} for reading", path); - } - - uint64_t remain = file_size; - while (remain > 0) { - uint8_t buf[READ_BUFFER_SIZE]; - size_t n = std::min(remain, static_cast(sizeof(buf))); - ssize_t bytes_read = read(*file, buf, n); - if (bytes_read == -1) { - if (errno == EINTR) { - continue; - } - throw Error("Error reading from {}: {}", path, strerror(errno)); - } - if (bytes_read == 0) { - throw Error("Error reading from {}: end of file", path); - } - writer.write(buf, bytes_read); - remain -= bytes_read; - } -} - -void -Result::Writer::write_raw_file_entry(const std::string& path, - uint32_t entry_number) -{ - const auto raw_file = get_raw_file_path(m_result_path, entry_number); - const auto old_stat = Stat::stat(raw_file); - try { - Util::clone_hard_link_or_copy_file(m_ctx, path, raw_file, true); - } catch (Error& e) { - throw Error( - "Failed to store {} as raw file {}: {}", path, raw_file, e.what()); - } - const auto new_stat = Stat::stat(raw_file); - m_ctx.counter_updates.increment( - Statistic::cache_size_kibibyte, - Util::size_change_kibibyte(old_stat, new_stat)); - m_ctx.counter_updates.increment(Statistic::files_in_cache, - (new_stat ? 1 : 0) - (old_stat ? 1 : 0)); -} - -} // namespace Result diff --git a/src/Result.hpp b/src/Result.hpp deleted file mode 100644 index 01068c9..0000000 --- a/src/Result.hpp +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "third_party/nonstd/optional.hpp" - -#include -#include -#include - -class CacheEntryReader; -class CacheEntryWriter; -class Context; - -namespace Result { - -extern const std::string k_file_suffix; -extern const uint8_t k_magic[4]; -extern const uint8_t k_version; - -extern const char* const k_unknown_file_type; - -using UnderlyingFileTypeInt = uint8_t; -enum class FileType : UnderlyingFileTypeInt { - // These values are written into the cache result file. This means they must - // never be changed or removed unless the result file version is incremented. - // Adding new values is OK. - - // The main output specified with -o or implicitly from the input filename. - object = 0, - - // Dependency file specified with -MF or implicitly from the output filename. - dependency = 1, - - // Text sent to standard output. - stderr_output = 2, - - // Coverage notes file generated by -ftest-coverage with filename in unmangled - // form, i.e. output file but with a .gcno extension. - coverage_unmangled = 3, - - // Stack usage file generated by -fstack-usage, i.e. output file but with a - // .su extension. - stackusage = 4, - - // Diagnostics output file specified by --serialize-diagnostics. - diagnostic = 5, - - // DWARF object file geenrated by -gsplit-dwarf, i.e. output file but with a - // .dwo extension. - dwarf_object = 6, - - // Coverage notes file generated by -ftest-coverage with filename in mangled - // form, i.e. full output file path but with a .gcno extension and with - // slashes replaced with hashes. - coverage_mangled = 7, -}; - -const char* file_type_to_string(FileType type); - -std::string gcno_file_in_mangled_form(const Context& ctx); -std::string gcno_file_in_unmangled_form(const Context& ctx); - -// This class knows how to read a result cache entry. -class Reader -{ -public: - Reader(const std::string& result_path); - - class Consumer - { - public: - virtual ~Consumer() = default; - - virtual void on_header(CacheEntryReader& cache_entry_reader) = 0; - virtual void on_entry_start(uint32_t entry_number, - FileType file_type, - uint64_t file_len, - nonstd::optional raw_file) = 0; - virtual void on_entry_data(const uint8_t* data, size_t size) = 0; - virtual void on_entry_end() = 0; - }; - - // Returns error message on error, otherwise nonstd::nullopt. - nonstd::optional read(Consumer& consumer); - -private: - const std::string m_result_path; - - bool read_result(Consumer& consumer); - void read_entry(CacheEntryReader& cache_entry_reader, - uint32_t entry_number, - Reader::Consumer& consumer); -}; - -// This class knows how to write a result cache entry. -class Writer -{ -public: - Writer(Context& ctx, const std::string& result_path); - - // Register a file to include in the result. Does not throw. - void write(FileType file_type, const std::string& file_path); - - // Write registered files to the result. Returns an error message on error. - nonstd::optional finalize(); - -private: - Context& m_ctx; - const std::string m_result_path; - std::vector> m_entries_to_write; - - void do_finalize(); - static void write_embedded_file_entry(CacheEntryWriter& writer, - const std::string& path, - uint64_t file_size); - void write_raw_file_entry(const std::string& path, uint32_t entry_number); -}; - -} // namespace Result diff --git a/src/ResultDumper.cpp b/src/ResultDumper.cpp deleted file mode 100644 index 1e3bac4..0000000 --- a/src/ResultDumper.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "ResultDumper.hpp" - -#include "CacheEntryReader.hpp" -#include "Context.hpp" -#include "Logging.hpp" -#include "fmtmacros.hpp" - -using nonstd::optional; - -ResultDumper::ResultDumper(FILE* stream) : m_stream(stream) -{ -} - -void -ResultDumper::on_header(CacheEntryReader& cache_entry_reader) -{ - cache_entry_reader.dump_header(m_stream); -} - -void -ResultDumper::on_entry_start(uint32_t entry_number, - Result::FileType file_type, - uint64_t file_len, - optional raw_file) -{ - PRINT(m_stream, - "{} file #{}: {} ({} bytes)\n", - raw_file ? "Raw" : "Embedded", - entry_number, - Result::file_type_to_string(file_type), - file_len); -} - -void -ResultDumper::on_entry_data(const uint8_t* /*data*/, size_t /*size*/) -{ -} - -void -ResultDumper::on_entry_end() -{ -} diff --git a/src/ResultDumper.hpp b/src/ResultDumper.hpp deleted file mode 100644 index 85602d6..0000000 --- a/src/ResultDumper.hpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Result.hpp" - -// This class dumps information about the result entry to `stream`. -class ResultDumper : public Result::Reader::Consumer -{ -public: - ResultDumper(FILE* stream); - - void on_header(CacheEntryReader& cache_entry_reader) override; - void on_entry_start(uint32_t entry_number, - Result::FileType file_type, - uint64_t file_len, - nonstd::optional raw_file) override; - void on_entry_data(const uint8_t* data, size_t size) override; - void on_entry_end() override; - -private: - FILE* m_stream; -}; diff --git a/src/ResultExtractor.cpp b/src/ResultExtractor.cpp deleted file mode 100644 index e18ae51..0000000 --- a/src/ResultExtractor.cpp +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "ResultExtractor.hpp" - -#include "Util.hpp" -#include "fmtmacros.hpp" - -ResultExtractor::ResultExtractor(const std::string& directory) - : m_directory(directory) -{ -} - -void -ResultExtractor::on_header(CacheEntryReader& /*cache_entry_reader*/) -{ -} - -void -ResultExtractor::on_entry_start(uint32_t /*entry_number*/, - Result::FileType file_type, - uint64_t /*file_len*/, - nonstd::optional raw_file) -{ - std::string suffix = Result::file_type_to_string(file_type); - if (suffix == Result::k_unknown_file_type) { - suffix = FMT(".type_{}", file_type); - } else if (suffix[0] == '<') { - suffix[0] = '.'; - suffix.resize(suffix.length() - 1); - } - - m_dest_path = FMT("{}/ccache-result{}", m_directory, suffix); - - if (!raw_file) { - m_dest_fd = Fd( - open(m_dest_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666)); - if (!m_dest_fd) { - throw Error( - "Failed to open {} for writing: {}", m_dest_path, strerror(errno)); - } - } else { - try { - Util::copy_file(*raw_file, m_dest_path, false); - } catch (Error& e) { - throw Error( - "Failed to copy {} to {}: {}", *raw_file, m_dest_path, e.what()); - } - } -} - -void -ResultExtractor::on_entry_data(const uint8_t* data, size_t size) -{ - ASSERT(m_dest_fd); - - try { - Util::write_fd(*m_dest_fd, data, size); - } catch (Error& e) { - throw Error("Failed to write to {}: {}", m_dest_path, e.what()); - } -} - -void -ResultExtractor::on_entry_end() -{ - if (m_dest_fd) { - m_dest_fd.close(); - } -} diff --git a/src/ResultExtractor.hpp b/src/ResultExtractor.hpp deleted file mode 100644 index f4b8f3b..0000000 --- a/src/ResultExtractor.hpp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Fd.hpp" -#include "Result.hpp" - -class Context; - -// This class extracts the parts of a result entry to a directory. -class ResultExtractor : public Result::Reader::Consumer -{ -public: - ResultExtractor(const std::string& directory); - - void on_header(CacheEntryReader& cache_entry_reader) override; - void on_entry_start(uint32_t entry_number, - Result::FileType file_type, - uint64_t file_len, - nonstd::optional raw_file) override; - void on_entry_data(const uint8_t* data, size_t size) override; - void on_entry_end() override; - -private: - const std::string m_directory; - Fd m_dest_fd; - std::string m_dest_path; -}; diff --git a/src/ResultRetriever.cpp b/src/ResultRetriever.cpp deleted file mode 100644 index 0218877..0000000 --- a/src/ResultRetriever.cpp +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright (C) 2020-2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "ResultRetriever.hpp" - -#include "Context.hpp" -#include "Depfile.hpp" -#include "Logging.hpp" - -using Result::FileType; - -ResultRetriever::ResultRetriever(Context& ctx, bool rewrite_dependency_target) - : m_ctx(ctx), - m_rewrite_dependency_target(rewrite_dependency_target) -{ -} - -void -ResultRetriever::on_header(CacheEntryReader& /*cache_entry_reader*/) -{ -} - -void -ResultRetriever::on_entry_start(uint32_t entry_number, - FileType file_type, - uint64_t file_len, - nonstd::optional raw_file) -{ - LOG("Reading {} entry #{} {} ({} bytes)", - raw_file ? "raw" : "embedded", - entry_number, - Result::file_type_to_string(file_type), - file_len); - - std::string dest_path; - m_dest_file_type = file_type; - - switch (file_type) { - case FileType::object: - dest_path = m_ctx.args_info.output_obj; - break; - - case FileType::dependency: - // Dependency file: Open destination file but accumulate data in m_dest_data - // and write it in on_entry_end. - if (m_ctx.args_info.generating_dependencies) { - dest_path = m_ctx.args_info.output_dep; - m_dest_data.reserve(file_len); - } - break; - - case FileType::stderr_output: - // Stderr data: Don't open a destination file. Instead accumulate it in - // m_dest_data and write it in on_entry_end. - m_dest_data.reserve(file_len); - break; - - case FileType::coverage_unmangled: - if (m_ctx.args_info.generating_coverage) { - dest_path = Util::change_extension(m_ctx.args_info.output_obj, ".gcno"); - } - break; - - case FileType::stackusage: - if (m_ctx.args_info.generating_stackusage) { - dest_path = m_ctx.args_info.output_su; - } - break; - - case FileType::diagnostic: - if (m_ctx.args_info.generating_diagnostics) { - dest_path = m_ctx.args_info.output_dia; - } - break; - - case FileType::dwarf_object: - if (m_ctx.args_info.seen_split_dwarf - && m_ctx.args_info.output_obj != "/dev/null") { - dest_path = m_ctx.args_info.output_dwo; - } - break; - - case FileType::coverage_mangled: - if (m_ctx.args_info.generating_coverage) { - dest_path = Result::gcno_file_in_mangled_form(m_ctx); - } - break; - } - - if (file_type == FileType::stderr_output) { - // Written in on_entry_end. - } else if (dest_path.empty()) { - LOG_RAW("Not writing"); - } else if (dest_path == "/dev/null") { - LOG_RAW("Not writing to /dev/null"); - } else if (raw_file) { - Util::clone_hard_link_or_copy_file(m_ctx, *raw_file, dest_path, false); - - // Update modification timestamp to save the file from LRU cleanup (and, if - // hard-linked, to make the object file newer than the source file). - Util::update_mtime(*raw_file); - } else { - LOG("Writing to {}", dest_path); - m_dest_fd = Fd( - open(dest_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666)); - if (!m_dest_fd) { - throw Error( - "Failed to open {} for writing: {}", dest_path, strerror(errno)); - } - m_dest_path = dest_path; - } -} - -void -ResultRetriever::on_entry_data(const uint8_t* data, size_t size) -{ - ASSERT(!(m_dest_file_type == FileType::stderr_output && m_dest_fd)); - - if (m_dest_file_type == FileType::stderr_output - || (m_dest_file_type == FileType::dependency && !m_dest_path.empty())) { - m_dest_data.append(reinterpret_cast(data), size); - } else if (m_dest_fd) { - try { - Util::write_fd(*m_dest_fd, data, size); - } catch (Error& e) { - throw Error("Failed to write to {}: {}", m_dest_path, e.what()); - } - } -} - -void -ResultRetriever::on_entry_end() -{ - if (m_dest_file_type == FileType::stderr_output) { - LOG("Writing to file descriptor {}", STDERR_FILENO); - Util::send_to_stderr(m_ctx, m_dest_data); - } else if (m_dest_file_type == FileType::dependency && !m_dest_path.empty()) { - write_dependency_file(); - } - - if (m_dest_fd) { - m_dest_fd.close(); - } - m_dest_path.clear(); - m_dest_data.clear(); -} - -void -ResultRetriever::write_dependency_file() -{ - try { - size_t start_pos = 0; - if (m_rewrite_dependency_target) { - size_t colon_pos = m_dest_data.find(':'); - if (colon_pos != std::string::npos) { - const auto escaped_output_obj = - Depfile::escape_filename(m_ctx.args_info.output_obj); - Util::write_fd( - *m_dest_fd, escaped_output_obj.data(), escaped_output_obj.length()); - start_pos = colon_pos; - } - } - - Util::write_fd(*m_dest_fd, - m_dest_data.data() + start_pos, - m_dest_data.length() - start_pos); - } catch (Error& e) { - throw Error("Failed to write to {}: {}", m_dest_path, e.what()); - } -} diff --git a/src/ResultRetriever.hpp b/src/ResultRetriever.hpp deleted file mode 100644 index 609f15f..0000000 --- a/src/ResultRetriever.hpp +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Fd.hpp" -#include "Result.hpp" - -class Context; - -// This class retrieves a result entry to the local file system. -class ResultRetriever : public Result::Reader::Consumer -{ -public: - ResultRetriever(Context& ctx, bool rewrite_dependency_target); - - void on_header(CacheEntryReader& cache_entry_reader) override; - void on_entry_start(uint32_t entry_number, - Result::FileType file_type, - uint64_t file_len, - nonstd::optional raw_file) override; - void on_entry_data(const uint8_t* data, size_t size) override; - void on_entry_end() override; - -private: - Context& m_ctx; - Result::FileType m_dest_file_type; - Fd m_dest_fd; - std::string m_dest_path; - - // Collects the full data of stderr output (since we want to potentially strip - // color codes which could span chunk boundaries) or dependency data (since we - // potentially want to rewrite the dependency target which in theory can span - // a chunk boundary). - std::string m_dest_data; - - // Whether to rewrite the first part of the dependency file data to the - // destination object file. - const bool m_rewrite_dependency_target; - - void write_dependency_file(); -}; diff --git a/src/SignalHandler.cpp b/src/SignalHandler.cpp index 8a07cc7..2d32ac6 100644 --- a/src/SignalHandler.cpp +++ b/src/SignalHandler.cpp @@ -24,6 +24,9 @@ # include "assertions.hpp" # include // NOLINT: sigaddset et al are defined in signal.h +# include +# include +# include namespace { @@ -68,6 +71,8 @@ SignalHandler::SignalHandler(Context& ctx) : m_ctx(ctx) # ifdef SIGQUIT register_signal_handler(SIGQUIT); # endif + + signal(SIGPIPE, SIG_IGN); // NOLINT: This is no error, clang-tidy } SignalHandler::~SignalHandler() diff --git a/src/SignalHandler.hpp b/src/SignalHandler.hpp index 3ef8847..603dbe2 100644 --- a/src/SignalHandler.hpp +++ b/src/SignalHandler.hpp @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - class Context; class SignalHandler diff --git a/src/Sloppiness.hpp b/src/Sloppiness.hpp deleted file mode 100644 index 62c0508..0000000 --- a/src/Sloppiness.hpp +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (C) 2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -enum Sloppiness { - SLOPPY_INCLUDE_FILE_MTIME = 1 << 0, - SLOPPY_INCLUDE_FILE_CTIME = 1 << 1, - SLOPPY_TIME_MACROS = 1 << 2, - SLOPPY_PCH_DEFINES = 1 << 3, - // Allow us to match files based on their stats (size, mtime, ctime), without - // looking at their contents. - SLOPPY_FILE_STAT_MATCHES = 1 << 4, - // Allow us to not include any system headers in the manifest include files, - // similar to -MM versus -M for dependencies. - SLOPPY_SYSTEM_HEADERS = 1 << 5, - // Allow us to ignore ctimes when comparing file stats, so we can fake mtimes - // if we want to (it is much harder to fake ctimes, requires changing clock) - SLOPPY_FILE_STAT_MATCHES_CTIME = 1 << 6, - // Allow us to not include the -index-store-path option in the manifest hash. - SLOPPY_CLANG_INDEX_STORE = 1 << 7, - // Ignore locale settings. - SLOPPY_LOCALE = 1 << 8, - // Allow caching even if -fmodules is used. - SLOPPY_MODULES = 1 << 9, - // Ignore virtual file system (VFS) overlay file. - SLOPPY_IVFSOVERLAY = 1 << 10, -}; diff --git a/src/Stat.cpp b/src/Stat.cpp index 244fd6f..356bd92 100644 --- a/src/Stat.cpp +++ b/src/Stat.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,14 +18,17 @@ #include "Stat.hpp" -#ifdef _WIN32 -# include "Win32Util.hpp" - -# include "third_party/win32/winerror_to_errno.h" -#endif - #include "Finalizer.hpp" #include "Logging.hpp" +#include "Win32Util.hpp" + +#include +#include +#include + +#ifdef _WIN32 +# include +#endif namespace { @@ -210,7 +213,7 @@ Stat::Stat(StatFunction stat_function, } else { m_errno = errno; if (on_error == OnError::throw_error) { - throw Error("failed to stat {}: {}", path, strerror(errno)); + throw core::Error(FMT("failed to stat {}: {}", path, strerror(errno))); } if (on_error == OnError::log) { LOG("Failed to stat {}: {}", path, strerror(errno)); diff --git a/src/Stat.hpp b/src/Stat.hpp index 5a046d2..5c81a5b 100644 --- a/src/Stat.hpp +++ b/src/Stat.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,12 +18,46 @@ #pragma once -#include "system.hpp" +#include +#include -#include "exceptions.hpp" +#include +#include +#include +#include #include +#ifdef _WIN32 +# ifndef S_IFIFO +# define S_IFIFO 0x1000 +# endif +# ifndef S_IFBLK +# define S_IFBLK 0x6000 +# endif +# ifndef S_IFLNK +# define S_IFLNK 0xA000 +# endif +# ifndef S_ISREG +# define S_ISREG(m) (((m)&S_IFMT) == S_IFREG) +# endif +# ifndef S_ISDIR +# define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR) +# endif +# ifndef S_ISFIFO +# define S_ISFIFO(m) (((m)&S_IFMT) == S_IFIFO) +# endif +# ifndef S_ISCHR +# define S_ISCHR(m) (((m)&S_IFMT) == S_IFCHR) +# endif +# ifndef S_ISLNK +# define S_ISLNK(m) (((m)&S_IFMT) == S_IFLNK) +# endif +# ifndef S_ISBLK +# define S_ISBLK(m) (((m)&S_IFMT) == S_IFBLK) +# endif +#endif + class Stat { public: @@ -53,9 +87,7 @@ public: uint32_t st_reparse_tag; }; #else - // Use of typedef needed to suppress a spurious 'declaration does not declare - // anything' warning in old GCC. - typedef struct ::stat stat_t; // NOLINT(modernize-use-using) + using stat_t = struct stat; #endif using dev_t = decltype(stat_t{}.st_dev); @@ -94,8 +126,9 @@ public: dev_t device() const; ino_t inode() const; mode_t mode() const; - time_t ctime() const; - time_t mtime() const; + util::TimePoint atime() const; + util::TimePoint ctime() const; + util::TimePoint mtime() const; uint64_t size() const; uint64_t size_on_disk() const; @@ -109,9 +142,6 @@ public: uint32_t reparse_tag() const; #endif - timespec ctim() const; - timespec mtim() const; - protected: using StatFunction = int (*)(const char*, stat_t*); @@ -137,7 +167,7 @@ inline Stat::operator bool() const inline bool Stat::same_inode_as(const Stat& other) const { - return device() == other.device() && inode() == other.inode(); + return m_errno == 0 && device() == other.device() && inode() == other.inode(); } inline int @@ -164,16 +194,40 @@ Stat::mode() const return m_stat.st_mode; } -inline time_t +inline util::TimePoint +Stat::atime() const +{ +#if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_ATIM) + return util::TimePoint(m_stat.st_atim); +#elif defined(HAVE_STRUCT_STAT_ST_ATIMESPEC) + return util::TimePoint(m_stat.st_atimespec); +#else + return util::TimePoint(m_stat.st_atime, 0); +#endif +} + +inline util::TimePoint Stat::ctime() const { - return ctim().tv_sec; +#if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_CTIM) + return util::TimePoint(m_stat.st_ctim); +#elif defined(HAVE_STRUCT_STAT_ST_CTIMESPEC) + return util::TimePoint(m_stat.st_ctimespec); +#else + return util::TimePoint(m_stat.st_ctime, 0); +#endif } -inline time_t +inline util::TimePoint Stat::mtime() const { - return mtim().tv_sec; +#if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_MTIM) + return util::TimePoint(m_stat.st_mtim); +#elif defined(HAVE_STRUCT_STAT_ST_CTIMESPEC) + return util::TimePoint(m_stat.st_mtimespec); +#else + return util::TimePoint(m_stat.st_mtime, 0); +#endif } inline uint64_t @@ -223,23 +277,3 @@ Stat::reparse_tag() const return m_stat.st_reparse_tag; } #endif - -inline timespec -Stat::ctim() const -{ -#if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_CTIM) - return m_stat.st_ctim; -#else - return {m_stat.st_ctime, 0}; -#endif -} - -inline timespec -Stat::mtim() const -{ -#if defined(_WIN32) || defined(HAVE_STRUCT_STAT_ST_MTIM) - return m_stat.st_mtim; -#else - return {m_stat.st_mtime, 0}; -#endif -} diff --git a/src/Statistic.hpp b/src/Statistic.hpp deleted file mode 100644 index cd6cda6..0000000 --- a/src/Statistic.hpp +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (C) 2021 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -// Statistics fields in storage order. -enum class Statistic { - none = 0, - compiler_produced_stdout = 1, - compile_failed = 2, - internal_error = 3, - cache_miss = 4, - preprocessor_error = 5, - could_not_find_compiler = 6, - missing_cache_file = 7, - preprocessed_cache_hit = 8, - bad_compiler_arguments = 9, - called_for_link = 10, - files_in_cache = 11, - cache_size_kibibyte = 12, - obsolete_max_files = 13, - obsolete_max_size = 14, - unsupported_source_language = 15, - bad_output_file = 16, - no_input_file = 17, - multiple_source_files = 18, - autoconf_test = 19, - unsupported_compiler_option = 20, - output_to_stdout = 21, - direct_cache_hit = 22, - compiler_produced_no_output = 23, - compiler_produced_empty_output = 24, - error_hashing_extra_file = 25, - compiler_check_failed = 26, - could_not_use_precompiled_header = 27, - called_for_preprocessing = 28, - cleanups_performed = 29, - unsupported_code_directive = 30, - stats_zeroed_timestamp = 31, - could_not_use_modules = 32, - - END -}; diff --git a/src/Statistics.cpp b/src/Statistics.cpp deleted file mode 100644 index 08ad892..0000000 --- a/src/Statistics.cpp +++ /dev/null @@ -1,355 +0,0 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "Statistics.hpp" - -#include "AtomicFile.hpp" -#include "Config.hpp" -#include "Lockfile.hpp" -#include "Logging.hpp" -#include "Util.hpp" -#include "exceptions.hpp" -#include "fmtmacros.hpp" - -const unsigned FLAG_NOZERO = 1; // don't zero with the -z option -const unsigned FLAG_ALWAYS = 2; // always show, even if zero -const unsigned FLAG_NEVER = 4; // never show - -using nonstd::nullopt; -using nonstd::optional; - -// Returns a formatted version of a statistics value, or the empty string if the -// statistics line shouldn't be printed. -using FormatFunction = std::string (*)(uint64_t value); - -static std::string -format_size(uint64_t size) -{ - return FMT("{:>11}", Util::format_human_readable_size(size)); -} - -static std::string -format_size_times_1024(uint64_t size) -{ - return format_size(size * 1024); -} - -static std::string -format_timestamp(uint64_t timestamp) -{ - if (timestamp > 0) { - const auto tm = Util::localtime(timestamp); - char buffer[100] = "?"; - if (tm) { - strftime(buffer, sizeof(buffer), "%c", &*tm); - } - return std::string(" ") + buffer; - } else { - return {}; - } -} - -static double -hit_rate(const Counters& counters) -{ - const uint64_t direct = counters.get(Statistic::direct_cache_hit); - const uint64_t preprocessed = counters.get(Statistic::preprocessed_cache_hit); - const uint64_t hit = direct + preprocessed; - const uint64_t miss = counters.get(Statistic::cache_miss); - const uint64_t total = hit + miss; - return total > 0 ? (100.0 * hit) / total : 0.0; -} - -static void -for_each_level_1_and_2_stats_file( - const std::string& cache_dir, - const std::function function) -{ - for (size_t level_1 = 0; level_1 <= 0xF; ++level_1) { - function(FMT("{}/{:x}/stats", cache_dir, level_1)); - for (size_t level_2 = 0; level_2 <= 0xF; ++level_2) { - function(FMT("{}/{:x}/{:x}/stats", cache_dir, level_1, level_2)); - } - } -} - -static std::pair -collect_counters(const Config& config) -{ - Counters counters; - uint64_t zero_timestamp = 0; - time_t last_updated = 0; - - // Add up the stats in each directory. - for_each_level_1_and_2_stats_file( - config.cache_dir(), [&](const std::string& path) { - counters.set(Statistic::stats_zeroed_timestamp, 0); // Don't add - counters.increment(Statistics::read(path)); - zero_timestamp = std::max(counters.get(Statistic::stats_zeroed_timestamp), - zero_timestamp); - last_updated = std::max(last_updated, Stat::stat(path).mtime()); - }); - - counters.set(Statistic::stats_zeroed_timestamp, zero_timestamp); - return std::make_pair(counters, last_updated); -} - -namespace { - -struct StatisticsField -{ - StatisticsField(Statistic statistic_, - const char* id_, - const char* message_, - unsigned flags_ = 0, - FormatFunction format_ = nullptr) - : statistic(statistic_), - id(id_), - message(message_), - flags(flags_), - format(format_) - { - } - - const Statistic statistic; - const char* const id; // for --print-stats - const char* const message; // for --show-stats - const unsigned flags; // bitmask of FLAG_* values - const FormatFunction format; // nullptr -> use plain integer format -}; - -} // namespace - -#define STATISTICS_FIELD(id, ...) \ - { \ - Statistic::id, #id, __VA_ARGS__ \ - } - -// Statistics fields in display order. -const StatisticsField k_statistics_fields[] = { - STATISTICS_FIELD( - stats_zeroed_timestamp, "stats zeroed", FLAG_ALWAYS, format_timestamp), - STATISTICS_FIELD(direct_cache_hit, "cache hit (direct)", FLAG_ALWAYS), - STATISTICS_FIELD( - preprocessed_cache_hit, "cache hit (preprocessed)", FLAG_ALWAYS), - STATISTICS_FIELD(cache_miss, "cache miss", FLAG_ALWAYS), - STATISTICS_FIELD(called_for_link, "called for link"), - STATISTICS_FIELD(called_for_preprocessing, "called for preprocessing"), - STATISTICS_FIELD(multiple_source_files, "multiple source files"), - STATISTICS_FIELD(compiler_produced_stdout, "compiler produced stdout"), - STATISTICS_FIELD(compiler_produced_no_output, "compiler produced no output"), - STATISTICS_FIELD(compiler_produced_empty_output, - "compiler produced empty output"), - STATISTICS_FIELD(compile_failed, "compile failed"), - STATISTICS_FIELD(internal_error, "ccache internal error"), - STATISTICS_FIELD(preprocessor_error, "preprocessor error"), - STATISTICS_FIELD(could_not_use_precompiled_header, - "can't use precompiled header"), - STATISTICS_FIELD(could_not_use_modules, "can't use modules"), - STATISTICS_FIELD(could_not_find_compiler, "couldn't find the compiler"), - STATISTICS_FIELD(missing_cache_file, "cache file missing"), - STATISTICS_FIELD(bad_compiler_arguments, "bad compiler arguments"), - STATISTICS_FIELD(unsupported_source_language, "unsupported source language"), - STATISTICS_FIELD(compiler_check_failed, "compiler check failed"), - STATISTICS_FIELD(autoconf_test, "autoconf compile/link"), - STATISTICS_FIELD(unsupported_compiler_option, "unsupported compiler option"), - STATISTICS_FIELD(unsupported_code_directive, "unsupported code directive"), - STATISTICS_FIELD(output_to_stdout, "output to stdout"), - STATISTICS_FIELD(bad_output_file, "could not write to output file"), - STATISTICS_FIELD(no_input_file, "no input file"), - STATISTICS_FIELD(error_hashing_extra_file, "error hashing extra file"), - STATISTICS_FIELD(cleanups_performed, "cleanups performed", FLAG_ALWAYS), - STATISTICS_FIELD(files_in_cache, "files in cache", FLAG_NOZERO | FLAG_ALWAYS), - STATISTICS_FIELD(cache_size_kibibyte, - "cache size", - FLAG_NOZERO | FLAG_ALWAYS, - format_size_times_1024), - STATISTICS_FIELD(obsolete_max_files, "OBSOLETE", FLAG_NOZERO | FLAG_NEVER), - STATISTICS_FIELD(obsolete_max_size, "OBSOLETE", FLAG_NOZERO | FLAG_NEVER), - STATISTICS_FIELD(none, nullptr), -}; - -namespace Statistics { - -Counters -read(const std::string& path) -{ - Counters counters; - - std::string data; - try { - data = Util::read_file(path); - } catch (const Error&) { - // Ignore. - return counters; - } - - size_t i = 0; - const char* str = data.c_str(); - while (true) { - char* end; - const uint64_t value = std::strtoull(str, &end, 10); - if (end == str) { - break; - } - counters.set_raw(i, value); - ++i; - str = end; - } - - return counters; -} - -optional -update(const std::string& path, - std::function function) -{ - Lockfile lock(path); - if (!lock.acquired()) { - LOG("Failed to acquire lock for {}", path); - return nullopt; - } - - auto counters = Statistics::read(path); - function(counters); - - AtomicFile file(path, AtomicFile::Mode::text); - for (size_t i = 0; i < counters.size(); ++i) { - file.write(FMT("{}\n", counters.get_raw(i))); - } - try { - file.commit(); - } catch (const Error& e) { - // Make failure to write a stats file a soft error since it's not - // important enough to fail whole the process and also because it is - // called in the Context destructor. - LOG("Error: {}", e.what()); - } - - return counters; -} - -optional -get_result(const Counters& counters) -{ - for (const auto& field : k_statistics_fields) { - if (counters.get(field.statistic) != 0 && !(field.flags & FLAG_NOZERO)) { - return field.message; - } - } - return nullopt; -} - -void -zero_all_counters(const Config& config) -{ - const time_t timestamp = time(nullptr); - - for_each_level_1_and_2_stats_file( - config.cache_dir(), [=](const std::string& path) { - Statistics::update(path, [=](Counters& cs) { - for (size_t i = 0; k_statistics_fields[i].message; ++i) { - if (!(k_statistics_fields[i].flags & FLAG_NOZERO)) { - cs.set(k_statistics_fields[i].statistic, 0); - } - } - cs.set(Statistic::stats_zeroed_timestamp, timestamp); - }); - }); -} - -std::string -format_human_readable(const Config& config) -{ - Counters counters; - time_t last_updated; - std::tie(counters, last_updated) = collect_counters(config); - std::string result; - - result += FMT("{:36}{}\n", "cache directory", config.cache_dir()); - result += FMT("{:36}{}\n", "primary config", config.primary_config_path()); - result += FMT( - "{:36}{}\n", "secondary config (readonly)", config.secondary_config_path()); - if (last_updated > 0) { - const auto tm = Util::localtime(last_updated); - char timestamp[100] = "?"; - if (tm) { - strftime(timestamp, sizeof(timestamp), "%c", &*tm); - } - result += FMT("{:36}{}\n", "stats updated", timestamp); - } - - // ...and display them. - for (size_t i = 0; k_statistics_fields[i].message; i++) { - const Statistic statistic = k_statistics_fields[i].statistic; - - if (k_statistics_fields[i].flags & FLAG_NEVER) { - continue; - } - if (counters.get(statistic) == 0 - && !(k_statistics_fields[i].flags & FLAG_ALWAYS)) { - continue; - } - - const std::string value = - k_statistics_fields[i].format - ? k_statistics_fields[i].format(counters.get(statistic)) - : FMT("{:8}", counters.get(statistic)); - if (!value.empty()) { - result += FMT("{:32}{}\n", k_statistics_fields[i].message, value); - } - - if (statistic == Statistic::cache_miss) { - double percent = hit_rate(counters); - result += FMT("{:34}{:6.2f} %\n", "cache hit rate", percent); - } - } - - if (config.max_files() != 0) { - result += FMT("{:32}{:8}\n", "max files", config.max_files()); - } - if (config.max_size() != 0) { - result += - FMT("{:32}{}\n", "max cache size", format_size(config.max_size())); - } - - return result; -} - -std::string -format_machine_readable(const Config& config) -{ - Counters counters; - time_t last_updated; - std::tie(counters, last_updated) = collect_counters(config); - std::string result; - - result += FMT("stats_updated_timestamp\t{}\n", last_updated); - - for (size_t i = 0; k_statistics_fields[i].message; i++) { - if (!(k_statistics_fields[i].flags & FLAG_NEVER)) { - result += FMT("{}\t{}\n", - k_statistics_fields[i].id, - counters.get(k_statistics_fields[i].statistic)); - } - } - - return result; -} - -} // namespace Statistics diff --git a/src/Statistics.hpp b/src/Statistics.hpp deleted file mode 100644 index 34a9982..0000000 --- a/src/Statistics.hpp +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Counters.hpp" -#include "Statistic.hpp" // Any reasonable use of Statistics requires the Statistic enum. - -#include "third_party/nonstd/optional.hpp" - -#include -#include - -class Config; - -namespace Statistics { - -// Read counters from `path`. No lock is acquired. -Counters read(const std::string& path); - -// Acquire a lock, read counters from `path`, call `function` with the counters, -// write the counters to `path` and release the lock. Returns the resulting -// counters or nullopt on error (e.g. if the lock could not be acquired). -nonstd::optional update(const std::string& path, - std::function); - -// Return a human-readable string representing the final ccache result, or -// nullopt if there was no result. -nonstd::optional get_result(const Counters& counters); - -// Zero all statistics counters except those tracking cache size and number of -// files in the cache. -void zero_all_counters(const Config& config); - -// Format cache statistics in human-readable format. -std::string format_human_readable(const Config& config); - -// Format cache statistics in machine-readable format. -std::string format_machine_readable(const Config& config); - -} // namespace Statistics diff --git a/src/StdMakeUnique.hpp b/src/StdMakeUnique.hpp deleted file mode 100644 index 829d2d5..0000000 --- a/src/StdMakeUnique.hpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (C) 2019 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -namespace std { - -#if __cplusplus < 201402L -template -inline unique_ptr -make_unique(TArgs&&... args) -{ - return unique_ptr(new T(std::forward(args)...)); -} -#endif - -} // namespace std diff --git a/src/TemporaryFile.cpp b/src/TemporaryFile.cpp index feaa5f1..2574812 100644 --- a/src/TemporaryFile.cpp +++ b/src/TemporaryFile.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020-2021 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -20,53 +20,48 @@ #include "Util.hpp" -#ifdef _WIN32 -# include "third_party/win32/mktemp.h" -#endif - -using nonstd::string_view; +#include +#include -namespace { +#include -#ifndef _WIN32 -mode_t -get_umask() -{ - static bool mask_retrieved = false; - static mode_t mask; - if (!mask_retrieved) { - mask = umask(0); - umask(mask); - mask_retrieved = true; - } - return mask; -} +#ifdef HAVE_UNISTD_H +# include #endif -} // namespace +#ifdef _WIN32 +# include "third_party/win32/mktemp.h" +#endif -TemporaryFile::TemporaryFile(string_view path_prefix) - : path(std::string(path_prefix) + ".XXXXXX") +TemporaryFile::TemporaryFile(std::string_view path_prefix, + std::string_view suffix) + : path(FMT("{}{}XXXXXX{}", path_prefix, tmp_file_infix, suffix)) { Util::ensure_dir_exists(Util::dir_name(path)); #ifdef _WIN32 - // MSVC lacks mkstemp() and Mingw-w64's implementation[1] is problematic, as + // MSVC lacks mkstemps() and Mingw-w64's implementation[1] is problematic, as // it can reuse the names of recently-deleted files unless the caller // remembers to call srand(). // [1]: - fd = Fd(bsd_mkstemp(&path[0])); + fd = Fd(bsd_mkstemps(&path[0], suffix.length())); #else - fd = Fd(mkstemp(&path[0])); + fd = Fd(mkstemps(&path[0], suffix.length())); #endif if (!fd) { - throw Fatal( - "Failed to create temporary file for {}: {}", path, strerror(errno)); + throw core::Fatal( + FMT("Failed to create temporary file for {}: {}", path, strerror(errno))); } Util::set_cloexec_flag(*fd); #ifndef _WIN32 - fchmod(*fd, 0666 & ~get_umask()); + fchmod(*fd, 0666 & ~Util::get_umask()); #endif } + +bool +TemporaryFile::is_tmp_file(std::string_view path) +{ + return Util::base_name(path).find(tmp_file_infix) != std::string::npos; +} diff --git a/src/TemporaryFile.hpp b/src/TemporaryFile.hpp index 0538d9a..b773c0a 100644 --- a/src/TemporaryFile.hpp +++ b/src/TemporaryFile.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -20,25 +20,26 @@ #include "Fd.hpp" -#include "third_party/nonstd/string_view.hpp" - #include +#include // This class represents a unique temporary file created by mkstemp. The file is // not deleted by the destructor. class TemporaryFile { public: + static constexpr char tmp_file_infix[] = ".tmp."; + // `path_prefix` is the base path. The resulting filename will be this path - // plus a unique suffix. If `path_prefix` refers to a nonexistent directory - // the directory will be created if possible.` - TemporaryFile(nonstd::string_view path_prefix); + // plus a unique string plus `suffix`. If `path_prefix` refers to a + // nonexistent directory the directory will be created if possible. + TemporaryFile(std::string_view path_prefix, std::string_view suffix = ".tmp"); TemporaryFile(TemporaryFile&& other) noexcept = default; - // Note: Should be declared noexcept, but since GCC 4.8 trips on it, don't do - // that for now. - TemporaryFile& operator=(TemporaryFile&& other) = default; + TemporaryFile& operator=(TemporaryFile&& other) noexcept = default; + + static bool is_tmp_file(std::string_view path); // The resulting open file descriptor in read/write mode. Unset on error. Fd fd; diff --git a/src/ThreadPool.cpp b/src/ThreadPool.cpp index a1d931a..d5da4d5 100644 --- a/src/ThreadPool.cpp +++ b/src/ThreadPool.cpp @@ -51,6 +51,10 @@ ThreadPool::shut_down() { { std::unique_lock lock(m_mutex); + if (m_shutting_down) { + // Already called shut_down. + return; + } m_shutting_down = true; } m_task_enqueued_or_shutting_down_condition.notify_all(); diff --git a/src/ThreadPool.hpp b/src/ThreadPool.hpp index 5eee381..fa46538 100644 --- a/src/ThreadPool.hpp +++ b/src/ThreadPool.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include #include #include diff --git a/src/UmaskScope.hpp b/src/UmaskScope.hpp index d96f448..d141ad3 100644 --- a/src/UmaskScope.hpp +++ b/src/UmaskScope.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,38 +18,56 @@ #pragma once -#include "system.hpp" +#include +#include -#include "third_party/nonstd/optional.hpp" +#include // This class sets a new (process-global) umask and restores the previous umask // when destructed. class UmaskScope { public: - UmaskScope(nonstd::optional new_umask); + UmaskScope(std::optional new_umask); ~UmaskScope(); + void release(); + private: - nonstd::optional m_saved_umask; + std::optional m_saved_umask = std::nullopt; }; -UmaskScope::UmaskScope(nonstd::optional new_umask) +inline UmaskScope::UmaskScope(std::optional new_umask) { #ifndef _WIN32 if (new_umask) { - m_saved_umask = umask(*new_umask); + m_saved_umask = Util::set_umask(*new_umask); } #else (void)new_umask; #endif } -UmaskScope::~UmaskScope() +inline UmaskScope::~UmaskScope() +{ + release(); +} + +inline void +UmaskScope::release() { #ifndef _WIN32 if (m_saved_umask) { - umask(*m_saved_umask); + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80635 +# if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +# endif + Util::set_umask(*m_saved_umask); +# if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic pop +# endif + m_saved_umask = std::nullopt; } #endif } diff --git a/src/Util.cpp b/src/Util.cpp index f49708f..e802a83 100644 --- a/src/Util.cpp +++ b/src/Util.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2021 Joel Rosdahl and other contributors +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -21,40 +21,38 @@ #include "Config.hpp" #include "Context.hpp" #include "Fd.hpp" -#include "FormatNonstdStringView.hpp" #include "Logging.hpp" #include "TemporaryFile.hpp" -#include "fmtmacros.hpp" +#include "Win32Util.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include // NOLINT: PATH_MAX is defined in limits.h extern "C" { #include "third_party/base32hex.h" } -#include -#include - -#ifndef HAVE_DIRENT_H -# include -#endif - -#ifdef HAVE_PWD_H -# include +#ifdef HAVE_DIRENT_H +# include #endif -#ifdef HAVE_SYS_TIME_H -# include +#ifdef HAVE_UNISTD_H +# include #endif -#ifdef HAVE_LINUX_FS_H -# include -# include -#elif defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) -# include -# include -#endif +#include -#ifdef _WIN32 -# include "Win32Util.hpp" +#ifdef HAVE_PWD_H +# include #endif #ifdef __linux__ @@ -79,12 +77,17 @@ extern "C" { # endif #endif -using nonstd::nullopt; -using nonstd::optional; -using nonstd::string_view; +using IncludeDelimiter = util::Tokenizer::IncludeDelimiter; namespace { +// Process umask, read and written by get_umask and set_umask. +mode_t g_umask = [] { + const mode_t mask = umask(0); + umask(mask); + return mask; +}(); + // Search for the first match of the following regular expression: // // \x1b\[[\x30-\x3f]*[\x20-\x2f]*[Km] @@ -92,8 +95,8 @@ namespace { // The primary reason for not using std::regex is that it's not available for // GCC 4.8. It's also a bit bloated. The reason for not using POSIX regex // functionality is that it's are not available in MinGW. -string_view -find_first_ansi_csi_seq(string_view string) +std::string_view +find_first_ansi_csi_seq(std::string_view string) { size_t pos = 0; while (pos < string.length() && string[pos] != 0x1b) { @@ -136,43 +139,38 @@ path_max(const std::string& path) template std::vector -split_at(string_view input, const char* separators) -{ - ASSERT(separators != nullptr && separators[0] != '\0'); +split_into(std::string_view string, + const char* separators, + util::Tokenizer::Mode mode, + IncludeDelimiter include_delimiter) +{ std::vector result; - - size_t start = 0; - while (start < input.size()) { - size_t end = input.find_first_of(separators, start); - - if (end == string_view::npos) { - result.emplace_back(input.data() + start, input.size() - start); - break; - } else if (start != end) { - result.emplace_back(input.data() + start, end - start); - } - - start = end + 1; + for (const auto token : + util::Tokenizer(string, separators, mode, include_delimiter)) { + result.emplace_back(token); } - return result; } std::string -rewrite_stderr_to_absolute_paths(string_view text) +rewrite_stderr_to_absolute_paths(std::string_view text) { static const std::string in_file_included_from = "In file included from "; std::string result; - for (auto line : Util::split_into_views(text, "\n")) { + using util::Tokenizer; + for (auto line : Tokenizer(text, + "\n", + Tokenizer::Mode::include_empty, + Tokenizer::IncludeDelimiter::yes)) { // Rewrite to in the following two cases, where X may // be optional ANSI CSI sequences: // // In file included from XX:1: // XX:1:2: ... - if (Util::starts_with(line, in_file_included_from)) { + if (util::starts_with(line, in_file_included_from)) { result += in_file_included_from; line = line.substr(in_file_included_from.length()); } @@ -182,7 +180,7 @@ rewrite_stderr_to_absolute_paths(string_view text) line = line.substr(csi_seq.length()); } size_t path_end = line.find(':'); - if (path_end == string_view::npos) { + if (path_end == std::string_view::npos) { result.append(line.data(), line.length()); } else { std::string path(line.substr(0, path_end)); @@ -194,7 +192,6 @@ rewrite_stderr_to_absolute_paths(string_view text) result.append(line.data(), line.length()); } } - result += '\n'; } return result; } @@ -203,8 +200,8 @@ rewrite_stderr_to_absolute_paths(string_view text) namespace Util { -string_view -base_name(string_view path) +std::string_view +base_name(std::string_view path) { #ifdef _WIN32 const char delim[] = "/\\"; @@ -216,9 +213,9 @@ base_name(string_view path) } std::string -change_extension(string_view path, string_view new_ext) +change_extension(std::string_view path, std::string_view new_ext) { - string_view without_ext = Util::remove_extension(path); + std::string_view without_ext = Util::remove_extension(path); return std::string(without_ext).append(new_ext.data(), new_ext.length()); } @@ -229,7 +226,7 @@ clone_file(const std::string& src, const std::string& dest, bool via_tmp_file) # if defined(__linux__) Fd src_fd(open(src.c_str(), O_RDONLY)); if (!src_fd) { - throw Error("{}: {}", src, strerror(errno)); + throw core::Error(FMT("{}: {}", src, strerror(errno))); } Fd dest_fd; @@ -242,12 +239,12 @@ clone_file(const std::string& src, const std::string& dest, bool via_tmp_file) dest_fd = Fd(open(dest.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666)); if (!dest_fd) { - throw Error("{}: {}", src, strerror(errno)); + throw core::Error(FMT("{}: {}", src, strerror(errno))); } } if (ioctl(*dest_fd, FICLONE, *src_fd) != 0) { - throw Error(strerror(errno)); + throw core::Error(strerror(errno)); } dest_fd.close(); @@ -259,46 +256,48 @@ clone_file(const std::string& src, const std::string& dest, bool via_tmp_file) # elif defined(__APPLE__) (void)via_tmp_file; if (clonefile(src.c_str(), dest.c_str(), CLONE_NOOWNERCOPY) != 0) { - throw Error(strerror(errno)); + throw core::Error(strerror(errno)); } # else (void)src; (void)dest; (void)via_tmp_file; - throw Error(strerror(EOPNOTSUPP)); + throw core::Error(strerror(EOPNOTSUPP)); # endif } #endif // FILE_CLONING_SUPPORTED void -clone_hard_link_or_copy_file(const Context& ctx, +clone_hard_link_or_copy_file(const Config& config, const std::string& source, const std::string& dest, bool via_tmp_file) { - if (ctx.config.file_clone()) { + if (config.file_clone()) { #ifdef FILE_CLONING_SUPPORTED LOG("Cloning {} to {}", source, dest); try { clone_file(source, dest, via_tmp_file); return; - } catch (Error& e) { + } catch (core::Error& e) { LOG("Failed to clone: {}", e.what()); } #else LOG("Not cloning {} to {} since it's unsupported", source, dest); #endif } - if (ctx.config.hard_link()) { + if (config.hard_link()) { LOG("Hard linking {} to {}", source, dest); try { Util::hard_link(source, dest); - if (chmod(dest.c_str(), 0444) != 0) { - LOG("Failed to chmod: {}", strerror(errno)); +#ifndef _WIN32 + if (chmod(dest.c_str(), 0444 & ~Util::get_umask()) != 0) { + LOG("Failed to chmod {}: {}", dest.c_str(), strerror(errno)); } +#endif return; - } catch (const Error& e) { - LOG_RAW(e.what()); + } catch (const core::Error& e) { + LOG("Failed to hard link {} to {}: {}", source, dest, e.what()); // Fall back to copying. } } @@ -308,7 +307,7 @@ clone_hard_link_or_copy_file(const Context& ctx, } size_t -common_dir_prefix_length(string_view dir, string_view path) +common_dir_prefix_length(std::string_view dir, std::string_view path) { if (dir.empty() || path.empty() || dir == "/" || path == "/") { return 0; @@ -340,18 +339,22 @@ common_dir_prefix_length(string_view dir, string_view path) void copy_fd(int fd_in, int fd_out) { - read_fd(fd_in, - [=](const void* data, size_t size) { write_fd(fd_out, data, size); }); + util::read_fd(fd_in, [=](const void* data, size_t size) { + util::write_fd(fd_out, data, size); + }); } void copy_file(const std::string& src, const std::string& dest, bool via_tmp_file) { - Fd src_fd(open(src.c_str(), O_RDONLY)); + Fd src_fd(open(src.c_str(), O_RDONLY | O_BINARY)); if (!src_fd) { - throw Error("{}: {}", src, strerror(errno)); + throw core::Error( + FMT("Failed to open {} for reading: {}", src, strerror(errno))); } + unlink(dest.c_str()); + Fd dest_fd; std::string tmp_file; if (via_tmp_file) { @@ -362,7 +365,8 @@ copy_file(const std::string& src, const std::string& dest, bool via_tmp_file) dest_fd = Fd(open(dest.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666)); if (!dest_fd) { - throw Error("{}: {}", dest, strerror(errno)); + throw core::Error( + FMT("Failed to open {} for writing: {}", dest, strerror(errno))); } } @@ -376,7 +380,7 @@ copy_file(const std::string& src, const std::string& dest, bool via_tmp_file) } bool -create_dir(string_view dir) +create_dir(std::string_view dir) { std::string dir_str(dir); auto st = Stat::stat(dir_str); @@ -403,8 +407,8 @@ create_dir(string_view dir) } } -string_view -dir_name(string_view path) +std::string_view +dir_name(std::string_view path) { #ifdef _WIN32 const char delim[] = "/\\"; @@ -434,7 +438,8 @@ expand_environment_variables(const std::string& str) { std::string result; const char* left = str.c_str(); - for (const char* right = left; *right; ++right) { + const char* right = left; + while (*right) { if (*right == '$') { result.append(left, right - left); @@ -448,7 +453,7 @@ expand_environment_variables(const std::string& str) ++right; } if (curly && *right != '}') { - throw Error("syntax error: missing '}}' after \"{}\"", left); + throw core::Error(FMT("syntax error: missing '}}' after \"{}\"", left)); } if (right == left) { // Special case: don't consider a single $ the left of a variable. @@ -458,7 +463,7 @@ expand_environment_variables(const std::string& str) std::string name(left, right - left); const char* value = getenv(name.c_str()); if (!value) { - throw Error("environment variable \"{}\" not set", name); + throw core::Error(FMT("environment variable \"{}\" not set", name)); } result += value; if (!curly) { @@ -467,6 +472,7 @@ expand_environment_variables(const std::string& str) left = right + 1; } } + ++right; } result += left; return result; @@ -476,8 +482,13 @@ int fallocate(int fd, long new_size) { #ifdef HAVE_POSIX_FALLOCATE - return posix_fallocate(fd, 0, new_size); -#else + const int posix_fallocate_err = posix_fallocate(fd, 0, new_size); + if (posix_fallocate_err == 0 || posix_fallocate_err != EINVAL) { + return posix_fallocate_err; + } + // the underlying filesystem does not support the operation so fallback to + // lseeks +#endif off_t saved_pos = lseek(fd, 0, SEEK_END); off_t old_size = lseek(fd, 0, SEEK_END); if (old_size == -1) { @@ -497,30 +508,13 @@ fallocate(int fd, long new_size) } int err = 0; try { - write_fd(fd, buf, bytes_to_write); - } catch (Error&) { + util::write_fd(fd, buf, bytes_to_write); + } catch (core::Error&) { err = errno; } lseek(fd, saved_pos, SEEK_SET); free(buf); return err; -#endif -} - -void -for_each_level_1_subdir(const std::string& cache_dir, - const SubdirVisitor& visitor, - const ProgressReceiver& progress_receiver) -{ - for (int i = 0; i <= 0xF; i++) { - double progress = 1.0 * i / 16; - progress_receiver(progress); - std::string subdir_path = FMT("{}/{:x}", cache_dir, i); - visitor(subdir_path, [&](double inner_progress) { - progress_receiver(progress + inner_progress / 16); - }); - } - progress_receiver(1.0); } std::string @@ -586,10 +580,11 @@ format_parsable_size_with_suffix(uint64_t size) } void -ensure_dir_exists(nonstd::string_view dir) +ensure_dir_exists(std::string_view dir) { if (!create_dir(dir)) { - throw Fatal("Failed to create directory {}: {}", dir, strerror(errno)); + throw core::Fatal( + FMT("Failed to create directory {}: {}", dir, strerror(errno))); } } @@ -617,25 +612,20 @@ get_apparent_cwd(const std::string& actual_cwd) return actual_cwd; #else auto pwd = getenv("PWD"); - if (!pwd) { + if (!pwd || !util::is_absolute_path(pwd)) { return actual_cwd; } auto pwd_stat = Stat::stat(pwd); auto cwd_stat = Stat::stat(actual_cwd); - if (!pwd_stat || !cwd_stat || !pwd_stat.same_inode_as(cwd_stat)) { - return actual_cwd; - } - std::string normalized_pwd = normalize_absolute_path(pwd); - return normalized_pwd == pwd - || Stat::stat(normalized_pwd).same_inode_as(pwd_stat) - ? normalized_pwd - : pwd; + return !pwd_stat || !cwd_stat || !pwd_stat.same_inode_as(cwd_stat) + ? actual_cwd + : normalize_concrete_absolute_path(pwd); #endif } -string_view -get_extension(string_view path) +std::string_view +get_extension(std::string_view path) { #ifndef _WIN32 const char stop_at_chars[] = "./"; @@ -643,7 +633,7 @@ get_extension(string_view path) const char stop_at_chars[] = "./\\"; #endif size_t pos = path.find_last_of(stop_at_chars); - if (pos == string_view::npos || path.at(pos) == '/') { + if (pos == std::string_view::npos || path.at(pos) == '/') { return {}; #ifdef _WIN32 } else if (path.at(pos) == '\\') { @@ -654,59 +644,31 @@ get_extension(string_view path) } } -std::vector -get_level_1_files(const std::string& dir, - const ProgressReceiver& progress_receiver) -{ - std::vector files; - - if (!Stat::stat(dir)) { - return files; - } - - size_t level_2_directories = 0; - - Util::traverse(dir, [&](const std::string& path, bool is_dir) { - auto name = Util::base_name(path); - if (name == "CACHEDIR.TAG" || name == "stats" || name.starts_with(".nfs")) { - return; - } - - if (!is_dir) { - files.emplace_back(path); - } else if (path != dir - && path.find('/', dir.size() + 1) == std::string::npos) { - ++level_2_directories; - progress_receiver(level_2_directories / 16.0); - } - }); - - progress_receiver(1.0); - return files; -} - std::string get_home_directory() { - const char* p = getenv("HOME"); - if (p) { +#ifdef _WIN32 + if (const char* p = getenv("USERPROFILE")) { return p; } -#ifdef _WIN32 - p = getenv("APPDATA"); - if (p) { + throw core::Fatal( + "The USERPROFILE environment variable must be set to your user profile " + "folder"); +#else + if (const char* p = getenv("HOME")) { return p; } -#endif -#ifdef HAVE_GETPWUID +# ifdef HAVE_GETPWUID { struct passwd* pwd = getpwuid(getuid()); if (pwd) { return pwd->pw_dir; } } +# endif + throw core::Fatal( + "Could not determine home directory from $HOME or getpwuid(3)"); #endif - throw Fatal("Could not determine home directory from $HOME or getpwuid(3)"); } const char* @@ -726,10 +688,10 @@ get_hostname() } std::string -get_relative_path(string_view dir, string_view path) +get_relative_path(std::string_view dir, std::string_view path) { - ASSERT(Util::is_absolute_path(dir)); - ASSERT(Util::is_absolute_path(path)); + ASSERT(util::is_absolute_path(dir)); + ASSERT(util::is_absolute_path(path)); #ifdef _WIN32 // Paths can be escaped by a slash for use with e.g. -isystem. @@ -769,25 +731,10 @@ get_relative_path(string_view dir, string_view path) return result.empty() ? "." : result; } -std::string -get_path_in_cache(string_view cache_dir, uint8_t level, string_view name) +mode_t +get_umask() { - ASSERT(level >= 1 && level <= 8); - ASSERT(name.length() >= level); - - std::string path(cache_dir); - path.reserve(path.size() + level * 2 + 1 + name.length() - level); - - for (uint8_t i = 0; i < level; ++i) { - path.push_back('/'); - path.push_back(name.at(i)); - } - - path.push_back('/'); - string_view name_remaining = name.substr(level); - path.append(name_remaining.data(), name_remaining.length()); - - return path; + return g_umask; } void @@ -800,72 +747,67 @@ hard_link(const std::string& oldpath, const std::string& newpath) #ifndef _WIN32 if (link(oldpath.c_str(), newpath.c_str()) != 0) { - throw Error( - "failed to link {} to {}: {}", oldpath, newpath, strerror(errno)); + throw core::Error(strerror(errno)); } #else if (!CreateHardLink(newpath.c_str(), oldpath.c_str(), nullptr)) { - DWORD error = GetLastError(); - throw Error("failed to link {} to {}: {}", - oldpath, - newpath, - Win32Util::error_message(error)); + throw core::Error(Win32Util::error_message(GetLastError())); } #endif } -bool -is_absolute_path(string_view path) +std::optional +is_absolute_path_with_prefix(std::string_view path) { #ifdef _WIN32 - if (path.length() >= 2 && path[1] == ':' - && (path[2] == '/' || path[2] == '\\')) { - return true; - } + const char delim[] = "/\\"; +#else + const char delim[] = "/"; #endif - return !path.empty() && path[0] == '/'; -} - -#if defined(HAVE_LINUX_FS_H) || defined(HAVE_STRUCT_STATFS_F_FSTYPENAME) -int -is_nfs_fd(int fd, bool* is_nfs) -{ - struct statfs buf; - if (fstatfs(fd, &buf) != 0) { - return errno; + auto split_pos = path.find_first_of(delim); + if (split_pos != std::string::npos) { +#ifdef _WIN32 + // -I/C:/foo and -I/c/foo will already be handled by delim_pos correctly + // resulting in -I and /C:/foo or /c/foo respectively. -IC:/foo will not as + // we would get -IC: and /foo. + if (split_pos > 0 && path[split_pos - 1] == ':') { + split_pos = split_pos - 2; + } +#endif + // This is not redundant on some platforms, so nothing to simplify. + // NOLINTNEXTLINE(readability-simplify-boolean-expr) + return split_pos; } -# ifdef HAVE_LINUX_FS_H - *is_nfs = buf.f_type == NFS_SUPER_MAGIC; -# else // Mac OS X and some other BSD flavors - *is_nfs = strcmp(buf.f_fstypename, "nfs") == 0; -# endif - return 0; + return std::nullopt; } -#else -int -is_nfs_fd(int /*fd*/, bool* /*is_nfs*/) + +bool +is_ccache_executable(const std::string_view path) { - return -1; -} + std::string name(Util::base_name(path)); +#ifdef _WIN32 + name = Util::to_lowercase(name); #endif + return util::starts_with(name, "ccache"); +} bool -is_precompiled_header(string_view path) +is_precompiled_header(std::string_view path) { - string_view ext = get_extension(path); + std::string_view ext = get_extension(path); return ext == ".gch" || ext == ".pch" || ext == ".pth" || get_extension(dir_name(path)) == ".gch"; } -optional -localtime(optional time) +std::optional +localtime(std::optional time) { - time_t timestamp = time ? *time : ::time(nullptr); + time_t timestamp = time ? time->sec() : util::TimePoint::now().sec(); tm result; if (localtime_r(×tamp, &result)) { return result; } else { - return nullopt; + return std::nullopt; } } @@ -873,9 +815,9 @@ std::string make_relative_path(const std::string& base_dir, const std::string& actual_cwd, const std::string& apparent_cwd, - nonstd::string_view path) + std::string_view path) { - if (base_dir.empty() || !Util::starts_with(path, base_dir)) { + if (base_dir.empty() || !util::path_starts_with(path, base_dir)) { return std::string(path); } @@ -907,8 +849,9 @@ make_relative_path(const std::string& base_dir, const auto path_suffix = std::string(original_path.substr(path.length())); const auto real_path = Util::real_path(std::string(path)); - const auto add_relpath_candidates = [&](nonstd::string_view path) { - const std::string normalized_path = Util::normalize_absolute_path(path); + const auto add_relpath_candidates = [&](auto path) { + const std::string normalized_path = + Util::normalize_abstract_absolute_path(path); relpath_candidates.push_back( Util::get_relative_path(actual_cwd, normalized_path)); if (apparent_cwd != actual_cwd) { @@ -924,7 +867,7 @@ make_relative_path(const std::string& base_dir, // Find best (i.e. shortest existing) match: std::sort(relpath_candidates.begin(), relpath_candidates.end(), - [](const std::string& path1, const std::string& path2) { + [](const auto& path1, const auto& path2) { return path1.length() < path2.length(); }); for (const auto& relpath : relpath_candidates) { @@ -938,35 +881,36 @@ make_relative_path(const std::string& base_dir, } std::string -make_relative_path(const Context& ctx, string_view path) +make_relative_path(const Context& ctx, std::string_view path) { return make_relative_path( ctx.config.base_dir(), ctx.actual_cwd, ctx.apparent_cwd, path); } bool -matches_dir_prefix_or_file(string_view dir_prefix_or_file, string_view path) +matches_dir_prefix_or_file(std::string_view dir_prefix_or_file, + std::string_view path) { return !dir_prefix_or_file.empty() && !path.empty() && dir_prefix_or_file.length() <= path.length() - && path.starts_with(dir_prefix_or_file) + && util::starts_with(path, dir_prefix_or_file) && (dir_prefix_or_file.length() == path.length() || is_dir_separator(path[dir_prefix_or_file.length()]) || is_dir_separator(dir_prefix_or_file.back())); } std::string -normalize_absolute_path(string_view path) +normalize_abstract_absolute_path(std::string_view path) { - if (!is_absolute_path(path)) { + if (!util::is_absolute_path(path)) { return std::string(path); } #ifdef _WIN32 - if (path.find("\\") != string_view::npos) { + if (path.find("\\") != std::string_view::npos) { std::string new_path(path); std::replace(new_path.begin(), new_path.end(), '\\', '/'); - return normalize_absolute_path(new_path); + return normalize_abstract_absolute_path(new_path); } std::string drive(path.substr(0, 2)); @@ -974,7 +918,7 @@ normalize_absolute_path(string_view path) #endif std::string result = "/"; - const size_t npos = string_view::npos; + const size_t npos = std::string_view::npos; size_t left = 1; while (true) { @@ -982,7 +926,8 @@ normalize_absolute_path(string_view path) break; } const auto right = path.find('/', left); - string_view part = path.substr(left, right == npos ? npos : right - left); + std::string_view part = + path.substr(left, right == npos ? npos : right - left); if (part == "..") { if (result.length() > 1) { // "/x/../part" -> "/part" @@ -1014,8 +959,17 @@ normalize_absolute_path(string_view path) #endif } +std::string +normalize_concrete_absolute_path(const std::string& path) +{ + const auto normalized_path = normalize_abstract_absolute_path(path); + return Stat::stat(normalized_path).same_inode_as(Stat::stat(path)) + ? normalized_path + : path; +} + uint64_t -parse_duration(const std::string& duration) +parse_duration(std::string_view duration) { uint64_t factor = 0; char last_ch = duration.empty() ? '\0' : duration[duration.length() - 1]; @@ -1028,40 +982,17 @@ parse_duration(const std::string& duration) factor = 1; break; default: - throw Error("invalid suffix (supported: d (day) and s (second)): \"{}\"", - duration); + throw core::Error(FMT( + "invalid suffix (supported: d (day) and s (second)): \"{}\"", duration)); } - return factor * parse_unsigned(duration.substr(0, duration.length() - 1)); -} - -int64_t -parse_signed(const std::string& value, - optional min_value, - optional max_value, - string_view description) -{ - std::string stripped_value = strip_whitespace(value); - - size_t end = 0; - long long result = 0; - bool failed = false; - try { - // Note: sizeof(long long) is guaranteed to be >= sizeof(int64_t) - result = std::stoll(stripped_value, &end, 10); - } catch (std::exception&) { - failed = true; - } - if (failed || end != stripped_value.size()) { - throw Error("invalid integer: \"{}\"", stripped_value); - } - - int64_t min = min_value ? *min_value : INT64_MIN; - int64_t max = max_value ? *max_value : INT64_MAX; - if (result < min || result > max) { - throw Error("{} must be between {} and {}", description, min, max); + const auto value = + util::parse_unsigned(duration.substr(0, duration.length() - 1)); + if (value) { + return factor * *value; + } else { + throw core::Error(value.error()); } - return result; } uint64_t @@ -1072,7 +1003,7 @@ parse_size(const std::string& value) char* p; double result = strtod(value.c_str(), &p); if (errno != 0 || result < 0 || p == value.c_str() || value.empty()) { - throw Error("invalid size: \"{}\"", value); + throw core::Error(FMT("invalid size: \"{}\"", value)); } while (isspace(*p)) { @@ -1084,19 +1015,19 @@ parse_size(const std::string& value) switch (*p) { case 'T': result *= multiplier; - // Fallthrough. + [[fallthrough]]; case 'G': result *= multiplier; - // Fallthrough. + [[fallthrough]]; case 'M': result *= multiplier; - // Fallthrough. + [[fallthrough]]; case 'K': case 'k': result *= multiplier; break; default: - throw Error("invalid size: \"{}\"", value); + throw core::Error(FMT("invalid size: \"{}\"", value)); } } else { // Default suffix: G. @@ -1105,113 +1036,13 @@ parse_size(const std::string& value) return static_cast(result); } -uint64_t -parse_unsigned(const std::string& value, - optional min_value, - optional max_value, - string_view description) -{ - std::string stripped_value = strip_whitespace(value); - - size_t end = 0; - unsigned long long result = 0; - bool failed = false; - if (Util::starts_with(stripped_value, "-")) { - failed = true; - } else { - try { - // Note: sizeof(unsigned long long) is guaranteed to be >= - // sizeof(uint64_t) - result = std::stoull(stripped_value, &end, 10); - } catch (std::exception&) { - failed = true; - } - } - if (failed || end != stripped_value.size()) { - throw Error("invalid unsigned integer: \"{}\"", stripped_value); - } - - uint64_t min = min_value ? *min_value : 0; - uint64_t max = max_value ? *max_value : UINT64_MAX; - if (result < min || result > max) { - throw Error("{} must be between {} and {}", description, min, max); - } - return result; -} - -bool -read_fd(int fd, DataReceiver data_receiver) -{ - ssize_t n; - char buffer[READ_BUFFER_SIZE]; - while ((n = read(fd, buffer, sizeof(buffer))) != 0) { - if (n == -1 && errno != EINTR) { - break; - } - if (n > 0) { - data_receiver(buffer, n); - } - } - return n >= 0; -} - -std::string -read_file(const std::string& path, size_t size_hint) -{ - if (size_hint == 0) { - auto stat = Stat::stat(path); - if (!stat) { - throw Error(strerror(errno)); - } - size_hint = stat.size(); - } - - // +1 to be able to detect EOF in the first read call - size_hint = (size_hint < 1024) ? 1024 : size_hint + 1; - - Fd fd(open(path.c_str(), O_RDONLY | O_BINARY)); - if (!fd) { - throw Error(strerror(errno)); - } - - ssize_t ret = 0; - size_t pos = 0; - std::string result; - result.resize(size_hint); - - while (true) { - if (pos == result.size()) { - result.resize(2 * result.size()); - } - const size_t max_read = result.size() - pos; - ret = read(*fd, &result[pos], max_read); - if (ret == 0 || (ret == -1 && errno != EINTR)) { - break; - } - if (ret > 0) { - pos += ret; - if (static_cast(ret) < max_read) { - break; - } - } - } - - if (ret == -1) { - LOG("Failed reading {}", path); - throw Error(strerror(errno)); - } - - result.resize(pos); - return result; -} - #ifndef _WIN32 std::string read_link(const std::string& path) { size_t buffer_size = path_max(path); std::unique_ptr buffer(new char[buffer_size]); - ssize_t len = readlink(path.c_str(), buffer.get(), buffer_size - 1); + const auto len = readlink(path.c_str(), buffer.get(), buffer_size - 1); if (len == -1) { return ""; } @@ -1261,8 +1092,8 @@ real_path(const std::string& path, bool return_empty_on_error) return resolved ? resolved : (return_empty_on_error ? "" : path); } -string_view -remove_extension(string_view path) +std::string_view +remove_extension(std::string_view path) { return path.substr(0, path.length() - get_extension(path).length()); } @@ -1272,8 +1103,8 @@ rename(const std::string& oldpath, const std::string& newpath) { #ifndef _WIN32 if (::rename(oldpath.c_str(), newpath.c_str()) != 0) { - throw Error( - "failed to rename {} to {}: {}", oldpath, newpath, strerror(errno)); + throw core::Error( + FMT("failed to rename {} to {}: {}", oldpath, newpath, strerror(errno))); } #else // Windows' rename() won't overwrite an existing file, so need to use @@ -1281,51 +1112,46 @@ rename(const std::string& oldpath, const std::string& newpath) if (!MoveFileExA( oldpath.c_str(), newpath.c_str(), MOVEFILE_REPLACE_EXISTING)) { DWORD error = GetLastError(); - throw Error("failed to rename {} to {}: {}", - oldpath, - newpath, - Win32Util::error_message(error)); + throw core::Error(FMT("failed to rename {} to {}: {}", + oldpath, + newpath, + Win32Util::error_message(error))); } #endif } -bool -same_program_name(nonstd::string_view program_name, - nonstd::string_view canonical_program_name) -{ -#ifdef _WIN32 - std::string lowercase_program_name = Util::to_lowercase(program_name); - return lowercase_program_name == canonical_program_name - || lowercase_program_name == FMT("{}.exe", canonical_program_name); -#else - return program_name == canonical_program_name; -#endif -} - void -send_to_stderr(const Context& ctx, const std::string& text) +send_to_fd(const Context& ctx, std::string_view text, int fd) { - const std::string* text_to_send = &text; + std::string_view text_to_send = text; std::string modified_text; +#ifdef _WIN32 + // stdout/stderr are normally opened in text mode, which would convert + // newlines a second time since we treat output as binary data. Make sure to + // switch to binary mode. + int oldmode = _setmode(fd, _O_BINARY); + Finalizer binary_mode_restorer([=] { _setmode(fd, oldmode); }); +#endif + if (ctx.args_info.strip_diagnostics_colors) { try { modified_text = strip_ansi_csi_seqs(text); - text_to_send = &modified_text; - } catch (const Error&) { - // Fall through + text_to_send = modified_text; + } catch (const core::Error&) { + // Ignore. } } if (ctx.config.absolute_paths_in_stderr()) { - modified_text = rewrite_stderr_to_absolute_paths(*text_to_send); - text_to_send = &modified_text; + modified_text = rewrite_stderr_to_absolute_paths(text_to_send); + text_to_send = modified_text; } - try { - write_fd(STDERR_FILENO, text_to_send->data(), text_to_send->length()); - } catch (Error& e) { - throw Error("Failed to write to stderr: {}", e.what()); + const auto result = + util::write_fd(fd, text_to_send.data(), text_to_send.length()); + if (!result) { + throw core::Error(FMT("Failed to write to {}: {}", fd, result.error())); } } @@ -1342,6 +1168,13 @@ set_cloexec_flag(int fd) #endif } +mode_t +set_umask(mode_t mask) +{ + g_umask = mask; + return umask(mask); +} + void setenv(const std::string& name, const std::string& value) { @@ -1354,20 +1187,27 @@ setenv(const std::string& name, const std::string& value) #endif } -std::vector -split_into_views(string_view input, const char* separators) +std::vector +split_into_views(std::string_view string, + const char* separators, + util::Tokenizer::Mode mode, + IncludeDelimiter include_delimiter) { - return split_at(input, separators); + return split_into( + string, separators, mode, include_delimiter); } std::vector -split_into_strings(string_view input, const char* separators) +split_into_strings(std::string_view string, + const char* separators, + util::Tokenizer::Mode mode, + IncludeDelimiter include_delimiter) { - return split_at(input, separators); + return split_into(string, separators, mode, include_delimiter); } std::string -strip_ansi_csi_seqs(string_view string) +strip_ansi_csi_seqs(std::string_view string) { size_t pos = 0; std::string result; @@ -1389,16 +1229,7 @@ strip_ansi_csi_seqs(string_view string) } std::string -strip_whitespace(string_view string) -{ - auto is_space = [](int ch) { return std::isspace(ch); }; - auto start = std::find_if_not(string.begin(), string.end(), is_space); - auto end = std::find_if_not(string.rbegin(), string.rend(), is_space).base(); - return start < end ? std::string(start, end) : std::string(); -} - -std::string -to_lowercase(string_view string) +to_lowercase(std::string_view string) { std::string result; result.resize(string.length()); @@ -1433,9 +1264,9 @@ traverse(const std::string& path, const TraverseVisitor& visitor) if (stat.error_number() == ENOENT || stat.error_number() == ESTALE) { continue; } - throw Error("failed to lstat {}: {}", - entry_path, - strerror(stat.error_number())); + throw core::Error(FMT("failed to lstat {}: {}", + entry_path, + strerror(stat.error_number()))); } is_dir = stat.is_directory(); } @@ -1450,7 +1281,8 @@ traverse(const std::string& path, const TraverseVisitor& visitor) } else if (errno == ENOTDIR) { visitor(path, false); } else { - throw Error("failed to open directory {}: {}", path, strerror(errno)); + throw core::Error( + FMT("failed to open directory {}: {}", path, strerror(errno))); } } @@ -1473,7 +1305,8 @@ traverse(const std::string& path, const TraverseVisitor& visitor) } else if (std::filesystem::exists(path)) { visitor(path, false); } else { - throw Error("failed to open directory {}: {}", path, strerror(errno)); + throw core::Error( + FMT("failed to open directory {}: {}", path, strerror(errno))); } } @@ -1487,12 +1320,13 @@ unlink_safe(const std::string& path, UnlinkLog unlink_log) // If path is on an NFS share, unlink isn't atomic, so we rename to a temp // file. We don't care if the temp file is trashed, so it's always safe to // unlink it first. - std::string tmp_name = path + ".ccache.rm.tmp"; + const std::string tmp_name = + FMT("{}.ccache{}unlink", path, TemporaryFile::tmp_file_infix); bool success = true; try { Util::rename(path, tmp_name); - } catch (Error&) { + } catch (core::Error&) { success = false; saved_errno = errno; } @@ -1538,21 +1372,13 @@ unsetenv(const std::string& name) { #ifdef HAVE_UNSETENV ::unsetenv(name.c_str()); +#elif defined(_WIN32) + SetEnvironmentVariable(name.c_str(), NULL); #else putenv(strdup(name.c_str())); // Leak to environment. #endif } -void -update_mtime(const std::string& path) -{ -#ifdef HAVE_UTIMES - utimes(path.c_str(), nullptr); -#else - utime(path.c_str(), nullptr); -#endif -} - void wipe_path(const std::string& path) { @@ -1562,46 +1388,12 @@ wipe_path(const std::string& path) traverse(path, [](const std::string& p, bool is_dir) { if (is_dir) { if (rmdir(p.c_str()) != 0 && errno != ENOENT && errno != ESTALE) { - throw Error("failed to rmdir {}: {}", p, strerror(errno)); + throw core::Error(FMT("failed to rmdir {}: {}", p, strerror(errno))); } } else if (unlink(p.c_str()) != 0 && errno != ENOENT && errno != ESTALE) { - throw Error("failed to unlink {}: {}", p, strerror(errno)); + throw core::Error(FMT("failed to unlink {}: {}", p, strerror(errno))); } }); } -void -write_fd(int fd, const void* data, size_t size) -{ - ssize_t written = 0; - do { - ssize_t count = - write(fd, static_cast(data) + written, size - written); - if (count == -1) { - if (errno != EAGAIN && errno != EINTR) { - throw Error(strerror(errno)); - } - } else { - written += count; - } - } while (static_cast(written) < size); -} - -void -write_file(const std::string& path, - const std::string& data, - std::ios_base::openmode open_mode) -{ - if (path.empty()) { - throw Error("No such file or directory"); - } - - open_mode |= std::ios::out; - std::ofstream file(path, open_mode); - if (!file) { - throw Error(strerror(errno)); - } - file << data; -} - } // namespace Util diff --git a/src/Util.hpp b/src/Util.hpp index 7db8d95..e29aff3 100644 --- a/src/Util.hpp +++ b/src/Util.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,36 +18,32 @@ #pragma once -#include "system.hpp" +#include +#include +#include -#include "CacheFile.hpp" - -#include "third_party/nonstd/optional.hpp" -#include "third_party/nonstd/string_view.hpp" - -#include +#include +#include #include -#include #include +#include #include +#include #include #include +class Config; class Context; namespace Util { -using DataReceiver = std::function; -using ProgressReceiver = std::function; -using SubdirVisitor = std::function; using TraverseVisitor = std::function; enum class UnlinkLog { log_failure, ignore_failure }; // Get base name of path. -nonstd::string_view base_name(nonstd::string_view path); +std::string_view base_name(std::string_view path); // Get an integer value from bytes in big endian order. // @@ -81,41 +77,32 @@ big_endian_to_int(const uint8_t* buffer, uint8_t& value) // Remove the extension via `remove_extension()`, then add `new_ext`. `new_ext` // should start with a dot, no extra dot is inserted. -std::string change_extension(nonstd::string_view path, - nonstd::string_view new_ext); - -// Return `value` adjusted to not be less than `min` and not more than `max`. -template -T -clamp(T value, T min, T max) -{ - return std::min(max, std::max(min, value)); -} +std::string change_extension(std::string_view path, std::string_view new_ext); // Clone a file from `src` to `dest`. If `via_tmp_file` is true, `src` is cloned -// to a temporary file and then renamed to `dest`. Throws `Error` on error. +// to a temporary file and then renamed to `dest`. Throws `core::Error` on +// error. void clone_file(const std::string& src, const std::string& dest, bool via_tmp_file = false); // Clone, hard link or copy a file from `source` to `dest` depending on settings // in `ctx`. If cloning or hard linking cannot and should not be done the file -// will be copied instead. Throws `Error` on error. -void clone_hard_link_or_copy_file(const Context& ctx, +// will be copied instead. Throws `core::Error` on error. +void clone_hard_link_or_copy_file(const Config& config, const std::string& source, const std::string& dest, bool via_tmp_file = false); // Compute the length of the longest directory path that is common to paths // `dir` (a directory) and `path` (any path). -size_t common_dir_prefix_length(nonstd::string_view dir, - nonstd::string_view path); +size_t common_dir_prefix_length(std::string_view dir, std::string_view path); -// Copy all data from `fd_in` to `fd_out`. Throws `Error` on error. +// Copy all data from `fd_in` to `fd_out`. Throws `core::Error` on error. void copy_fd(int fd_in, int fd_out); // Copy a file from `src` to `dest`. If via_tmp_file is true, `src` is copied to -// a temporary file and then renamed to dest. Throws `Error` on error. +// a temporary file and then renamed to dest. Throws `core::Error` on error. void copy_file(const std::string& src, const std::string& dest, bool via_tmp_file = false); @@ -123,23 +110,16 @@ void copy_file(const std::string& src, // Create a directory if needed, including its parents if needed. // // Returns true if the directory exists or could be created, otherwise false. -bool create_dir(nonstd::string_view dir); +bool create_dir(std::string_view dir); // Get directory name of path. -nonstd::string_view dir_name(nonstd::string_view path); - -// Return true if `suffix` is a suffix of `string`. -inline bool -ends_with(nonstd::string_view string, nonstd::string_view suffix) -{ - return string.ends_with(suffix); -} +std::string_view dir_name(std::string_view path); // Like create_dir but throws Fatal on error. -void ensure_dir_exists(nonstd::string_view dir); +void ensure_dir_exists(std::string_view dir); // Expand all instances of $VAR or ${VAR}, where VAR is an environment variable, -// in `str`. Throws `Error` if one of the environment variables. +// in `str`. Throws `core::Error` if one of the environment variables. [[nodiscard]] std::string expand_environment_variables(const std::string& str); // Extends file size to at least new_size by calling posix_fallocate() if @@ -151,17 +131,6 @@ void ensure_dir_exists(nonstd::string_view dir); // Returns 0 on success, an error number otherwise. int fallocate(int fd, long new_size); -// Call a function for each subdir (0-9a-f) in the cache. -// -// Parameters: -// - cache_dir: Path to the cache directory. -// - visitor: Function to call with directory path and progress_receiver as -// arguments. -// - progress_receiver: Function that will be called for progress updates. -void for_each_level_1_subdir(const std::string& cache_dir, - const SubdirVisitor& visitor, - const ProgressReceiver& progress_receiver); - // Format `argv` as a simple string for logging purposes. That is, the result is // not intended to be machine parsable. `argv` must be terminated by a nullptr. std::string format_argv_for_logging(const char* const* argv); @@ -192,25 +161,7 @@ std::string get_apparent_cwd(const std::string& actual_cwd); // Return the file extension (including the dot) as a view into `path`. If // `path` has no file extension, an empty string_view is returned. -nonstd::string_view get_extension(nonstd::string_view path); - -// Get a list of files in a level 1 subdirectory of the cache. -// -// The function works under the assumption that directory entries with one -// character names (except ".") are subdirectories and that there are no other -// subdirectories. -// -// Files ignored: -// - CACHEDIR.TAG -// - stats -// - .nfs* (temporary NFS files that may be left for open but deleted files). -// -// Parameters: -// - dir: The directory to traverse recursively. -// - progress_receiver: Function that will be called for progress updates. -std::vector -get_level_1_files(const std::string& dir, - const ProgressReceiver& progress_receiver); +std::string_view get_extension(std::string_view path); // Return the current user's home directory, or throw `Fatal` if it can't // be determined. @@ -223,17 +174,12 @@ const char* get_hostname(); // `path` (an absolute path). Assumes that both `dir` and `path` are normalized. // The algorithm does *not* follow symlinks, so the result may not actually // resolve to the same file as `path`. -std::string get_relative_path(nonstd::string_view dir, - nonstd::string_view path); +std::string get_relative_path(std::string_view dir, std::string_view path); -// Join `cache_dir`, a '/' and `name` into a single path and return it. -// Additionally, `level` single-character, '/'-separated subpaths are split from -// the beginning of `name` before joining them all. -std::string get_path_in_cache(nonstd::string_view cache_dir, - uint8_t level, - nonstd::string_view name); +// Get process umask. +mode_t get_umask(); -// Hard-link `oldpath` to `newpath`. Throws `Error` on error. +// Hard-link `oldpath` to `newpath`. Throws `core::Error` on error. void hard_link(const std::string& oldpath, const std::string& newpath); // Write bytes in big endian order from an integer value. @@ -265,16 +211,12 @@ int_to_big_endian(int8_t value, uint8_t* buffer) buffer[0] = value; } -// Return whether `path` is absolute. -bool is_absolute_path(nonstd::string_view path); +// Determine if `path` is an absolute path with prefix, returning the split +// point. +std::optional is_absolute_path_with_prefix(std::string_view path); -// Test if a file is on nfs. -// -// Sets is_nfs to the result if fstatfs is available and no error occurred. -// -// Returns 0 if is_nfs was set, -1 if fstatfs is not available or errno if an -// error occurred. -int is_nfs_fd(int fd, bool* is_nfs); +// Detmine if `path` refers to a ccache executable. +bool is_ccache_executable(std::string_view path); // Return whether `ch` is a directory separator, i.e. '/' on POSIX systems and // '/' or '\\' on Windows systems. @@ -288,93 +230,64 @@ is_dir_separator(char ch) ; } -// Return whether `path` is a full path. -inline bool -is_full_path(nonstd::string_view path) -{ -#ifdef _WIN32 - if (path.find('\\') != nonstd::string_view::npos) { - return true; - } -#endif - return path.find('/') != nonstd::string_view::npos; -} - // Return whether `path` represents a precompiled header (see "Precompiled // Headers" in GCC docs). -bool is_precompiled_header(nonstd::string_view path); +bool is_precompiled_header(std::string_view path); // Thread-safe version of `localtime(3)`. If `time` is not specified the current // time of day is used. -nonstd::optional localtime(nonstd::optional time = {}); +std::optional localtime(std::optional time = {}); + +// Construct a normalized native path. +// +// Example: +// +// std::string path = Util::make_path("usr", "local", "bin"); +template +std::string +make_path(const T&... args) +{ + return (std::filesystem::path{} / ... / args).lexically_normal().string(); +} // Make a relative path from current working directory (either `actual_cwd` or // `apparent_cwd`) to `path` if `path` is under `base_dir`. std::string make_relative_path(const std::string& base_dir, const std::string& actual_cwd, const std::string& apparent_cwd, - nonstd::string_view path); + std::string_view path); // Like above but with base directory and apparent/actual CWD taken from `ctx`. -std::string make_relative_path(const Context& ctx, nonstd::string_view path); +std::string make_relative_path(const Context& ctx, std::string_view path); // Return whether `path` is equal to `dir_prefix_or_file` or if // `dir_prefix_or_file` is a directory prefix of `path`. -bool matches_dir_prefix_or_file(nonstd::string_view dir_prefix_or_file, - nonstd::string_view path); +bool matches_dir_prefix_or_file(std::string_view dir_prefix_or_file, + std::string_view path); // Normalize absolute path `path`, not taking symlinks into account. // // Normalization here means syntactically removing redundant slashes and // resolving "." and ".." parts. The algorithm does however *not* follow -// symlinks, so the result may not actually resolve to `path`. +// symlinks, so the result may not actually resolve to the same filesystem entry +// as `path` (nor to any existing file system entry for that matter). // // On Windows: Backslashes are replaced with forward slashes. -std::string normalize_absolute_path(nonstd::string_view path); +std::string normalize_abstract_absolute_path(std::string_view path); -// Parse `duration`, an unsigned integer with d (days) or s (seconds) suffix, -// into seconds. Throws `Error` on error. -uint64_t parse_duration(const std::string& duration); +// Like normalize_abstract_absolute_path, but returns `path` unchanged if the +// normalized result doesn't resolve to the same file system entry as `path`. +std::string normalize_concrete_absolute_path(const std::string& path); -// Parse a string into a signed integer. -// -// Throws `Error` if `value` cannot be parsed as an int64_t or if the value -// falls out of the range [`min_value`, `max_value`]. `min_value` and -// `max_value` default to min and max values of int64_t. `description` is -// included in the error message for range violations. -int64_t parse_signed(const std::string& value, - nonstd::optional min_value = nonstd::nullopt, - nonstd::optional max_value = nonstd::nullopt, - nonstd::string_view description = "integer"); +// Parse `duration`, an unsigned integer with d (days) or s (seconds) suffix, +// into seconds. Throws `core::Error` on error. +uint64_t parse_duration(std::string_view duration); // Parse a "size value", i.e. a string that can end in k, M, G, T (10-based // suffixes) or Ki, Mi, Gi, Ti (2-based suffixes). For backward compatibility, K -// is also recognized as a synonym of k. Throws `Error` on parse error. +// is also recognized as a synonym of k. Throws `core::Error` on parse error. uint64_t parse_size(const std::string& value); -// Parse a string into an unsigned integer. -// -// Throws `Error` if `value` cannot be parsed as an uint64_t or if the value -// falls out of the range [`min_value`, `max_value`]. `min_value` and -// `max_value` default to min and max values of uint64_t. `description` is -// included in the error message for range violations. -uint64_t parse_unsigned(const std::string& value, - nonstd::optional min_value = nonstd::nullopt, - nonstd::optional max_value = nonstd::nullopt, - nonstd::string_view description = "integer"); - -// Read data from `fd` until end of file and call `data_receiver` with the read -// data. Returns whether reading was successful, i.e. whether the read(2) call -// did not return -1. -bool read_fd(int fd, DataReceiver data_receiver); - -// Return `path`'s content as a string. If `size_hint` is not 0 then assume that -// `path` has this size (this saves system calls). -// -// Throws `Error` on error. The description contains the error message without -// the path. -std::string read_file(const std::string& path, size_t size_hint = 0); - #ifndef _WIN32 // Like readlink(2) but returns the string (or the empty string on failure). std::string read_link(const std::string& path); @@ -388,26 +301,24 @@ std::string real_path(const std::string& path, // Return a view into `path` containing the given path without the filename // extension as determined by `get_extension()`. -nonstd::string_view remove_extension(nonstd::string_view path); +std::string_view remove_extension(std::string_view path); -// Rename `oldpath` to `newpath` (deleting `newpath`). Throws `Error` on error. +// Rename `oldpath` to `newpath` (deleting `newpath`). Throws `core::Error` on +// error. void rename(const std::string& oldpath, const std::string& newpath); -// Detmine if `program_name` is equal to `canonical_program_name`. On Windows, -// this means performing a case insensitive equality check with and without a -// ".exe" suffix. On non-Windows, it is a simple equality check. -bool same_program_name(nonstd::string_view program_name, - nonstd::string_view canonical_program_name); - -// Send `text` to STDERR_FILENO, optionally stripping ANSI color sequences if -// `ctx.args_info.strip_diagnostics_colors` is true and rewriting paths to -// absolute if `ctx.config.absolute_paths_in_stderr` is true. Throws `Error` on -// error. -void send_to_stderr(const Context& ctx, const std::string& text); +// Send `text` to file descriptor `fd`, optionally stripping ANSI color +// sequences if `ctx.args_info.strip_diagnostics_colors` is true and rewriting +// paths to absolute if `ctx.config.absolute_paths_in_stderr` is true. Throws +// `core::Error` on error. +void send_to_fd(const Context& ctx, std::string_view text, int fd); // Set the FD_CLOEXEC on file descriptor `fd`. This is a NOP on Windows. void set_cloexec_flag(int fd); +// Set process umask. Returns the previous mask. +mode_t set_umask(mode_t mask); + // Set environment variable `name` to `value`. void setenv(const std::string& name, const std::string& value); @@ -420,51 +331,40 @@ size_change_kibibyte(const Stat& old_stat, const Stat& new_stat) / 1024; } -// Split `input` into words at any of the characters listed in `separators`. -// These words are a view into `input`; empty words are omitted. `separators` -// must neither be the empty string nor a nullptr. -std::vector split_into_views(nonstd::string_view input, - const char* separators); - -// Same as `split_into_views` but the words are copied from `input`. -std::vector split_into_strings(nonstd::string_view input, - const char* separators); - -// Return true if `prefix` is a prefix of `string`. -inline bool -starts_with(const char* string, nonstd::string_view prefix) -{ - // Optimized version of starts_with(string_view, string_view): avoid computing - // the length of the string argument. - return strncmp(string, prefix.data(), prefix.length()) == 0; -} - -// Return true if `prefix` is a prefix of `string`. -inline bool -starts_with(nonstd::string_view string, nonstd::string_view prefix) -{ - return string.starts_with(prefix); -} +// Split `string` into tokens at any of the characters in `separators`. These +// tokens are views into `string`. `separators` must neither be the empty string +// nor a nullptr. +std::vector +split_into_views(std::string_view string, + const char* separators, + util::Tokenizer::Mode mode = util::Tokenizer::Mode::skip_empty, + util::Tokenizer::IncludeDelimiter include_delimiter = + util::Tokenizer::IncludeDelimiter::no); + +// Same as `split_into_views` but the tokens are copied from `string`. +std::vector split_into_strings( + std::string_view string, + const char* separators, + util::Tokenizer::Mode mode = util::Tokenizer::Mode::skip_empty, + util::Tokenizer::IncludeDelimiter include_delimiter = + util::Tokenizer::IncludeDelimiter::no); // Returns a copy of string with the specified ANSI CSI sequences removed. -[[nodiscard]] std::string strip_ansi_csi_seqs(nonstd::string_view string); - -// Strip whitespace from left and right side of a string. -[[nodiscard]] std::string strip_whitespace(nonstd::string_view string); +[[nodiscard]] std::string strip_ansi_csi_seqs(std::string_view string); // Convert a string to lowercase. -[[nodiscard]] std::string to_lowercase(nonstd::string_view string); +[[nodiscard]] std::string to_lowercase(std::string_view string); // Traverse `path` recursively (postorder, i.e. files are visited before their // parent directory). // -// Throws Error on error. +// Throws core::Error on error. void traverse(const std::string& path, const TraverseVisitor& visitor); // Remove `path` (non-directory), NFS safe. Logs according to `unlink_log`. // -// Returns whether removal was successful. A nonexistent `path` is considered -// successful. +// Returns whether removal was successful. A nonexistent `path` is considered a +// failure. bool unlink_safe(const std::string& path, UnlinkLog unlink_log = UnlinkLog::log_failure); @@ -479,26 +379,10 @@ bool unlink_tmp(const std::string& path, // Unset environment variable `name`. void unsetenv(const std::string& name); -// Set mtime of `path` to the current timestamp. -void update_mtime(const std::string& path); - // Remove `path` (and its contents if it's a directory). A nonexistent path is // not considered an error. // -// Throws Error on error. +// Throws core::Error on error. void wipe_path(const std::string& path); -// Write `size` bytes from `data` to `fd`. Throws `Error` on error. -void write_fd(int fd, const void* data, size_t size); - -// Write `data` to `path`. The file will be opened according to `open_mode`, -// which always will include `std::ios::out` even if not specified at the call -// site. -// -// Throws `Error` on error. The description contains the error message without -// the path. -void write_file(const std::string& path, - const std::string& data, - std::ios_base::openmode open_mode = std::ios::binary); - } // namespace Util diff --git a/src/Win32Util.cpp b/src/Win32Util.cpp index 1a9de6a..2769f48 100644 --- a/src/Win32Util.cpp +++ b/src/Win32Util.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -70,6 +70,10 @@ error_message(DWORD error_code) 0, nullptr); std::string message(buffer, size); + while (!message.empty() + && (message.back() == '\n' || message.back() == '\r')) { + message.pop_back(); + } LocalFree(buffer); return message; } @@ -93,10 +97,12 @@ argv_to_string(const char* const* argv, ++bs; break; } - // Fallthrough. + [[fallthrough]]; + case '"': bs = (bs << 1) + 1; - // Fallthrough. + [[fallthrough]]; + default: while (bs > 0) { result += '\\'; @@ -127,28 +133,6 @@ get_last_ntstatus() } // namespace Win32Util -// From: https://stackoverflow.com/a/58162122/262458 -#ifdef _MSC_VER -int -gettimeofday(struct timeval* tp, struct timezone* /*tzp*/) -{ - namespace sc = std::chrono; - sc::system_clock::duration d = sc::system_clock::now().time_since_epoch(); - sc::seconds s = sc::duration_cast(d); - tp->tv_sec = static_cast(s.count()); - tp->tv_usec = - static_cast(sc::duration_cast(d - s).count()); - - return 0; -} -#endif - -void -usleep(int64_t usec) -{ - std::this_thread::sleep_for(std::chrono::microseconds(usec)); -} - struct tm* localtime_r(time_t* _clock, struct tm* _result) { diff --git a/src/Win32Util.hpp b/src/Win32Util.hpp index 0a2f1a1..2611759 100644 --- a/src/Win32Util.hpp +++ b/src/Win32Util.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020-2021 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,9 +18,17 @@ #pragma once -#include "system.hpp" +#ifdef _WIN32 -#include +# include + +# include + +struct tm* localtime_r(time_t* _clock, struct tm* _result); + +# ifdef _MSC_VER +int asprintf(char** strp, const char* fmt, ...); +# endif namespace Win32Util { @@ -44,3 +52,5 @@ std::string error_message(DWORD error_code); NTSTATUS get_last_ntstatus(); } // namespace Win32Util + +#endif diff --git a/src/ZstdCompressor.cpp b/src/ZstdCompressor.cpp deleted file mode 100644 index 39f6bb2..0000000 --- a/src/ZstdCompressor.cpp +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "ZstdCompressor.hpp" - -#include "Logging.hpp" -#include "assertions.hpp" -#include "exceptions.hpp" - -#include - -ZstdCompressor::ZstdCompressor(FILE* stream, int8_t compression_level) - : m_stream(stream), - m_zstd_stream(ZSTD_createCStream()) -{ - if (compression_level == 0) { - compression_level = default_compression_level; - LOG("Using default compression level {}", compression_level); - } - - // libzstd 1.3.4 and newer support negative levels. However, the query - // function ZSTD_minCLevel did not appear until 1.3.6, so perform detection - // based on version instead. - if (ZSTD_versionNumber() < 10304 && compression_level < 1) { - LOG( - "Using compression level 1 (minimum level supported by libzstd) instead" - " of {}", - compression_level); - compression_level = 1; - } - - m_compression_level = std::min(compression_level, ZSTD_maxCLevel()); - if (m_compression_level != compression_level) { - LOG("Using compression level {} (max libzstd level) instead of {}", - m_compression_level, - compression_level); - } - - size_t ret = ZSTD_initCStream(m_zstd_stream, m_compression_level); - if (ZSTD_isError(ret)) { - ZSTD_freeCStream(m_zstd_stream); - throw Error("error initializing zstd compression stream"); - } -} - -ZstdCompressor::~ZstdCompressor() -{ - ZSTD_freeCStream(m_zstd_stream); -} - -int8_t -ZstdCompressor::actual_compression_level() const -{ - return m_compression_level; -} - -void -ZstdCompressor::write(const void* data, size_t count) -{ - m_zstd_in.src = data; - m_zstd_in.size = count; - m_zstd_in.pos = 0; - - int flush = data ? 0 : 1; - - size_t ret; - while (m_zstd_in.pos < m_zstd_in.size) { - uint8_t buffer[READ_BUFFER_SIZE]; - m_zstd_out.dst = buffer; - m_zstd_out.size = sizeof(buffer); - m_zstd_out.pos = 0; - ret = ZSTD_compressStream(m_zstd_stream, &m_zstd_out, &m_zstd_in); - ASSERT(!(ZSTD_isError(ret))); - size_t compressed_bytes = m_zstd_out.pos; - if (fwrite(buffer, 1, compressed_bytes, m_stream) != compressed_bytes - || ferror(m_stream)) { - throw Error("failed to write to zstd output stream "); - } - } - ret = flush; - while (ret > 0) { - uint8_t buffer[READ_BUFFER_SIZE]; - m_zstd_out.dst = buffer; - m_zstd_out.size = sizeof(buffer); - m_zstd_out.pos = 0; - ret = ZSTD_endStream(m_zstd_stream, &m_zstd_out); - size_t compressed_bytes = m_zstd_out.pos; - if (fwrite(buffer, 1, compressed_bytes, m_stream) != compressed_bytes - || ferror(m_stream)) { - throw Error("failed to write to zstd output stream"); - } - } -} - -void -ZstdCompressor::finalize() -{ - write(nullptr, 0); -} diff --git a/src/ZstdCompressor.hpp b/src/ZstdCompressor.hpp deleted file mode 100644 index 58ca453..0000000 --- a/src/ZstdCompressor.hpp +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Compressor.hpp" -#include "NonCopyable.hpp" - -#include - -// A compressor of a Zstandard stream. -class ZstdCompressor : public Compressor, NonCopyable -{ -public: - // Parameters: - // - stream: The file to write data to. - // - compression_level: Desired compression level. - ZstdCompressor(FILE* stream, int8_t compression_level); - - ~ZstdCompressor() override; - - int8_t actual_compression_level() const override; - void write(const void* data, size_t count) override; - void finalize() override; - - constexpr static uint8_t default_compression_level = 1; - -private: - FILE* m_stream; - ZSTD_CStream* m_zstd_stream; - ZSTD_inBuffer m_zstd_in; - ZSTD_outBuffer m_zstd_out; - int8_t m_compression_level; -}; diff --git a/src/ZstdDecompressor.cpp b/src/ZstdDecompressor.cpp deleted file mode 100644 index c08d0f9..0000000 --- a/src/ZstdDecompressor.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "ZstdDecompressor.hpp" - -#include "assertions.hpp" -#include "exceptions.hpp" - -ZstdDecompressor::ZstdDecompressor(FILE* stream) - : m_stream(stream), - m_input_size(0), - m_input_consumed(0), - m_zstd_stream(ZSTD_createDStream()), - m_reached_stream_end(false) -{ - size_t ret = ZSTD_initDStream(m_zstd_stream); - if (ZSTD_isError(ret)) { - ZSTD_freeDStream(m_zstd_stream); - throw Error("failed to initialize zstd decompression stream"); - } -} - -ZstdDecompressor::~ZstdDecompressor() -{ - ZSTD_freeDStream(m_zstd_stream); -} - -void -ZstdDecompressor::read(void* data, size_t count) -{ - size_t bytes_read = 0; - while (bytes_read < count) { - ASSERT(m_input_size >= m_input_consumed); - if (m_input_size == m_input_consumed) { - m_input_size = fread(m_input_buffer, 1, sizeof(m_input_buffer), m_stream); - if (m_input_size == 0) { - throw Error("failed to read from zstd input stream"); - } - m_input_consumed = 0; - } - - m_zstd_in.src = (m_input_buffer + m_input_consumed); - m_zstd_in.size = m_input_size - m_input_consumed; - m_zstd_in.pos = 0; - - m_zstd_out.dst = static_cast(data) + bytes_read; - m_zstd_out.size = count - bytes_read; - m_zstd_out.pos = 0; - size_t ret = ZSTD_decompressStream(m_zstd_stream, &m_zstd_out, &m_zstd_in); - if (ZSTD_isError(ret)) { - throw Error("failed to read from zstd input stream"); - } - if (ret == 0) { - m_reached_stream_end = true; - break; - } - bytes_read += m_zstd_out.pos; - m_input_consumed += m_zstd_in.pos; - } -} - -void -ZstdDecompressor::finalize() -{ - if (!m_reached_stream_end) { - throw Error("garbage data at end of zstd input stream"); - } -} diff --git a/src/ZstdDecompressor.hpp b/src/ZstdDecompressor.hpp deleted file mode 100644 index 8f85ce6..0000000 --- a/src/ZstdDecompressor.hpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Decompressor.hpp" - -#include -#include - -// A decompressor of a Zstandard stream. -class ZstdDecompressor : public Decompressor -{ -public: - // Parameters: - // - stream: The file to read data from. - explicit ZstdDecompressor(FILE* stream); - - ~ZstdDecompressor() override; - - void read(void* data, size_t count) override; - void finalize() override; - -private: - FILE* m_stream; - char m_input_buffer[READ_BUFFER_SIZE]; - size_t m_input_size; - size_t m_input_consumed; - ZSTD_DStream* m_zstd_stream; - ZSTD_inBuffer m_zstd_in; - ZSTD_outBuffer m_zstd_out; - bool m_reached_stream_end; -}; diff --git a/src/argprocessing.cpp b/src/argprocessing.cpp index 92b929e..0bc37e5 100644 --- a/src/argprocessing.cpp +++ b/src/argprocessing.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020-2021 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -19,23 +19,39 @@ #include "argprocessing.hpp" #include "Context.hpp" -#include "FormatNonstdStringView.hpp" #include "Logging.hpp" #include "assertions.hpp" #include "compopt.hpp" #include "fmtmacros.hpp" #include "language.hpp" +#include +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + #include -using nonstd::nullopt; -using nonstd::optional; -using nonstd::string_view; +using core::Statistic; namespace { enum class ColorDiagnostics : int8_t { never, automatic, always }; +// The dependency target in the dependency file is taken from the highest +// priority source. +enum class OutputDepOrigin : uint8_t { + // Not set + none = 0, + // From -MF target + mf = 1, + // From -Wp,-MD,target or -Wp,-MMD,target + wp = 2 +}; + struct ArgumentProcessingState { bool found_c_opt = false; @@ -43,19 +59,23 @@ struct ArgumentProcessingState bool found_S_opt = false; bool found_pch = false; bool found_fpch_preprocess = false; + bool found_Yu = false; + bool found_valid_Fp = false; + bool found_syntax_only = false; ColorDiagnostics color_diagnostics = ColorDiagnostics::automatic; bool found_directives_only = false; bool found_rewrite_includes = false; + std::optional found_xarch_arch; + bool found_mf_opt = false; + bool found_wp_md_or_mmd_opt = false; + bool found_md_or_mmd_opt = false; + bool found_Wa_a_opt = false; std::string explicit_language; // As specified with -x. std::string input_charset_option; // -finput-charset=... - // Is the dependency makefile name overridden with -MF? - bool dependency_filename_specified = false; - - // Is the dependency target name implicitly specified using - // DEPENDENCIES_OUTPUT or SUNPRO_DEPENDENCIES? - bool dependency_implicit_target_specified = false; + // Is the dependency file set via -Wp,-M[M]D,target or -MFtarget? + OutputDepOrigin output_dep_origin = OutputDepOrigin::none; // Is the compiler being asked to output debug info on level 3? bool generating_debuginfo_level_3 = false; @@ -87,30 +107,62 @@ struct ArgumentProcessingState // Whether to include the full command line in the hash. bool hash_full_command_line = false; + + // Whether to include the actual CWD in the hash. + bool hash_actual_cwd = false; }; bool color_output_possible() { const char* term_env = getenv("TERM"); - return isatty(STDERR_FILENO) && term_env && strcasecmp(term_env, "DUMB") != 0; + return isatty(STDERR_FILENO) && term_env + && Util::to_lowercase(term_env) != "dumb"; } bool -detect_pch(Context& ctx, - const std::string& option, +detect_pch(const std::string& option, const std::string& arg, + std::string& included_pch_file, bool is_cc1_option, - bool* found_pch) + ArgumentProcessingState& state) { - ASSERT(found_pch); - // Try to be smart about detecting precompiled headers. // If the option is an option for Clang (is_cc1_option), don't accept // anything just because it has a corresponding precompiled header, // because Clang doesn't behave that way either. std::string pch_file; - if (option == "-include-pch" || option == "-include-pth") { + if (option == "-Yu") { + state.found_Yu = true; + if (state.found_valid_Fp) { // Use file set by -Fp. + LOG("Detected use of precompiled header: {}", included_pch_file); + pch_file = included_pch_file; + included_pch_file.clear(); // reset pch file set from /Fp + } else { + std::string file = Util::change_extension(arg, ".pch"); + if (Stat::stat(file)) { + LOG("Detected use of precompiled header: {}", file); + pch_file = file; + } + } + } else if (option == "-Fp") { + std::string file = arg; + if (Util::get_extension(file).empty()) { + file += ".pch"; + } + if (Stat::stat(file)) { + state.found_valid_Fp = true; + if (!state.found_Yu) { + LOG("Precompiled header file specified: {}", file); + included_pch_file = file; // remember file + return true; // -Fp does not turn on PCH + } + LOG("Detected use of precompiled header: {}", file); + pch_file = file; + included_pch_file.clear(); // reset pch file set from /Yu + // continue and set as if the file was passed to -Yu + } + } else if (option == "-include-pch" || option == "-include-pth") { if (Stat::stat(arg)) { LOG("Detected use of precompiled header: {}", arg); pch_file = arg; @@ -126,20 +178,22 @@ detect_pch(Context& ctx, } if (!pch_file.empty()) { - if (!ctx.included_pch_file.empty()) { + if (!included_pch_file.empty()) { LOG("Multiple precompiled headers used: {} and {}", - ctx.included_pch_file, + included_pch_file, pch_file); return false; } - ctx.included_pch_file = pch_file; - *found_pch = true; + included_pch_file = pch_file; + state.found_pch = true; } return true; } bool -process_profiling_option(Context& ctx, const std::string& arg) +process_profiling_option(const Context& ctx, + ArgsInfo& args_info, + const std::string& arg) { static const std::vector known_simple_options = { "-fprofile-correction", @@ -156,31 +210,31 @@ process_profiling_option(Context& ctx, const std::string& arg) std::string new_profile_path; bool new_profile_use = false; - if (Util::starts_with(arg, "-fprofile-dir=")) { + if (util::starts_with(arg, "-fprofile-dir=")) { new_profile_path = arg.substr(arg.find('=') + 1); } else if (arg == "-fprofile-generate" || arg == "-fprofile-instr-generate") { - ctx.args_info.profile_generate = true; - if (ctx.config.compiler_type() == CompilerType::clang) { + args_info.profile_generate = true; + if (ctx.config.is_compiler_group_clang()) { new_profile_path = "."; } else { // GCC uses $PWD/$(basename $obj). new_profile_path = ctx.apparent_cwd; } - } else if (Util::starts_with(arg, "-fprofile-generate=") - || Util::starts_with(arg, "-fprofile-instr-generate=")) { - ctx.args_info.profile_generate = true; + } else if (util::starts_with(arg, "-fprofile-generate=") + || util::starts_with(arg, "-fprofile-instr-generate=")) { + args_info.profile_generate = true; new_profile_path = arg.substr(arg.find('=') + 1); } else if (arg == "-fprofile-use" || arg == "-fprofile-instr-use" || arg == "-fprofile-sample-use" || arg == "-fbranch-probabilities" || arg == "-fauto-profile") { new_profile_use = true; - if (ctx.args_info.profile_path.empty()) { + if (args_info.profile_path.empty()) { new_profile_path = "."; } - } else if (Util::starts_with(arg, "-fprofile-use=") - || Util::starts_with(arg, "-fprofile-instr-use=") - || Util::starts_with(arg, "-fprofile-sample-use=") - || Util::starts_with(arg, "-fauto-profile=")) { + } else if (util::starts_with(arg, "-fprofile-use=") + || util::starts_with(arg, "-fprofile-instr-use=") + || util::starts_with(arg, "-fprofile-sample-use=") + || util::starts_with(arg, "-fauto-profile=")) { new_profile_use = true; new_profile_path = arg.substr(arg.find('=') + 1); } else { @@ -189,19 +243,19 @@ process_profiling_option(Context& ctx, const std::string& arg) } if (new_profile_use) { - if (ctx.args_info.profile_use) { + if (args_info.profile_use) { LOG_RAW("Multiple profiling options not supported"); return false; } - ctx.args_info.profile_use = true; + args_info.profile_use = true; } if (!new_profile_path.empty()) { - ctx.args_info.profile_path = new_profile_path; - LOG("Set profile directory to {}", ctx.args_info.profile_path); + args_info.profile_path = new_profile_path; + LOG("Set profile directory to {}", args_info.profile_path); } - if (ctx.args_info.profile_generate && ctx.args_info.profile_use) { + if (args_info.profile_generate && args_info.profile_use) { // Too hard to figure out what the compiler will do. LOG_RAW("Both generating and using profile info, giving up"); return false; @@ -210,17 +264,17 @@ process_profiling_option(Context& ctx, const std::string& arg) return true; } -optional -process_arg(Context& ctx, - Args& args, - size_t& args_index, - ArgumentProcessingState& state) +// Returns std::nullopt if the option wasn't recognized, otherwise the error +// code (with Statistic::none for "no error"). +std::optional +process_option_arg(const Context& ctx, + ArgsInfo& args_info, + Config& config, + Args& args, + size_t& args_index, + ArgumentProcessingState& state) { - ArgsInfo& args_info = ctx.args_info; - Config& config = ctx.config; - size_t& i = args_index; - // The user knows best: just swallow the next arg. if (args[i] == "--ccache-skip") { i++; @@ -229,12 +283,20 @@ process_arg(Context& ctx, return Statistic::bad_compiler_arguments; } state.common_args.push_back(args[i]); - return nullopt; + return Statistic::none; + } + + bool changed_from_slash = false; + if (ctx.config.is_compiler_group_msvc() && util::starts_with(args[i], "/")) { + // MSVC understands both /option and -option, so convert all /option to + // -option to simplify our handling. + args[i][0] = '-'; + changed_from_slash = true; } // Ignore clang -ivfsoverlay to not detect multiple input files. if (args[i] == "-ivfsoverlay" - && !(config.sloppiness() & SLOPPY_IVFSOVERLAY)) { + && !(config.sloppiness().is_enabled(core::Sloppy::ivfsoverlay))) { LOG_RAW( "You have to specify \"ivfsoverlay\" sloppiness when using" " -ivfsoverlay to get hits"); @@ -245,15 +307,22 @@ process_arg(Context& ctx, if (args[i] == "-E") { return Statistic::called_for_preprocessing; } + // MSVC -P is -E with output to a file. + if (args[i] == "-P" && ctx.config.is_compiler_group_msvc()) { + return Statistic::called_for_preprocessing; + } // Handle "@file" argument. - if (Util::starts_with(args[i], "@") || Util::starts_with(args[i], "-@")) { + if (util::starts_with(args[i], "@") || util::starts_with(args[i], "-@")) { const char* argpath = args[i].c_str() + 1; if (argpath[-1] == '-') { ++argpath; } - auto file_args = Args::from_gcc_atfile(argpath); + auto file_args = Args::from_atfile(argpath, + config.is_compiler_group_msvc() + ? Args::AtFileFormat::msvc + : Args::AtFileFormat::gcc); if (!file_args) { LOG("Couldn't read arg file {}", argpath); return Statistic::bad_compiler_arguments; @@ -261,7 +330,7 @@ process_arg(Context& ctx, args.replace(i, *file_args); i--; - return nullopt; + return Statistic::none; } // Handle cuda "-optf" and "--options-file" argument. @@ -276,7 +345,7 @@ process_arg(Context& ctx, // Argument is a comma-separated list of files. auto paths = Util::split_into_strings(args[i], ","); for (auto it = paths.rbegin(); it != paths.rend(); ++it) { - auto file_args = Args::from_gcc_atfile(*it); + auto file_args = Args::from_atfile(*it); if (!file_args) { LOG("Couldn't read CUDA options file {}", *it); return Statistic::bad_compiler_arguments; @@ -285,12 +354,13 @@ process_arg(Context& ctx, args.insert(i + 1, *file_args); } - return nullopt; + return Statistic::none; } // These are always too hard. - if (compopt_too_hard(args[i]) || Util::starts_with(args[i], "-fdump-") - || Util::starts_with(args[i], "-MJ")) { + if (compopt_too_hard(args[i]) || util::starts_with(args[i], "-fdump-") + || util::starts_with(args[i], "-MJ") + || util::starts_with(args[i], "-Yc")) { LOG("Compiler option {} is unsupported", args[i]); return Statistic::unsupported_compiler_option; } @@ -302,9 +372,20 @@ process_arg(Context& ctx, } // -Xarch_* options are too hard. - if (Util::starts_with(args[i], "-Xarch_")) { - LOG("Unsupported compiler option: {}", args[i]); - return Statistic::unsupported_compiler_option; + if (util::starts_with(args[i], "-Xarch_")) { + if (i == args.size() - 1) { + LOG("Missing argument to {}", args[i]); + return Statistic::bad_compiler_arguments; + } + const auto arch = args[i].substr(7); + if (!state.found_xarch_arch) { + state.found_xarch_arch = arch; + } else if (*state.found_xarch_arch != arch) { + LOG_RAW("Multiple different -Xarch_* options not supported"); + return Statistic::unsupported_compiler_option; + } + state.common_args.push_back(args[i]); + return Statistic::none; } // Handle -arch options. @@ -314,13 +395,13 @@ process_arg(Context& ctx, if (args_info.arch_args.size() == 2) { config.set_run_second_cpp(true); } - return nullopt; + return Statistic::none; } // Some arguments that clang passes directly to cc1 (related to precompiled // headers) need the usual ccache handling. In those cases, the -Xclang // prefix is skipped and the cc1 argument is handled instead. - if (args[i] == "-Xclang" && i < args.size() - 1 + if (args[i] == "-Xclang" && i + 1 < args.size() && (args[i + 1] == "-emit-pch" || args[i + 1] == "-emit-pth" || args[i + 1] == "-include-pch" || args[i + 1] == "-include-pth" || args[i + 1] == "-fno-pch-timestamp")) { @@ -334,6 +415,24 @@ process_arg(Context& ctx, ++i; } + if (util::starts_with(args[i], "-Wa,")) { + for (const auto part : util::Tokenizer(&args[i][4], ",")) { + if (util::starts_with(part, "-a")) { + if (state.found_Wa_a_opt) { + LOG_RAW( + "Multiple assembler listing options (-Wa,-a) are not supported"); + return Statistic::unsupported_compiler_option; + } + state.found_Wa_a_opt = true; + + const auto eq_pos = part.find('='); + if (eq_pos != std::string_view::npos) { + args_info.output_al = part.substr(eq_pos + 1); + } + } + } + } + // Handle options that should not be passed to the preprocessor. if (compopt_affects_compiler_output(args[i])) { state.compiler_only_args.push_back(args[i]); @@ -347,11 +446,11 @@ process_arg(Context& ctx, state.compiler_only_args.push_back(args[i + 1]); ++i; } - return nullopt; + return Statistic::none; } if (compopt_prefix_affects_compiler_output(args[i])) { state.compiler_only_args.push_back(args[i]); - return nullopt; + return Statistic::none; } // Modules are handled on demand as necessary in the background, so there is @@ -367,7 +466,7 @@ process_arg(Context& ctx, LOG("Compiler option {} is unsupported without direct depend mode", args[i]); return Statistic::could_not_use_modules; - } else if (!(config.sloppiness() & SLOPPY_MODULES)) { + } else if (!(config.sloppiness().is_enabled(core::Sloppy::modules))) { LOG_RAW( "You have to specify \"modules\" sloppiness when using" " -fmodules to get hits"); @@ -378,31 +477,37 @@ process_arg(Context& ctx, // We must have -c. if (args[i] == "-c") { state.found_c_opt = true; - return nullopt; + return Statistic::none; + } + + // MSVC -Fo with no space. + if (util::starts_with(args[i], "-Fo") && config.is_compiler_group_msvc()) { + args_info.output_obj = args[i].substr(3); + return Statistic::none; } // when using nvcc with separable compilation, -dc implies -c if ((args[i] == "-dc" || args[i] == "--device-c") && config.compiler_type() == CompilerType::nvcc) { state.found_dc_opt = true; - return nullopt; + return Statistic::none; } // -S changes the default extension. if (args[i] == "-S") { state.common_args.push_back(args[i]); state.found_S_opt = true; - return nullopt; + return Statistic::none; } - if (Util::starts_with(args[i], "-x")) { + if (util::starts_with(args[i], "-x")) { if (args[i].length() >= 3 && !islower(args[i][2])) { // -xCODE (where CODE can be e.g. Host or CORE-AVX2, always starting with // an uppercase letter) is an ordinary Intel compiler option, not a // language specification. (GCC's "-x" language argument is always // lowercase.) state.common_args.push_back(args[i]); - return nullopt; + return Statistic::none; } // Special handling for -x: remember the last specified language before the @@ -416,14 +521,14 @@ process_arg(Context& ctx, state.explicit_language = args[i + 1]; } i++; - return nullopt; + return Statistic::none; } DEBUG_ASSERT(args[i].length() >= 3); if (args_info.input_file.empty()) { state.explicit_language = args[i].substr(2); } - return nullopt; + return Statistic::none; } // We need to work out where the output was meant to go. @@ -432,42 +537,45 @@ process_arg(Context& ctx, LOG("Missing argument to {}", args[i]); return Statistic::bad_compiler_arguments; } - args_info.output_obj = Util::make_relative_path(ctx, args[i + 1]); + args_info.output_obj = args[i + 1]; i++; - return nullopt; + return Statistic::none; } // Alternate form of -o with no space. Nvcc does not support this. - if (Util::starts_with(args[i], "-o") - && config.compiler_type() != CompilerType::nvcc) { - args_info.output_obj = - Util::make_relative_path(ctx, string_view(args[i]).substr(2)); - return nullopt; - } - - if (Util::starts_with(args[i], "-fdebug-prefix-map=") - || Util::starts_with(args[i], "-ffile-prefix-map=")) { + // Cl does support it as deprecated, but also has -openmp or -link -out + // which can confuse this and cause incorrect output_obj (and thus + // ccache debug file location), so better ignore it. + if (util::starts_with(args[i], "-o") + && config.compiler_type() != CompilerType::nvcc + && config.compiler_type() != CompilerType::msvc) { + args_info.output_obj = args[i].substr(2); + return Statistic::none; + } + + if (util::starts_with(args[i], "-fdebug-prefix-map=") + || util::starts_with(args[i], "-ffile-prefix-map=")) { std::string map = args[i].substr(args[i].find('=') + 1); args_info.debug_prefix_maps.push_back(map); state.common_args.push_back(args[i]); - return nullopt; + return Statistic::none; } // Debugging is handled specially, so that we know if we can strip line // number info. - if (Util::starts_with(args[i], "-g")) { + if (util::starts_with(args[i], "-g")) { state.common_args.push_back(args[i]); - if (Util::starts_with(args[i], "-gdwarf")) { + if (util::starts_with(args[i], "-gdwarf")) { // Selection of DWARF format (-gdwarf or -gdwarf-) enables // debug info on level 2. args_info.generating_debuginfo = true; - return nullopt; + return Statistic::none; } - if (Util::starts_with(args[i], "-gz")) { + if (util::starts_with(args[i], "-gz")) { // -gz[=type] neither disables nor enables debug info. - return nullopt; + return Statistic::none; } char last_char = args[i].back(); @@ -484,20 +592,21 @@ process_arg(Context& ctx, args_info.seen_split_dwarf = true; } } - return nullopt; + return Statistic::none; } // These options require special handling, because they behave differently // with gcc -E, when the output file is not specified. - if (args[i] == "-MD" || args[i] == "-MMD") { + if ((args[i] == "-MD" || args[i] == "-MMD") + && !config.is_compiler_group_msvc()) { + state.found_md_or_mmd_opt = true; args_info.generating_dependencies = true; - args_info.seen_MD_MMD = true; state.dep_args.push_back(args[i]); - return nullopt; + return Statistic::none; } - if (Util::starts_with(args[i], "-MF")) { - state.dependency_filename_specified = true; + if (util::starts_with(args[i], "-MF")) { + state.found_mf_opt = true; std::string dep_file; bool separate_argument = (args[i].size() == 3); @@ -513,7 +622,11 @@ process_arg(Context& ctx, // -MFarg or -MF=arg (EDG-based compilers) dep_file = args[i].substr(args[i][3] == '=' ? 4 : 3); } - args_info.output_dep = Util::make_relative_path(ctx, dep_file); + + if (state.output_dep_origin <= OutputDepOrigin::mf) { + state.output_dep_origin = OutputDepOrigin::mf; + args_info.output_dep = Util::make_relative_path(ctx, dep_file); + } // Keep the format of the args the same. if (separate_argument) { state.dep_args.push_back("-MF"); @@ -521,12 +634,14 @@ process_arg(Context& ctx, } else { state.dep_args.push_back("-MF" + args_info.output_dep); } - return nullopt; + return Statistic::none; } - if (Util::starts_with(args[i], "-MQ") || Util::starts_with(args[i], "-MT")) { - ctx.args_info.dependency_target_specified = true; + if ((util::starts_with(args[i], "-MQ") || util::starts_with(args[i], "-MT")) + && !config.is_compiler_group_msvc()) { + const bool is_mq = args[i][2] == 'Q'; + std::string_view dep_target; if (args[i].size() == 3) { // -MQ arg or -MT arg if (i == args.size() - 1) { @@ -534,40 +649,69 @@ process_arg(Context& ctx, return Statistic::bad_compiler_arguments; } state.dep_args.push_back(args[i]); - std::string relpath = Util::make_relative_path(ctx, args[i + 1]); - state.dep_args.push_back(relpath); + state.dep_args.push_back(args[i + 1]); + dep_target = args[i + 1]; i++; } else { - auto arg_opt = string_view(args[i]).substr(0, 3); - auto option = string_view(args[i]).substr(3); - auto relpath = Util::make_relative_path(ctx, option); - state.dep_args.push_back(FMT("{}{}", arg_opt, relpath)); + // -MQarg or -MTarg + const std::string_view arg_view(args[i]); + const auto arg_opt = arg_view.substr(0, 3); + dep_target = arg_view.substr(3); + state.dep_args.push_back(FMT("{}{}", arg_opt, dep_target)); + } + + if (args_info.dependency_target) { + args_info.dependency_target->push_back(' '); + } else { + args_info.dependency_target = ""; } - return nullopt; + *args_info.dependency_target += + is_mq ? Depfile::escape_filename(dep_target) : dep_target; + + return Statistic::none; + } + + // MSVC -MD[d], -MT[d] and -LT[d] options are something different than GCC's + // -MD etc. + if (config.is_compiler_group_msvc() + && (util::starts_with(args[i], "-MD") || util::starts_with(args[i], "-MT") + || util::starts_with(args[i], "-LD"))) { + // These affect compiler but also #define some things. + state.cpp_args.push_back(args[i]); + state.common_args.push_back(args[i]); + return Statistic::none; + } + + if (args[i] == "-showIncludes") { + args_info.generating_includes = true; + state.dep_args.push_back(args[i]); + return Statistic::none; } if (args[i] == "-fprofile-arcs") { args_info.profile_arcs = true; state.common_args.push_back(args[i]); - return nullopt; + return Statistic::none; } if (args[i] == "-ftest-coverage") { args_info.generating_coverage = true; state.common_args.push_back(args[i]); - return nullopt; + return Statistic::none; } if (args[i] == "-fstack-usage") { args_info.generating_stackusage = true; state.common_args.push_back(args[i]); - return nullopt; + return Statistic::none; } - if (args[i] == "-fsyntax-only") { + // -Zs is MSVC's -fsyntax-only equivalent + if (args[i] == "-fsyntax-only" || args[i] == "-Zs") { args_info.expect_output_obj = false; state.compiler_only_args.push_back(args[i]); - return nullopt; + state.found_syntax_only = true; + return Statistic::none; } if (args[i] == "--coverage" // = -fprofile-arcs -ftest-coverage @@ -575,31 +719,40 @@ process_arg(Context& ctx, args_info.profile_arcs = true; args_info.generating_coverage = true; state.common_args.push_back(args[i]); - return nullopt; + return Statistic::none; + } + + if (args[i] == "-fprofile-abs-path") { + if (!config.sloppiness().is_enabled(core::Sloppy::gcno_cwd)) { + // -fprofile-abs-path makes the compiler include absolute paths based on + // the actual CWD in the .gcno file. + state.hash_actual_cwd = true; + } + return Statistic::none; } - if (Util::starts_with(args[i], "-fprofile-") - || Util::starts_with(args[i], "-fauto-profile") + if (util::starts_with(args[i], "-fprofile-") + || util::starts_with(args[i], "-fauto-profile") || args[i] == "-fbranch-probabilities") { - if (!process_profiling_option(ctx, args[i])) { + if (!process_profiling_option(ctx, args_info, args[i])) { // The failure is logged by process_profiling_option. return Statistic::unsupported_compiler_option; } state.common_args.push_back(args[i]); - return nullopt; + return Statistic::none; } - if (Util::starts_with(args[i], "-fsanitize-blacklist=")) { + if (util::starts_with(args[i], "-fsanitize-blacklist=")) { args_info.sanitize_blacklists.emplace_back(args[i].substr(21)); state.common_args.push_back(args[i]); - return nullopt; + return Statistic::none; } - if (Util::starts_with(args[i], "--sysroot=")) { - auto path = string_view(args[i]).substr(10); + if (util::starts_with(args[i], "--sysroot=")) { + auto path = std::string_view(args[i]).substr(10); auto relpath = Util::make_relative_path(ctx, path); state.common_args.push_back("--sysroot=" + relpath); - return nullopt; + return Statistic::none; } // Alternate form of specifying sysroot without = @@ -612,7 +765,7 @@ process_arg(Context& ctx, auto relpath = Util::make_relative_path(ctx, args[i + 1]); state.common_args.push_back(relpath); i++; - return nullopt; + return Statistic::none; } // Alternate form of specifying target without = @@ -624,7 +777,7 @@ process_arg(Context& ctx, state.common_args.push_back(args[i]); state.common_args.push_back(args[i + 1]); i++; - return nullopt; + return Statistic::none; } if (args[i] == "-P" || args[i] == "-Wp,-P") { @@ -633,44 +786,48 @@ process_arg(Context& ctx, state.compiler_only_args.push_back(args[i]); LOG("{} used; not compiling preprocessed code", args[i]); config.set_run_second_cpp(true); - return nullopt; + return Statistic::none; } - if (Util::starts_with(args[i], "-Wp,")) { + if (util::starts_with(args[i], "-Wp,")) { if (args[i].find(",-P,") != std::string::npos - || Util::ends_with(args[i], ",-P")) { - // -P together with other preprocessor options is just too hard. + || util::ends_with(args[i], ",-P")) { + LOG("-P together with other preprocessor options is too hard: {}", + args[i]); return Statistic::unsupported_compiler_option; - } else if (Util::starts_with(args[i], "-Wp,-MD,") + } else if (util::starts_with(args[i], "-Wp,-MD,") && args[i].find(',', 8) == std::string::npos) { + state.found_wp_md_or_mmd_opt = true; args_info.generating_dependencies = true; - state.dependency_filename_specified = true; - args_info.output_dep = - Util::make_relative_path(ctx, string_view(args[i]).substr(8)); + if (state.output_dep_origin <= OutputDepOrigin::wp) { + state.output_dep_origin = OutputDepOrigin::wp; + args_info.output_dep = args[i].substr(8); + } state.dep_args.push_back(args[i]); - return nullopt; - } else if (Util::starts_with(args[i], "-Wp,-MMD,") + return Statistic::none; + } else if (util::starts_with(args[i], "-Wp,-MMD,") && args[i].find(',', 9) == std::string::npos) { + state.found_wp_md_or_mmd_opt = true; args_info.generating_dependencies = true; - state.dependency_filename_specified = true; - args_info.output_dep = - Util::make_relative_path(ctx, string_view(args[i]).substr(9)); + if (state.output_dep_origin <= OutputDepOrigin::wp) { + state.output_dep_origin = OutputDepOrigin::wp; + args_info.output_dep = args[i].substr(9); + } state.dep_args.push_back(args[i]); - return nullopt; - } else if (Util::starts_with(args[i], "-Wp,-D") + return Statistic::none; + } else if (util::starts_with(args[i], "-Wp,-D") && args[i].find(',', 6) == std::string::npos) { // Treat it like -D. state.cpp_args.push_back(args[i].substr(4)); - return nullopt; + return Statistic::none; } else if (args[i] == "-Wp,-MP" - || (args[i].size() > 8 && Util::starts_with(args[i], "-Wp,-M") + || (args[i].size() > 8 && util::starts_with(args[i], "-Wp,-M") && args[i][7] == ',' && (args[i][6] == 'F' || args[i][6] == 'Q' || args[i][6] == 'T') && args[i].find(',', 8) == std::string::npos)) { - // TODO: Make argument to MF/MQ/MT relative. state.dep_args.push_back(args[i]); - return nullopt; + return Statistic::none; } else if (config.direct_mode()) { // -Wp, can be used to pass too hard options to the preprocessor. // Hence, disable direct mode. @@ -680,18 +837,18 @@ process_arg(Context& ctx, // Any other -Wp,* arguments are only relevant for the preprocessor. state.cpp_args.push_back(args[i]); - return nullopt; + return Statistic::none; } if (args[i] == "-MP") { state.dep_args.push_back(args[i]); - return nullopt; + return Statistic::none; } // Input charset needs to be handled specially. - if (Util::starts_with(args[i], "-finput-charset=")) { + if (util::starts_with(args[i], "-finput-charset=")) { state.input_charset_option = args[i]; - return nullopt; + return Statistic::none; } if (args[i] == "--serialize-diagnostics") { @@ -702,73 +859,69 @@ process_arg(Context& ctx, args_info.generating_diagnostics = true; args_info.output_dia = Util::make_relative_path(ctx, args[i + 1]); i++; - return nullopt; - } - - if (config.compiler_type() == CompilerType::gcc - && (args[i] == "-fcolor-diagnostics" - || args[i] == "-fno-color-diagnostics")) { - // Special case: If a GCC compiler gets -f(no-)color-diagnostics we'll bail - // out and just execute the compiler. The reason is that we don't include - // -f(no-)color-diagnostics in the hash so there can be a false cache hit in - // the following scenario: - // - // 1. ccache gcc -c example.c # adds a cache entry - // 2. ccache gcc -c example.c -fcolor-diagnostics # unexpectedly succeeds - return Statistic::unsupported_compiler_option; - } - - // In the "-Xclang -fcolor-diagnostics" form, -Xclang is skipped and the - // -fcolor-diagnostics argument which is passed to cc1 is handled below. - if (args[i] == "-Xclang" && i < args.size() - 1 - && args[i + 1] == "-fcolor-diagnostics") { - state.compiler_only_args_no_hash.push_back(args[i]); - ++i; - } - - if (args[i] == "-fcolor-diagnostics" || args[i] == "-fdiagnostics-color" - || args[i] == "-fdiagnostics-color=always") { - state.color_diagnostics = ColorDiagnostics::always; - state.compiler_only_args_no_hash.push_back(args[i]); - return nullopt; - } - if (args[i] == "-fno-color-diagnostics" || args[i] == "-fno-diagnostics-color" - || args[i] == "-fdiagnostics-color=never") { - state.color_diagnostics = ColorDiagnostics::never; - state.compiler_only_args_no_hash.push_back(args[i]); - return nullopt; - } - if (args[i] == "-fdiagnostics-color=auto") { - state.color_diagnostics = ColorDiagnostics::automatic; - state.compiler_only_args_no_hash.push_back(args[i]); - return nullopt; + return Statistic::none; + } + + if (config.compiler_type() == CompilerType::gcc) { + if (args[i] == "-fdiagnostics-color" + || args[i] == "-fdiagnostics-color=always") { + state.color_diagnostics = ColorDiagnostics::always; + state.compiler_only_args_no_hash.push_back(args[i]); + return Statistic::none; + } else if (args[i] == "-fno-diagnostics-color" + || args[i] == "-fdiagnostics-color=never") { + state.color_diagnostics = ColorDiagnostics::never; + state.compiler_only_args_no_hash.push_back(args[i]); + return Statistic::none; + } else if (args[i] == "-fdiagnostics-color=auto") { + state.color_diagnostics = ColorDiagnostics::automatic; + state.compiler_only_args_no_hash.push_back(args[i]); + return Statistic::none; + } + } else if (config.is_compiler_group_clang()) { + // In the "-Xclang -fcolor-diagnostics" form, -Xclang is skipped and the + // -fcolor-diagnostics argument which is passed to cc1 is handled below. + if (args[i] == "-Xclang" && i + 1 < args.size() + && args[i + 1] == "-fcolor-diagnostics") { + state.compiler_only_args_no_hash.push_back(args[i]); + ++i; + } + if (args[i] == "-fcolor-diagnostics") { + state.color_diagnostics = ColorDiagnostics::always; + state.compiler_only_args_no_hash.push_back(args[i]); + return Statistic::none; + } else if (args[i] == "-fno-color-diagnostics") { + state.color_diagnostics = ColorDiagnostics::never; + state.compiler_only_args_no_hash.push_back(args[i]); + return Statistic::none; + } } // GCC if (args[i] == "-fdirectives-only") { state.found_directives_only = true; - return nullopt; + return Statistic::none; } // Clang if (args[i] == "-frewrite-includes") { state.found_rewrite_includes = true; - return nullopt; + return Statistic::none; } if (args[i] == "-fno-pch-timestamp") { args_info.fno_pch_timestamp = true; state.common_args.push_back(args[i]); - return nullopt; + return Statistic::none; } if (args[i] == "-fpch-preprocess") { state.found_fpch_preprocess = true; state.common_args.push_back(args[i]); - return nullopt; + return Statistic::none; } - if (config.sloppiness() & SLOPPY_CLANG_INDEX_STORE + if (config.sloppiness().is_enabled(core::Sloppy::clang_index_store) && args[i] == "-index-store-path") { // Xcode 9 or later calls Clang with this option. The given path includes a // UUID that might lead to cache misses, especially when cache is shared @@ -777,16 +930,19 @@ process_arg(Context& ctx, if (i <= args.size() - 1) { LOG("Skipping argument -index-store-path {}", args[i]); } - return nullopt; + return Statistic::none; } if (args[i] == "-frecord-gcc-switches") { state.hash_full_command_line = true; } - // Options taking an argument that we may want to rewrite to relative paths to - // get better hit rate. A secondary effect is that paths in the standard error - // output produced by the compiler will be normalized. + // MSVC -u is something else than GCC -u, handle it specially. + if (args[i] == "-u" && ctx.config.is_compiler_group_msvc()) { + state.cpp_args.push_back(args[i]); + return Statistic::none; + } + if (compopt_takes_path(args[i])) { if (i == args.size() - 1) { LOG("Missing argument to {}", args[i]); @@ -795,16 +951,19 @@ process_arg(Context& ctx, // In the -Xclang -include-(pch/pth) -Xclang case, the path is one // index further behind. - int next = 1; - if (args[i + 1] == "-Xclang" && i + 2 < args.size()) { - next = 2; - } + const size_t next = args[i + 1] == "-Xclang" && i + 2 < args.size() ? 2 : 1; - if (!detect_pch( - ctx, args[i], args[i + next], next == 2, &state.found_pch)) { + if (!detect_pch(args[i], + args[i + next], + args_info.included_pch_file, + next == 2, + state)) { return Statistic::bad_compiler_arguments; } + // Potentially rewrite path argument to relative path to get better hit + // rate. A secondary effect is that paths in the standard error output + // produced by the compiler will be normalized. std::string relpath = Util::make_relative_path(ctx, args[i + next]); auto& dest_args = compopt_affects_cpp_output(args[i]) ? state.cpp_args : state.common_args; @@ -815,25 +974,40 @@ process_arg(Context& ctx, dest_args.push_back(relpath); i += next; - return nullopt; + return Statistic::none; + } + + // Detect PCH for options with concatenated path (relative or absolute). + if (util::starts_with(args[i], "-include") + || util::starts_with(args[i], "-Fp") + || util::starts_with(args[i], "-Yu")) { + const size_t path_pos = util::starts_with(args[i], "-include") ? 8 : 3; + if (!detect_pch(args[i].substr(0, path_pos), + args[i].substr(path_pos), + args_info.included_pch_file, + false, + state)) { + return Statistic::bad_compiler_arguments; + } + + // Fall through to the next section, so intentionally not returning here. } - // Same as above but options with concatenated argument beginning with a - // slash. + // Potentially rewrite concatenated absolute path argument to relative. if (args[i][0] == '-') { - size_t slash_pos = args[i].find('/'); - if (slash_pos != std::string::npos) { - std::string option = args[i].substr(0, slash_pos); + const auto path_pos = Util::is_absolute_path_with_prefix(args[i]); + if (path_pos) { + const std::string option = args[i].substr(0, *path_pos); if (compopt_takes_concat_arg(option) && compopt_takes_path(option)) { - auto relpath = - Util::make_relative_path(ctx, string_view(args[i]).substr(slash_pos)); + const auto relpath = Util::make_relative_path( + ctx, std::string_view(args[i]).substr(*path_pos)); std::string new_option = option + relpath; if (compopt_affects_cpp_output(option)) { state.cpp_args.push_back(new_option); } else { state.common_args.push_back(new_option); } - return nullopt; + return Statistic::none; } } } @@ -854,7 +1028,12 @@ process_arg(Context& ctx, } i++; - return nullopt; + return Statistic::none; + } + + if (args[i] == "--") { + args_info.seen_double_dash = true; + return Statistic::none; } // Other options. @@ -865,9 +1044,34 @@ process_arg(Context& ctx, } else { state.common_args.push_back(args[i]); } - return nullopt; + return Statistic::none; + } + + // It was not a known option. + if (changed_from_slash) { + args[i][0] = '/'; + } + + return std::nullopt; +} + +Statistic +process_arg(const Context& ctx, + ArgsInfo& args_info, + Config& config, + Args& args, + size_t& args_index, + ArgumentProcessingState& state) +{ + const auto processed = + process_option_arg(ctx, args_info, config, args, args_index, state); + if (processed) { + const auto& error = *processed; + return error; } + size_t& i = args_index; + // If an argument isn't a plain file then assume its an option, not an input // file. This allows us to cope better with unusual compiler options. // @@ -878,7 +1082,7 @@ process_arg(Context& ctx, if (!st || !st.is_regular()) { LOG("{} is not a regular file, not considering as input file", args[i]); state.common_args.push_back(args[i]); - return nullopt; + return Statistic::none; } } @@ -899,72 +1103,19 @@ process_arg(Context& ctx, } } - // The source code file path gets put into the notes. - if (args_info.generating_coverage) { - args_info.input_file = args[i]; - return nullopt; - } - // Rewrite to relative to increase hit rate. + args_info.orig_input_file = args[i]; args_info.input_file = Util::make_relative_path(ctx, args[i]); + args_info.normalized_input_file = + Util::normalize_concrete_absolute_path(args_info.input_file); - return nullopt; + return Statistic::none; } -void -handle_dependency_environment_variables(Context& ctx, - ArgumentProcessingState& state) +const char* +get_default_object_file_extension(const Config& config) { - ArgsInfo& args_info = ctx.args_info; - - // See . - // Contrary to what the documentation seems to imply the compiler still - // creates object files with these defined (confirmed with GCC 8.2.1), i.e. - // they work as -MMD/-MD, not -MM/-M. These environment variables do nothing - // on Clang. - const char* dependencies_env = getenv("DEPENDENCIES_OUTPUT"); - bool using_sunpro_dependencies = false; - if (!dependencies_env) { - dependencies_env = getenv("SUNPRO_DEPENDENCIES"); - using_sunpro_dependencies = true; - } - if (!dependencies_env) { - return; - } - - args_info.generating_dependencies = true; - state.dependency_filename_specified = true; - - auto dependencies = Util::split_into_views(dependencies_env, " "); - - if (!dependencies.empty()) { - auto abspath_file = dependencies[0]; - args_info.output_dep = Util::make_relative_path(ctx, abspath_file); - } - - // Specifying target object is optional. - if (dependencies.size() > 1) { - // It's the "file target" form. - ctx.args_info.dependency_target_specified = true; - string_view abspath_obj = dependencies[1]; - std::string relpath_obj = Util::make_relative_path(ctx, abspath_obj); - // Ensure that the compiler gets a relative path. - std::string relpath_both = FMT("{} {}", args_info.output_dep, relpath_obj); - if (using_sunpro_dependencies) { - Util::setenv("SUNPRO_DEPENDENCIES", relpath_both); - } else { - Util::setenv("DEPENDENCIES_OUTPUT", relpath_both); - } - } else { - // It's the "file" form. - state.dependency_implicit_target_specified = true; - // Ensure that the compiler gets a relative path. - if (using_sunpro_dependencies) { - Util::setenv("SUNPRO_DEPENDENCIES", args_info.output_dep); - } else { - Util::setenv("DEPENDENCIES_OUTPUT", args_info.output_dep); - } - } + return config.is_compiler_group_msvc() ? ".obj" : ".o"; } } // namespace @@ -985,14 +1136,24 @@ process_args(Context& ctx) state.common_args.push_back(args[0]); // Compiler - optional argument_error; + std::optional argument_error; for (size_t i = 1; i < args.size(); i++) { - const auto error = process_arg(ctx, args, i, state); - if (error && !argument_error) { + const auto error = process_arg(ctx, args_info, ctx.config, args, i, state); + if (error != Statistic::none && !argument_error) { argument_error = error; } } + // Bail out on too hard combinations of options. + if (state.found_mf_opt && state.found_wp_md_or_mmd_opt) { + // GCC and Clang behave differently when "-Wp,-M[M]D,wp.d" and "-MF mf.d" + // are used: GCC writes to wp.d but Clang writes to mf.d. We could + // potentially support this by behaving differently depending on the + // compiler type, but let's just bail out for now. + LOG_RAW("-Wp,-M[M]D in combination with -MF is not supported"); + return Statistic::unsupported_compiler_option; + } + // Don't try to second guess the compiler's heuristics for stdout handling. if (args_info.output_obj == "-") { LOG_RAW("Output file is -"); @@ -1000,13 +1161,31 @@ process_args(Context& ctx) } // Determine output object file. - const bool implicit_output_obj = args_info.output_obj.empty(); - if (implicit_output_obj && !args_info.input_file.empty()) { - string_view extension = state.found_S_opt ? ".s" : ".o"; - args_info.output_obj = + bool output_obj_by_source = args_info.output_obj.empty(); + if (!output_obj_by_source && ctx.config.is_compiler_group_msvc()) { + if (*args_info.output_obj.rbegin() == '\\') { + output_obj_by_source = true; + } else { + auto st = Stat::stat(args_info.output_obj); + if (st && st.is_directory()) { + args_info.output_obj.append("\\"); + output_obj_by_source = true; + } + } + } + + if (output_obj_by_source && !args_info.input_file.empty()) { + std::string_view extension = + state.found_S_opt ? ".s" : get_default_object_file_extension(ctx.config); + args_info.output_obj += Util::change_extension(Util::base_name(args_info.input_file), extension); } + args_info.orig_output_obj = args_info.output_obj; + args_info.output_obj = Util::make_relative_path(ctx, args_info.output_obj); + + // Determine output dependency file. + // On argument processing error, return now since we have determined // args_info.output_obj which is needed to determine the log filename in // CCACHE_DEBUG mode. @@ -1030,8 +1209,6 @@ process_args(Context& ctx) } #endif - handle_dependency_environment_variables(ctx, state); - if (args_info.input_file.empty()) { LOG_RAW("No input file found"); return Statistic::no_input_file; @@ -1039,7 +1216,7 @@ process_args(Context& ctx) if (state.found_pch || state.found_fpch_preprocess) { args_info.using_precompiled_header = true; - if (!(config.sloppiness() & SLOPPY_TIME_MACROS)) { + if (!(config.sloppiness().is_enabled(core::Sloppy::time_macros))) { LOG_RAW( "You have to specify \"time_macros\" sloppiness when using" " precompiled headers to get direct hits"); @@ -1070,19 +1247,23 @@ process_args(Context& ctx) args_info.actual_language.find("-header") != std::string::npos || Util::is_precompiled_header(args_info.output_obj); - if (args_info.output_is_precompiled_header && implicit_output_obj) { - args_info.output_obj = args_info.input_file + ".gch"; + if (args_info.output_is_precompiled_header && output_obj_by_source) { + args_info.orig_output_obj = args_info.orig_input_file + ".gch"; + args_info.output_obj = + Util::make_relative_path(ctx, args_info.orig_output_obj); } if (args_info.output_is_precompiled_header - && !(config.sloppiness() & SLOPPY_PCH_DEFINES)) { + && !(config.sloppiness().is_enabled(core::Sloppy::pch_defines))) { LOG_RAW( "You have to specify \"pch_defines,time_macros\" sloppiness when" " creating precompiled headers"); return Statistic::could_not_use_precompiled_header; } - if (!state.found_c_opt && !state.found_dc_opt && !state.found_S_opt) { + // -fsyntax-only/-Zs does not need -c + if (!state.found_c_opt && !state.found_dc_opt && !state.found_S_opt + && !state.found_syntax_only) { if (args_info.output_is_precompiled_header) { state.common_args.push_back("-c"); } else { @@ -1100,6 +1281,11 @@ process_args(Context& ctx) return Statistic::unsupported_source_language; } + if (args_info.actual_language == "assembler") { + // -MD/-MMD for assembler file does not produce a dependency file. + args_info.generating_dependencies = false; + } + if (!config.run_second_cpp() && (args_info.actual_language == "cu" || args_info.actual_language == "cuda")) { @@ -1151,12 +1337,12 @@ process_args(Context& ctx) // Some options shouldn't be passed to the real compiler when it compiles // preprocessed code: // - // -finput-charset=XXX (otherwise conversion happens twice) - // -x XXX (otherwise the wrong language is selected) + // -finput-charset=CHARSET (otherwise conversion happens twice) + // -x CHARSET (otherwise the wrong language is selected) if (!state.input_charset_option.empty()) { state.cpp_args.push_back(state.input_charset_option); } - if (state.found_pch) { + if (state.found_pch && ctx.config.compiler_type() != CompilerType::msvc) { state.cpp_args.push_back("-fpch-preprocess"); } if (!state.explicit_language.empty()) { @@ -1171,8 +1357,8 @@ process_args(Context& ctx) // Since output is redirected, compilers will not color their output by // default, so force it explicitly. - nonstd::optional diagnostics_color_arg; - if (config.compiler_type() == CompilerType::clang) { + std::optional diagnostics_color_arg; + if (config.is_compiler_group_clang()) { // Don't pass -fcolor-diagnostics when compiling assembler to avoid an // "argument unused during compilation" warning. if (args_info.actual_language != "assembler") { @@ -1186,29 +1372,54 @@ process_args(Context& ctx) } if (args_info.generating_dependencies) { - if (!state.dependency_filename_specified) { - auto default_depfile_name = - Util::change_extension(args_info.output_obj, ".d"); - args_info.output_dep = - Util::make_relative_path(ctx, default_depfile_name); + if (state.output_dep_origin == OutputDepOrigin::none) { + args_info.output_dep = Util::change_extension(args_info.output_obj, ".d"); if (!config.run_second_cpp()) { // If we're compiling preprocessed code we're sending dep_args to the // preprocessor so we need to use -MF to write to the correct .d file // location since the preprocessor doesn't know the final object path. state.dep_args.push_back("-MF"); - state.dep_args.push_back(default_depfile_name); + state.dep_args.push_back(args_info.output_dep); } } - if (!ctx.args_info.dependency_target_specified - && !state.dependency_implicit_target_specified - && !config.run_second_cpp()) { + if (!args_info.dependency_target && !config.run_second_cpp()) { // If we're compiling preprocessed code we're sending dep_args to the // preprocessor so we need to use -MQ to get the correct target object // file in the .d file. state.dep_args.push_back("-MQ"); state.dep_args.push_back(args_info.output_obj); } + + if (!args_info.dependency_target) { + std::string dep_target = args_info.orig_output_obj; + + // GCC and Clang behave differently when "-Wp,-M[M]D,wp.d" is used with + // "-o" but with neither "-MMD" nor "-MT"/"-MQ": GCC uses a dependency + // target based on the source filename but Clang bases it on the output + // filename. + if (state.found_wp_md_or_mmd_opt && !args_info.output_obj.empty() + && !state.found_md_or_mmd_opt) { + if (config.compiler_type() == CompilerType::clang) { + // Clang does the sane thing: the dependency target is the output file + // so that the dependency file actually makes sense. + } else if (config.compiler_type() == CompilerType::gcc) { + // GCC strangely uses the base name of the source file but with a .o + // extension. + dep_target = Util::change_extension( + Util::base_name(args_info.orig_input_file), + get_default_object_file_extension(ctx.config)); + } else { + // How other compilers behave is currently unknown, so bail out. + LOG_RAW( + "-Wp,-M[M]D with -o without -MMD, -MQ or -MT is only supported for" + " GCC or Clang"); + return Statistic::unsupported_compiler_option; + } + } + + args_info.dependency_target = Depfile::escape_filename(dep_target); + } } if (args_info.generating_stackusage) { @@ -1254,6 +1465,17 @@ process_args(Context& ctx) compiler_args.push_back("-dc"); } + if (state.found_xarch_arch && !args_info.arch_args.empty()) { + if (args_info.arch_args.size() > 1) { + LOG_RAW( + "Multiple -arch options in combination with -Xarch_* not supported"); + return Statistic::unsupported_compiler_option; + } else if (args_info.arch_args[0] != *state.found_xarch_arch) { + LOG_RAW("-arch option not matching -Xarch_* option not supported"); + return Statistic::unsupported_compiler_option; + } + } + for (const auto& arch : args_info.arch_args) { compiler_args.push_back("-arch"); compiler_args.push_back(arch); @@ -1291,9 +1513,21 @@ process_args(Context& ctx) } if (ctx.config.depend_mode()) { // The compiler is invoked with the original arguments in the depend mode. - ctx.args_info.depend_extra_args.push_back(*diagnostics_color_arg); + args_info.depend_extra_args.push_back(*diagnostics_color_arg); } } - return {preprocessor_args, extra_args_to_hash, compiler_args}; + if (ctx.config.depend_mode() && !args_info.generating_includes + && ctx.config.compiler_type() == CompilerType::msvc) { + ctx.auto_depend_mode = true; + args_info.generating_includes = true; + args_info.depend_extra_args.push_back("/showIncludes"); + } + + return { + preprocessor_args, + extra_args_to_hash, + compiler_args, + state.hash_actual_cwd, + }; } diff --git a/src/argprocessing.hpp b/src/argprocessing.hpp index a8e8f3a..2f5c2f7 100644 --- a/src/argprocessing.hpp +++ b/src/argprocessing.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -19,22 +19,24 @@ #pragma once #include "Args.hpp" -#include "Statistic.hpp" -#include "third_party/nonstd/optional.hpp" +#include + +#include class Context; struct ProcessArgsResult { - ProcessArgsResult(Statistic error_); + ProcessArgsResult(core::Statistic error_); ProcessArgsResult(const Args& preprocessor_args_, const Args& extra_args_to_hash_, - const Args& compiler_args_); + const Args& compiler_args_, + bool hash_actual_cwd_); // nullopt on success, otherwise the statistics counter that should be // incremented. - nonstd::optional error; + std::optional error; // Arguments (except -E) to send to the preprocessor. Args preprocessor_args; @@ -44,18 +46,24 @@ struct ProcessArgsResult // Arguments to send to the real compiler. Args compiler_args; + + // Whether to include the actual CWD in the hash. + bool hash_actual_cwd; }; -inline ProcessArgsResult::ProcessArgsResult(Statistic error_) : error(error_) +inline ProcessArgsResult::ProcessArgsResult(core::Statistic error_) + : error(error_) { } inline ProcessArgsResult::ProcessArgsResult(const Args& preprocessor_args_, const Args& extra_args_to_hash_, - const Args& compiler_args_) + const Args& compiler_args_, + bool hash_actual_cwd_) : preprocessor_args(preprocessor_args_), extra_args_to_hash(extra_args_to_hash_), - compiler_args(compiler_args_) + compiler_args(compiler_args_), + hash_actual_cwd(hash_actual_cwd_) { } diff --git a/src/assertions.hpp b/src/assertions.hpp index 040c3a5..d88cb10 100644 --- a/src/assertions.hpp +++ b/src/assertions.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,7 +18,7 @@ #pragma once -#include "system.hpp" +#include #ifdef _MSC_VER # define CCACHE_FUNCTION __func__ diff --git a/src/ccache.cpp b/src/ccache.cpp index 761b9ab..c063a58 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -1,5 +1,5 @@ // Copyright (C) 2002-2007 Andrew Tridgell -// Copyright (C) 2009-2021 Joel Rosdahl and other contributors +// Copyright (C) 2009-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -21,205 +21,115 @@ #include "Args.hpp" #include "ArgsInfo.hpp" -#include "Checksum.hpp" -#include "Compression.hpp" #include "Context.hpp" #include "Depfile.hpp" #include "Fd.hpp" #include "File.hpp" #include "Finalizer.hpp" -#include "FormatNonstdStringView.hpp" #include "Hash.hpp" -#include "Lockfile.hpp" #include "Logging.hpp" -#include "Manifest.hpp" #include "MiniTrace.hpp" -#include "ProgressBar.hpp" -#include "Result.hpp" -#include "ResultDumper.hpp" -#include "ResultExtractor.hpp" -#include "ResultRetriever.hpp" #include "SignalHandler.hpp" -#include "Statistics.hpp" -#include "StdMakeUnique.hpp" #include "TemporaryFile.hpp" #include "UmaskScope.hpp" #include "Util.hpp" +#include "Win32Util.hpp" #include "argprocessing.hpp" -#include "cleanup.hpp" #include "compopt.hpp" -#include "compress.hpp" -#include "exceptions.hpp" #include "execute.hpp" #include "fmtmacros.hpp" #include "hashutil.hpp" #include "language.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include "third_party/fmt/core.h" -#include "third_party/nonstd/optional.hpp" -#include "third_party/nonstd/string_view.hpp" -#ifdef HAVE_GETOPT_LONG -# include -#elif defined(_WIN32) -# include "third_party/win32/getopt.h" -#else -extern "C" { -# include "third_party/getopt_long.h" -} -#endif +#include -#ifdef _WIN32 -# include "Win32Util.hpp" +#include +#include + +#ifdef HAVE_UNISTD_H +# include #endif #include #include #include +#include -#ifndef MYNAME -# define MYNAME "ccache" -#endif -const char CCACHE_NAME[] = MYNAME; - -using nonstd::nullopt; -using nonstd::optional; -using nonstd::string_view; - -constexpr const char VERSION_TEXT[] = - R"({} version {} - -Copyright (C) 2002-2007 Andrew Tridgell -Copyright (C) 2009-2021 Joel Rosdahl and other contributors - -See for a complete list of contributors. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; either version 3 of the License, or (at your option) any later -version. -)"; - -constexpr const char USAGE_TEXT[] = - R"(Usage: - {} [options] - {} compiler [compiler options] - compiler [compiler options] (via symbolic link) - -Common options: - -c, --cleanup delete old files and recalculate size counters - (normally not needed as this is done - automatically) - -C, --clear clear the cache completely (except configuration) - --config-path PATH operate on configuration file PATH instead of the - default - -d, --directory PATH operate on cache directory PATH instead of the - default - --evict-older-than AGE remove files older than AGE (unsigned integer - with a d (days) or s (seconds) suffix) - -F, --max-files NUM set maximum number of files in cache to NUM (use - 0 for no limit) - -M, --max-size SIZE set maximum size of cache to SIZE (use 0 for no - limit); available suffixes: k, M, G, T (decimal) - and Ki, Mi, Gi, Ti (binary); default suffix: G - -X, --recompress LEVEL recompress the cache to level LEVEL (integer or - "uncompressed") using the Zstandard algorithm; - see "Cache compression" in the manual for details - -o, --set-config KEY=VAL set configuration item KEY to value VAL - -x, --show-compression show compression statistics - -p, --show-config show current configuration options in - human-readable format - -s, --show-stats show summary of configuration and statistics - counters in human-readable format - -z, --zero-stats zero statistics counters - - -h, --help print this help text - -V, --version print version and copyright information - -Options for scripting or debugging: - --checksum-file PATH print the checksum (64 bit XXH3) of the file at - PATH - --dump-manifest PATH dump manifest file at PATH in text format - --dump-result PATH dump result file at PATH in text format - --extract-result PATH extract data stored in result file at PATH to the - current working directory - -k, --get-config KEY print the value of configuration key KEY - --hash-file PATH print the hash (160 bit BLAKE3) of the file at - PATH - --print-stats print statistics counter IDs and corresponding - values in machine-parsable format - -See also the manual on . -)"; - -// How often (in seconds) to scan $CCACHE_DIR/tmp for left-over temporary -// files. -const int k_tempdir_cleanup_interval = 2 * 24 * 60 * 60; // 2 days - -// Maximum files per cache directory. This constant is somewhat arbitrarily -// chosen to be large enough to avoid unnecessary cache levels but small enough -// not to make esoteric file systems (with bad performance for large -// directories) too slow. It could be made configurable, but hopefully there -// will be no need to do that. -const uint64_t k_max_cache_files_per_directory = 2000; - -// Minimum number of cache levels ($CCACHE_DIR/1/2/stored_file). -const uint8_t k_min_cache_levels = 2; - -// Maximum number of cache levels ($CCACHE_DIR/1/2/3/stored_file). -// -// On a cache miss, (k_max_cache_levels - k_min_cache_levels + 1) cache lookups -// (i.e. stat system calls) will be performed for a cache entry. -// -// An assumption made here is that if a cache is so large that it holds more -// than 16^4 * k_max_cache_files_per_directory files then we can assume that the -// file system is sane enough to handle more than -// k_max_cache_files_per_directory. -const uint8_t k_max_cache_levels = 4; +using core::Statistic; // This is a string that identifies the current "version" of the hash sum // computed by ccache. If, for any reason, we want to force the hash sum to be // different for the same input in a new ccache version, we can just change // this string. A typical example would be if the format of one of the files // stored in the cache changes in a backwards-incompatible way. -const char HASH_PREFIX[] = "3"; +const char HASH_PREFIX[] = "4"; namespace { -// Throw a Failure if ccache did not succeed in getting or putting a result in -// the cache. If `exit_code` is set, just exit with that code directly, -// otherwise execute the real compiler and exit with its exit code. Also updates -// statistics counter `statistic` if it's not `Statistic::none`. -class Failure : public std::exception +// Return nonstd::make_unexpected if ccache did not succeed in getting +// or putting a result in the cache. If `exit_code` is set, ccache will just +// exit with that code directly, otherwise execute the real compiler and exit +// with its exit code. Statistics counters will also be incremented. +class Failure { public: - Failure(Statistic statistic, - nonstd::optional exit_code = nonstd::nullopt); + Failure(Statistic statistic); + Failure(std::initializer_list statistics); - nonstd::optional exit_code() const; - Statistic statistic() const; + const core::StatisticsCounters& counters() const; + std::optional exit_code() const; + void set_exit_code(int exit_code); private: - Statistic m_statistic; - nonstd::optional m_exit_code; + core::StatisticsCounters m_counters; + std::optional m_exit_code; }; -inline Failure::Failure(Statistic statistic, nonstd::optional exit_code) - : m_statistic(statistic), - m_exit_code(exit_code) +inline Failure::Failure(const Statistic statistic) : m_counters({statistic}) +{ +} + +inline Failure::Failure(const std::initializer_list statistics) + : m_counters(statistics) +{ +} + +inline const core::StatisticsCounters& +Failure::counters() const { + return m_counters; } -inline nonstd::optional +inline std::optional Failure::exit_code() const { return m_exit_code; } -inline Statistic -Failure::statistic() const +inline void +Failure::set_exit_code(const int exit_code) { - return m_statistic; + m_exit_code = exit_code; } } // namespace @@ -233,9 +143,9 @@ add_prefix(const Context& ctx, Args& args, const std::string& prefix_command) Args prefix; for (const auto& word : Util::split_into_strings(prefix_command, " ")) { - std::string path = find_executable(ctx, word, CCACHE_NAME); + std::string path = find_executable(ctx, word, ctx.orig_args[0]); if (path.empty()) { - throw Fatal("{}: {}", word, strerror(errno)); + throw core::Fatal(FMT("{}: {}", word, strerror(errno))); } prefix.push_back(path); @@ -247,63 +157,53 @@ add_prefix(const Context& ctx, Args& args, const std::string& prefix_command) } } -static void -clean_up_internal_tempdir(const Config& config) -{ - time_t now = time(nullptr); - auto dir_st = Stat::stat(config.cache_dir(), Stat::OnError::log); - if (!dir_st || dir_st.mtime() + k_tempdir_cleanup_interval >= now) { - // No cleanup needed. - return; - } - - Util::update_mtime(config.cache_dir()); - - const std::string& temp_dir = config.temporary_dir(); - if (!Stat::lstat(temp_dir)) { - return; - } - - Util::traverse(temp_dir, [now](const std::string& path, bool is_dir) { - if (is_dir) { - return; - } - auto st = Stat::lstat(path, Stat::OnError::log); - if (st && st.mtime() + k_tempdir_cleanup_interval < now) { - Util::unlink_tmp(path); - } - }); -} - static std::string prepare_debug_path(const std::string& debug_dir, + const util::TimePoint& time_of_invocation, const std::string& output_obj, - string_view suffix) + std::string_view suffix) { - const std::string prefix = - debug_dir.empty() ? output_obj : debug_dir + Util::real_path(output_obj); - try { - Util::ensure_dir_exists(Util::dir_name(prefix)); - } catch (Error&) { - // Ignore since we can't handle an error in another way in this context. The - // caller takes care of logging when trying to open the path for writing. - } - return FMT("{}.ccache-{}", prefix, suffix); + auto prefix = debug_dir.empty() + ? output_obj + : debug_dir + util::to_absolute_path_no_drive(output_obj); + + // Ignore any error from create_dir since we can't handle an error in another + // way in this context. The caller takes care of logging when trying to open + // the path for writing. + Util::create_dir(Util::dir_name(prefix)); + + char timestamp[100]; + const auto tm = Util::localtime(time_of_invocation); + if (tm) { + strftime(timestamp, sizeof(timestamp), "%Y%m%d_%H%M%S", &*tm); + } else { + snprintf(timestamp, + sizeof(timestamp), + "%llu", + static_cast(time_of_invocation.sec())); + } + return FMT("{}.{}_{:06}.ccache-{}", + prefix, + timestamp, + time_of_invocation.nsec_decimal_part() / 1000, + suffix); } static void init_hash_debug(Context& ctx, Hash& hash, char type, - string_view section_name, + std::string_view section_name, FILE* debug_text_file) { if (!ctx.config.debug()) { return; } - const auto path = prepare_debug_path( - ctx.config.debug_dir(), ctx.args_info.output_obj, FMT("input-{}", type)); + const auto path = prepare_debug_path(ctx.config.debug_dir(), + ctx.time_of_invocation, + ctx.args_info.output_obj, + FMT("input-{}", type)); File debug_binary_file(path, "wb"); if (debug_binary_file) { hash.enable_debug(section_name, debug_binary_file.get(), debug_text_file); @@ -314,7 +214,7 @@ init_hash_debug(Context& ctx, } CompilerType -guess_compiler(string_view path) +guess_compiler(std::string_view path) { std::string compiler_path(path); @@ -326,7 +226,7 @@ guess_compiler(string_view path) if (symlink_value.empty()) { break; } - if (Util::is_absolute_path(symlink_value)) { + if (util::is_absolute_path(symlink_value)) { compiler_path = symlink_value; } else { compiler_path = @@ -335,16 +235,21 @@ guess_compiler(string_view path) } #endif - const string_view name = Util::base_name(compiler_path); - if (name.find("clang") != nonstd::string_view::npos) { + const auto name = + Util::to_lowercase(Util::remove_extension(Util::base_name(compiler_path))); + if (name.find("clang-cl") != std::string_view::npos) { + return CompilerType::clang_cl; + } else if (name.find("clang") != std::string_view::npos) { return CompilerType::clang; - } else if (name.find("gcc") != nonstd::string_view::npos - || name.find("g++") != nonstd::string_view::npos) { + } else if (name.find("gcc") != std::string_view::npos + || name.find("g++") != std::string_view::npos) { return CompilerType::gcc; - } else if (name.find("nvcc") != nonstd::string_view::npos) { + } else if (name.find("nvcc") != std::string_view::npos) { return CompilerType::nvcc; - } else if (name == "pump" || name == "distcc-pump") { - return CompilerType::pump; + } else if (name == "icl") { + return CompilerType::icl; + } else if (name == "cl") { + return CompilerType::msvc; } else { return CompilerType::other; } @@ -358,14 +263,14 @@ include_file_too_new(const Context& ctx, // The comparison using >= is intentional, due to a possible race between // starting compilation and writing the include file. See also the notes under // "Performance" in doc/MANUAL.adoc. - if (!(ctx.config.sloppiness() & SLOPPY_INCLUDE_FILE_MTIME) + if (!(ctx.config.sloppiness().is_enabled(core::Sloppy::include_file_mtime)) && path_stat.mtime() >= ctx.time_of_compilation) { LOG("Include file {} too new", path); return true; } // The same >= logic as above applies to the change time of the file. - if (!(ctx.config.sloppiness() & SLOPPY_INCLUDE_FILE_CTIME) + if (!(ctx.config.sloppiness().is_enabled(core::Sloppy::include_file_ctime)) && path_stat.ctime() >= ctx.time_of_compilation) { LOG("Include file {} ctime too new", path); return true; @@ -389,18 +294,19 @@ do_remember_include_file(Context& ctx, return true; } - if (path == ctx.args_info.input_file) { + if (path == ctx.args_info.normalized_input_file) { // Don't remember the input file. return true; } - if (system && (ctx.config.sloppiness() & SLOPPY_SYSTEM_HEADERS)) { + if (system + && (ctx.config.sloppiness().is_enabled(core::Sloppy::system_headers))) { // Don't remember this system header. return true; } // Canonicalize path for comparison; Clang uses ./header.h. - if (Util::starts_with(path, "./")) { + if (util::starts_with(path, "./")) { path.erase(0, 2); } @@ -459,10 +365,10 @@ do_remember_include_file(Context& ctx, } // Let's hash the include file content. - Hash fhash; + Digest file_digest; if (is_pch) { - if (ctx.included_pch_file.empty()) { + if (ctx.args_info.included_pch_file.empty()) { LOG("Detected use of precompiled header: {}", path); } bool using_pch_sum = false; @@ -477,28 +383,27 @@ do_remember_include_file(Context& ctx, } } - if (!hash_binary_file(ctx, fhash, path)) { + if (!hash_binary_file(ctx, file_digest, path)) { return false; } cpp_hash.hash_delimiter(using_pch_sum ? "pch_sum_hash" : "pch_hash"); - cpp_hash.hash(fhash.digest().to_string()); + cpp_hash.hash(file_digest.to_string()); } if (ctx.config.direct_mode()) { if (!is_pch) { // else: the file has already been hashed. - int result = hash_source_code_file(ctx, fhash, path); + int result = hash_source_code_file(ctx, file_digest, path); if (result & HASH_SOURCE_CODE_ERROR || result & HASH_SOURCE_CODE_FOUND_TIME) { return false; } } - Digest d = fhash.digest(); - ctx.included_files.emplace(path, d); + ctx.included_files.emplace(path, file_digest); if (depend_mode_hash) { depend_mode_hash->hash_delimiter("include"); - depend_mode_hash->hash(d.to_string()); + depend_mode_hash->hash(file_digest.to_string()); } } @@ -532,8 +437,8 @@ remember_include_file(Context& ctx, static void print_included_files(const Context& ctx, FILE* fp) { - for (const auto& item : ctx.included_files) { - PRINT(fp, "{}\n", item.first); + for (const auto& [path, digest] : ctx.included_files) { + PRINT(fp, "{}\n", path); } } @@ -543,36 +448,32 @@ print_included_files(const Context& ctx, FILE* fp) // - Makes include file paths for which the base directory is a prefix relative // when computing the hash sum. // - Stores the paths and hashes of included files in ctx.included_files. -// -// Returns Statistic::none on success, otherwise a statistics counter to be -// incremented. -static Statistic -process_preprocessed_file(Context& ctx, - Hash& hash, - const std::string& path, - bool pump) +static nonstd::expected +process_preprocessed_file(Context& ctx, Hash& hash, const std::string& path) { - std::string data; - try { - data = Util::read_file(path); - } catch (Error&) { - return Statistic::internal_error; + auto data = util::read_file(path); + if (!data) { + LOG("Failed to read {}: {}", path, data.error()); + return nonstd::make_unexpected(Statistic::internal_error); } // Bytes between p and q are pending to be hashed. - const char* p = &data[0]; - char* q = &data[0]; - const char* end = p + data.length(); + char* q = &(*data)[0]; + const char* p = q; + const char* end = p + data->length(); // There must be at least 7 characters (# 1 "x") left to potentially find an // include file path. while (q < end - 7) { - static const string_view pragma_gcc_pch_preprocess = + static const std::string_view pragma_gcc_pch_preprocess = "pragma GCC pch_preprocess "; - static const string_view hash_31_command_line_newline = + static const std::string_view hash_31_command_line_newline = "# 31 \"\"\n"; - static const string_view hash_32_command_line_2_newline = + static const std::string_view hash_32_command_line_2_newline = "# 32 \"\" 2\n"; + // Note: Intentionally not using the string form to avoid false positive + // match by ccache itself. + static const char incbin_directive[] = {'.', 'i', 'n', 'c', 'b', 'i', 'n'}; // Check if we look at a line containing the file name of an included file. // At least the following formats exist (where N is a positive integer): @@ -598,14 +499,14 @@ process_preprocessed_file(Context& ctx, // GCC: && ((q[1] == ' ' && q[2] >= '0' && q[2] <= '9') // GCC precompiled header: - || Util::starts_with(&q[1], pragma_gcc_pch_preprocess) + || util::starts_with(&q[1], pragma_gcc_pch_preprocess) // HP/AIX: || (q[1] == 'l' && q[2] == 'i' && q[3] == 'n' && q[4] == 'e' && q[5] == ' ')) - && (q == data.data() || q[-1] == '\n')) { + && (q == data->data() || q[-1] == '\n')) { // Workarounds for preprocessor linemarker bugs in GCC version 6. if (q[2] == '3') { - if (Util::starts_with(q, hash_31_command_line_newline)) { + if (util::starts_with(q, hash_31_command_line_newline)) { // Bogus extra line with #31, after the regular #1: Ignore the whole // line, and continue parsing. hash.hash(p, q - p); @@ -615,7 +516,7 @@ process_preprocessed_file(Context& ctx, q++; p = q; continue; - } else if (Util::starts_with(q, hash_32_command_line_2_newline)) { + } else if (util::starts_with(q, hash_32_command_line_2_newline)) { // Bogus wrong line with #32, instead of regular #1: Replace the line // number with the usual one. hash.hash(p, q - p); @@ -637,7 +538,7 @@ process_preprocessed_file(Context& ctx, q++; if (q >= end) { LOG_RAW("Failed to parse included file path"); - return Statistic::internal_error; + return nonstd::make_unexpected(Statistic::internal_error); } // q points to the beginning of an include file path hash.hash(p, q - p); @@ -645,6 +546,10 @@ process_preprocessed_file(Context& ctx, while (q < end && *q != '"') { q++; } + if (p == q) { + // Skip empty file name. + continue; + } // Look for preprocessor flags, after the "filename". bool system = false; const char* r = q + 1; @@ -656,49 +561,39 @@ process_preprocessed_file(Context& ctx, } // p and q span the include file path. std::string inc_path(p, q - p); - if (!ctx.has_absolute_include_headers) { - ctx.has_absolute_include_headers = Util::is_absolute_path(inc_path); - } + inc_path = Util::normalize_concrete_absolute_path(inc_path); inc_path = Util::make_relative_path(ctx, inc_path); - bool should_hash_inc_path = true; - if (!ctx.config.hash_dir()) { - if (Util::starts_with(inc_path, ctx.apparent_cwd) - && Util::ends_with(inc_path, "//")) { - // When compiling with -g or similar, GCC adds the absolute path to - // CWD like this: - // - // # 1 "CWD//" - // - // If the user has opted out of including the CWD in the hash, don't - // hash it. See also how debug_prefix_map is handled. - should_hash_inc_path = false; - } - } - if (should_hash_inc_path) { + if ((inc_path != ctx.apparent_cwd) || ctx.config.hash_dir()) { hash.hash(inc_path); } if (remember_include_file(ctx, inc_path, hash, system, nullptr) == RememberIncludeFileResult::cannot_use_pch) { - return Statistic::could_not_use_precompiled_header; + return nonstd::make_unexpected( + Statistic::could_not_use_precompiled_header); } p = q; // Everything of interest between p and q has been hashed now. - } else if (q[0] == '.' && q[1] == 'i' && q[2] == 'n' && q[3] == 'c' - && q[4] == 'b' && q[5] == 'i' && q[6] == 'n') { + } else if (strncmp(q, incbin_directive, sizeof(incbin_directive)) == 0 + && ((q[7] == ' ' + && (q[8] == '"' || (q[8] == '\\' && q[9] == '"'))) + || q[7] == '"')) { // An assembler .inc bin (without the space) statement, which could be // part of inline assembly, refers to an external file. If the file // changes, the hash should change as well, but finding out what file to // hash is too hard for ccache, so just bail out. LOG_RAW( - "Found unsupported .inc" + "Found potential unsupported .inc" "bin directive in source code"); - throw Failure(Statistic::unsupported_code_directive); - } else if (pump && strncmp(q, "_________", 9) == 0) { + return nonstd::make_unexpected( + Failure(Statistic::unsupported_code_directive)); + } else if (strncmp(q, "___________", 10) == 0 + && (q == data->data() || q[-1] == '\n')) { // Unfortunately the distcc-pump wrapper outputs standard output lines: // __________Using distcc-pump from /usr/bin // __________Using # distcc servers in pump mode // __________Shutting down distcc-pump include server + hash.hash(p, q - p); while (q < end && *q != '\n') { q++; } @@ -716,8 +611,9 @@ process_preprocessed_file(Context& ctx, // Explicitly check the .gch/.pch/.pth file as Clang does not include any // mention of it in the preprocessed output. - if (!ctx.included_pch_file.empty()) { - std::string pch_path = Util::make_relative_path(ctx, ctx.included_pch_file); + if (!ctx.args_info.included_pch_file.empty()) { + std::string pch_path = + Util::make_relative_path(ctx, ctx.args_info.included_pch_file); hash.hash(pch_path); remember_include_file(ctx, pch_path, hash, false, nullptr); } @@ -727,38 +623,36 @@ process_preprocessed_file(Context& ctx, print_included_files(ctx, stdout); } - return Statistic::none; + return {}; } // Extract the used includes from the dependency file. Note that we cannot // distinguish system headers from other includes here. -static optional -result_name_from_depfile(Context& ctx, Hash& hash) +static std::optional +result_key_from_depfile(Context& ctx, Hash& hash) { - std::string file_content; - try { - file_content = Util::read_file(ctx.args_info.output_dep); - } catch (const Error& e) { - LOG( - "Cannot open dependency file {}: {}", ctx.args_info.output_dep, e.what()); - return nullopt; + const auto file_content = + util::read_file(ctx.args_info.output_dep); + if (!file_content) { + LOG("Failed to read dependency file {}: {}", + ctx.args_info.output_dep, + file_content.error()); + return std::nullopt; } - for (string_view token : Depfile::tokenize(file_content)) { - if (token.ends_with(":")) { + for (std::string_view token : Depfile::tokenize(*file_content)) { + if (util::ends_with(token, ":")) { continue; } - if (!ctx.has_absolute_include_headers) { - ctx.has_absolute_include_headers = Util::is_absolute_path(token); - } std::string path = Util::make_relative_path(ctx, token); remember_include_file(ctx, path, hash, false, &hash); } // Explicitly check the .gch/.pch/.pth file as it may not be mentioned in the // dependencies output. - if (!ctx.included_pch_file.empty()) { - std::string pch_path = Util::make_relative_path(ctx, ctx.included_pch_file); + if (!ctx.args_info.included_pch_file.empty()) { + std::string pch_path = + Util::make_relative_path(ctx, ctx.args_info.included_pch_file); hash.hash(pch_path); remember_include_file(ctx, pch_path, hash, false, nullptr); } @@ -770,13 +664,68 @@ result_name_from_depfile(Context& ctx, Hash& hash) return hash.digest(); } + +struct GetTmpFdResult +{ + Fd fd; + std::string path; +}; + +static GetTmpFdResult +get_tmp_fd(Context& ctx, + const std::string_view description, + const bool capture_output) +{ + if (capture_output) { + TemporaryFile tmp_stdout( + FMT("{}/{}", ctx.config.temporary_dir(), description)); + ctx.register_pending_tmp_file(tmp_stdout.path); + return {std::move(tmp_stdout.fd), std::move(tmp_stdout.path)}; + } else { + const auto dev_null_path = util::get_dev_null_path(); + return {Fd(open(dev_null_path, O_WRONLY | O_BINARY)), dev_null_path}; + } +} + +struct DoExecuteResult +{ + int exit_status; + util::Bytes stdout_data; + util::Bytes stderr_data; +}; + +// Extract the used includes from /showIncludes output in stdout. Note that we +// cannot distinguish system headers from other includes here. +static std::optional +result_key_from_includes(Context& ctx, Hash& hash, std::string_view stdout_data) +{ + for (std::string_view token : core::MsvcShowIncludesOutput::get_includes( + stdout_data, ctx.config.msvc_dep_prefix())) { + const std::string path = Util::make_relative_path(ctx, token); + remember_include_file(ctx, path, hash, false, &hash); + } + + // Explicitly check the .pch file as it is not mentioned in the + // includes output. + if (!ctx.args_info.included_pch_file.empty()) { + std::string pch_path = + Util::make_relative_path(ctx, ctx.args_info.included_pch_file); + hash.hash(pch_path); + remember_include_file(ctx, pch_path, hash, false, nullptr); + } + + const bool debug_included = getenv("CCACHE_DEBUG_INCLUDED"); + if (debug_included) { + print_included_files(ctx, stdout); + } + + return hash.digest(); +} + // Execute the compiler/preprocessor, with logic to retry without requesting // colored diagnostics messages if that fails. -static int -do_execute(Context& ctx, - Args& args, - TemporaryFile&& tmp_stdout, - TemporaryFile&& tmp_stderr) +static nonstd::expected +do_execute(Context& ctx, Args& args, const bool capture_stdout = true) { UmaskScope umask_scope(ctx.original_umask); @@ -784,14 +733,18 @@ do_execute(Context& ctx, DEBUG_ASSERT(ctx.config.compiler_type() == CompilerType::gcc); args.erase_last("-fdiagnostics-color"); } + + auto tmp_stdout = get_tmp_fd(ctx, "stdout", capture_stdout); + auto tmp_stderr = get_tmp_fd(ctx, "stderr", true); + int status = execute(ctx, args.to_argv().data(), std::move(tmp_stdout.fd), std::move(tmp_stderr.fd)); if (status != 0 && !ctx.diagnostics_color_failed && ctx.config.compiler_type() == CompilerType::gcc) { - auto errors = Util::read_file(tmp_stderr.path); - if (errors.find("fdiagnostics-color") != std::string::npos) { + const auto errors = util::read_file(tmp_stderr.path); + if (errors && errors->find("fdiagnostics-color") != std::string::npos) { // GCC versions older than 4.9 don't understand -fdiagnostics-color, and // non-GCC compilers misclassified as CompilerType::gcc might not do it // either. We assume that if the error message contains @@ -802,117 +755,89 @@ do_execute(Context& ctx, // error message.) LOG_RAW("-fdiagnostics-color is unsupported; trying again without it"); - tmp_stdout.fd = Fd(open( - tmp_stdout.path.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_BINARY, 0600)); - if (!tmp_stdout.fd) { - LOG("Failed to truncate {}: {}", tmp_stdout.path, strerror(errno)); - throw Failure(Statistic::internal_error); - } - - tmp_stderr.fd = Fd(open( - tmp_stderr.path.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_BINARY, 0600)); - if (!tmp_stderr.fd) { - LOG("Failed to truncate {}: {}", tmp_stderr.path, strerror(errno)); - throw Failure(Statistic::internal_error); - } - ctx.diagnostics_color_failed = true; - return do_execute( - ctx, args, std::move(tmp_stdout), std::move(tmp_stderr)); + return do_execute(ctx, args, capture_stdout); } } - return status; -} - -struct LookUpCacheFileResult -{ - std::string path; - Stat stat; - uint8_t level; -}; - -static LookUpCacheFileResult -look_up_cache_file(const std::string& cache_dir, - const Digest& name, - nonstd::string_view suffix) -{ - const auto name_string = FMT("{}{}", name.to_string(), suffix); - for (uint8_t level = k_min_cache_levels; level <= k_max_cache_levels; - ++level) { - const auto path = Util::get_path_in_cache(cache_dir, level, name_string); - const auto stat = Stat::stat(path); - if (stat) { - return {path, stat, level}; + util::Bytes stdout_data; + if (capture_stdout) { + auto stdout_data_result = util::read_file(tmp_stdout.path); + if (!stdout_data_result) { + LOG("Failed to read {} (cleanup in progress?): {}", + tmp_stdout.path, + stdout_data_result.error()); + return nonstd::make_unexpected(Statistic::missing_cache_file); } + stdout_data = *stdout_data_result; + } + + auto stderr_data_result = util::read_file(tmp_stderr.path); + if (!stderr_data_result) { + LOG("Failed to read {} (cleanup in progress?): {}", + tmp_stderr.path, + stderr_data_result.error()); + return nonstd::make_unexpected(Statistic::missing_cache_file); } - const auto shallowest_path = - Util::get_path_in_cache(cache_dir, k_min_cache_levels, name_string); - return {shallowest_path, Stat(), k_min_cache_levels}; + return DoExecuteResult{status, stdout_data, *stderr_data_result}; } -// Create or update the manifest file. static void -update_manifest_file(Context& ctx) +read_manifest(Context& ctx, nonstd::span cache_entry_data) { - if (!ctx.config.direct_mode() || ctx.config.read_only() - || ctx.config.read_only_direct()) { - return; + try { + core::CacheEntry cache_entry(cache_entry_data); + cache_entry.verify_checksum(); + ctx.manifest.read(cache_entry.payload()); + } catch (const core::Error& e) { + LOG("Error reading manifest: {}", e.what()); } +} - ASSERT(ctx.manifest_path()); - ASSERT(ctx.result_path()); +static void +update_manifest(Context& ctx, + const Digest& manifest_key, + const Digest& result_key) +{ + if (ctx.config.read_only() || ctx.config.read_only_direct()) { + return; + } - MTR_BEGIN("manifest", "manifest_put"); + ASSERT(ctx.config.direct_mode()); - const auto old_stat = Stat::stat(*ctx.manifest_path()); + MTR_SCOPE("manifest", "manifest_put"); - // See comment in get_file_hash_index for why saving of timestamps is forced - // for precompiled headers. + // ctime() may be 0, so we have to check time_of_compilation against + // MAX(mtime, ctime). + // + // ccache only reads mtime/ctime if file_stat_matches sloppiness is enabled, + // so mtimes/ctimes are stored as a dummy value (-1) if not enabled. This + // reduces the number of file_info entries for the common case. const bool save_timestamp = - (ctx.config.sloppiness() & SLOPPY_FILE_STAT_MATCHES) + (ctx.config.sloppiness().is_enabled(core::Sloppy::file_stat_matches)) || ctx.args_info.output_is_precompiled_header; - LOG("Adding result name to {}", *ctx.manifest_path()); - if (!Manifest::put(ctx.config, - *ctx.manifest_path(), - *ctx.result_name(), - ctx.included_files, - ctx.time_of_compilation, - save_timestamp)) { - LOG("Failed to add result name to {}", *ctx.manifest_path()); + const bool added = ctx.manifest.add_result( + result_key, ctx.included_files, [&](const std::string& path) { + auto stat = Stat::stat(path, Stat::OnError::log); + bool cache_time = + save_timestamp + && ctx.time_of_compilation > std::max(stat.mtime(), stat.ctime()); + return core::Manifest::FileStats{ + stat.size(), + stat && cache_time ? stat.mtime() : util::TimePoint(), + stat && cache_time ? stat.ctime() : util::TimePoint(), + }; + }); + if (added) { + LOG("Added result key to manifest {}", manifest_key.to_string()); + core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest); + ctx.storage.put(manifest_key, + core::CacheEntryType::manifest, + core::CacheEntry::serialize(header, ctx.manifest)); } else { - const auto new_stat = Stat::stat(*ctx.manifest_path(), Stat::OnError::log); - ctx.manifest_counter_updates.increment( - Statistic::cache_size_kibibyte, - Util::size_change_kibibyte(old_stat, new_stat)); - ctx.manifest_counter_updates.increment(Statistic::files_in_cache, - !old_stat && new_stat ? 1 : 0); - } - MTR_END("manifest", "manifest_put"); -} - -static void -create_cachedir_tag(const Context& ctx) -{ - constexpr char cachedir_tag[] = - "Signature: 8a477f597d28d172789f06886806bc55\n" - "# This file is a cache directory tag created by ccache.\n" - "# For information about cache directory tags, see:\n" - "#\thttp://www.brynosaurus.com/cachedir/\n"; - - const std::string path = FMT("{}/{}/CACHEDIR.TAG", - ctx.config.cache_dir(), - ctx.result_name()->to_string()[0]); - const auto stat = Stat::stat(path); - if (stat) { - return; - } - try { - Util::write_file(path, cachedir_tag); - } catch (const Error& e) { - LOG("Failed to create {}: {}", path, e.what()); + LOG("Did not add result key to manifest {}", manifest_key.to_string()); } } @@ -930,8 +855,8 @@ find_coverage_file(const Context& ctx) // (in CWD) if -fprofile-dir=DIR is present (regardless of DIR) instead of the // traditional /dir/to/example.gcno. - std::string mangled_form = Result::gcno_file_in_mangled_form(ctx); - std::string unmangled_form = Result::gcno_file_in_unmangled_form(ctx); + std::string mangled_form = core::Result::gcno_file_in_mangled_form(ctx); + std::string unmangled_form = core::Result::gcno_file_in_unmangled_form(ctx); std::string found_file; if (Stat::stat(mangled_form)) { LOG("Found coverage file {}", mangled_form); @@ -954,15 +879,149 @@ find_coverage_file(const Context& ctx) return {true, found_file, found_file == mangled_form}; } -// Run the real compiler and put the result in cache. -static void +[[nodiscard]] static bool +write_result(Context& ctx, + const Digest& result_key, + const Stat& obj_stat, + const util::Bytes& stdout_data, + const util::Bytes& stderr_data) +{ + core::Result::Serializer serializer(ctx.config); + + if (!stderr_data.empty()) { + serializer.add_data(core::Result::FileType::stderr_output, stderr_data); + } + // Write stdout only after stderr (better with MSVC), as ResultRetriever + // will later print process them in the order they are read. + if (!stdout_data.empty()) { + serializer.add_data(core::Result::FileType::stdout_output, stdout_data); + } + if (obj_stat + && !serializer.add_file(core::Result::FileType::object, + ctx.args_info.output_obj)) { + LOG("Object file {} missing", ctx.args_info.output_obj); + return false; + } + if (ctx.args_info.generating_dependencies + && !serializer.add_file(core::Result::FileType::dependency, + ctx.args_info.output_dep)) { + LOG("Dependency file {} missing", ctx.args_info.output_dep); + return false; + } + if (ctx.args_info.generating_coverage) { + const auto coverage_file = find_coverage_file(ctx); + if (!coverage_file.found) { + LOG_RAW("Coverage file not found"); + return false; + } + if (!serializer.add_file(coverage_file.mangled + ? core::Result::FileType::coverage_mangled + : core::Result::FileType::coverage_unmangled, + coverage_file.path)) { + LOG("Coverage file {} missing", coverage_file.path); + return false; + } + } + if (ctx.args_info.generating_stackusage + && !serializer.add_file(core::Result::FileType::stackusage, + ctx.args_info.output_su)) { + LOG("Stack usage file {} missing", ctx.args_info.output_su); + return false; + } + if (ctx.args_info.generating_diagnostics + && !serializer.add_file(core::Result::FileType::diagnostic, + ctx.args_info.output_dia)) { + LOG("Diagnostics file {} missing", ctx.args_info.output_dia); + return false; + } + if (ctx.args_info.seen_split_dwarf + // Only store .dwo file if it was created by the compiler (GCC and Clang + // behave differently e.g. for "-gsplit-dwarf -g1"). + && Stat::stat(ctx.args_info.output_dwo) + && !serializer.add_file(core::Result::FileType::dwarf_object, + ctx.args_info.output_dwo)) { + LOG("Split dwarf file {} missing", ctx.args_info.output_dwo); + return false; + } + if (!ctx.args_info.output_al.empty() + && !serializer.add_file(core::Result::FileType::assembler_listing, + ctx.args_info.output_al)) { + LOG("Assembler listing file {} missing", ctx.args_info.output_al); + return false; + } + + core::CacheEntry::Header header(ctx.config, core::CacheEntryType::result); + const auto cache_entry_data = core::CacheEntry::serialize(header, serializer); + + if (!ctx.config.remote_only()) { + const auto& raw_files = serializer.get_raw_files(); + if (!raw_files.empty()) { + ctx.storage.local.put_raw_files(result_key, raw_files); + } + } + + ctx.storage.put(result_key, core::CacheEntryType::result, cache_entry_data); + + return true; +} + +static util::Bytes +rewrite_stdout_from_compiler(const Context& ctx, util::Bytes&& stdout_data) +{ + using util::Tokenizer; + using Mode = Tokenizer::Mode; + using IncludeDelimiter = Tokenizer::IncludeDelimiter; + if (!stdout_data.empty()) { + util::Bytes new_stdout_data; + for (const auto line : Tokenizer(util::to_string_view(stdout_data), + "\n", + Mode::include_empty, + IncludeDelimiter::yes)) { + if (util::starts_with(line, "__________")) { + Util::send_to_fd(ctx, line, STDOUT_FILENO); + } + // Ninja uses the lines with 'Note: including file: ' to determine the + // used headers. Headers within basedir need to be changed into relative + // paths because otherwise Ninja will use the abs path to original header + // to check if a file needs to be recompiled. + else if (ctx.config.compiler_type() == CompilerType::msvc + && !ctx.config.base_dir().empty() + && util::starts_with(line, ctx.config.msvc_dep_prefix())) { + std::string orig_line(line.data(), line.length()); + std::string abs_inc_path = + util::replace_first(orig_line, ctx.config.msvc_dep_prefix(), ""); + abs_inc_path = util::strip_whitespace(abs_inc_path); + std::string rel_inc_path = Util::make_relative_path( + ctx, Util::normalize_concrete_absolute_path(abs_inc_path)); + std::string line_with_rel_inc = + util::replace_first(orig_line, abs_inc_path, rel_inc_path); + new_stdout_data.insert(new_stdout_data.begin(), + line_with_rel_inc.data(), + line_with_rel_inc.size()); + } else { + new_stdout_data.insert(new_stdout_data.end(), line.data(), line.size()); + } + } + return new_stdout_data; + } else { + return std::move(stdout_data); + } +} + +// Run the real compiler and put the result in cache. Returns the result key. +static nonstd::expected to_cache(Context& ctx, Args& args, + std::optional result_key, const Args& depend_extra_args, Hash* depend_mode_hash) { - args.push_back("-o"); - args.push_back(ctx.args_info.output_obj); + if (ctx.config.is_compiler_group_msvc()) { + args.push_back(fmt::format("-Fo{}", ctx.args_info.output_obj)); + } else { + args.push_back("-o"); + args.push_back(ctx.args_info.output_obj); + } if (ctx.config.hard_link() && ctx.args_info.output_obj != "/dev/null") { // Workaround for Clang bug where it overwrites an existing object file @@ -976,12 +1035,9 @@ to_cache(Context& ctx, args.push_back(ctx.args_info.output_dia); } - // Turn off DEPENDENCIES_OUTPUT when running cc1, because otherwise it will - // emit a line like this: - // - // tmp.stdout.vexed.732.o: /home/mbp/.ccache/tmp.stdout.vexed.732.i - Util::unsetenv("DEPENDENCIES_OUTPUT"); - Util::unsetenv("SUNPRO_DEPENDENCIES"); + if (ctx.args_info.seen_double_dash) { + args.push_back("--"); + } if (ctx.config.run_second_cpp()) { args.push_back(ctx.args_info.input_file); @@ -997,25 +1053,16 @@ to_cache(Context& ctx, if (unlink(ctx.args_info.output_dwo.c_str()) != 0 && errno != ENOENT && errno != ESTALE) { LOG("Failed to unlink {}: {}", ctx.args_info.output_dwo, strerror(errno)); - throw Failure(Statistic::bad_output_file); + return nonstd::make_unexpected(Statistic::bad_output_file); } } LOG_RAW("Running real compiler"); MTR_BEGIN("execute", "compiler"); - TemporaryFile tmp_stdout(FMT("{}/tmp.stdout", ctx.config.temporary_dir())); - ctx.register_pending_tmp_file(tmp_stdout.path); - std::string tmp_stdout_path = tmp_stdout.path; - - TemporaryFile tmp_stderr(FMT("{}/tmp.stderr", ctx.config.temporary_dir())); - ctx.register_pending_tmp_file(tmp_stderr.path); - std::string tmp_stderr_path = tmp_stderr.path; - - int status; + nonstd::expected result; if (!ctx.config.depend_mode()) { - status = - do_execute(ctx, args, std::move(tmp_stdout), std::move(tmp_stderr)); + result = do_execute(ctx, args); args.pop_back(3); } else { // Use the original arguments (including dependency options) in depend @@ -1025,51 +1072,62 @@ to_cache(Context& ctx, depend_mode_args.push_back(depend_extra_args); add_prefix(ctx, depend_mode_args, ctx.config.prefix_command()); - ctx.time_of_compilation = time(nullptr); - status = do_execute( - ctx, depend_mode_args, std::move(tmp_stdout), std::move(tmp_stderr)); + ctx.time_of_compilation = util::TimePoint::now(); + result = do_execute(ctx, depend_mode_args); } MTR_END("execute", "compiler"); - auto st = Stat::stat(tmp_stdout_path, Stat::OnError::log); - if (!st) { - // The stdout file was removed - cleanup in progress? Better bail out. - throw Failure(Statistic::missing_cache_file); - } - - // distcc-pump outputs lines like this: - // __________Using # distcc servers in pump mode - if (st.size() != 0 && ctx.config.compiler_type() != CompilerType::pump) { - LOG_RAW("Compiler produced stdout"); - throw Failure(Statistic::compiler_produced_stdout); + if (!result) { + return nonstd::make_unexpected(result.error()); } // Merge stderr from the preprocessor (if any) and stderr from the real - // compiler into tmp_stderr. - if (!ctx.cpp_stderr.empty()) { - std::string combined_stderr = - Util::read_file(ctx.cpp_stderr) + Util::read_file(tmp_stderr_path); - Util::write_file(tmp_stderr_path, combined_stderr); + // compiler. + if (!ctx.cpp_stderr_data.empty()) { + result->stderr_data.insert(result->stderr_data.begin(), + ctx.cpp_stderr_data.begin(), + ctx.cpp_stderr_data.end()); } - if (status != 0) { - LOG("Compiler gave exit status {}", status); + result->stdout_data = + rewrite_stdout_from_compiler(ctx, std::move(result->stdout_data)); + + if (result->exit_status != 0) { + LOG("Compiler gave exit status {}", result->exit_status); // We can output stderr immediately instead of rerunning the compiler. - Util::send_to_stderr(ctx, Util::read_file(tmp_stderr_path)); + Util::send_to_fd( + ctx, util::to_string_view(result->stderr_data), STDERR_FILENO); + Util::send_to_fd( + ctx, + util::to_string_view(core::MsvcShowIncludesOutput::strip_includes( + ctx, std::move(result->stdout_data))), + STDOUT_FILENO); - throw Failure(Statistic::compile_failed, status); + auto failure = Failure(Statistic::compile_failed); + failure.set_exit_code(result->exit_status); + return nonstd::make_unexpected(failure); } if (ctx.config.depend_mode()) { ASSERT(depend_mode_hash); - auto result_name = result_name_from_depfile(ctx, *depend_mode_hash); - if (!result_name) { - throw Failure(Statistic::internal_error); + if (ctx.args_info.generating_dependencies) { + result_key = result_key_from_depfile(ctx, *depend_mode_hash); + } else if (ctx.args_info.generating_includes) { + result_key = result_key_from_includes( + ctx, *depend_mode_hash, util::to_string_view(result->stdout_data)); + } else { + ASSERT(false); + } + if (!result_key) { + return nonstd::make_unexpected(Statistic::internal_error); } - ctx.set_result_name(*result_name); + LOG_RAW("Got result key from dependency file"); + LOG("Result key: {}", result_key->to_string()); } + ASSERT(result_key); + bool produce_dep_file = ctx.args_info.generating_dependencies && ctx.args_info.output_dep != "/dev/null"; @@ -1077,173 +1135,116 @@ to_cache(Context& ctx, Depfile::make_paths_relative_in_output_dep(ctx); } - const auto obj_stat = Stat::stat(ctx.args_info.output_obj); - if (!obj_stat) { - if (ctx.args_info.expect_output_obj) { - LOG_RAW("Compiler didn't produce an object file (unexpected)"); - throw Failure(Statistic::compiler_produced_no_output); - } else { - LOG_RAW("Compiler didn't produce an object file (expected)"); - } - } else if (obj_stat.size() == 0) { - LOG_RAW("Compiler produced an empty object file"); - throw Failure(Statistic::compiler_produced_empty_output); - } - - const auto stderr_stat = Stat::stat(tmp_stderr_path, Stat::OnError::log); - if (!stderr_stat) { - throw Failure(Statistic::internal_error); - } - - const auto result_file = look_up_cache_file( - ctx.config.cache_dir(), *ctx.result_name(), Result::k_file_suffix); - ctx.set_result_path(result_file.path); - Result::Writer result_writer(ctx, result_file.path); - - if (stderr_stat.size() > 0) { - result_writer.write(Result::FileType::stderr_output, tmp_stderr_path); - } - if (obj_stat) { - result_writer.write(Result::FileType::object, ctx.args_info.output_obj); - } - if (ctx.args_info.generating_dependencies) { - result_writer.write(Result::FileType::dependency, ctx.args_info.output_dep); - } - if (ctx.args_info.generating_coverage) { - const auto coverage_file = find_coverage_file(ctx); - if (!coverage_file.found) { - throw Failure(Statistic::internal_error); - } - result_writer.write(coverage_file.mangled - ? Result::FileType::coverage_mangled - : Result::FileType::coverage_unmangled, - coverage_file.path); - } - if (ctx.args_info.generating_stackusage) { - result_writer.write(Result::FileType::stackusage, ctx.args_info.output_su); - } - if (ctx.args_info.generating_diagnostics) { - result_writer.write(Result::FileType::diagnostic, ctx.args_info.output_dia); - } - if (ctx.args_info.seen_split_dwarf && Stat::stat(ctx.args_info.output_dwo)) { - // Only store .dwo file if it was created by the compiler (GCC and Clang - // behave differently e.g. for "-gsplit-dwarf -g1"). - result_writer.write(Result::FileType::dwarf_object, - ctx.args_info.output_dwo); - } - - auto error = result_writer.finalize(); - if (error) { - LOG("Error: {}", *error); + Stat obj_stat; + if (!ctx.args_info.expect_output_obj) { + // Don't probe for object file when we don't expect one since we otherwise + // will be fooled by an already existing object file. + LOG_RAW("Compiler not expected to produce an object file"); } else { - LOG("Stored in cache: {}", result_file.path); + obj_stat = Stat::stat(ctx.args_info.output_obj); + if (!obj_stat) { + LOG_RAW("Compiler didn't produce an object file"); + return nonstd::make_unexpected(Statistic::compiler_produced_no_output); + } else if (obj_stat.size() == 0) { + LOG_RAW("Compiler produced an empty object file"); + return nonstd::make_unexpected(Statistic::compiler_produced_empty_output); + } } - auto new_result_stat = Stat::stat(result_file.path, Stat::OnError::log); - if (!new_result_stat) { - throw Failure(Statistic::internal_error); + MTR_BEGIN("result", "result_put"); + if (!write_result( + ctx, *result_key, obj_stat, result->stdout_data, result->stderr_data)) { + return nonstd::make_unexpected(Statistic::compiler_produced_no_output); } - ctx.counter_updates.increment( - Statistic::cache_size_kibibyte, - Util::size_change_kibibyte(result_file.stat, new_result_stat)); - ctx.counter_updates.increment(Statistic::files_in_cache, - result_file.stat ? 0 : 1); - - MTR_END("file", "file_put"); - - // Make sure we have a CACHEDIR.TAG in the cache part of cache_dir. This can - // be done almost anywhere, but we might as well do it near the end as we save - // the stat call if we exit early. - create_cachedir_tag(ctx); + MTR_END("result", "result_put"); // Everything OK. - Util::send_to_stderr(ctx, Util::read_file(tmp_stderr_path)); + Util::send_to_fd( + ctx, util::to_string_view(result->stderr_data), STDERR_FILENO); + // Send stdout after stderr, it makes the output clearer with MSVC. + Util::send_to_fd( + ctx, + util::to_string_view(core::MsvcShowIncludesOutput::strip_includes( + ctx, std::move(result->stdout_data))), + STDOUT_FILENO); + + return *result_key; } -// Find the result name by running the compiler in preprocessor mode and +// Find the result key by running the compiler in preprocessor mode and // hashing the result. -static Digest -get_result_name_from_cpp(Context& ctx, Args& args, Hash& hash) +static nonstd::expected +get_result_key_from_cpp(Context& ctx, Args& args, Hash& hash) { - ctx.time_of_compilation = time(nullptr); + ctx.time_of_compilation = util::TimePoint::now(); + + std::string preprocessed_path; + util::Bytes cpp_stderr_data; - std::string stderr_path; - std::string stdout_path; - int status; if (ctx.args_info.direct_i_file) { // We are compiling a .i or .ii file - that means we can skip the cpp stage // and directly form the correct i_tmpfile. - stdout_path = ctx.args_info.input_file; - status = 0; + preprocessed_path = ctx.args_info.input_file; } else { // Run cpp on the input file to obtain the .i. - TemporaryFile tmp_stdout( - FMT("{}/tmp.cpp_stdout", ctx.config.temporary_dir())); - ctx.register_pending_tmp_file(tmp_stdout.path); - - // stdout_path needs the proper cpp_extension for the compiler to do its - // thing correctly. - stdout_path = FMT("{}.{}", tmp_stdout.path, ctx.config.cpp_extension()); - Util::hard_link(tmp_stdout.path, stdout_path); - ctx.register_pending_tmp_file(stdout_path); + // preprocessed_path needs the proper cpp_extension for the compiler to do + // its thing correctly. + TemporaryFile tmp_stdout(FMT("{}/cpp_stdout", ctx.config.temporary_dir()), + FMT(".{}", ctx.config.cpp_extension())); + preprocessed_path = tmp_stdout.path; + tmp_stdout.fd.close(); // We're only using the path. + ctx.register_pending_tmp_file(preprocessed_path); - TemporaryFile tmp_stderr( - FMT("{}/tmp.cpp_stderr", ctx.config.temporary_dir())); - stderr_path = tmp_stderr.path; - ctx.register_pending_tmp_file(stderr_path); + const size_t orig_args_size = args.size(); - size_t args_added = 2; - args.push_back("-E"); - if (ctx.args_info.actual_language == "hip") { - args.push_back("-o"); - args.push_back("-"); - args_added += 2; - } if (ctx.config.keep_comments_cpp()) { args.push_back("-C"); - args_added++; } + + // Send preprocessor output to a file instead of stdout to work around + // compilers that don't exit with a proper status on write error to stdout. + // See also . + if (ctx.config.is_compiler_group_msvc()) { + args.push_back("-P"); + args.push_back(FMT("-Fi{}", preprocessed_path)); + } else { + args.push_back("-E"); + args.push_back("-o"); + args.push_back(preprocessed_path); + } + args.push_back(ctx.args_info.input_file); + add_prefix(ctx, args, ctx.config.prefix_command_cpp()); LOG_RAW("Running preprocessor"); MTR_BEGIN("execute", "preprocessor"); - status = - do_execute(ctx, args, std::move(tmp_stdout), std::move(tmp_stderr)); + const auto result = do_execute(ctx, args, false); MTR_END("execute", "preprocessor"); - args.pop_back(args_added); - } + args.pop_back(args.size() - orig_args_size); + + if (!result) { + return nonstd::make_unexpected(result.error()); + } else if (result->exit_status != 0) { + LOG("Preprocessor gave exit status {}", result->exit_status); + return nonstd::make_unexpected(Statistic::preprocessor_error); + } - if (status != 0) { - LOG("Preprocessor gave exit status {}", status); - throw Failure(Statistic::preprocessor_error); + cpp_stderr_data = result->stderr_data; } hash.hash_delimiter("cpp"); - const bool is_pump = ctx.config.compiler_type() == CompilerType::pump; - const Statistic error = - process_preprocessed_file(ctx, hash, stdout_path, is_pump); - if (error != Statistic::none) { - throw Failure(error); - } + TRY(process_preprocessed_file(ctx, hash, preprocessed_path)); hash.hash_delimiter("cppstderr"); - if (!ctx.args_info.direct_i_file && !hash.hash_file(stderr_path)) { - // Somebody removed the temporary file? - LOG("Failed to open {}: {}", stderr_path, strerror(errno)); - throw Failure(Statistic::internal_error); - } + hash.hash(util::to_string_view(cpp_stderr_data)); - if (ctx.args_info.direct_i_file) { - ctx.i_tmpfile = ctx.args_info.input_file; - } else { - ctx.i_tmpfile = stdout_path; - } + ctx.i_tmpfile = preprocessed_path; if (!ctx.config.run_second_cpp()) { // If we are using the CPP trick, we need to remember this stderr data and // output it just before the main stderr from the compiler pass. - ctx.cpp_stderr = stderr_path; + ctx.cpp_stderr_data = std::move(cpp_stderr_data); hash.hash_delimiter("runsecondcpp"); hash.hash("false"); } @@ -1253,7 +1254,7 @@ get_result_name_from_cpp(Context& ctx, Args& args, Hash& hash) // Hash mtime or content of a file, or the output of a command, according to // the CCACHE_COMPILERCHECK setting. -static void +static nonstd::expected hash_compiler(const Context& ctx, Hash& hash, const Stat& st, @@ -1265,8 +1266,8 @@ hash_compiler(const Context& ctx, } else if (ctx.config.compiler_check() == "mtime") { hash.hash_delimiter("cc_mtime"); hash.hash(st.size()); - hash.hash(st.mtime()); - } else if (Util::starts_with(ctx.config.compiler_check(), "string:")) { + hash.hash(st.mtime().nsec()); + } else if (util::starts_with(ctx.config.compiler_check(), "string:")) { hash.hash_delimiter("cc_hash"); hash.hash(&ctx.config.compiler_check()[7]); } else if (ctx.config.compiler_check() == "content" || !allow_command) { @@ -1277,9 +1278,10 @@ hash_compiler(const Context& ctx, hash, ctx.config.compiler_check(), ctx.orig_args[0])) { LOG("Failure running compiler check command: {}", ctx.config.compiler_check()); - throw Failure(Statistic::compiler_check_failed); + return nonstd::make_unexpected(Statistic::compiler_check_failed); } } + return {}; } // Hash the host compiler(s) invoked by nvcc. @@ -1287,7 +1289,7 @@ hash_compiler(const Context& ctx, // If `ccbin_st` and `ccbin` are set, they refer to a directory or compiler set // with -ccbin/--compiler-bindir. If `ccbin_st` is nullptr or `ccbin` is the // empty string, the compilers are looked up in PATH instead. -static void +static nonstd::expected hash_nvcc_host_compiler(const Context& ctx, Hash& hash, const Stat* ccbin_st = nullptr, @@ -1318,29 +1320,25 @@ hash_nvcc_host_compiler(const Context& ctx, std::string path = FMT("{}/{}", ccbin, compiler); auto st = Stat::stat(path); if (st) { - hash_compiler(ctx, hash, st, path, false); + TRY(hash_compiler(ctx, hash, st, path, false)); } } else { - std::string path = find_executable(ctx, compiler, CCACHE_NAME); + std::string path = find_executable(ctx, compiler, ctx.orig_args[0]); if (!path.empty()) { auto st = Stat::stat(path, Stat::OnError::log); - hash_compiler(ctx, hash, st, ccbin, false); + TRY(hash_compiler(ctx, hash, st, ccbin, false)); } } } } else { - hash_compiler(ctx, hash, *ccbin_st, ccbin, false); + TRY(hash_compiler(ctx, hash, *ccbin_st, ccbin, false)); } -} -static bool -should_rewrite_dependency_target(const ArgsInfo& args_info) -{ - return !args_info.dependency_target_specified && args_info.seen_MD_MMD; + return {}; } // update a hash with information common for the direct and preprocessor modes. -static void +static nonstd::expected hash_common_info(const Context& ctx, const Args& args, Hash& hash, @@ -1348,6 +1346,11 @@ hash_common_info(const Context& ctx, { hash.hash(HASH_PREFIX); + if (!ctx.config.namespace_().empty()) { + hash.hash_delimiter("namespace"); + hash.hash(ctx.config.namespace_()); + } + // We have to hash the extension, as a .i file isn't treated the same by the // compiler as a .ii file. hash.hash_delimiter("ext"); @@ -1361,11 +1364,11 @@ hash_common_info(const Context& ctx, auto st = Stat::stat(compiler_path, Stat::OnError::log); if (!st) { - throw Failure(Statistic::could_not_find_compiler); + return nonstd::make_unexpected(Statistic::could_not_find_compiler); } // Hash information about the compiler. - hash_compiler(ctx, hash, st, compiler_path, true); + TRY(hash_compiler(ctx, hash, st, compiler_path, true)); // Also hash the compiler name as some compilers use hard links and behave // differently depending on the real name. @@ -1374,11 +1377,23 @@ hash_common_info(const Context& ctx, // Hash variables that may affect the compilation. const char* always_hash_env_vars[] = { - // From : + // From + // (note: SOURCE_DATE_EPOCH is handled in hash_source_code_string()): "COMPILER_PATH", "GCC_COMPARE_DEBUG", "GCC_EXEC_PREFIX", - // Note: SOURCE_DATE_EPOCH is handled in hash_source_code_string(). + // Variables that affect which underlying compiler ICC uses. Reference: + // + "I_MPI_CC", + "I_MPI_CXX", +#ifdef __APPLE__ + // On macOS, /usr/bin/clang is a compiler wrapper that switches compiler + // based on at least these variables: + "DEVELOPER_DIR", + "MACOSX_DEPLOYMENT_TARGET", +#endif }; for (const char* name : always_hash_env_vars) { const char* value = getenv(name); @@ -1388,7 +1403,7 @@ hash_common_info(const Context& ctx, } } - if (!(ctx.config.sloppiness() & SLOPPY_LOCALE)) { + if (!(ctx.config.sloppiness().is_enabled(core::Sloppy::locale))) { // Hash environment variables that may affect localization of compiler // warning messages. const char* envvars[] = { @@ -1414,7 +1429,7 @@ hash_common_info(const Context& ctx, old_path, new_path, ctx.apparent_cwd); - if (Util::starts_with(ctx.apparent_cwd, old_path)) { + if (util::starts_with(ctx.apparent_cwd, old_path)) { dir_to_hash = new_path + ctx.apparent_cwd.substr(old_path.size()); } } @@ -1424,12 +1439,20 @@ hash_common_info(const Context& ctx, hash.hash(dir_to_hash); } - if ((!should_rewrite_dependency_target(ctx.args_info) - && ctx.args_info.generating_dependencies) - || ctx.args_info.seen_split_dwarf || ctx.args_info.profile_arcs) { - // If generating dependencies: The output object file name is part of the .d - // file, so include the path in the hash. - // + // The object file produced by MSVC includes the full path to the source file + // even without debug flags. Hashing the directory should be enough since the + // filename is included in the hash anyway. + if (ctx.config.is_compiler_group_msvc() && ctx.config.hash_dir()) { + const std::string output_obj_dir = + util::is_absolute_path(args_info.output_obj) + ? std::string(Util::dir_name(args_info.output_obj)) + : ctx.actual_cwd; + LOG("Hashing object file directory {}", output_obj_dir); + hash.hash_delimiter("source path"); + hash.hash(output_obj_dir); + } + + if (ctx.args_info.seen_split_dwarf || ctx.args_info.profile_arcs) { // When using -gsplit-dwarf: Object files include a link to the // corresponding .dwo file based on the target object filename, so hashing // the object file path will do it, although just hashing the object file @@ -1441,6 +1464,16 @@ hash_common_info(const Context& ctx, hash.hash(ctx.args_info.output_obj); } + if (ctx.args_info.generating_coverage + && !(ctx.config.sloppiness().is_enabled(core::Sloppy::gcno_cwd))) { + // GCC 9+ includes $PWD in the .gcno file. Since we don't have knowledge + // about compiler version we always (unless sloppiness is wanted) include + // the directory in the hash for now. + LOG_RAW("Hashing apparent CWD due to generating a .gcno file"); + hash.hash_delimiter("CWD in .gcno"); + hash.hash(ctx.apparent_cwd); + } + // Possibly hash the coverage data file path. if (ctx.args_info.generating_coverage && ctx.args_info.profile_arcs) { std::string dir; @@ -1450,7 +1483,7 @@ hash_common_info(const Context& ctx, dir = Util::real_path(std::string(Util::dir_name(ctx.args_info.output_obj))); } - string_view stem = + std::string_view stem = Util::remove_extension(Util::base_name(ctx.args_info.output_obj)); std::string gcda_path = FMT("{}/{}.gcda", dir, stem); LOG("Hashing coverage path {}", gcda_path); @@ -1461,19 +1494,19 @@ hash_common_info(const Context& ctx, // Possibly hash the sanitize blacklist file path. for (const auto& sanitize_blacklist : args_info.sanitize_blacklists) { LOG("Hashing sanitize blacklist {}", sanitize_blacklist); - hash.hash("sanitizeblacklist"); + hash.hash_delimiter("sanitizeblacklist"); if (!hash_binary_file(ctx, hash, sanitize_blacklist)) { - throw Failure(Statistic::error_hashing_extra_file); + return nonstd::make_unexpected(Statistic::error_hashing_extra_file); } } if (!ctx.config.extra_files_to_hash().empty()) { - for (const std::string& path : Util::split_into_strings( - ctx.config.extra_files_to_hash(), PATH_DELIM)) { + for (const std::string& path : + util::split_path_list(ctx.config.extra_files_to_hash())) { LOG("Hashing extra file {}", path); hash.hash_delimiter("extrafile"); if (!hash_binary_file(ctx, hash, path)) { - throw Failure(Statistic::error_hashing_extra_file); + return nonstd::make_unexpected(Statistic::error_hashing_extra_file); } } } @@ -1486,253 +1519,345 @@ hash_common_info(const Context& ctx, hash.hash(gcc_colors); } } + + return {}; } static bool -hash_profile_data_file(const Context& ctx, Hash& hash) +option_should_be_ignored(const std::string& arg, + const std::vector& patterns) { - const std::string& profile_path = ctx.args_info.profile_path; - string_view base_name = Util::remove_extension(ctx.args_info.output_obj); - std::string hashified_cwd = ctx.apparent_cwd; - std::replace(hashified_cwd.begin(), hashified_cwd.end(), '/', '#'); + return std::any_of( + patterns.cbegin(), patterns.cend(), [&arg](const auto& pattern) { + const auto& prefix = + std::string_view(pattern).substr(0, pattern.length() - 1); + return ( + pattern == arg + || (util::ends_with(pattern, "*") && util::starts_with(arg, prefix))); + }); +} - std::vector paths_to_try{ - // -fprofile-use[=dir]/-fbranch-probabilities (GCC <9) - FMT("{}/{}.gcda", profile_path, base_name), - // -fprofile-use[=dir]/-fbranch-probabilities (GCC >=9) - FMT("{}/{}#{}.gcda", profile_path, hashified_cwd, base_name), - // -fprofile(-instr|-sample)-use=file (Clang), -fauto-profile=file (GCC >=5) - profile_path, - // -fprofile(-instr|-sample)-use=dir (Clang) - FMT("{}/default.profdata", profile_path), - // -fauto-profile (GCC >=5) - "fbdata.afdo", // -fprofile-dir is not used - }; +static std::tuple, + std::optional> +get_option_and_value(std::string_view option, const Args& args, size_t& i) +{ + if (args[i] == option) { + if (i + 1 < args.size()) { + ++i; + return {option, args[i]}; + } else { + return {std::nullopt, std::nullopt}; + } + } else if (util::starts_with(args[i], option)) { + return {option, std::string_view(args[i]).substr(option.length())}; + } else { + return {std::nullopt, std::nullopt}; + } +} - bool found = false; - for (const std::string& p : paths_to_try) { - LOG("Checking for profile data file {}", p); - auto st = Stat::stat(p); - if (st && !st.is_directory()) { - LOG("Adding profile data {} to the hash", p); - hash.hash_delimiter("-fprofile-use"); - if (hash_binary_file(ctx, hash, p)) { - found = true; - } +static nonstd::expected +hash_argument(const Context& ctx, + const Args& args, + size_t& i, + Hash& hash, + const bool is_clang, + const bool direct_mode, + bool& found_ccbin) +{ + // Trust the user if they've said we should not hash a given option. + if (option_should_be_ignored(args[i], ctx.ignore_options())) { + LOG("Not hashing ignored option: {}", args[i]); + if (i + 1 < args.size() && compopt_takes_arg(args[i])) { + i++; + LOG("Not hashing argument of ignored option: {}", args[i]); } + return {}; } - return found; -} - -static bool -option_should_be_ignored(const std::string& arg, - const std::vector& patterns) -{ - return std::any_of( - patterns.cbegin(), patterns.cend(), [&arg](const std::string& pattern) { - const auto& prefix = string_view(pattern).substr(0, pattern.length() - 1); - return ( - pattern == arg - || (Util::ends_with(pattern, "*") && Util::starts_with(arg, prefix))); - }); -} - -// Update a hash sum with information specific to the direct and preprocessor -// modes and calculate the result name. Returns the result name on success, -// otherwise nullopt. -static optional -calculate_result_name(Context& ctx, - const Args& args, - Args& preprocessor_args, - Hash& hash, - bool direct_mode) -{ - bool found_ccbin = false; - - hash.hash_delimiter("result version"); - hash.hash(Result::k_version); + // -L doesn't affect compilation (except for clang). + if (i < args.size() - 1 && args[i] == "-L" && !is_clang) { + i++; + return {}; + } + if (util::starts_with(args[i], "-L") && !is_clang) { + return {}; + } - if (direct_mode) { - hash.hash_delimiter("manifest version"); - hash.hash(Manifest::k_version); + // -Wl,... doesn't affect compilation (except for clang). + if (util::starts_with(args[i], "-Wl,") && !is_clang) { + return {}; } - // clang will emit warnings for unused linker flags, so we shouldn't skip - // those arguments. - int is_clang = ctx.config.compiler_type() == CompilerType::clang - || ctx.config.compiler_type() == CompilerType::other; + if (util::starts_with(args[i], "-Wa,")) { + // We have to distinguish between three cases: + // + // Case 1: -Wa,-a (write to stdout) + // Case 2: -Wa,-a= (write to stdout and stderr) + // Case 3: -Wa,-a=file (write to file) + // + // No need to include the file part in case 3 in the hash since the filename + // is not part of the output. - // First the arguments. - for (size_t i = 1; i < args.size(); i++) { - // Trust the user if they've said we should not hash a given option. - if (option_should_be_ignored(args[i], ctx.ignore_options())) { - LOG("Not hashing ignored option: {}", args[i]); - if (i + 1 < args.size() && compopt_takes_arg(args[i])) { - i++; - LOG("Not hashing argument of ignored option: {}", args[i]); + hash.hash_delimiter("arg"); + bool first = true; + for (const auto part : + util::Tokenizer(args[i], ",", util::Tokenizer::Mode::include_empty)) { + if (first) { + first = false; + } else { + hash.hash(","); } - continue; - } - - // -L doesn't affect compilation (except for clang). - if (i < args.size() - 1 && args[i] == "-L" && !is_clang) { - i++; - continue; - } - if (Util::starts_with(args[i], "-L") && !is_clang) { - continue; + if (util::starts_with(part, "-a")) { + const auto eq_pos = part.find('='); + if (eq_pos < part.size() - 1) { + // Case 3: + hash.hash(part.substr(0, eq_pos + 1)); + hash.hash("file"); + continue; + } + } + // Case 1 and 2: + hash.hash(part); } + return {}; + } - // -Wl,... doesn't affect compilation (except for clang). - if (Util::starts_with(args[i], "-Wl,") && !is_clang) { - continue; - } + // The -fdebug-prefix-map option may be used in combination with + // CCACHE_BASEDIR to reuse results across different directories. Skip using + // the value of the option from hashing but still hash the existence of the + // option. + if (util::starts_with(args[i], "-fdebug-prefix-map=")) { + hash.hash_delimiter("arg"); + hash.hash("-fdebug-prefix-map="); + return {}; + } + if (util::starts_with(args[i], "-ffile-prefix-map=")) { + hash.hash_delimiter("arg"); + hash.hash("-ffile-prefix-map="); + return {}; + } + if (util::starts_with(args[i], "-fmacro-prefix-map=")) { + hash.hash_delimiter("arg"); + hash.hash("-fmacro-prefix-map="); + return {}; + } - // The -fdebug-prefix-map option may be used in combination with - // CCACHE_BASEDIR to reuse results across different directories. Skip using - // the value of the option from hashing but still hash the existence of the - // option. - if (Util::starts_with(args[i], "-fdebug-prefix-map=")) { - hash.hash_delimiter("arg"); - hash.hash("-fdebug-prefix-map="); - continue; - } - if (Util::starts_with(args[i], "-ffile-prefix-map=")) { - hash.hash_delimiter("arg"); - hash.hash("-ffile-prefix-map="); - continue; - } - if (Util::starts_with(args[i], "-fmacro-prefix-map=")) { - hash.hash_delimiter("arg"); - hash.hash("-fmacro-prefix-map="); - continue; - } + if (util::starts_with(args[i], "-frandom-seed=") + && ctx.config.sloppiness().is_enabled(core::Sloppy::random_seed)) { + LOG("Ignoring {} since random_seed sloppiness is requested", args[i]); + return {}; + } - // When using the preprocessor, some arguments don't contribute to the - // hash. The theory is that these arguments will change the output of -E if - // they are going to have any effect at all. For precompiled headers this - // might not be the case. - if (!direct_mode && !ctx.args_info.output_is_precompiled_header - && !ctx.args_info.using_precompiled_header) { - if (compopt_affects_cpp_output(args[i])) { - if (compopt_takes_arg(args[i])) { - i++; - } - continue; - } - if (compopt_affects_cpp_output(args[i].substr(0, 2))) { - continue; + // When using the preprocessor, some arguments don't contribute to the hash. + // The theory is that these arguments will change the output of -E if they are + // going to have any effect at all. For precompiled headers this might not be + // the case. + if (!direct_mode && !ctx.args_info.output_is_precompiled_header + && !ctx.args_info.using_precompiled_header) { + if (compopt_affects_cpp_output(args[i])) { + if (compopt_takes_arg(args[i])) { + i++; } + return {}; } + if (compopt_affects_cpp_output(args[i].substr(0, 2))) { + return {}; + } + } - // If we're generating dependencies, we make sure to skip the filename of - // the dependency file, since it doesn't impact the output. - if (ctx.args_info.generating_dependencies) { - if (Util::starts_with(args[i], "-Wp,")) { - if (Util::starts_with(args[i], "-Wp,-MD,") - && args[i].find(',', 8) == std::string::npos) { - hash.hash(args[i].data(), 8); - continue; - } else if (Util::starts_with(args[i], "-Wp,-MMD,") - && args[i].find(',', 9) == std::string::npos) { - hash.hash(args[i].data(), 9); - continue; - } - } else if (Util::starts_with(args[i], "-MF")) { - // In either case, hash the "-MF" part. - hash.hash_delimiter("arg"); - hash.hash(args[i].data(), 3); - - if (ctx.args_info.output_dep != "/dev/null") { - bool separate_argument = (args[i].size() == 3); - if (separate_argument) { - // Next argument is dependency name, so skip it. - i++; - } - } - continue; + if (ctx.args_info.generating_dependencies) { + std::optional option; + std::optional value; + + if (util::starts_with(args[i], "-Wp,")) { + // Skip the dependency filename since it doesn't impact the output. + if (util::starts_with(args[i], "-Wp,-MD,") + && args[i].find(',', 8) == std::string::npos) { + hash.hash(args[i].data(), 8); + return {}; + } else if (util::starts_with(args[i], "-Wp,-MMD,") + && args[i].find(',', 9) == std::string::npos) { + hash.hash(args[i].data(), 9); + return {}; } + } else if (std::tie(option, value) = get_option_and_value("-MF", args, i); + option) { + // Skip the dependency filename since it doesn't impact the output. + hash.hash(*option); + return {}; + } else if (std::tie(option, value) = get_option_and_value("-MQ", args, i); + option) { + hash.hash(*option); + // No need to hash the dependency target since we always calculate it on + // a cache hit. + return {}; + } else if (std::tie(option, value) = get_option_and_value("-MT", args, i); + option) { + hash.hash(*option); + // No need to hash the dependency target since we always calculate it on + // a cache hit. + return {}; } + } - if (Util::starts_with(args[i], "-specs=") - || Util::starts_with(args[i], "--specs=") - || (args[i] == "-specs" || args[i] == "--specs")) { - std::string path; - size_t eq_pos = args[i].find('='); - if (eq_pos == std::string::npos) { - if (i + 1 >= args.size()) { - LOG("missing argument for \"{}\"", args[i]); - throw Failure(Statistic::bad_compiler_arguments); - } - path = args[i + 1]; - i++; - } else { - path = args[i].substr(eq_pos + 1); - } - auto st = Stat::stat(path, Stat::OnError::log); - if (st) { - // If given an explicit specs file, then hash that file, but don't - // include the path to it in the hash. - hash.hash_delimiter("specs"); - hash_compiler(ctx, hash, st, path, false); - continue; + if (util::starts_with(args[i], "-specs=") + || util::starts_with(args[i], "--specs=") + || (args[i] == "-specs" || args[i] == "--specs") + || args[i] == "--config") { + std::string path; + size_t eq_pos = args[i].find('='); + if (eq_pos == std::string::npos) { + if (i + 1 >= args.size()) { + LOG("missing argument for \"{}\"", args[i]); + return nonstd::make_unexpected(Statistic::bad_compiler_arguments); } + path = args[i + 1]; + i++; + } else { + path = args[i].substr(eq_pos + 1); + } + auto st = Stat::stat(path, Stat::OnError::log); + if (st) { + // If given an explicit specs file, then hash that file, but don't + // include the path to it in the hash. + hash.hash_delimiter("specs"); + TRY(hash_compiler(ctx, hash, st, path, false)); + return {}; } + } - if (Util::starts_with(args[i], "-fplugin=")) { - auto st = Stat::stat(&args[i][9], Stat::OnError::log); - if (st) { - hash.hash_delimiter("plugin"); - hash_compiler(ctx, hash, st, &args[i][9], false); - continue; - } + if (util::starts_with(args[i], "-fplugin=")) { + auto st = Stat::stat(&args[i][9], Stat::OnError::log); + if (st) { + hash.hash_delimiter("plugin"); + TRY(hash_compiler(ctx, hash, st, &args[i][9], false)); + return {}; } + } - if (args[i] == "-Xclang" && i + 3 < args.size() && args[i + 1] == "-load" - && args[i + 2] == "-Xclang") { - auto st = Stat::stat(args[i + 3], Stat::OnError::log); - if (st) { - hash.hash_delimiter("plugin"); - hash_compiler(ctx, hash, st, args[i + 3], false); - i += 3; - continue; - } + if (args[i] == "-Xclang" && i + 3 < args.size() && args[i + 1] == "-load" + && args[i + 2] == "-Xclang") { + auto st = Stat::stat(args[i + 3], Stat::OnError::log); + if (st) { + hash.hash_delimiter("plugin"); + TRY(hash_compiler(ctx, hash, st, args[i + 3], false)); + i += 3; + return {}; } + } - if ((args[i] == "-ccbin" || args[i] == "--compiler-bindir") - && i + 1 < args.size()) { - auto st = Stat::stat(args[i + 1]); - if (st) { - found_ccbin = true; - hash.hash_delimiter("ccbin"); - hash_nvcc_host_compiler(ctx, hash, &st, args[i + 1]); - i++; - continue; - } + if ((args[i] == "-ccbin" || args[i] == "--compiler-bindir") + && i + 1 < args.size()) { + auto st = Stat::stat(args[i + 1]); + if (st) { + found_ccbin = true; + hash.hash_delimiter("ccbin"); + TRY(hash_nvcc_host_compiler(ctx, hash, &st, args[i + 1])); + i++; + return {}; } + } - // All other arguments are included in the hash. + // All other arguments are included in the hash. + hash.hash_delimiter("arg"); + hash.hash(args[i]); + if (i + 1 < args.size() && compopt_takes_arg(args[i])) { + i++; hash.hash_delimiter("arg"); hash.hash(args[i]); - if (i + 1 < args.size() && compopt_takes_arg(args[i])) { - i++; - hash.hash_delimiter("arg"); - hash.hash(args[i]); - } } - // Make results with dependency file /dev/null different from those without - // it. - if (ctx.args_info.generating_dependencies - && ctx.args_info.output_dep == "/dev/null") { - hash.hash_delimiter("/dev/null dependency file"); + return {}; +} + +static nonstd::expected, Failure> +get_manifest_key(Context& ctx, Hash& hash) +{ + // Hash environment variables that affect the preprocessor output. + const char* envvars[] = {"CPATH", + "C_INCLUDE_PATH", + "CPLUS_INCLUDE_PATH", + "OBJC_INCLUDE_PATH", + "OBJCPLUS_INCLUDE_PATH", // clang + nullptr}; + for (const char** p = envvars; *p; ++p) { + const char* v = getenv(*p); + if (v) { + hash.hash_delimiter(*p); + hash.hash(v); + } + } + + // Make sure that the direct mode hash is unique for the input file path. If + // this would not be the case: + // + // * A false cache hit may be produced. Scenario: + // - a/r.h exists. + // - a/x.c has #include "r.h". + // - b/x.c is identical to a/x.c. + // - Compiling a/x.c records a/r.h in the manifest. + // - Compiling b/x.c results in a false cache hit since a/x.c and b/x.c + // share manifests and a/r.h exists. + // * The expansion of __FILE__ may be incorrect. + hash.hash_delimiter("inputfile"); + hash.hash(ctx.args_info.input_file); + + hash.hash_delimiter("sourcecode hash"); + Digest input_file_digest; + int result = + hash_source_code_file(ctx, input_file_digest, ctx.args_info.input_file); + if (result & HASH_SOURCE_CODE_ERROR) { + return nonstd::make_unexpected(Statistic::internal_error); + } + if (result & HASH_SOURCE_CODE_FOUND_TIME) { + LOG_RAW("Disabling direct mode"); + ctx.config.set_direct_mode(false); + return {}; } + hash.hash(input_file_digest.to_string()); + return hash.digest(); +} - if (!found_ccbin && ctx.args_info.actual_language == "cu") { - hash_nvcc_host_compiler(ctx, hash); +static bool +hash_profile_data_file(const Context& ctx, Hash& hash) +{ + const std::string& profile_path = ctx.args_info.profile_path; + std::string_view base_name = Util::remove_extension(ctx.args_info.output_obj); + std::string hashified_cwd = ctx.apparent_cwd; + std::replace(hashified_cwd.begin(), hashified_cwd.end(), '/', '#'); + + std::vector paths_to_try{ + // -fprofile-use[=dir]/-fbranch-probabilities (GCC <9) + FMT("{}/{}.gcda", profile_path, base_name), + // -fprofile-use[=dir]/-fbranch-probabilities (GCC >=9) + FMT("{}/{}#{}.gcda", profile_path, hashified_cwd, base_name), + // -fprofile(-instr|-sample)-use=file (Clang), -fauto-profile=file (GCC >=5) + profile_path, + // -fprofile(-instr|-sample)-use=dir (Clang) + FMT("{}/default.profdata", profile_path), + // -fauto-profile (GCC >=5) + "fbdata.afdo", // -fprofile-dir is not used + }; + + bool found = false; + for (const std::string& p : paths_to_try) { + LOG("Checking for profile data file {}", p); + auto st = Stat::stat(p); + if (st && !st.is_directory()) { + LOG("Adding profile data {} to the hash", p); + hash.hash_delimiter("-fprofile-use"); + if (hash_binary_file(ctx, hash, p)) { + found = true; + } + } } + return found; +} + +static nonstd::expected +hash_profiling_related_data(const Context& ctx, Hash& hash) +{ // For profile generation (-fprofile(-instr)-generate[=path]) // - hash profile path // @@ -1748,118 +1873,172 @@ calculate_result_name(Context& ctx, if (ctx.args_info.profile_generate) { ASSERT(!ctx.args_info.profile_path.empty()); - LOG("Adding profile directory {} to our hash", ctx.args_info.profile_path); + + // For a relative profile directory D the compiler stores $PWD/D as part of + // the profile filename so we need to include the same information in the + // hash. + const std::string profile_path = + util::is_absolute_path(ctx.args_info.profile_path) + ? ctx.args_info.profile_path + : FMT("{}/{}", ctx.apparent_cwd, ctx.args_info.profile_path); + LOG("Adding profile directory {} to our hash", profile_path); hash.hash_delimiter("-fprofile-dir"); - hash.hash(ctx.args_info.profile_path); + hash.hash(profile_path); } if (ctx.args_info.profile_use && !hash_profile_data_file(ctx, hash)) { LOG_RAW("No profile data file found"); - throw Failure(Statistic::no_input_file); + return nonstd::make_unexpected(Statistic::no_input_file); } - // Adding -arch to hash since cpp output is affected. - for (const auto& arch : ctx.args_info.arch_args) { - hash.hash_delimiter("-arch"); - hash.hash(arch); + return {}; +} + +static std::optional +get_result_key_from_manifest(Context& ctx, const Digest& manifest_key) +{ + MTR_BEGIN("manifest", "manifest_get"); + std::optional result_key; + size_t read_manifests = 0; + ctx.storage.get( + manifest_key, core::CacheEntryType::manifest, [&](util::Bytes&& value) { + try { + read_manifest(ctx, value); + ++read_manifests; + result_key = ctx.manifest.look_up_result_digest(ctx); + } catch (const core::Error& e) { + LOG("Failed to look up result key in manifest: {}", e.what()); + } + if (result_key) { + LOG_RAW("Got result key from manifest"); + return true; + } else { + LOG_RAW("Did not find result key in manifest"); + return false; + } + }); + MTR_END("manifest", "manifest_get"); + if (read_manifests > 1 && !ctx.config.remote_only()) { + MTR_SCOPE("manifest", "merge"); + LOG("Storing merged manifest {} locally", manifest_key.to_string()); + core::CacheEntry::Header header(ctx.config, core::CacheEntryType::manifest); + ctx.storage.local.put(manifest_key, + core::CacheEntryType::manifest, + core::CacheEntry::serialize(header, ctx.manifest)); } - optional result_name; + return result_key; +} + +// Update a hash sum with information specific to the direct and preprocessor +// modes and calculate the result key. Returns the result key on success, and +// if direct_mode is true also the manifest key. +static nonstd::expected, std::optional>, + Failure> +calculate_result_and_manifest_key(Context& ctx, + const Args& args, + Args& preprocessor_args, + Hash& hash, + bool direct_mode) +{ + bool found_ccbin = false; + + hash.hash_delimiter("cache entry version"); + hash.hash(core::CacheEntry::k_format_version); + + hash.hash_delimiter("result version"); + hash.hash(core::Result::k_format_version); + if (direct_mode) { - // Hash environment variables that affect the preprocessor output. - const char* envvars[] = {"CPATH", - "C_INCLUDE_PATH", - "CPLUS_INCLUDE_PATH", - "OBJC_INCLUDE_PATH", - "OBJCPLUS_INCLUDE_PATH", // clang - nullptr}; - for (const char** p = envvars; *p; ++p) { - const char* v = getenv(*p); - if (v) { - hash.hash_delimiter(*p); - hash.hash(v); - } - } + hash.hash_delimiter("manifest version"); + hash.hash(core::Manifest::k_format_version); + } - // Make sure that the direct mode hash is unique for the input file path. - // If this would not be the case: - // - // * An false cache hit may be produced. Scenario: - // - a/r.h exists. - // - a/x.c has #include "r.h". - // - b/x.c is identical to a/x.c. - // - Compiling a/x.c records a/r.h in the manifest. - // - Compiling b/x.c results in a false cache hit since a/x.c and b/x.c - // share manifests and a/r.h exists. - // * The expansion of __FILE__ may be incorrect. - hash.hash_delimiter("inputfile"); - hash.hash(ctx.args_info.input_file); - - hash.hash_delimiter("sourcecode"); - int result = hash_source_code_file(ctx, hash, ctx.args_info.input_file); - if (result & HASH_SOURCE_CODE_ERROR) { - throw Failure(Statistic::internal_error); - } - if (result & HASH_SOURCE_CODE_FOUND_TIME) { - LOG_RAW("Disabling direct mode"); - ctx.config.set_direct_mode(false); - return nullopt; - } + // clang will emit warnings for unused linker flags, so we shouldn't skip + // those arguments. + int is_clang = ctx.config.is_compiler_group_clang() + || ctx.config.compiler_type() == CompilerType::other; - const auto manifest_name = hash.digest(); - ctx.set_manifest_name(manifest_name); + // First the arguments. + for (size_t i = 1; i < args.size(); i++) { + TRY(hash_argument(ctx, args, i, hash, is_clang, direct_mode, found_ccbin)); + } - const auto manifest_file = look_up_cache_file( - ctx.config.cache_dir(), manifest_name, Manifest::k_file_suffix); - ctx.set_manifest_path(manifest_file.path); + // Make results with dependency file /dev/null different from those without + // it. + if (ctx.args_info.generating_dependencies + && ctx.args_info.output_dep == "/dev/null") { + hash.hash_delimiter("/dev/null dependency file"); + } - if (manifest_file.stat) { - LOG("Looking for result name in {}", manifest_file.path); - MTR_BEGIN("manifest", "manifest_get"); - result_name = Manifest::get(ctx, manifest_file.path); - MTR_END("manifest", "manifest_get"); - if (result_name) { - LOG_RAW("Got result name from manifest"); - } else { - LOG_RAW("Did not find result name in manifest"); - } - } else { - LOG("No manifest with name {} in the cache", manifest_name.to_string()); - } + if (!found_ccbin && ctx.args_info.actual_language == "cu") { + TRY(hash_nvcc_host_compiler(ctx, hash)); + } + + TRY(hash_profiling_related_data(ctx, hash)); + + // Adding -arch to hash since cpp output is affected. + for (const auto& arch : ctx.args_info.arch_args) { + hash.hash_delimiter("-arch"); + hash.hash(arch); + } + + std::optional result_key; + std::optional manifest_key; + + if (direct_mode) { + const auto manifest_key_result = get_manifest_key(ctx, hash); + if (!manifest_key_result) { + return nonstd::make_unexpected(manifest_key_result.error()); + } + manifest_key = *manifest_key_result; + if (manifest_key) { + LOG("Manifest key: {}", manifest_key->to_string()); + result_key = get_result_key_from_manifest(ctx, *manifest_key); + } + } else if (ctx.args_info.arch_args.empty()) { + const auto digest = get_result_key_from_cpp(ctx, preprocessor_args, hash); + if (!digest) { + return nonstd::make_unexpected(digest.error()); + } + result_key = *digest; + LOG_RAW("Got result key from preprocessor"); } else { - if (ctx.args_info.arch_args.empty()) { - result_name = get_result_name_from_cpp(ctx, preprocessor_args, hash); - LOG_RAW("Got result name from preprocessor"); - } else { - preprocessor_args.push_back("-arch"); - for (size_t i = 0; i < ctx.args_info.arch_args.size(); ++i) { - preprocessor_args.push_back(ctx.args_info.arch_args[i]); - result_name = get_result_name_from_cpp(ctx, preprocessor_args, hash); - LOG("Got result name from preprocessor with -arch {}", - ctx.args_info.arch_args[i]); - if (i != ctx.args_info.arch_args.size() - 1) { - result_name = nullopt; - } - preprocessor_args.pop_back(); + preprocessor_args.push_back("-arch"); + for (size_t i = 0; i < ctx.args_info.arch_args.size(); ++i) { + preprocessor_args.push_back(ctx.args_info.arch_args[i]); + const auto digest = get_result_key_from_cpp(ctx, preprocessor_args, hash); + if (!digest) { + return nonstd::make_unexpected(digest.error()); + } + result_key = *digest; + LOG("Got result key from preprocessor with -arch {}", + ctx.args_info.arch_args[i]); + if (i != ctx.args_info.arch_args.size() - 1) { + result_key = std::nullopt; } preprocessor_args.pop_back(); } + preprocessor_args.pop_back(); } - return result_name; + if (result_key) { + LOG("Result key: {}", result_key->to_string()); + } + return std::make_pair(result_key, manifest_key); } enum class FromCacheCallMode { direct, cpp }; // Try to return the compile result from cache. -static optional -from_cache(Context& ctx, FromCacheCallMode mode) +static nonstd::expected +from_cache(Context& ctx, FromCacheCallMode mode, const Digest& result_key) { UmaskScope umask_scope(ctx.original_umask); // The user might be disabling cache hits. if (ctx.config.recache()) { - return nullopt; + return false; } // If we're using Clang, we can't trust a precompiled header object based on @@ -1869,47 +2048,49 @@ from_cache(Context& ctx, FromCacheCallMode mode) // // file 'foo.h' has been modified since the precompiled header 'foo.pch' // was built - if ((ctx.config.compiler_type() == CompilerType::clang + if ((ctx.config.is_compiler_group_clang() || ctx.config.compiler_type() == CompilerType::other) && ctx.args_info.output_is_precompiled_header - && !ctx.args_info.fno_pch_timestamp && mode == FromCacheCallMode::cpp) { + && mode == FromCacheCallMode::cpp) { LOG_RAW("Not considering cached precompiled header in preprocessor mode"); - return nullopt; + return false; } - MTR_BEGIN("cache", "from_cache"); + MTR_SCOPE("cache", "from_cache"); // Get result from cache. - const auto result_file = look_up_cache_file( - ctx.config.cache_dir(), *ctx.result_name(), Result::k_file_suffix); - if (!result_file.stat) { - LOG("No result with name {} in the cache", ctx.result_name()->to_string()); - return nullopt; + util::Bytes cache_entry_data; + ctx.storage.get( + result_key, core::CacheEntryType::result, [&](util::Bytes&& value) { + cache_entry_data = std::move(value); + return true; + }); + if (cache_entry_data.empty()) { + return false; } - ctx.set_result_path(result_file.path); - Result::Reader result_reader(result_file.path); - ResultRetriever result_retriever( - ctx, should_rewrite_dependency_target(ctx.args_info)); - auto error = result_reader.read(result_retriever); - MTR_END("cache", "from_cache"); - if (error) { - LOG("Failed to get result from cache: {}", *error); - return nullopt; + try { + core::CacheEntry cache_entry(cache_entry_data); + cache_entry.verify_checksum(); + core::Result::Deserializer deserializer(cache_entry.payload()); + core::ResultRetriever result_retriever(ctx, result_key); + deserializer.visit(result_retriever); + } catch (core::ResultRetriever::WriteError& e) { + LOG("Write error when retrieving result from {}: {}", + result_key.to_string(), + e.what()); + return nonstd::make_unexpected(Statistic::bad_output_file); + } catch (core::Error& e) { + LOG("Failed to get result from {}: {}", result_key.to_string(), e.what()); + return false; } - // Update modification timestamp to save file from LRU cleanup. - Util::update_mtime(*ctx.result_path()); - LOG_RAW("Succeeded getting cached result"); - - return mode == FromCacheCallMode::direct ? Statistic::direct_cache_hit - : Statistic::preprocessed_cache_hit; + return true; } // Find the real compiler and put it into ctx.orig_args[0]. We just search the -// PATH to find an executable of the same name that isn't a link to ourselves. -// Pass find_executable function as second parameter. +// PATH to find an executable of the same name that isn't ourselves. void find_compiler(Context& ctx, const FindExecutableFunction& find_executable_function) @@ -1919,8 +2100,7 @@ find_compiler(Context& ctx, // ccache ccache gcc --> 2 size_t compiler_pos = 0; while (compiler_pos < ctx.orig_args.size() - && Util::same_program_name( - Util::base_name(ctx.orig_args[compiler_pos]), CCACHE_NAME)) { + && Util::is_ccache_executable(ctx.orig_args[compiler_pos])) { ++compiler_pos; } @@ -1934,152 +2114,35 @@ find_compiler(Context& ctx, : ctx.orig_args[compiler_pos]); const std::string resolved_compiler = - Util::is_full_path(compiler) + util::is_full_path(compiler) ? compiler - : find_executable_function(ctx, compiler, CCACHE_NAME); + : find_executable_function(ctx, compiler, ctx.orig_args[0]); if (resolved_compiler.empty()) { - throw Fatal("Could not find compiler \"{}\" in PATH", compiler); + throw core::Fatal(FMT("Could not find compiler \"{}\" in PATH", compiler)); } - if (Util::same_program_name(Util::base_name(resolved_compiler), - CCACHE_NAME)) { - throw Fatal( - "Recursive invocation (the name of the ccache binary must be \"{}\")", - CCACHE_NAME); + if (Util::is_ccache_executable(resolved_compiler)) { + throw core::Fatal("Recursive invocation of ccache"); } ctx.orig_args.pop_front(compiler_pos); ctx.orig_args[0] = resolved_compiler; } -static std::string -default_cache_dir(const std::string& home_dir) -{ -#ifdef _WIN32 - return home_dir + "/ccache"; -#elif defined(__APPLE__) - return home_dir + "/Library/Caches/ccache"; -#else - return home_dir + "/.cache/ccache"; -#endif -} - -static std::string -default_config_dir(const std::string& home_dir) -{ -#ifdef _WIN32 - return home_dir + "/ccache"; -#elif defined(__APPLE__) - return home_dir + "/Library/Preferences/ccache"; -#else - return home_dir + "/.config/ccache"; -#endif -} - -// Read config file(s), populate variables, create configuration file in cache -// directory if missing, etc. -static void -set_up_config(Config& config) -{ - const std::string home_dir = Util::get_home_directory(); - const std::string legacy_ccache_dir = home_dir + "/.ccache"; - const bool legacy_ccache_dir_exists = - Stat::stat(legacy_ccache_dir).is_directory(); - const char* const env_xdg_cache_home = getenv("XDG_CACHE_HOME"); - const char* const env_xdg_config_home = getenv("XDG_CONFIG_HOME"); - - const char* env_ccache_configpath = getenv("CCACHE_CONFIGPATH"); - if (env_ccache_configpath) { - config.set_primary_config_path(env_ccache_configpath); - } else { - // Only used for ccache tests: - const char* const env_ccache_configpath2 = getenv("CCACHE_CONFIGPATH2"); - - config.set_secondary_config_path(env_ccache_configpath2 - ? env_ccache_configpath2 - : FMT("{}/ccache.conf", SYSCONFDIR)); - MTR_BEGIN("config", "conf_read_secondary"); - // A missing config file in SYSCONFDIR is OK so don't check return value. - config.update_from_file(config.secondary_config_path()); - MTR_END("config", "conf_read_secondary"); - - const char* const env_ccache_dir = getenv("CCACHE_DIR"); - std::string primary_config_dir; - if (env_ccache_dir && *env_ccache_dir) { - primary_config_dir = env_ccache_dir; - } else if (!config.cache_dir().empty() && !env_ccache_dir) { - primary_config_dir = config.cache_dir(); - } else if (legacy_ccache_dir_exists) { - primary_config_dir = legacy_ccache_dir; - } else if (env_xdg_config_home) { - primary_config_dir = FMT("{}/ccache", env_xdg_config_home); - } else { - primary_config_dir = default_config_dir(home_dir); - } - config.set_primary_config_path(primary_config_dir + "/ccache.conf"); - } - - const std::string& cache_dir_before_primary_config = config.cache_dir(); - - MTR_BEGIN("config", "conf_read_primary"); - config.update_from_file(config.primary_config_path()); - MTR_END("config", "conf_read_primary"); - - // Ignore cache_dir set in primary config. - config.set_cache_dir(cache_dir_before_primary_config); - - MTR_BEGIN("config", "conf_update_from_environment"); - config.update_from_environment(); - // (config.cache_dir is set above if CCACHE_DIR is set.) - MTR_END("config", "conf_update_from_environment"); - - if (config.cache_dir().empty()) { - if (legacy_ccache_dir_exists) { - config.set_cache_dir(legacy_ccache_dir); - } else if (env_xdg_cache_home) { - config.set_cache_dir(FMT("{}/ccache", env_xdg_cache_home)); - } else { - config.set_cache_dir(default_cache_dir(home_dir)); - } - } - // else: cache_dir was set explicitly via environment or via secondary config. - - // We have now determined config.cache_dir and populated the rest of config in - // prio order (1. environment, 2. primary config, 3. secondary config). -} - -static void -set_up_context(Context& ctx, int argc, const char* const* argv) -{ - ctx.orig_args = Args::from_argv(argc, argv); - ctx.ignore_header_paths = Util::split_into_strings( - ctx.config.ignore_headers_in_manifest(), PATH_DELIM); - ctx.set_ignore_options( - Util::split_into_strings(ctx.config.ignore_options(), " ")); -} - -// Initialize ccache, must be called once before anything else is run. +// Initialize ccache. Must be called once before anything else is run. static void initialize(Context& ctx, int argc, const char* const* argv) { - set_up_config(ctx.config); - set_up_context(ctx, argc, argv); - Logging::init(ctx.config); - - // Set default umask for all files created by ccache from now on (if - // configured to). This is intentionally done after calling init_log so that - // the log file won't be affected by the umask but before creating the initial - // configuration file. The intention is that all files and directories in the - // cache directory should be affected by the configured umask and that no - // other files and directories should. - if (ctx.config.umask() != std::numeric_limits::max()) { - ctx.original_umask = umask(ctx.config.umask()); - } + ctx.orig_args = Args::from_argv(argc, argv); + ctx.storage.initialize(); LOG("=== CCACHE {} STARTED =========================================", CCACHE_VERSION); + LOG("Configuration file: {}", ctx.config.config_path()); + LOG("System configuration file: {}", ctx.config.system_config_path()); + if (getenv("CCACHE_INTERNAL_TRACE")) { #ifdef MTR_ENABLED ctx.mini_trace = std::make_unique(ctx.args_info); @@ -2091,205 +2154,80 @@ initialize(Context& ctx, int argc, const char* const* argv) // Make a copy of stderr that will not be cached, so things like distcc can // send networking errors to it. -static void +static nonstd::expected set_up_uncached_err() { int uncached_fd = dup(STDERR_FILENO); // The file descriptor is intentionally leaked. if (uncached_fd == -1) { LOG("dup(2) failed: {}", strerror(errno)); - throw Failure(Statistic::internal_error); + return nonstd::make_unexpected(Statistic::internal_error); } Util::setenv("UNCACHED_ERR_FD", FMT("{}", uncached_fd)); -} - -static void -configuration_logger(const std::string& key, - const std::string& value, - const std::string& origin) -{ - BULK_LOG("Config: ({}) {} = {}", origin, key, value); -} - -static void -configuration_printer(const std::string& key, - const std::string& value, - const std::string& origin) -{ - PRINT(stdout, "({}) {} = {}\n", origin, key, value); + return {}; } static int cache_compilation(int argc, const char* const* argv); -static Statistic do_cache_compilation(Context& ctx, const char* const* argv); -static uint8_t -calculate_wanted_cache_level(uint64_t files_in_level_1) -{ - uint64_t files_per_directory = files_in_level_1 / 16; - for (uint8_t i = k_min_cache_levels; i <= k_max_cache_levels; ++i) { - if (files_per_directory < k_max_cache_files_per_directory) { - return i; - } - files_per_directory /= 16; - } - return k_max_cache_levels; -} - -static optional -update_stats_and_maybe_move_cache_file(const Context& ctx, - const Digest& name, - const std::string& current_path, - const Counters& counter_updates, - const std::string& file_suffix) -{ - if (counter_updates.all_zero()) { - return nullopt; - } - - // Use stats file in the level one subdirectory for cache bookkeeping counters - // since cleanup is performed on level one. Use stats file in the level two - // subdirectory for other counters to reduce lock contention. - const bool use_stats_on_level_1 = - counter_updates.get(Statistic::cache_size_kibibyte) != 0 - || counter_updates.get(Statistic::files_in_cache) != 0; - std::string level_string = FMT("{:x}", name.bytes()[0] >> 4); - if (!use_stats_on_level_1) { - level_string += FMT("/{:x}", name.bytes()[0] & 0xF); - } - const auto stats_file = - FMT("{}/{}/stats", ctx.config.cache_dir(), level_string); - - auto counters = - Statistics::update(stats_file, [&counter_updates](Counters& cs) { - cs.increment(counter_updates); - }); - if (!counters) { - return nullopt; - } - - if (use_stats_on_level_1) { - // Only consider moving the cache file to another level when we have read - // the level 1 stats file since it's only then we know the proper - // files_in_cache value. - const auto wanted_level = - calculate_wanted_cache_level(counters->get(Statistic::files_in_cache)); - const auto wanted_path = Util::get_path_in_cache( - ctx.config.cache_dir(), wanted_level, name.to_string() + file_suffix); - if (current_path != wanted_path) { - Util::ensure_dir_exists(Util::dir_name(wanted_path)); - LOG("Moving {} to {}", current_path, wanted_path); - try { - Util::rename(current_path, wanted_path); - } catch (const Error&) { - // Two ccache processes may move the file at the same time, so failure - // to rename is OK. - } - } - } - return counters; -} +static nonstd::expected +do_cache_compilation(Context& ctx, const char* const* argv); static void -finalize_stats_and_trigger_cleanup(Context& ctx) +log_result_to_debug_log(Context& ctx) { - const auto& config = ctx.config; - - if (config.disable()) { - // Just log result, don't update statistics. - LOG_RAW("Result: disabled"); + if (ctx.config.log_file().empty() && !ctx.config.debug()) { return; } - if (!config.log_file().empty() || config.debug()) { - const auto result = Statistics::get_result(ctx.counter_updates); - if (result) { - LOG("Result: {}", *result); - } - } - - if (!config.stats()) { - return; + core::Statistics statistics(ctx.storage.local.get_statistics_updates()); + for (const auto& message : statistics.get_statistics_ids()) { + LOG("Result: {}", message); } +} - if (!ctx.result_path()) { - ASSERT(ctx.counter_updates.get(Statistic::cache_size_kibibyte) == 0); - ASSERT(ctx.counter_updates.get(Statistic::files_in_cache) == 0); - - // Context::set_result_path hasn't been called yet, so we just choose one of - // the stats files in the 256 level 2 directories. - const auto bucket = getpid() % 256; - const auto stats_file = - FMT("{}/{:x}/{:x}/stats", config.cache_dir(), bucket / 16, bucket % 16); - Statistics::update( - stats_file, [&ctx](Counters& cs) { cs.increment(ctx.counter_updates); }); +static void +log_result_to_stats_log(Context& ctx) +{ + if (ctx.config.stats_log().empty()) { return; } - if (ctx.manifest_path()) { - update_stats_and_maybe_move_cache_file(ctx, - *ctx.manifest_name(), - *ctx.manifest_path(), - ctx.manifest_counter_updates, - Manifest::k_file_suffix); - } - - const auto counters = - update_stats_and_maybe_move_cache_file(ctx, - *ctx.result_name(), - *ctx.result_path(), - ctx.counter_updates, - Result::k_file_suffix); - if (!counters) { + core::Statistics statistics(ctx.storage.local.get_statistics_updates()); + const auto ids = statistics.get_statistics_ids(); + if (ids.empty()) { return; } - const auto subdir = - FMT("{}/{:x}", config.cache_dir(), ctx.result_name()->bytes()[0] >> 4); - bool need_cleanup = false; - - if (config.max_files() != 0 - && counters->get(Statistic::files_in_cache) > config.max_files() / 16) { - LOG("Need to clean up {} since it holds {} files (limit: {} files)", - subdir, - counters->get(Statistic::files_in_cache), - config.max_files() / 16); - need_cleanup = true; - } - if (config.max_size() != 0 - && counters->get(Statistic::cache_size_kibibyte) - > config.max_size() / 1024 / 16) { - LOG("Need to clean up {} since it holds {} KiB (limit: {} KiB)", - subdir, - counters->get(Statistic::cache_size_kibibyte), - config.max_size() / 1024 / 16); - need_cleanup = true; - } - - if (need_cleanup) { - const double factor = config.limit_multiple() / 16; - const uint64_t max_size = round(config.max_size() * factor); - const uint32_t max_files = round(config.max_files() * factor); - const time_t max_age = 0; - clean_up_dir( - subdir, max_size, max_files, max_age, [](double /*progress*/) {}); - } + core::StatsLog(ctx.config.stats_log()) + .log_result(ctx.args_info.input_file, ids); } static void finalize_at_exit(Context& ctx) { try { - finalize_stats_and_trigger_cleanup(ctx); - } catch (const ErrorBase& e) { + if (ctx.config.disable()) { + // Just log result, don't update statistics. + LOG_RAW("Result: disabled"); + return; + } + + log_result_to_debug_log(ctx); + log_result_to_stats_log(ctx); + + ctx.storage.finalize(); + } catch (const core::ErrorBase& e) { // finalize_at_exit must not throw since it's called by a destructor. LOG("Error while finalizing stats: {}", e.what()); } // Dump log buffer last to not lose any logs. if (ctx.config.debug() && !ctx.args_info.output_obj.empty()) { - Logging::dump_log(prepare_debug_path( - ctx.config.debug_dir(), ctx.args_info.output_obj, "log")); + Logging::dump_log(prepare_debug_path(ctx.config.debug_dir(), + ctx.time_of_invocation, + ctx.args_info.output_obj, + "log")); } } @@ -2301,11 +2239,12 @@ cache_compilation(int argc, const char* const* argv) bool fall_back_to_original_compiler = false; Args saved_orig_args; - nonstd::optional original_umask; + std::optional original_umask; std::string saved_temp_dir; { Context ctx; + ctx.initialize(); SignalHandler signal_handler(ctx); Finalizer finalizer([&ctx] { finalize_at_exit(ctx); }); @@ -2315,16 +2254,12 @@ cache_compilation(int argc, const char* const* argv) find_compiler(ctx, &find_executable); MTR_END("main", "find_compiler"); - try { - Statistic statistic = do_cache_compilation(ctx, argv); - ctx.counter_updates.increment(statistic); - } catch (const Failure& e) { - if (e.statistic() != Statistic::none) { - ctx.counter_updates.increment(e.statistic()); - } - - if (e.exit_code()) { - return *e.exit_code(); + const auto result = do_cache_compilation(ctx, argv); + const auto& counters = result ? *result : result.error().counters(); + ctx.storage.local.increment_statistics(counters); + if (!result) { + if (result.error().exit_code()) { + return *result.error().exit_code(); } // Else: Fall back to running the real compiler. fall_back_to_original_compiler = true; @@ -2349,37 +2284,40 @@ cache_compilation(int argc, const char* const* argv) if (fall_back_to_original_compiler) { if (original_umask) { - umask(*original_umask); + Util::set_umask(*original_umask); } auto execv_argv = saved_orig_args.to_argv(); execute_noreturn(execv_argv.data(), saved_temp_dir); - throw Fatal( - "execute_noreturn of {} failed: {}", execv_argv[0], strerror(errno)); + throw core::Fatal( + FMT("execute_noreturn of {} failed: {}", execv_argv[0], strerror(errno))); } return EXIT_SUCCESS; } -static Statistic +static nonstd::expected do_cache_compilation(Context& ctx, const char* const* argv) { if (ctx.actual_cwd.empty()) { LOG("Unable to determine current working directory: {}", strerror(errno)); - throw Failure(Statistic::internal_error); + return nonstd::make_unexpected(Statistic::internal_error); } - MTR_BEGIN("main", "clean_up_internal_tempdir"); - if (ctx.config.temporary_dir() == ctx.config.cache_dir() + "/tmp") { - clean_up_internal_tempdir(ctx.config); - } - MTR_END("main", "clean_up_internal_tempdir"); - if (!ctx.config.log_file().empty() || ctx.config.debug()) { - ctx.config.visit_items(configuration_logger); + ctx.config.visit_items([&ctx](const std::string& key, + const std::string& value, + const std::string& origin) { + const auto& log_value = + key == "remote_storage" + ? ctx.storage.get_remote_storage_config_for_logging() + : value; + BULK_LOG("Config: ({}) {} = {}", origin, key, log_value); + }); } // Guess compiler after logging the config value in order to be able to - // display "compiler_type = auto" before overwriting the value with the guess. + // display "compiler_type = auto" before overwriting the value with the + // guess. if (ctx.config.compiler_type() == CompilerType::auto_guess) { ctx.config.set_compiler_type(guess_compiler(ctx.orig_args[0])); } @@ -2387,8 +2325,7 @@ do_cache_compilation(Context& ctx, const char* const* argv) if (ctx.config.disable()) { LOG_RAW("ccache is disabled"); - // Statistic::cache_miss is a dummy to trigger stats_flush. - throw Failure(Statistic::cache_miss); + return nonstd::make_unexpected(Statistic::none); } LOG("Command line: {}", Util::format_argv_for_logging(argv)); @@ -2400,22 +2337,68 @@ do_cache_compilation(Context& ctx, const char* const* argv) LOG("Compiler type: {}", compiler_type_to_string(ctx.config.compiler_type())); + // Set CCACHE_DISABLE so no process ccache executes from now on will risk + // calling ccache second time. For instance, if the real compiler is a wrapper + // script that calls "ccache $compiler ..." we want that inner ccache call to + // be disabled. + Util::setenv("CCACHE_DISABLE", "1"); + MTR_BEGIN("main", "process_args"); ProcessArgsResult processed = process_args(ctx); MTR_END("main", "process_args"); if (processed.error) { - throw Failure(*processed.error); + return nonstd::make_unexpected(*processed.error); + } + + TRY(set_up_uncached_err()); + + // VS_UNICODE_OUTPUT prevents capturing stdout/stderr, as the output is sent + // directly to Visual Studio. + if (ctx.config.compiler_type() == CompilerType::msvc) { + Util::unsetenv("VS_UNICODE_OUTPUT"); + } + + for (const auto& name : {"DEPENDENCIES_OUTPUT", "SUNPRO_DEPENDENCIES"}) { + if (getenv(name)) { + LOG("Unsupported environment variable: {}", name); + return Statistic::unsupported_environment_variable; + } } - set_up_uncached_err(); + if (ctx.config.is_compiler_group_msvc()) { + for (const auto& name : {"CL", "_CL_"}) { + if (getenv(name)) { + LOG("Unsupported environment variable: {}", name); + return Statistic::unsupported_environment_variable; + } + } + } + + if (!ctx.config.run_second_cpp() && ctx.config.is_compiler_group_msvc()) { + LOG_RAW("Second preprocessor cannot be disabled"); + ctx.config.set_run_second_cpp(true); + } + + if (ctx.config.depend_mode()) { + const bool deps = ctx.args_info.generating_dependencies + && ctx.args_info.output_dep != "/dev/null"; + const bool includes = ctx.args_info.generating_includes; + if (!ctx.config.run_second_cpp() || (!deps && !includes)) { + LOG_RAW("Disabling depend mode"); + ctx.config.set_depend_mode(false); + } + } - if (ctx.config.depend_mode() - && (!ctx.args_info.generating_dependencies - || ctx.args_info.output_dep == "/dev/null" - || !ctx.config.run_second_cpp())) { - LOG_RAW("Disabling depend mode"); - ctx.config.set_depend_mode(false); + if (ctx.storage.has_remote_storage()) { + if (ctx.config.file_clone()) { + LOG_RAW("Disabling file clone mode since remote storage is enabled"); + ctx.config.set_file_clone(false); + } + if (ctx.config.hard_link()) { + LOG_RAW("Disabling hard link mode since remote storage is enabled"); + ctx.config.set_hard_link(false); + } } LOG("Source file: {}", ctx.args_info.input_file); @@ -2439,8 +2422,10 @@ do_cache_compilation(Context& ctx, const char* const* argv) MTR_META_THREAD_NAME(ctx.args_info.output_obj.c_str()); if (ctx.config.debug()) { - const auto path = prepare_debug_path( - ctx.config.debug_dir(), ctx.args_info.output_obj, "input-text"); + const auto path = prepare_debug_path(ctx.config.debug_dir(), + ctx.time_of_invocation, + ctx.args_info.orig_output_obj, + "input-text"); File debug_text_file(path, "w"); if (debug_text_file) { ctx.hash_debug_files.push_back(std::move(debug_text_file)); @@ -2456,10 +2441,16 @@ do_cache_compilation(Context& ctx, const char* const* argv) Hash common_hash; init_hash_debug(ctx, common_hash, 'c', "COMMON", debug_text_file); - MTR_BEGIN("hash", "common_hash"); - hash_common_info( - ctx, processed.preprocessor_args, common_hash, ctx.args_info); - MTR_END("hash", "common_hash"); + { + MTR_SCOPE("hash", "common_hash"); + TRY(hash_common_info( + ctx, processed.preprocessor_args, common_hash, ctx.args_info)); + } + + if (processed.hash_actual_cwd) { + common_hash.hash_delimiter("actual_cwd"); + common_hash.hash(ctx.actual_cwd); + } // Try to find the hash using the manifest. Hash direct_hash = common_hash; @@ -2469,38 +2460,49 @@ do_cache_compilation(Context& ctx, const char* const* argv) args_to_hash.push_back(processed.extra_args_to_hash); bool put_result_in_manifest = false; - optional result_name; - optional result_name_from_manifest; + std::optional result_key; + std::optional result_key_from_manifest; + std::optional manifest_key; + if (ctx.config.direct_mode()) { LOG_RAW("Trying direct lookup"); - MTR_BEGIN("hash", "direct_hash"); Args dummy_args; - result_name = - calculate_result_name(ctx, args_to_hash, dummy_args, direct_hash, true); + MTR_BEGIN("hash", "direct_hash"); + const auto result_and_manifest_key = calculate_result_and_manifest_key( + ctx, args_to_hash, dummy_args, direct_hash, true); MTR_END("hash", "direct_hash"); - if (result_name) { - ctx.set_result_name(*result_name); - + if (!result_and_manifest_key) { + return nonstd::make_unexpected(result_and_manifest_key.error()); + } + std::tie(result_key, manifest_key) = *result_and_manifest_key; + if (result_key) { // If we can return from cache at this point then do so. - auto result = from_cache(ctx, FromCacheCallMode::direct); - if (result) { - return *result; + const auto from_cache_result = + from_cache(ctx, FromCacheCallMode::direct, *result_key); + if (!from_cache_result) { + return nonstd::make_unexpected(from_cache_result.error()); + } else if (*from_cache_result) { + return Statistic::direct_cache_hit; } // Wasn't able to return from cache at this point. However, the result // was already found in manifest, so don't re-add it later. put_result_in_manifest = false; - result_name_from_manifest = result_name; + result_key_from_manifest = result_key; } else { // Add result to manifest later. put_result_in_manifest = true; } + + if (!ctx.config.recache()) { + ctx.storage.local.increment_statistic(Statistic::direct_cache_miss); + } } if (ctx.config.read_only_direct()) { LOG_RAW("Read-only direct mode; running real compiler"); - throw Failure(Statistic::cache_miss); + return nonstd::make_unexpected(Statistic::cache_miss); } if (!ctx.config.depend_mode()) { @@ -2510,21 +2512,19 @@ do_cache_compilation(Context& ctx, const char* const* argv) init_hash_debug(ctx, cpp_hash, 'p', "PREPROCESSOR MODE", debug_text_file); MTR_BEGIN("hash", "cpp_hash"); - result_name = calculate_result_name( + const auto result_and_manifest_key = calculate_result_and_manifest_key( ctx, args_to_hash, processed.preprocessor_args, cpp_hash, false); MTR_END("hash", "cpp_hash"); + if (!result_and_manifest_key) { + return nonstd::make_unexpected(result_and_manifest_key.error()); + } + result_key = result_and_manifest_key->first; - // calculate_result_name does not return nullopt if the last (direct_mode) - // argument is false. - ASSERT(result_name); - ctx.set_result_name(*result_name); - - if (result_name_from_manifest && result_name_from_manifest != result_name) { - // manifest_path is guaranteed to be set when calculate_result_name - // returns a non-nullopt result in direct mode, i.e. when - // result_name_from_manifest is set. - ASSERT(ctx.manifest_path()); + // calculate_result_and_manifest_key always returns a non-nullopt result_key + // if the last argument (direct_mode) is false. + ASSERT(result_key); + if (result_key_from_manifest && result_key_from_manifest != result_key) { // The hash from manifest differs from the hash of the preprocessor // output. This could be because: // @@ -2540,24 +2540,32 @@ do_cache_compilation(Context& ctx, const char* const* argv) LOG_RAW("Hash from manifest doesn't match preprocessor output"); LOG_RAW("Likely reason: different CCACHE_BASEDIRs used"); LOG_RAW("Removing manifest as a safety measure"); - Util::unlink_safe(*ctx.manifest_path()); + ctx.storage.remove(*result_key, core::CacheEntryType::result); put_result_in_manifest = true; } // If we can return from cache at this point then do. - auto result = from_cache(ctx, FromCacheCallMode::cpp); - if (result) { - if (put_result_in_manifest) { - update_manifest_file(ctx); + const auto from_cache_result = + from_cache(ctx, FromCacheCallMode::cpp, *result_key); + if (!from_cache_result) { + return nonstd::make_unexpected(from_cache_result.error()); + } else if (*from_cache_result) { + if (ctx.config.direct_mode() && manifest_key && put_result_in_manifest) { + MTR_SCOPE("cache", "update_manifest"); + update_manifest(ctx, *manifest_key, *result_key); } - return *result; + return Statistic::preprocessed_cache_hit; + } + + if (!ctx.config.recache()) { + ctx.storage.local.increment_statistic(Statistic::preprocessed_cache_miss); } } if (ctx.config.read_only()) { LOG_RAW("Read-only mode; running real compiler"); - throw Failure(Statistic::cache_miss); + return nonstd::make_unexpected(Statistic::cache_miss); } add_prefix(ctx, processed.compiler_args, ctx.config.prefix_command()); @@ -2567,281 +2575,43 @@ do_cache_compilation(Context& ctx, const char* const* argv) // Run real compiler, sending output to cache. MTR_BEGIN("cache", "to_cache"); - to_cache(ctx, - processed.compiler_args, - ctx.args_info.depend_extra_args, - depend_mode_hash); - update_manifest_file(ctx); + const auto digest = to_cache(ctx, + processed.compiler_args, + result_key, + ctx.args_info.depend_extra_args, + depend_mode_hash); MTR_END("cache", "to_cache"); - - return Statistic::cache_miss; -} - -// The main program when not doing a compile. -static int -handle_main_options(int argc, const char* const* argv) -{ - enum longopts { - CHECKSUM_FILE, - CONFIG_PATH, - DUMP_MANIFEST, - DUMP_RESULT, - EVICT_OLDER_THAN, - EXTRACT_RESULT, - HASH_FILE, - PRINT_STATS, - }; - static const struct option options[] = { - {"checksum-file", required_argument, nullptr, CHECKSUM_FILE}, - {"cleanup", no_argument, nullptr, 'c'}, - {"clear", no_argument, nullptr, 'C'}, - {"config-path", required_argument, nullptr, CONFIG_PATH}, - {"directory", required_argument, nullptr, 'd'}, - {"dump-manifest", required_argument, nullptr, DUMP_MANIFEST}, - {"dump-result", required_argument, nullptr, DUMP_RESULT}, - {"evict-older-than", required_argument, nullptr, EVICT_OLDER_THAN}, - {"extract-result", required_argument, nullptr, EXTRACT_RESULT}, - {"get-config", required_argument, nullptr, 'k'}, - {"hash-file", required_argument, nullptr, HASH_FILE}, - {"help", no_argument, nullptr, 'h'}, - {"max-files", required_argument, nullptr, 'F'}, - {"max-size", required_argument, nullptr, 'M'}, - {"print-stats", no_argument, nullptr, PRINT_STATS}, - {"recompress", required_argument, nullptr, 'X'}, - {"set-config", required_argument, nullptr, 'o'}, - {"show-compression", no_argument, nullptr, 'x'}, - {"show-config", no_argument, nullptr, 'p'}, - {"show-stats", no_argument, nullptr, 's'}, - {"version", no_argument, nullptr, 'V'}, - {"zero-stats", no_argument, nullptr, 'z'}, - {nullptr, 0, nullptr, 0}}; - - Context ctx; - initialize(ctx, argc, argv); - - int c; - while ((c = getopt_long(argc, - const_cast(argv), - "cCd:k:hF:M:po:sVxX:z", - options, - nullptr)) - != -1) { - std::string arg = optarg ? optarg : std::string(); - - switch (c) { - case CHECKSUM_FILE: { - Checksum checksum; - Fd fd(arg == "-" ? STDIN_FILENO : open(arg.c_str(), O_RDONLY)); - Util::read_fd(*fd, [&checksum](const void* data, size_t size) { - checksum.update(data, size); - }); - PRINT(stdout, "{:016x}\n", checksum.digest()); - break; - } - - case CONFIG_PATH: - Util::setenv("CCACHE_CONFIGPATH", arg); - break; - - case DUMP_MANIFEST: - return Manifest::dump(arg, stdout) ? 0 : 1; - - case DUMP_RESULT: { - ResultDumper result_dumper(stdout); - Result::Reader result_reader(arg); - auto error = result_reader.read(result_dumper); - if (error) { - PRINT(stderr, "Error: {}\n", *error); - } - return error ? EXIT_FAILURE : EXIT_SUCCESS; - } - - case EVICT_OLDER_THAN: { - auto seconds = Util::parse_duration(arg); - ProgressBar progress_bar("Evicting..."); - clean_old( - ctx, [&](double progress) { progress_bar.update(progress); }, seconds); - if (isatty(STDOUT_FILENO)) { - PRINT_RAW(stdout, "\n"); - } - break; - } - - case EXTRACT_RESULT: { - ResultExtractor result_extractor("."); - Result::Reader result_reader(arg); - auto error = result_reader.read(result_extractor); - if (error) { - PRINT(stderr, "Error: {}\n", *error); - } - return error ? EXIT_FAILURE : EXIT_SUCCESS; - } - - case HASH_FILE: { - Hash hash; - if (arg == "-") { - hash.hash_fd(STDIN_FILENO); - } else { - hash.hash_file(arg); - } - PRINT(stdout, "{}\n", hash.digest().to_string()); - break; - } - - case PRINT_STATS: - PRINT_RAW(stdout, Statistics::format_machine_readable(ctx.config)); - break; - - case 'c': // --cleanup - { - ProgressBar progress_bar("Cleaning..."); - clean_up_all(ctx.config, - [&](double progress) { progress_bar.update(progress); }); - if (isatty(STDOUT_FILENO)) { - PRINT_RAW(stdout, "\n"); - } - break; - } - - case 'C': // --clear - { - ProgressBar progress_bar("Clearing..."); - wipe_all(ctx, [&](double progress) { progress_bar.update(progress); }); - if (isatty(STDOUT_FILENO)) { - PRINT_RAW(stdout, "\n"); - } - break; - } - - case 'd': // --directory - Util::setenv("CCACHE_DIR", arg); - break; - - case 'h': // --help - PRINT(stdout, USAGE_TEXT, CCACHE_NAME, CCACHE_NAME); - exit(EXIT_SUCCESS); - - case 'k': // --get-config - PRINT(stdout, "{}\n", ctx.config.get_string_value(arg)); - break; - - case 'F': { // --max-files - auto files = Util::parse_unsigned(arg); - Config::set_value_in_file( - ctx.config.primary_config_path(), "max_files", arg); - if (files == 0) { - PRINT_RAW(stdout, "Unset cache file limit\n"); - } else { - PRINT(stdout, "Set cache file limit to {}\n", files); - } - break; - } - - case 'M': { // --max-size - uint64_t size = Util::parse_size(arg); - Config::set_value_in_file( - ctx.config.primary_config_path(), "max_size", arg); - if (size == 0) { - PRINT_RAW(stdout, "Unset cache size limit\n"); - } else { - PRINT(stdout, - "Set cache size limit to {}\n", - Util::format_human_readable_size(size)); - } - break; - } - - case 'o': { // --set-config - // Start searching for equal sign at position 1 to improve error message - // for the -o=K=V case (key "=K" and value "V"). - size_t eq_pos = arg.find('=', 1); - if (eq_pos == std::string::npos) { - throw Error("missing equal sign in \"{}\"", arg); - } - std::string key = arg.substr(0, eq_pos); - std::string value = arg.substr(eq_pos + 1); - Config::set_value_in_file(ctx.config.primary_config_path(), key, value); - break; - } - - case 'p': // --show-config - ctx.config.visit_items(configuration_printer); - break; - - case 's': // --show-stats - PRINT_RAW(stdout, Statistics::format_human_readable(ctx.config)); - break; - - case 'V': // --version - PRINT(VERSION_TEXT, CCACHE_NAME, CCACHE_VERSION); - exit(EXIT_SUCCESS); - - case 'x': // --show-compression - { - ProgressBar progress_bar("Scanning..."); - compress_stats(ctx.config, - [&](double progress) { progress_bar.update(progress); }); - break; - } - - case 'X': // --recompress - { - optional wanted_level; - if (arg == "uncompressed") { - wanted_level = nullopt; - } else { - wanted_level = - Util::parse_signed(arg, INT8_MIN, INT8_MAX, "compression level"); - } - - ProgressBar progress_bar("Recompressing..."); - compress_recompress(ctx, wanted_level, [&](double progress) { - progress_bar.update(progress); - }); - break; - } - - case 'z': // --zero-stats - Statistics::zero_all_counters(ctx.config); - PRINT_RAW(stdout, "Statistics zeroed\n"); - break; - - default: - PRINT(stderr, USAGE_TEXT, CCACHE_NAME, CCACHE_NAME); - exit(EXIT_FAILURE); - } - - // Some of the above switches might have changed config settings, so run the - // setup again. - ctx.config = Config(); - set_up_config(ctx.config); + if (!digest) { + return nonstd::make_unexpected(digest.error()); + } + result_key = *digest; + if (ctx.config.direct_mode()) { + ASSERT(manifest_key); + MTR_SCOPE("cache", "update_manifest"); + update_manifest(ctx, *manifest_key, *result_key); } - return 0; + return ctx.config.recache() ? Statistic::recache : Statistic::cache_miss; } -int ccache_main(int argc, const char* const* argv); - int ccache_main(int argc, const char* const* argv) { try { - // Check if we are being invoked as "ccache". - std::string program_name(Util::base_name(argv[0])); - if (Util::same_program_name(program_name, CCACHE_NAME)) { + if (Util::is_ccache_executable(argv[0])) { if (argc < 2) { - PRINT(stderr, USAGE_TEXT, CCACHE_NAME, CCACHE_NAME); + PRINT_RAW(stderr, core::get_usage_text(Util::base_name(argv[0]))); exit(EXIT_FAILURE); } - // If the first argument isn't an option, then assume we are being passed - // a compiler name and options. + // If the first argument isn't an option, then assume we are being + // passed a compiler name and options. if (argv[1][0] == '-') { - return handle_main_options(argc, argv); + return core::process_main_options(argc, argv); } } return cache_compilation(argc, argv); - } catch (const ErrorBase& e) { + } catch (const core::ErrorBase& e) { PRINT(stderr, "ccache: error: {}\n", e.what()); return EXIT_FAILURE; } diff --git a/src/ccache.hpp b/src/ccache.hpp index bf34cb0..510563a 100644 --- a/src/ccache.hpp +++ b/src/ccache.hpp @@ -1,5 +1,5 @@ // Copyright (C) 2002-2007 Andrew Tridgell -// Copyright (C) 2009-2020 Joel Rosdahl and other contributors +// Copyright (C) 2009-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -19,14 +19,11 @@ #pragma once -#include "system.hpp" - #include "Config.hpp" -#include "third_party/nonstd/string_view.hpp" - #include #include +#include class Context; @@ -35,9 +32,11 @@ extern const char CCACHE_VERSION[]; using FindExecutableFunction = std::function; + const std::string& exclude_path)>; + +int ccache_main(int argc, const char* const* argv); // Tested by unit tests. void find_compiler(Context& ctx, const FindExecutableFunction& find_executable_function); -CompilerType guess_compiler(nonstd::string_view path); +CompilerType guess_compiler(std::string_view path); diff --git a/src/cleanup.cpp b/src/cleanup.cpp deleted file mode 100644 index 5c76ebb..0000000 --- a/src/cleanup.cpp +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright (C) 2002-2006 Andrew Tridgell -// Copyright (C) 2009-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "cleanup.hpp" - -#include "CacheFile.hpp" -#include "Config.hpp" -#include "Context.hpp" -#include "Logging.hpp" -#include "Statistics.hpp" -#include "Util.hpp" - -#ifdef INODE_CACHE_SUPPORTED -# include "InodeCache.hpp" -#endif - -#include - -static void -delete_file(const std::string& path, - uint64_t size, - uint64_t* cache_size, - uint64_t* files_in_cache) -{ - bool deleted = Util::unlink_safe(path, Util::UnlinkLog::ignore_failure); - if (!deleted && errno != ENOENT && errno != ESTALE) { - LOG("Failed to unlink {} ({})", path, strerror(errno)); - } else if (cache_size && files_in_cache) { - // The counters are intentionally subtracted even if there was no file to - // delete since the final cache size calculation will be incorrect if they - // aren't. (This can happen when there are several parallel ongoing - // cleanups of the same directory.) - *cache_size -= size; - --*files_in_cache; - } -} - -static void -update_counters(const std::string& dir, - uint64_t files_in_cache, - uint64_t cache_size, - bool cleanup_performed) -{ - const std::string stats_file = dir + "/stats"; - Statistics::update(stats_file, [=](Counters& cs) { - if (cleanup_performed) { - cs.increment(Statistic::cleanups_performed); - } - cs.set(Statistic::files_in_cache, files_in_cache); - cs.set(Statistic::cache_size_kibibyte, cache_size / 1024); - }); -} - -void -clean_old(const Context& ctx, - const Util::ProgressReceiver& progress_receiver, - uint64_t max_age) -{ - Util::for_each_level_1_subdir( - ctx.config.cache_dir(), - [&](const std::string& subdir, - const Util::ProgressReceiver& sub_progress_receiver) { - clean_up_dir(subdir, 0, 0, max_age, sub_progress_receiver); - }, - progress_receiver); -} - -// Clean up one cache subdirectory. -void -clean_up_dir(const std::string& subdir, - uint64_t max_size, - uint64_t max_files, - uint64_t max_age, - const Util::ProgressReceiver& progress_receiver) -{ - LOG("Cleaning up cache directory {}", subdir); - - std::vector files = Util::get_level_1_files( - subdir, [&](double progress) { progress_receiver(progress / 3); }); - - uint64_t cache_size = 0; - uint64_t files_in_cache = 0; - time_t current_time = time(nullptr); - - for (size_t i = 0; i < files.size(); - ++i, progress_receiver(1.0 / 3 + 1.0 * i / files.size() / 3)) { - const auto& file = files[i]; - - if (!file.lstat().is_regular()) { - // Not a file or missing file. - continue; - } - - // Delete any tmp files older than 1 hour right away. - if (file.lstat().mtime() + 3600 < current_time - && Util::base_name(file.path()).find(".tmp.") != std::string::npos) { - Util::unlink_tmp(file.path()); - continue; - } - - cache_size += file.lstat().size_on_disk(); - files_in_cache += 1; - } - - // Sort according to modification time, oldest first. - std::sort( - files.begin(), files.end(), [](const CacheFile& f1, const CacheFile& f2) { - return f1.lstat().mtime() < f2.lstat().mtime(); - }); - - LOG("Before cleanup: {:.0f} KiB, {:.0f} files", - static_cast(cache_size) / 1024, - static_cast(files_in_cache)); - - bool cleaned = false; - for (size_t i = 0; i < files.size(); - ++i, progress_receiver(2.0 / 3 + 1.0 * i / files.size() / 3)) { - const auto& file = files[i]; - - if (!file.lstat() || file.lstat().is_directory()) { - continue; - } - - if ((max_size == 0 || cache_size <= max_size) - && (max_files == 0 || files_in_cache <= max_files) - && (max_age == 0 - || file.lstat().mtime() - > (current_time - static_cast(max_age)))) { - break; - } - - if (Util::ends_with(file.path(), ".stderr")) { - // In order to be nice to legacy ccache versions, make sure that the .o - // file is deleted before .stderr, because if the ccache process gets - // killed after deleting the .stderr but before deleting the .o, the - // cached result will be inconsistent. (.stderr is the only file that is - // optional for legacy ccache versions; any other file missing from the - // cache will be detected.) - std::string o_file = file.path().substr(0, file.path().size() - 6) + "o"; - - // Don't subtract this extra deletion from the cache size; that - // bookkeeping will be done when the loop reaches the .o file. If the - // loop doesn't reach the .o file since the target limits have been - // reached, the bookkeeping won't happen, but that small counter - // discrepancy won't do much harm and it will correct itself in the next - // cleanup. - delete_file(o_file, 0, nullptr, nullptr); - } - - delete_file( - file.path(), file.lstat().size_on_disk(), &cache_size, &files_in_cache); - cleaned = true; - } - - LOG("After cleanup: {:.0f} KiB, {:.0f} files", - static_cast(cache_size) / 1024, - static_cast(files_in_cache)); - - if (cleaned) { - LOG("Cleaned up cache directory {}", subdir); - } - - update_counters(subdir, files_in_cache, cache_size, cleaned); -} - -// Clean up all cache subdirectories. -void -clean_up_all(const Config& config, - const Util::ProgressReceiver& progress_receiver) -{ - Util::for_each_level_1_subdir( - config.cache_dir(), - [&](const std::string& subdir, - const Util::ProgressReceiver& sub_progress_receiver) { - clean_up_dir(subdir, - config.max_size() / 16, - config.max_files() / 16, - 0, - sub_progress_receiver); - }, - progress_receiver); -} - -// Wipe one cache subdirectory. -static void -wipe_dir(const std::string& subdir, - const Util::ProgressReceiver& progress_receiver) -{ - LOG("Clearing out cache directory {}", subdir); - - const std::vector files = Util::get_level_1_files( - subdir, [&](double progress) { progress_receiver(progress / 2); }); - - for (size_t i = 0; i < files.size(); ++i) { - Util::unlink_safe(files[i].path()); - progress_receiver(0.5 + 0.5 * i / files.size()); - } - - const bool cleared = !files.empty(); - if (cleared) { - LOG("Cleared out cache directory {}", subdir); - } - update_counters(subdir, 0, 0, cleared); -} - -// Wipe all cached files in all subdirectories. -void -wipe_all(const Context& ctx, const Util::ProgressReceiver& progress_receiver) -{ - Util::for_each_level_1_subdir( - ctx.config.cache_dir(), wipe_dir, progress_receiver); -#ifdef INODE_CACHE_SUPPORTED - ctx.inode_cache.drop(); -#endif -} diff --git a/src/cleanup.hpp b/src/cleanup.hpp deleted file mode 100644 index 053eb53..0000000 --- a/src/cleanup.hpp +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Util.hpp" - -#include - -class Config; -class Context; - -void clean_old(const Context& ctx, - const Util::ProgressReceiver& progress_receiver, - uint64_t max_age); - -void clean_up_dir(const std::string& subdir, - uint64_t max_size, - uint64_t max_files, - uint64_t max_age, - const Util::ProgressReceiver& progress_receiver); - -void clean_up_all(const Config& config, - const Util::ProgressReceiver& progress_receiver); - -void wipe_all(const Context& ctx, - const Util::ProgressReceiver& progress_receiver); diff --git a/src/compopt.cpp b/src/compopt.cpp index 3ed4f5b..867d8c4 100644 --- a/src/compopt.cpp +++ b/src/compopt.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2010-2021 Joel Rosdahl and other contributors +// Copyright (C) 2010-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -45,6 +45,8 @@ // The option only affects compilation; not passed to the preprocessor. #define AFFECTS_COMP (1 << 6) +#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0])) + struct CompOpt { const char* name; @@ -52,11 +54,13 @@ struct CompOpt }; const CompOpt compopts[] = { - {"--Werror", TAKES_ARG}, // nvcc - {"--analyze", TOO_HARD}, // Clang - {"--compiler-bindir", AFFECTS_CPP | TAKES_ARG}, // nvcc - {"--libdevice-directory", AFFECTS_CPP | TAKES_ARG}, // nvcc - {"--output-directory", AFFECTS_CPP | TAKES_ARG}, // nvcc + {"--Werror", TAKES_ARG}, // nvcc + {"--analyze", TOO_HARD}, // Clang + {"--compiler-bindir", AFFECTS_CPP | TAKES_ARG}, // nvcc + {"--config", TAKES_ARG}, // Clang + {"--gcc-toolchain=", TAKES_CONCAT_ARG | TAKES_PATH}, // Clang + {"--libdevice-directory", AFFECTS_CPP | TAKES_ARG}, // nvcc + {"--output-directory", AFFECTS_CPP | TAKES_ARG}, // nvcc {"--param", TAKES_ARG}, {"--save-temps", TOO_HARD}, {"--save-temps=cwd", TOO_HARD}, @@ -64,10 +68,15 @@ const CompOpt compopts[] = { {"--serialize-diagnostics", TAKES_ARG | TAKES_PATH}, {"--specs", TAKES_ARG}, {"-A", TAKES_ARG}, + {"-AI", TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, // msvc {"-B", TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-D", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG}, {"-E", TOO_HARD}, + {"-EP", TOO_HARD}, // msvc {"-F", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, + {"-FI", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, // msvc + {"-FU", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, // msvc + {"-Fp", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, // msvc {"-G", TAKES_ARG}, {"-I", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-L", TAKES_ARG}, @@ -87,6 +96,10 @@ const CompOpt compopts[] = { {"-Xclang", TAKES_ARG}, {"-Xlinker", TAKES_ARG | TAKES_CONCAT_ARG | AFFECTS_COMP}, {"-Xpreprocessor", AFFECTS_CPP | TOO_HARD_DIRECT | TAKES_ARG}, + {"-Yc", TAKES_ARG | TOO_HARD}, // msvc + {"-Yu", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, // msvc + {"-ZI", TOO_HARD}, // msvc + {"-Zi", TOO_HARD}, // msvc {"-all_load", AFFECTS_COMP}, {"-analyze", TOO_HARD}, // Clang {"-arch", TAKES_ARG}, @@ -97,16 +110,20 @@ const CompOpt compopts[] = { {"-ccbin", AFFECTS_CPP | TAKES_ARG}, // nvcc {"-emit-pch", AFFECTS_COMP}, // Clang {"-emit-pth", AFFECTS_COMP}, // Clang + {"-external:I", + AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, // msvc {"-fno-working-directory", AFFECTS_CPP}, {"-fplugin=libcc1plugin", TOO_HARD}, // interaction with GDB {"-frepo", TOO_HARD}, {"-ftime-trace", TOO_HARD}, // Clang {"-fworking-directory", AFFECTS_CPP}, + {"-gcc-toolchain", TAKES_ARG | TAKES_PATH}, // Clang {"-gen-cdb-fragment-path", TAKES_ARG | TOO_HARD}, // Clang {"-gtoggle", TOO_HARD}, {"-idirafter", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-iframework", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-imacros", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, + {"-imsvc", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-imultilib", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-include", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-include-pch", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, @@ -121,6 +138,7 @@ const CompOpt compopts[] = { {"-iwithprefixbefore", AFFECTS_CPP | TAKES_ARG | TAKES_CONCAT_ARG | TAKES_PATH}, {"-ldir", AFFECTS_CPP | TAKES_ARG}, // nvcc + {"-link", TOO_HARD}, // msvc {"-nolibc", AFFECTS_COMP}, {"-nostdinc", AFFECTS_CPP}, {"-nostdinc++", AFFECTS_CPP}, @@ -137,6 +155,7 @@ const CompOpt compopts[] = { {"-stdlib=", AFFECTS_CPP | TAKES_CONCAT_ARG}, {"-trigraphs", AFFECTS_CPP}, {"-u", TAKES_ARG | TAKES_CONCAT_ARG}, + {"-z", TAKES_ARG | TAKES_CONCAT_ARG | AFFECTS_COMP}, }; static int diff --git a/src/compopt.hpp b/src/compopt.hpp index a016928..29a3354 100644 --- a/src/compopt.hpp +++ b/src/compopt.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2010-2020 Joel Rosdahl and other contributors +// Copyright (C) 2010-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include bool compopt_short(bool (*fn)(const std::string& option), diff --git a/src/compress.cpp b/src/compress.cpp deleted file mode 100644 index 1164b79..0000000 --- a/src/compress.cpp +++ /dev/null @@ -1,372 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#include "compress.hpp" - -#include "AtomicFile.hpp" -#include "CacheEntryReader.hpp" -#include "CacheEntryWriter.hpp" -#include "Context.hpp" -#include "File.hpp" -#include "Logging.hpp" -#include "Manifest.hpp" -#include "Result.hpp" -#include "Statistics.hpp" -#include "StdMakeUnique.hpp" -#include "ThreadPool.hpp" -#include "ZstdCompressor.hpp" -#include "assertions.hpp" -#include "fmtmacros.hpp" - -#include "third_party/fmt/core.h" - -#include -#include - -using nonstd::optional; - -namespace { - -class RecompressionStatistics -{ -public: - void update(uint64_t content_size, - uint64_t old_size, - uint64_t new_size, - uint64_t incompressible_size); - uint64_t content_size() const; - uint64_t old_size() const; - uint64_t new_size() const; - uint64_t incompressible_size() const; - -private: - mutable std::mutex m_mutex; - uint64_t m_content_size = 0; - uint64_t m_old_size = 0; - uint64_t m_new_size = 0; - uint64_t m_incompressible_size = 0; -}; - -void -RecompressionStatistics::update(uint64_t content_size, - uint64_t old_size, - uint64_t new_size, - uint64_t incompressible_size) -{ - std::unique_lock lock(m_mutex); - m_incompressible_size += incompressible_size; - m_content_size += content_size; - m_old_size += old_size; - m_new_size += new_size; -} - -uint64_t -RecompressionStatistics::content_size() const -{ - std::unique_lock lock(m_mutex); - return m_content_size; -} - -uint64_t -RecompressionStatistics::old_size() const -{ - std::unique_lock lock(m_mutex); - return m_old_size; -} - -uint64_t -RecompressionStatistics::new_size() const -{ - std::unique_lock lock(m_mutex); - return m_new_size; -} - -uint64_t -RecompressionStatistics::incompressible_size() const -{ - std::unique_lock lock(m_mutex); - return m_incompressible_size; -} - -File -open_file(const std::string& path, const char* mode) -{ - File f(path, mode); - if (!f) { - throw Error("failed to open {} for reading: {}", path, strerror(errno)); - } - return f; -} - -std::unique_ptr -create_reader(const CacheFile& cache_file, FILE* stream) -{ - if (cache_file.type() == CacheFile::Type::unknown) { - throw Error("unknown file type for {}", cache_file.path()); - } - - switch (cache_file.type()) { - case CacheFile::Type::result: - return std::make_unique( - stream, Result::k_magic, Result::k_version); - - case CacheFile::Type::manifest: - return std::make_unique( - stream, Manifest::k_magic, Manifest::k_version); - - case CacheFile::Type::unknown: - ASSERT(false); // Handled at function entry. - } - - ASSERT(false); -} - -std::unique_ptr -create_writer(FILE* stream, - const CacheEntryReader& reader, - Compression::Type compression_type, - int8_t compression_level) -{ - return std::make_unique(stream, - reader.magic(), - reader.version(), - compression_type, - compression_level, - reader.payload_size()); -} - -void -recompress_file(RecompressionStatistics& statistics, - const std::string& stats_file, - const CacheFile& cache_file, - optional level) -{ - auto file = open_file(cache_file.path(), "rb"); - auto reader = create_reader(cache_file, file.get()); - - auto old_stat = Stat::stat(cache_file.path(), Stat::OnError::log); - uint64_t content_size = reader->content_size(); - int8_t wanted_level = - level ? (*level == 0 ? ZstdCompressor::default_compression_level : *level) - : 0; - - if (reader->compression_level() == wanted_level) { - statistics.update(content_size, old_stat.size(), old_stat.size(), 0); - return; - } - - LOG("Recompressing {} to {}", - cache_file.path(), - level ? FMT("level {}", wanted_level) : "uncompressed"); - AtomicFile atomic_new_file(cache_file.path(), AtomicFile::Mode::binary); - auto writer = - create_writer(atomic_new_file.stream(), - *reader, - level ? Compression::Type::zstd : Compression::Type::none, - wanted_level); - - char buffer[READ_BUFFER_SIZE]; - size_t bytes_left = reader->payload_size(); - while (bytes_left > 0) { - size_t bytes_to_read = std::min(bytes_left, sizeof(buffer)); - reader->read(buffer, bytes_to_read); - writer->write(buffer, bytes_to_read); - bytes_left -= bytes_to_read; - } - reader->finalize(); - writer->finalize(); - - file.close(); - - atomic_new_file.commit(); - auto new_stat = Stat::stat(cache_file.path(), Stat::OnError::log); - - Statistics::update(stats_file, [=](Counters& cs) { - cs.increment(Statistic::cache_size_kibibyte, - Util::size_change_kibibyte(old_stat, new_stat)); - }); - - statistics.update(content_size, old_stat.size(), new_stat.size(), 0); - - LOG("Recompression of {} done", cache_file.path()); -} - -} // namespace - -void -compress_stats(const Config& config, - const Util::ProgressReceiver& progress_receiver) -{ - uint64_t on_disk_size = 0; - uint64_t compr_size = 0; - uint64_t content_size = 0; - uint64_t incompr_size = 0; - - Util::for_each_level_1_subdir( - config.cache_dir(), - [&](const std::string& subdir, - const Util::ProgressReceiver& sub_progress_receiver) { - const std::vector files = Util::get_level_1_files( - subdir, [&](double progress) { sub_progress_receiver(progress / 2); }); - - for (size_t i = 0; i < files.size(); ++i) { - const auto& cache_file = files[i]; - on_disk_size += cache_file.lstat().size_on_disk(); - - try { - auto file = open_file(cache_file.path(), "rb"); - auto reader = create_reader(cache_file, file.get()); - compr_size += cache_file.lstat().size(); - content_size += reader->content_size(); - } catch (Error&) { - incompr_size += cache_file.lstat().size(); - } - - sub_progress_receiver(1.0 / 2 + 1.0 * i / files.size() / 2); - } - }, - progress_receiver); - - if (isatty(STDOUT_FILENO)) { - PRINT_RAW(stdout, "\n\n"); - } - - double ratio = - compr_size > 0 ? static_cast(content_size) / compr_size : 0.0; - double savings = ratio > 0.0 ? 100.0 - (100.0 / ratio) : 0.0; - - std::string on_disk_size_str = Util::format_human_readable_size(on_disk_size); - std::string cache_size_str = - Util::format_human_readable_size(compr_size + incompr_size); - std::string compr_size_str = Util::format_human_readable_size(compr_size); - std::string content_size_str = Util::format_human_readable_size(content_size); - std::string incompr_size_str = Util::format_human_readable_size(incompr_size); - - PRINT(stdout, - "Total data: {:>8s} ({} disk blocks)\n", - cache_size_str, - on_disk_size_str); - PRINT(stdout, - "Compressed data: {:>8s} ({:.1f}% of original size)\n", - compr_size_str, - 100.0 - savings); - PRINT(stdout, " - Original data: {:>8s}\n", content_size_str); - PRINT(stdout, - " - Compression ratio: {:>5.3f} x ({:.1f}% space savings)\n", - ratio, - savings); - PRINT(stdout, "Incompressible data: {:>8s}\n", incompr_size_str); -} - -void -compress_recompress(Context& ctx, - optional level, - const Util::ProgressReceiver& progress_receiver) -{ - const size_t threads = std::thread::hardware_concurrency(); - const size_t read_ahead = 2 * threads; - ThreadPool thread_pool(threads, read_ahead); - RecompressionStatistics statistics; - - Util::for_each_level_1_subdir( - ctx.config.cache_dir(), - [&](const std::string& subdir, - const Util::ProgressReceiver& sub_progress_receiver) { - std::vector files = - Util::get_level_1_files(subdir, [&](double progress) { - sub_progress_receiver(0.1 * progress); - }); - - auto stats_file = subdir + "/stats"; - - for (size_t i = 0; i < files.size(); ++i) { - const auto& file = files[i]; - - if (file.type() != CacheFile::Type::unknown) { - thread_pool.enqueue([&statistics, stats_file, file, level] { - try { - recompress_file(statistics, stats_file, file, level); - } catch (Error&) { - // Ignore for now. - } - }); - } else { - statistics.update(0, 0, 0, file.lstat().size()); - } - - sub_progress_receiver(0.1 + 0.9 * i / files.size()); - } - - if (Util::ends_with(subdir, "f")) { - // Wait here instead of after Util::for_each_level_1_subdir to avoid - // updating the progress bar to 100% before all work is done. - thread_pool.shut_down(); - } - }, - progress_receiver); - - if (isatty(STDOUT_FILENO)) { - PRINT_RAW(stdout, "\n\n"); - } - - double old_ratio = - statistics.old_size() > 0 - ? static_cast(statistics.content_size()) / statistics.old_size() - : 0.0; - double old_savings = old_ratio > 0.0 ? 100.0 - (100.0 / old_ratio) : 0.0; - double new_ratio = - statistics.new_size() > 0 - ? static_cast(statistics.content_size()) / statistics.new_size() - : 0.0; - double new_savings = new_ratio > 0.0 ? 100.0 - (100.0 / new_ratio) : 0.0; - int64_t size_difference = static_cast(statistics.new_size()) - - static_cast(statistics.old_size()); - - std::string old_compr_size_str = - Util::format_human_readable_size(statistics.old_size()); - std::string new_compr_size_str = - Util::format_human_readable_size(statistics.new_size()); - std::string content_size_str = - Util::format_human_readable_size(statistics.content_size()); - std::string incompr_size_str = - Util::format_human_readable_size(statistics.incompressible_size()); - std::string size_difference_str = - FMT("{}{}", - size_difference < 0 ? "-" : (size_difference > 0 ? "+" : " "), - Util::format_human_readable_size( - size_difference < 0 ? -size_difference : size_difference)); - - PRINT(stdout, "Original data: {:>8s}\n", content_size_str); - PRINT(stdout, - "Old compressed data: {:>8s} ({:.1f}% of original size)\n", - old_compr_size_str, - 100.0 - old_savings); - PRINT(stdout, - " - Compression ratio: {:>5.3f} x ({:.1f}% space savings)\n", - old_ratio, - old_savings); - PRINT(stdout, - "New compressed data: {:>8s} ({:.1f}% of original size)\n", - new_compr_size_str, - 100.0 - new_savings); - PRINT(stdout, - " - Compression ratio: {:>5.3f} x ({:.1f}% space savings)\n", - new_ratio, - new_savings); - PRINT(stdout, "Size change: {:>9s}\n", size_difference_str); -} diff --git a/src/compress.hpp b/src/compress.hpp deleted file mode 100644 index 91eb04d..0000000 --- a/src/compress.hpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "Util.hpp" - -#include "third_party/nonstd/optional.hpp" - -class Config; -class Context; - -void compress_stats(const Config& config, - const Util::ProgressReceiver& progress_receiver); - -// Recompress the cache. -// -// Arguments: -// - ctx: The context. -// - level: Target compression level (positive or negative value for actual -// level, 0 for default level and nonstd::nullopt for no compression). -// - progress_receiver: Function that will be called for progress updates. -void compress_recompress(Context& ctx, - nonstd::optional level, - const Util::ProgressReceiver& progress_receiver); diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt new file mode 100644 index 0000000..0a448b9 --- /dev/null +++ b/src/core/CMakeLists.txt @@ -0,0 +1,20 @@ +set( + sources + CacheEntry.cpp + Manifest.cpp + MsvcShowIncludesOutput.cpp + Result.cpp + ResultExtractor.cpp + ResultInspector.cpp + ResultRetriever.cpp + Statistics.cpp + StatisticsCounters.cpp + StatsLog.cpp + mainoptions.cpp + types.cpp +) + +file(GLOB headers *.hpp) +list(APPEND sources ${headers}) + +target_sources(ccache_framework PRIVATE ${sources}) diff --git a/src/core/CacheEntry.cpp b/src/core/CacheEntry.cpp new file mode 100644 index 0000000..2f4730e --- /dev/null +++ b/src/core/CacheEntry.cpp @@ -0,0 +1,337 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "CacheEntry.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace { + +const size_t k_static_header_fields_size = + sizeof(core::CacheEntry::Header::magic) + + sizeof(core::CacheEntry::Header::entry_format_version) + + sizeof(core::CacheEntry::Header::entry_type) + + sizeof(core::CacheEntry::Header::compression_type) + + sizeof(core::CacheEntry::Header::compression_level) + + sizeof(core::CacheEntry::Header::self_contained) + + sizeof(core::CacheEntry::Header::creation_time) + + sizeof(core::CacheEntry::Header::entry_size) + // ccache_version length field: + + 1 + // namespace_ length field: + + 1; + +const size_t k_epilogue_fields_size = sizeof(uint64_t) + sizeof(uint64_t); + +core::CacheEntryType +cache_entry_type_from_int(const uint8_t entry_type) +{ + switch (entry_type) { + case 0: + return core::CacheEntryType::result; + break; + case 1: + return core::CacheEntryType::manifest; + break; + default: + throw core::Error(FMT("Unknown entry type: {}", entry_type)); + } +} + +} // namespace + +namespace core { + +// Version 0: +// - First version. +// Version 1: +// - Added self_contained field. +// - The checksum is now for the (potentially) compressed payload instead of +// the uncompressed payload, and the checksum is now always stored +// uncompressed. +const uint8_t CacheEntry::k_format_version = 1; + +CacheEntry::Header::Header(const Config& config, + core::CacheEntryType entry_type) + : magic(k_ccache_magic), + entry_format_version(k_format_version), + entry_type(entry_type), + compression_type(compression_type_from_config(config)), + compression_level(compression_level_from_config(config)), + self_contained(entry_type != CacheEntryType::result + || !core::Result::Serializer::use_raw_files(config)), + creation_time(util::TimePoint::now().sec()), + ccache_version(CCACHE_VERSION), + namespace_(config.namespace_()), + entry_size(0) +{ + if (compression_level == 0) { + compression_level = default_compression_level; + LOG("Using default compression level {}", compression_level); + } +} + +CacheEntry::Header::Header(nonstd::span data) +{ + parse(data); +} + +CacheEntry::Header::Header(const std::string& path) +{ + const auto data = util::read_file_part(path, 0, 1000); + if (!data) { + throw core::Error(data.error()); + } + parse(*data); +} + +std::string +CacheEntry::Header::inspect() const +{ + std::string result; + result += FMT("Magic: {:04x}\n", magic); + result += FMT("Entry format version: {}\n", entry_format_version); + result += FMT("Entry type: {} ({})\n", + static_cast(entry_type), + to_string(entry_type)); + result += FMT("Compression type: {}\n", to_string(compression_type)); + result += FMT("Compression level: {}\n", compression_level); + result += FMT("Self-contained: {}\n", self_contained ? "yes" : "no"); + result += FMT("Creation time: {}\n", creation_time); + result += FMT("Ccache version: {}\n", ccache_version); + result += FMT("Namespace: {}\n", namespace_); + result += FMT("Entry size: {}\n", entry_size); + return result; +} + +void +CacheEntry::Header::parse(nonstd::span data) +{ + CacheEntryDataReader reader(data); + reader.read_int(magic); + if (magic != core::k_ccache_magic) { + throw core::Error(FMT("Bad magic value: 0x{:04x}", magic)); + } + + reader.read_int(entry_format_version); + if (entry_format_version != k_format_version) { + throw core::Error( + FMT("Unknown entry format version: {}", entry_format_version)); + } + + entry_type = cache_entry_type_from_int(reader.read_int()); + compression_type = compression_type_from_int(reader.read_int()); + reader.read_int(compression_level); + self_contained = bool(reader.read_int()); + reader.read_int(creation_time); + ccache_version = reader.read_str(reader.read_int()); + namespace_ = reader.read_str(reader.read_int()); + reader.read_int(entry_size); +} + +size_t +CacheEntry::Header::serialized_size() const +{ + return k_static_header_fields_size + ccache_version.length() + + namespace_.length(); +} + +void +CacheEntry::Header::serialize(util::Bytes& output) const +{ + core::CacheEntryDataWriter writer(output); + writer.write_int(magic); + writer.write_int(entry_format_version); + writer.write_int(static_cast(entry_type)); + writer.write_int(static_cast(compression_type)); + writer.write_int(compression_level); + writer.write_int(self_contained); + writer.write_int(creation_time); + writer.write_int(ccache_version.length()); + writer.write_str(ccache_version); + writer.write_int(namespace_.length()); + writer.write_str(namespace_); + writer.write_int(entry_size); +} + +uint32_t +CacheEntry::Header::uncompressed_payload_size() const +{ + return entry_size - serialized_size() - k_epilogue_fields_size; +} + +CacheEntry::CacheEntry(nonstd::span data) : m_header(data) +{ + const size_t non_payload_size = + m_header.serialized_size() + k_epilogue_fields_size; + if (data.size() <= non_payload_size) { + throw core::Error("CacheEntry data underflow"); + } + m_payload = + data.subspan(m_header.serialized_size(), data.size() - non_payload_size); + m_checksum = data.last(k_epilogue_fields_size); + + switch (m_header.compression_type) { + case CompressionType::none: + break; + + case CompressionType::zstd: + m_uncompressed_payload.reserve(m_header.uncompressed_payload_size()); + util::throw_on_error( + util::zstd_decompress( + m_payload, m_uncompressed_payload, m_uncompressed_payload.capacity()), + "Cache entry payload decompression error: "); + + break; + } +} + +void +CacheEntry::verify_checksum() const +{ + util::Bytes header_data; + m_header.serialize(header_data); + + util::XXH3_128 checksum; + checksum.update(header_data); + checksum.update(m_payload); + const auto actual = checksum.digest(); + + if (actual != m_checksum) { + throw core::Error( + FMT("Incorrect checksum (actual {}, expected {})", + Util::format_base16(actual.data(), actual.size()), + Util::format_base16(m_checksum.data(), m_checksum.size()))); + } +} + +const CacheEntry::Header& +CacheEntry::header() const +{ + return m_header; +} + +nonstd::span +CacheEntry::payload() const +{ + return m_header.compression_type == CompressionType::none + ? m_payload + : nonstd::span(m_uncompressed_payload); +} + +util::Bytes +CacheEntry::serialize(const CacheEntry::Header& header, + Serializer& payload_serializer) +{ + return do_serialize( + header, + payload_serializer.serialized_size(), + [&payload_serializer](util::Bytes& result, const CacheEntry::Header& hdr) { + switch (hdr.compression_type) { + case CompressionType::none: + payload_serializer.serialize(result); + break; + + case CompressionType::zstd: + util::Bytes payload; + payload_serializer.serialize(payload); + util::throw_on_error( + util::zstd_compress(payload, result, hdr.compression_level), + "Cache entry payload compression error: "); + break; + } + }); +} + +util::Bytes +CacheEntry::serialize(const CacheEntry::Header& header, + nonstd::span payload) +{ + return do_serialize( + header, + payload.size(), + [&payload](util::Bytes& result, const CacheEntry::Header& hdr) { + switch (hdr.compression_type) { + case CompressionType::none: + result.insert(result.end(), payload.begin(), payload.end()); + break; + + case CompressionType::zstd: + util::throw_on_error( + util::zstd_compress(payload, result, hdr.compression_level), + "Cache entry payload compression error: "); + break; + } + }); +} + +util::Bytes +CacheEntry::do_serialize( + const CacheEntry::Header& header, + size_t serialized_payload_size, + std::function serialize_payload) +{ + CacheEntry::Header hdr(header); + const size_t non_payload_size = + hdr.serialized_size() + k_epilogue_fields_size; + hdr.entry_size = non_payload_size + serialized_payload_size; + + if (hdr.compression_type == CompressionType::zstd) { + const auto [level, explanation] = + util::zstd_supported_compression_level(hdr.compression_level); + if (!explanation.empty()) { + LOG("Using ZSTD compression level {} ({}) instead of {}", + level, + explanation, + hdr.compression_level); + } + hdr.compression_level = level; + } + + const size_t max_serialized_size = + hdr.compression_type == CompressionType::zstd + ? (non_payload_size + util::zstd_compress_bound(serialized_payload_size)) + : hdr.entry_size; + util::Bytes result; + result.reserve(max_serialized_size); + + hdr.serialize(result); + serialize_payload(result, hdr); + + util::XXH3_128 checksum; + checksum.update(result); + const auto digest = checksum.digest(); + result.insert(result.end(), digest.begin(), digest.end()); + + return result; +} + +} // namespace core diff --git a/src/core/CacheEntry.hpp b/src/core/CacheEntry.hpp new file mode 100644 index 0000000..f3c958a --- /dev/null +++ b/src/core/CacheEntry.hpp @@ -0,0 +1,127 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include +#include + +// Cache entry format +// ================== +// +// Integers are big-endian. +// +// ::=
+//
::= +// +// +// ::= uint16_t (0xccac) +// ::= uint8_t +// ::= | +// ::= 0 (uint8_t) +// ::= 1 (uint8_t) +// ::= 0/1 (uint8_t) ; whether suitable for remote storage +// ::= | +// ::= 0 (uint8_t) +// ::= 1 (uint8_t) +// ::= int8_t +// ::= uint64_t (Unix epoch time when entry was created) +// ::= string length (uint8_t) + string data +// ::= string length (uint8_t) + string data +// ::= uint64_t ; = size of entry in uncompressed form +// ::= depends on entry_type; potentially compressed +// ::= +// ::= uint64_t ; XXH3-128 (high bits) of
+ +// ::= uint64_t ; XXH3-128 (low bits) of
+ + +class Config; + +namespace core { + +const uint16_t k_ccache_magic = 0xccac; + +class CacheEntry +{ +public: + static const uint8_t k_format_version; + constexpr static uint8_t default_compression_level = 1; + + class Header + { + public: + Header(const Config& config, CacheEntryType entry_type); + explicit Header(nonstd::span data); + explicit Header(const std::string& path); + + std::string inspect() const; + + uint16_t magic; + uint8_t entry_format_version; + CacheEntryType entry_type; + CompressionType compression_type; + int8_t compression_level; + bool self_contained; + uint64_t creation_time; + std::string ccache_version; + std::string namespace_; + uint64_t entry_size; + + size_t serialized_size() const; + void serialize(util::Bytes& output) const; + uint32_t uncompressed_payload_size() const; + + private: + void parse(nonstd::span data); + }; + + explicit CacheEntry(nonstd::span data); + + void verify_checksum() const; + const Header& header() const; + + // Return uncompressed payload. + nonstd::span payload() const; + + static util::Bytes serialize(const Header& header, + Serializer& payload_serializer); + static util::Bytes serialize(const Header& header, + nonstd::span payload); + +private: + Header m_header; + nonstd::span m_payload; // Potentially compressed + util::Bytes m_checksum; + + mutable util::Bytes m_uncompressed_payload; + + static util::Bytes + do_serialize(const Header& header, + size_t serialized_payload_size, + std::function + serialize_payload); +}; + +} // namespace core diff --git a/src/core/CacheEntryDataReader.hpp b/src/core/CacheEntryDataReader.hpp new file mode 100644 index 0000000..b889a00 --- /dev/null +++ b/src/core/CacheEntryDataReader.hpp @@ -0,0 +1,106 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include + +namespace core { + +class CacheEntryDataReader +{ +public: + CacheEntryDataReader(nonstd::span data); + + // Read `size` bytes. Throws `core::Error` on failure. + nonstd::span read_bytes(size_t size); + + // Read and copy `buffer.size()` bytes into `buffer`. Throws `core::Error` on + // failure. + void read_and_copy_bytes(nonstd::span buffer); + + // Read a string of length `length`. Throws `core::Error` on failure. + std::string_view read_str(size_t length); + + // Read an integer. Throws `core::Error` on failure. + template T read_int(); + + // Read an integer into `value`. Throws `core::Error` on failure. + template void read_int(T& value); + +private: + nonstd::span m_data; +}; + +inline CacheEntryDataReader::CacheEntryDataReader( + nonstd::span data) + : m_data(data) +{ +} + +inline nonstd::span +CacheEntryDataReader::read_bytes(size_t size) +{ + if (size > m_data.size()) { + throw core::Error(FMT("CacheEntryDataReader: data underflow of {} bytes", + size - m_data.size())); + } + const auto bytes = m_data.first(size); + m_data = m_data.subspan(size); + return bytes; +} + +inline void +CacheEntryDataReader::read_and_copy_bytes(nonstd::span buffer) +{ + const auto span = read_bytes(buffer.size()); + memcpy(buffer.data(), span.data(), span.size()); +} + +inline std::string_view +CacheEntryDataReader::read_str(const size_t length) +{ + return util::to_string_view(read_bytes(length)); +} + +template +inline T +CacheEntryDataReader::read_int() +{ + const auto buffer = read_bytes(sizeof(T)); + T value; + Util::big_endian_to_int(buffer.data(), value); + return value; +} + +template +inline void +CacheEntryDataReader::read_int(T& value) +{ + value = read_int(); +} + +} // namespace core diff --git a/src/core/CacheEntryDataWriter.hpp b/src/core/CacheEntryDataWriter.hpp new file mode 100644 index 0000000..92890f0 --- /dev/null +++ b/src/core/CacheEntryDataWriter.hpp @@ -0,0 +1,78 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace core { + +class CacheEntryDataWriter +{ +public: + CacheEntryDataWriter(util::Bytes& output); + + // Write `data`. Throws `core::Error` on failure. + void write_bytes(nonstd::span data); + + // Write `data`. Throws `core::Error` on failure. + void write_str(std::string_view data); + + // Write integer `value`. Throws `core::Error` on failure. + template void write_int(T value); + +private: + util::Bytes& m_output; +}; + +inline CacheEntryDataWriter::CacheEntryDataWriter(util::Bytes& output) + : m_output(output) +{ +} + +inline void +CacheEntryDataWriter::write_bytes(nonstd::span data) +{ + m_output.insert(m_output.end(), data.begin(), data.end()); +} + +template +inline void +CacheEntryDataWriter::write_int(const T value) +{ + uint8_t buffer[sizeof(T)]; + Util::int_to_big_endian(value, buffer); + write_bytes(buffer); +} + +inline void +CacheEntryDataWriter::write_str(std::string_view value) +{ + write_bytes(util::to_span(value)); +} + +} // namespace core diff --git a/src/core/Manifest.cpp b/src/core/Manifest.cpp new file mode 100644 index 0000000..19093c0 --- /dev/null +++ b/src/core/Manifest.cpp @@ -0,0 +1,469 @@ +// Copyright (C) 2009-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "Manifest.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Manifest data format +// ==================== +// +// Integers are big-endian. +// +// ::= +// ::= uint8_t +// ::= * +// ::= uint32_t +// ::= +// ::= uint16_t +// ::= path_len bytes +// ::= * +// ::= uint32_t +// ::= +// ::= uint32_t +// ::= Digest::size() bytes +// ::= uint64_t ; file size +// ::= int64_t ; modification time (ns), 0 = not recorded +// ::= int64_t ; status change time (ns), 0 = not recorded +// ::= * +// ::= uint32_t +// ::= * +// ::= uint32_t +// ::= uint32_t +// ::= Digest::size() bytes + +const uint32_t k_max_manifest_entries = 100; +const uint32_t k_max_manifest_file_info_entries = 10000; + +namespace std { + +template<> struct hash +{ + size_t + operator()(const core::Manifest::FileInfo& file_info) const + { + static_assert(sizeof(file_info) == 48); // No padding. + util::XXH3_64 hash; + hash.update(&file_info, sizeof(file_info)); + return hash.digest(); + } +}; + +} // namespace std + +namespace core { + +// Format version history: +// +// Version 0: +// - First version. +// Version 1: +// - mtime and ctime are now stored with nanoseconds resolution. +const uint8_t Manifest::k_format_version = 1; + +void +Manifest::read(nonstd::span data) +{ + std::vector files; + std::vector file_infos; + std::vector results; + + core::CacheEntryDataReader reader(data); + + const auto format_version = reader.read_int(); + if (format_version != k_format_version) { + throw core::Error(FMT("Unknown manifest format version: {} != {}", + format_version, + k_format_version)); + } + + const auto file_count = reader.read_int(); + for (uint32_t i = 0; i < file_count; ++i) { + files.emplace_back(reader.read_str(reader.read_int())); + } + + const auto file_info_count = reader.read_int(); + for (uint32_t i = 0; i < file_info_count; ++i) { + file_infos.emplace_back(); + auto& entry = file_infos.back(); + + reader.read_int(entry.index); + reader.read_and_copy_bytes({entry.digest.bytes(), Digest::size()}); + reader.read_int(entry.fsize); + entry.mtime.set_nsec(reader.read_int()); + entry.ctime.set_nsec(reader.read_int()); + } + + const auto result_count = reader.read_int(); + for (uint32_t i = 0; i < result_count; ++i) { + results.emplace_back(); + auto& entry = results.back(); + + const auto file_info_index_count = reader.read_int(); + for (uint32_t j = 0; j < file_info_index_count; ++j) { + entry.file_info_indexes.push_back(reader.read_int()); + } + reader.read_and_copy_bytes({entry.key.bytes(), Digest::size()}); + } + + if (m_results.empty()) { + m_files = std::move(files); + m_file_infos = std::move(file_infos); + m_results = std::move(results); + } else { + for (const auto& result : results) { + std::unordered_map included_files; + std::unordered_map included_files_stats; + for (auto file_info_index : result.file_info_indexes) { + const auto& file_info = file_infos[file_info_index]; + included_files.emplace(files[file_info.index], file_info.digest); + included_files_stats.emplace( + files[file_info.index], + FileStats{file_info.fsize, file_info.mtime, file_info.ctime}); + } + add_result(result.key, included_files, [&](const std::string& path) { + return included_files_stats[path]; + }); + } + } +} + +std::optional +Manifest::look_up_result_digest(const Context& ctx) const +{ + std::unordered_map stated_files; + std::unordered_map hashed_files; + + // Check newest result first since it's a more likely to match. + for (size_t i = m_results.size(); i > 0; i--) { + const auto& result = m_results[i - 1]; + if (result_matches(ctx, result, stated_files, hashed_files)) { + return result.key; + } + } + + return std::nullopt; +} + +bool +Manifest::add_result( + const Digest& result_key, + const std::unordered_map& included_files, + const FileStater& stat_file_function) +{ + if (m_results.size() > k_max_manifest_entries) { + // Normally, there shouldn't be many result entries in the manifest since + // new entries are added only if an include file has changed but not the + // source file, and you typically change source files more often than header + // files. However, it's certainly possible to imagine cases where the + // manifest will grow large (for instance, a generated header file that + // changes for every build), and this must be taken care of since processing + // an ever growing manifest eventually will take too much time. A good way + // of solving this would be to maintain the result entries in LRU order and + // discarding the old ones. An easy way is to throw away all entries when + // there are too many. Let's do that for now. + LOG("More than {} entries in manifest file; discarding", + k_max_manifest_entries); + clear(); + } else if (m_file_infos.size() > k_max_manifest_file_info_entries) { + // Rarely, FileInfo entries can grow large in pathological cases where many + // included files change, but the main file does not. This also puts an + // upper bound on the number of FileInfo entries. + LOG("More than {} FileInfo entries in manifest file; discarding", + k_max_manifest_file_info_entries); + clear(); + } + + std::unordered_map mf_files; + for (uint32_t i = 0; i < m_files.size(); ++i) { + mf_files.emplace(m_files[i], i); + } + + std::unordered_map mf_file_infos; + for (uint32_t i = 0; i < m_file_infos.size(); ++i) { + mf_file_infos.emplace(m_file_infos[i], i); + } + + std::vector file_info_indexes; + file_info_indexes.reserve(included_files.size()); + + for (const auto& [path, digest] : included_files) { + file_info_indexes.push_back(get_file_info_index( + path, digest, mf_files, mf_file_infos, stat_file_function)); + } + + ResultEntry entry{std::move(file_info_indexes), result_key}; + if (std::find(m_results.begin(), m_results.end(), entry) == m_results.end()) { + m_results.push_back(std::move(entry)); + return true; + } else { + return false; + } +} + +uint32_t +Manifest::serialized_size() const +{ + uint64_t size = 0; + + size += 1; // format_ver + size += 4; // n_files + for (const auto& file : m_files) { + size += 2 + file.length(); + } + size += 4; // n_file_infos + size += m_file_infos.size() * (4 + Digest::size() + 8 + 8 + 8); + size += 4; // n_results + for (const auto& result : m_results) { + size += 4; // n_file_info_indexes + size += result.file_info_indexes.size() * 4; + size += Digest::size(); + } + + // In order to support 32-bit ccache builds, restrict size to uint32_t for + // now. This restriction can be lifted when we drop 32-bit support. + const auto max = std::numeric_limits::max(); + if (size > max) { + throw core::Error( + FMT("Serialized manifest too large ({} > {})", size, max)); + } + return size; +} + +void +Manifest::serialize(util::Bytes& output) +{ + core::CacheEntryDataWriter writer(output); + + writer.write_int(k_format_version); + writer.write_int(m_files.size()); + for (const auto& file : m_files) { + writer.write_int(file.length()); + writer.write_str(file); + } + + writer.write_int(m_file_infos.size()); + for (const auto& file_info : m_file_infos) { + writer.write_int(file_info.index); + writer.write_bytes({file_info.digest.bytes(), Digest::size()}); + writer.write_int(file_info.fsize); + writer.write_int(file_info.mtime.nsec()); + writer.write_int(file_info.ctime.nsec()); + } + + writer.write_int(m_results.size()); + for (const auto& result : m_results) { + writer.write_int(result.file_info_indexes.size()); + for (auto index : result.file_info_indexes) { + writer.write_int(index); + } + writer.write_bytes({result.key.bytes(), Digest::size()}); + } +} + +bool +Manifest::FileInfo::operator==(const FileInfo& other) const +{ + return index == other.index && digest == other.digest && fsize == other.fsize + && mtime == other.mtime && ctime == other.ctime; +} + +bool +Manifest::ResultEntry::operator==(const ResultEntry& other) const +{ + return file_info_indexes == other.file_info_indexes && key == other.key; +} + +void +Manifest::clear() +{ + m_files.clear(); + m_file_infos.clear(); + m_results.clear(); +} + +uint32_t +Manifest::get_file_info_index( + const std::string& path, + const Digest& digest, + const std::unordered_map& mf_files, + const std::unordered_map& mf_file_infos, + const FileStater& file_stater) +{ + FileInfo fi; + + const auto f_it = mf_files.find(path); + if (f_it != mf_files.end()) { + fi.index = f_it->second; + } else { + m_files.push_back(path); + fi.index = m_files.size() - 1; + } + + fi.digest = digest; + + const auto file_stat = file_stater(path); + fi.mtime = file_stat.mtime; + fi.ctime = file_stat.ctime; + fi.fsize = file_stat.size; + + const auto fi_it = mf_file_infos.find(fi); + if (fi_it != mf_file_infos.end()) { + return fi_it->second; + } else { + m_file_infos.push_back(fi); + return m_file_infos.size() - 1; + } +} + +bool +Manifest::result_matches( + const Context& ctx, + const ResultEntry& result, + std::unordered_map& stated_files, + std::unordered_map& hashed_files) const +{ + for (uint32_t file_info_index : result.file_info_indexes) { + const auto& fi = m_file_infos[file_info_index]; + const auto& path = m_files[fi.index]; + + auto stated_files_iter = stated_files.find(path); + if (stated_files_iter == stated_files.end()) { + auto file_stat = Stat::stat(path, Stat::OnError::log); + if (!file_stat) { + return false; + } + FileStats st; + st.size = file_stat.size(); + st.mtime = file_stat.mtime(); + st.ctime = file_stat.ctime(); + stated_files_iter = stated_files.emplace(path, st).first; + } + const FileStats& fs = stated_files_iter->second; + + if (fi.fsize != fs.size) { + return false; + } + + // Clang stores the mtime of the included files in the precompiled header, + // and will error out if that header is later used without rebuilding. + if ((ctx.config.compiler_type() == CompilerType::clang + || ctx.config.compiler_type() == CompilerType::other) + && ctx.args_info.output_is_precompiled_header + && !ctx.args_info.fno_pch_timestamp && fi.mtime != fs.mtime) { + LOG("Precompiled header includes {}, which has a new mtime", path); + return false; + } + + if (ctx.config.sloppiness().is_enabled(core::Sloppy::file_stat_matches)) { + if (!(ctx.config.sloppiness().is_enabled( + core::Sloppy::file_stat_matches_ctime))) { + if (fi.mtime == fs.mtime && fi.ctime == fs.ctime) { + LOG("mtime/ctime hit for {}", path); + continue; + } else { + LOG("mtime/ctime miss for {}", path); + } + } else { + if (fi.mtime == fs.mtime) { + LOG("mtime hit for {}", path); + continue; + } else { + LOG("mtime miss for {}", path); + } + } + } + + auto hashed_files_iter = hashed_files.find(path); + if (hashed_files_iter == hashed_files.end()) { + Digest actual_digest; + int ret = hash_source_code_file(ctx, actual_digest, path, fs.size); + if (ret & HASH_SOURCE_CODE_ERROR) { + LOG("Failed hashing {}", path); + return false; + } + if (ret & HASH_SOURCE_CODE_FOUND_TIME) { + return false; + } + + hashed_files_iter = hashed_files.emplace(path, actual_digest).first; + } + + if (fi.digest != hashed_files_iter->second) { + return false; + } + } + + return true; +} + +void +Manifest::inspect(FILE* const stream) const +{ + PRINT(stream, "Manifest format version: {}\n", k_format_version); + + PRINT(stream, "File paths ({}):\n", m_files.size()); + for (size_t i = 0; i < m_files.size(); ++i) { + PRINT(stream, " {}: {}\n", i, m_files[i]); + } + + PRINT(stream, "File infos ({}):\n", m_file_infos.size()); + for (size_t i = 0; i < m_file_infos.size(); ++i) { + PRINT(stream, " {}:\n", i); + PRINT(stream, " Path index: {}\n", m_file_infos[i].index); + PRINT(stream, " Hash: {}\n", m_file_infos[i].digest.to_string()); + PRINT(stream, " File size: {}\n", m_file_infos[i].fsize); + if (m_file_infos[i].mtime == util::TimePoint()) { + PRINT_RAW(stream, " Mtime: -\n"); + } else { + PRINT(stream, + " Mtime: {}.{:09}\n", + m_file_infos[i].mtime.sec(), + m_file_infos[i].mtime.nsec_decimal_part()); + } + if (m_file_infos[i].ctime == util::TimePoint()) { + PRINT_RAW(stream, " Ctime: -\n"); + } else { + PRINT(stream, + " Ctime: {}.{:09}\n", + m_file_infos[i].ctime.sec(), + m_file_infos[i].ctime.nsec_decimal_part()); + } + } + + PRINT(stream, "Results ({}):\n", m_results.size()); + for (size_t i = 0; i < m_results.size(); ++i) { + PRINT(stream, " {}:\n", i); + PRINT_RAW(stream, " File info indexes:"); + for (uint32_t file_info_index : m_results[i].file_info_indexes) { + PRINT(stream, " {}", file_info_index); + } + PRINT_RAW(stream, "\n"); + PRINT(stream, " Key: {}\n", m_results[i].key.to_string()); + } +} + +} // namespace core diff --git a/src/core/Manifest.hpp b/src/core/Manifest.hpp new file mode 100644 index 0000000..a8f3b67 --- /dev/null +++ b/src/core/Manifest.hpp @@ -0,0 +1,110 @@ +// Copyright (C) 2009-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +class Context; + +namespace core { + +class Manifest : public Serializer +{ +public: + static const uint8_t k_format_version; + + struct FileStats + { + uint64_t size; + util::TimePoint mtime; + util::TimePoint ctime; + }; + + using FileStater = std::function; + + Manifest() = default; + + void read(nonstd::span data); + + std::optional look_up_result_digest(const Context& ctx) const; + + bool add_result(const Digest& result_key, + const std::unordered_map& included_files, + const FileStater& stat_file); + + // core::Serializer + uint32_t serialized_size() const override; + void serialize(util::Bytes& output) override; + + void inspect(FILE* stream) const; + +private: + struct FileInfo + { + uint32_t index; // Index to m_files. + Digest digest; // Digest of referenced file. + uint64_t fsize; // Size of referenced file. + util::TimePoint mtime; // mtime of referenced file. + util::TimePoint ctime; // ctime of referenced file. + + bool operator==(const FileInfo& other) const; + }; + + friend std::hash; + + struct ResultEntry + { + std::vector file_info_indexes; // Indexes to m_file_infos. + Digest key; // Key of the result. + + bool operator==(const ResultEntry& other) const; + }; + + std::vector m_files; // Names of referenced include files. + std::vector m_file_infos; // Info about referenced include files. + std::vector m_results; + + void clear(); + + uint32_t get_file_info_index( + const std::string& path, + const Digest& digest, + const std::unordered_map& mf_files, + const std::unordered_map& mf_file_infos, + const FileStater& file_state); + + bool + result_matches(const Context& ctx, + const ResultEntry& result, + std::unordered_map& stated_files, + std::unordered_map& hashed_files) const; +}; + +} // namespace core diff --git a/src/core/MsvcShowIncludesOutput.cpp b/src/core/MsvcShowIncludesOutput.cpp new file mode 100644 index 0000000..311a1bc --- /dev/null +++ b/src/core/MsvcShowIncludesOutput.cpp @@ -0,0 +1,76 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "MsvcShowIncludesOutput.hpp" + +#include +#include +#include + +namespace core::MsvcShowIncludesOutput { + +std::vector +get_includes(std::string_view file_content, std::string_view prefix) +{ + // /showIncludes output is written to stdout together with other messages. + // Every line of it is " " where the prefix is "Note: + // including file:" in English but can be localized. + + std::vector result; + // This will split at each \r or \n, but that simply means there will be empty + // "lines". + for (std::string_view line : Util::split_into_views(file_content, "\r\n")) { + if (util::starts_with(line, prefix)) { + size_t pos = prefix.size(); + while (pos < line.size() && isspace(line[pos])) { + ++pos; + } + std::string_view include = line.substr(pos); + if (!include.empty()) { + result.push_back(include); + } + } + } + return result; +} + +util::Bytes +strip_includes(const Context& ctx, util::Bytes&& stdout_data) +{ + using util::Tokenizer; + using Mode = Tokenizer::Mode; + using IncludeDelimiter = Tokenizer::IncludeDelimiter; + + if (stdout_data.empty() || !ctx.auto_depend_mode + || ctx.config.compiler_type() != CompilerType::msvc) { + return std::move(stdout_data); + } + + util::Bytes new_stdout_data; + for (const auto line : Tokenizer(util::to_string_view(stdout_data), + "\n", + Mode::include_empty, + IncludeDelimiter::yes)) { + if (!util::starts_with(line, ctx.config.msvc_dep_prefix())) { + new_stdout_data.insert(new_stdout_data.end(), line.data(), line.size()); + } + } + return new_stdout_data; +} + +} // namespace core::MsvcShowIncludesOutput diff --git a/src/core/MsvcShowIncludesOutput.hpp b/src/core/MsvcShowIncludesOutput.hpp new file mode 100644 index 0000000..97c59c0 --- /dev/null +++ b/src/core/MsvcShowIncludesOutput.hpp @@ -0,0 +1,35 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include +#include + +class Context; + +namespace core::MsvcShowIncludesOutput { + +std::vector get_includes(std::string_view file_content, + std::string_view prefix); + +util::Bytes strip_includes(const Context& ctx, util::Bytes&& stdout_data); + +} // namespace core::MsvcShowIncludesOutput diff --git a/src/core/Result.cpp b/src/core/Result.cpp new file mode 100644 index 0000000..b51b912 --- /dev/null +++ b/src/core/Result.cpp @@ -0,0 +1,333 @@ +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "Result.hpp" + +#include "Config.hpp" +#include "Context.hpp" +#include "Fd.hpp" +#include "File.hpp" +#include "Logging.hpp" +#include "Stat.hpp" +#include "Util.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + +#include + +// Result data format +// ================== +// +// Integers are big-endian. +// +// ::= * +// ::= uint8_t +// ::= uint8_t +// ::= | +// ::= +// +// ::= 0 (uint8_t) +// ::= uint8_t (see Result::FileType) +// ::= uint64_t +// ::= file_size bytes +// ::= +// ::= 1 (uint8_t) +// ::= uint64_t + +namespace { + +// File data stored inside the result file. +const uint8_t k_embedded_file_marker = 0; + +// File stored as-is in the file system. +const uint8_t k_raw_file_marker = 1; + +const uint8_t k_max_raw_file_entries = 10; + +bool +should_store_raw_file(const Config& config, core::Result::FileType type) +{ + if (!core::Result::Serializer::use_raw_files(config)) { + return false; + } + + // Only store object files as raw files since there are several problems with + // storing other file types: + // + // 1. The compiler unlinks object files before writing to them but it doesn't + // unlink .d files, so it's possible to corrupt .d files just by running + // the compiler (see ccache issue 599). + // 2. .d files cause trouble for automake if hard-linked (see ccache issue + // 378). + // 3. It's unknown how the compiler treats other file types, so better safe + // than sorry. + // + // It would be possible to store all files in raw form for the file_clone case + // and only hard link object files. However, most likely it's only object + // files that become large enough that it's of interest to clone or hard link + // them, so we keep things simple for now. This will also save i-nodes in the + // cache. + return type == core::Result::FileType::object; +} + +} // namespace + +namespace core::Result { + +const uint8_t k_format_version = 0; + +const char* const k_unknown_file_type = ""; + +const char* +file_type_to_string(FileType type) +{ + switch (type) { + case FileType::object: + return ".o"; + + case FileType::dependency: + return ".d"; + + case FileType::stderr_output: + return ""; + + case FileType::coverage_unmangled: + return ".gcno-unmangled"; + + case FileType::stackusage: + return ".su"; + + case FileType::diagnostic: + return ".dia"; + + case FileType::dwarf_object: + return ".dwo"; + + case FileType::coverage_mangled: + return ".gcno-mangled"; + + case FileType::stdout_output: + return ""; + + case FileType::assembler_listing: + return ".al"; + } + + return k_unknown_file_type; +} + +std::string +gcno_file_in_mangled_form(const Context& ctx) +{ + const auto& output_obj = ctx.args_info.output_obj; + const std::string abs_output_obj = + util::is_absolute_path(output_obj) + ? output_obj + : FMT("{}/{}", ctx.apparent_cwd, output_obj); + std::string hashified_obj = abs_output_obj; + std::replace(hashified_obj.begin(), hashified_obj.end(), '/', '#'); + return Util::change_extension(hashified_obj, ".gcno"); +} + +std::string +gcno_file_in_unmangled_form(const Context& ctx) +{ + return Util::change_extension(ctx.args_info.output_obj, ".gcno"); +} + +Deserializer::Deserializer(nonstd::span data) : m_data(data) +{ +} + +void +Deserializer::visit(Deserializer::Visitor& visitor) const +{ + Header header; + + CacheEntryDataReader reader(m_data); + header.format_version = reader.read_int(); + if (header.format_version != k_format_version) { + visitor.on_header(header); + throw Error(FMT("Unknown result format version: {} != {}", + header.format_version, + k_format_version)); + } + + header.n_files = reader.read_int(); + if (header.n_files >= k_max_raw_file_entries) { + visitor.on_header(header); + throw Error(FMT("Too many raw file entries: {} > {}", + header.n_files, + k_max_raw_file_entries)); + } + + visitor.on_header(header); + + uint8_t file_number; + for (file_number = 0; file_number < header.n_files; ++file_number) { + const auto marker = reader.read_int(); + switch (marker) { + case k_embedded_file_marker: + case k_raw_file_marker: + break; + + default: + throw Error(FMT("Unknown entry type: {}", marker)); + } + + const auto type = reader.read_int(); + const auto file_type = FileType(type); + const auto file_size = reader.read_int(); + + if (marker == k_embedded_file_marker) { + visitor.on_embedded_file( + file_number, file_type, reader.read_bytes(file_size)); + } else { + ASSERT(marker == k_raw_file_marker); + visitor.on_raw_file(file_number, file_type, file_size); + } + } + + if (file_number != header.n_files) { + throw Error(FMT( + "Too few entries (read {}, expected {})", file_number, header.n_files)); + } +} + +Serializer::Serializer(const Config& config) + : m_config(config), + m_serialized_size(1 + 1) // format_ver + n_files +{ +} + +void +Serializer::add_data(const FileType file_type, nonstd::span data) +{ + m_serialized_size += 1 + 1 + 8; // marker + file_type + file_size + m_serialized_size += data.size(); + m_file_entries.push_back(FileEntry{file_type, data}); +} + +bool +Serializer::add_file(const FileType file_type, const std::string& path) +{ + m_serialized_size += 1 + 1 + 8; // marker + file_type + file_size + if (!should_store_raw_file(m_config, file_type)) { + auto st = Stat::stat(path); + if (!st) { + return false; + } + m_serialized_size += st.size(); + } + m_file_entries.push_back(FileEntry{file_type, path}); + return true; +} + +uint32_t +Serializer::serialized_size() const +{ + // In order to support 32-bit ccache builds, restrict size to uint32_t for + // now. This restriction can be lifted when we drop 32-bit support. + const auto max = std::numeric_limits::max(); + if (m_serialized_size > max) { + throw Error( + FMT("Serialized result too large ({} > {})", m_serialized_size, max)); + } + return m_serialized_size; +} + +void +Serializer::serialize(util::Bytes& output) +{ + CacheEntryDataWriter writer(output); + + writer.write_int(k_format_version); + writer.write_int(m_file_entries.size()); + + uint8_t file_number = 0; + for (const auto& entry : m_file_entries) { + const bool is_file_entry = std::holds_alternative(entry.data); + const bool store_raw = + is_file_entry && should_store_raw_file(m_config, entry.file_type); + const uint64_t file_size = + is_file_entry ? Stat::stat(std::get(entry.data), + Stat::OnError::throw_error) + .size() + : std::get>(entry.data).size(); + + LOG("Storing {} entry #{} {} ({} bytes){}", + store_raw ? "raw" : "embedded", + file_number, + file_type_to_string(entry.file_type), + file_size, + is_file_entry ? FMT(" from {}", std::get(entry.data)) + : ""); + + writer.write_int(store_raw ? k_raw_file_marker + : k_embedded_file_marker); + writer.write_int(UnderlyingFileTypeInt(entry.file_type)); + writer.write_int(file_size); + + if (store_raw) { + m_raw_files.push_back( + RawFile{file_number, std::get(entry.data)}); + } else if (is_file_entry) { + const auto& path = std::get(entry.data); + const auto data = util::value_or_throw( + util::read_file(path), FMT("Failed to read {}: ", path)); + writer.write_bytes(data); + } else { + writer.write_bytes(std::get>(entry.data)); + } + + ++file_number; + } +} + +bool +Serializer::use_raw_files(const Config& config) +{ + return config.file_clone() || config.hard_link(); +} + +const std::vector& +Serializer::get_raw_files() const +{ + return m_raw_files; +} + +} // namespace core::Result diff --git a/src/core/Result.hpp b/src/core/Result.hpp new file mode 100644 index 0000000..3080f71 --- /dev/null +++ b/src/core/Result.hpp @@ -0,0 +1,178 @@ +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +#include + +#include +#include +#include +#include + +class Config; +class Context; + +namespace core { + +class CacheEntryDataParser; + +namespace Result { + +extern const uint8_t k_format_version; + +extern const char* const k_unknown_file_type; + +using UnderlyingFileTypeInt = uint8_t; +enum class FileType : UnderlyingFileTypeInt { + // These values are written into the cache result file. This means they must + // never be changed or removed unless the result file version is incremented. + // Adding new values is OK. + + // The main output specified with -o or implicitly from the input filename. + object = 0, + + // Dependency file specified with -MF or implicitly from the output filename. + dependency = 1, + + // Text sent to standard error output. + stderr_output = 2, + + // Coverage notes file generated by -ftest-coverage with filename in unmangled + // form, i.e. output file but with a .gcno extension. + coverage_unmangled = 3, + + // Stack usage file generated by -fstack-usage, i.e. output file but with a + // .su extension. + stackusage = 4, + + // Diagnostics output file specified by --serialize-diagnostics. + diagnostic = 5, + + // DWARF object file generated by -gsplit-dwarf, i.e. output file but with a + // .dwo extension. + dwarf_object = 6, + + // Coverage notes file generated by -ftest-coverage with filename in mangled + // form, i.e. full output file path but with a .gcno extension and with + // slashes replaced with hashes. + coverage_mangled = 7, + + // Text sent to standard output. + stdout_output = 8, + + // Assembler listing file from -Wa,-a=file. + assembler_listing = 9, +}; + +const char* file_type_to_string(FileType type); + +std::string gcno_file_in_mangled_form(const Context& ctx); +std::string gcno_file_in_unmangled_form(const Context& ctx); + +// This class knows how to deserializer a result cache entry. +class Deserializer +{ +public: + // Read a result from `data`. + Deserializer(nonstd::span data); + + struct Header + { + uint8_t format_version = 0; + uint8_t n_files = 0; + }; + + class Visitor + { + public: + virtual ~Visitor() = default; + + virtual void on_header(const Header& header); + + virtual void on_embedded_file(uint8_t file_number, + FileType file_type, + nonstd::span data) = 0; + virtual void on_raw_file(uint8_t file_number, + FileType file_type, + uint64_t file_size) = 0; + }; + + // Throws core::Error on error. + void visit(Visitor& visitor) const; + +private: + nonstd::span m_data; + + void parse_file_entry(CacheEntryDataParser& parser, + uint8_t file_number) const; +}; + +inline void +Deserializer::Visitor::on_header(const Header& /*header*/) +{ +} + +// This class knows how to serialize a result cache entry. +class Serializer : public core::Serializer +{ +public: + Serializer(const Config& config); + + // Register data to include in the result. The data must live until + // serialize() has been called. + void add_data(FileType file_type, nonstd::span data); + + // Register a file path whose content should be included in the result. + [[nodiscard]] bool add_file(FileType file_type, const std::string& path); + + // core::Serializer + uint32_t serialized_size() const override; + void serialize(util::Bytes& output) override; + + static bool use_raw_files(const Config& config); + + struct RawFile + { + uint8_t file_number; + std::string path; + }; + + // Get raw files to store in local storage. + const std::vector& get_raw_files() const; + +private: + const Config& m_config; + uint64_t m_serialized_size; + + struct FileEntry + { + FileType file_type; + std::variant, std::string> data; + }; + std::vector m_file_entries; + + std::vector m_raw_files; +}; + +} // namespace Result + +} // namespace core diff --git a/src/core/ResultExtractor.cpp b/src/core/ResultExtractor.cpp new file mode 100644 index 0000000..8649ede --- /dev/null +++ b/src/core/ResultExtractor.cpp @@ -0,0 +1,89 @@ +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "ResultExtractor.hpp" + +#include "Util.hpp" +#include "fmtmacros.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +namespace core { + +ResultExtractor::ResultExtractor( + const std::string& output_directory, + std::optional get_raw_file_path) + : m_output_directory(output_directory), + m_get_raw_file_path(get_raw_file_path) +{ +} + +void +ResultExtractor::on_embedded_file(uint8_t /*file_number*/, + Result::FileType file_type, + nonstd::span data) +{ + std::string suffix = Result::file_type_to_string(file_type); + if (suffix == Result::k_unknown_file_type) { + suffix = + FMT(".type_{}", static_cast(file_type)); + } else if (suffix[0] == '<') { + suffix[0] = '.'; + suffix.resize(suffix.length() - 1); + } + + const auto dest_path = FMT("{}/ccache-result{}", m_output_directory, suffix); + util::throw_on_error(util::write_file(dest_path, data), + FMT("Failed to write to {}: ", dest_path)); +} + +void +ResultExtractor::on_raw_file(uint8_t file_number, + Result::FileType file_type, + uint64_t file_size) +{ + if (!m_get_raw_file_path) { + throw Error("Raw entry for non-local result"); + } + const auto raw_file_path = (*m_get_raw_file_path)(file_number); + const auto st = Stat::stat(raw_file_path, Stat::OnError::throw_error); + if (st.size() != file_size) { + throw Error(FMT("Bad file size of {} (actual {} bytes, expected {} bytes)", + raw_file_path, + st.size(), + file_size)); + } + + const auto data = util::value_or_throw( + util::read_file(raw_file_path, file_size), + FMT("Failed to read {}: ", raw_file_path)); + on_embedded_file(file_number, file_type, data); +} + +} // namespace core diff --git a/src/core/ResultExtractor.hpp b/src/core/ResultExtractor.hpp new file mode 100644 index 0000000..916bea7 --- /dev/null +++ b/src/core/ResultExtractor.hpp @@ -0,0 +1,55 @@ +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Fd.hpp" + +#include + +#include +#include +#include + +namespace core { + +// This class extracts the parts of a result entry to a directory. +class ResultExtractor : public Result::Deserializer::Visitor +{ +public: + using GetRawFilePathFunction = std::function; + + //`result_path` should be the path to the local result entry file if the + // result comes from local storage. + ResultExtractor( + const std::string& output_directory, + std::optional get_raw_file_path = std::nullopt); + + void on_embedded_file(uint8_t file_number, + Result::FileType file_type, + nonstd::span data) override; + void on_raw_file(uint8_t file_number, + Result::FileType file_type, + uint64_t file_size) override; + +private: + std::string m_output_directory; + std::optional m_get_raw_file_path; +}; + +} // namespace core diff --git a/src/core/ResultInspector.cpp b/src/core/ResultInspector.cpp new file mode 100644 index 0000000..b27a3f1 --- /dev/null +++ b/src/core/ResultInspector.cpp @@ -0,0 +1,62 @@ +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "ResultInspector.hpp" + +#include "Context.hpp" +#include "Logging.hpp" +#include "fmtmacros.hpp" + +namespace core { + +ResultInspector::ResultInspector(FILE* stream) : m_stream(stream) +{ +} + +void +ResultInspector::on_header(const Result::Deserializer::Header& header) +{ + PRINT(m_stream, "Result format version: {}\n", header.format_version); + PRINT(m_stream, "Number of files: {}\n", header.n_files); +} + +void +ResultInspector::on_embedded_file(uint8_t file_number, + Result::FileType file_type, + nonstd::span data) +{ + PRINT(m_stream, + "Embedded file #{}: {} ({} bytes)\n", + file_number, + Result::file_type_to_string(file_type), + data.size()); +} + +void +ResultInspector::on_raw_file(uint8_t file_number, + Result::FileType file_type, + uint64_t file_size) +{ + PRINT(m_stream, + "Raw file #{}: {} ({} bytes)\n", + file_number, + Result::file_type_to_string(file_type), + file_size); +} + +} // namespace core diff --git a/src/core/ResultInspector.hpp b/src/core/ResultInspector.hpp new file mode 100644 index 0000000..9fdd9d3 --- /dev/null +++ b/src/core/ResultInspector.hpp @@ -0,0 +1,47 @@ +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include +#include + +namespace core { + +// This class writes information about the result entry to `stream`. +class ResultInspector : public Result::Deserializer::Visitor +{ +public: + ResultInspector(FILE* stream); + + void on_header(const Result::Deserializer::Header& header) override; + + void on_embedded_file(uint8_t file_number, + Result::FileType file_type, + nonstd::span data) override; + void on_raw_file(uint8_t file_number, + Result::FileType file_type, + uint64_t file_size) override; + +private: + FILE* m_stream; +}; + +} // namespace core diff --git a/src/core/ResultRetriever.cpp b/src/core/ResultRetriever.cpp new file mode 100644 index 0000000..0aecc17 --- /dev/null +++ b/src/core/ResultRetriever.cpp @@ -0,0 +1,224 @@ +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "ResultRetriever.hpp" + +#include "Context.hpp" +#include "Depfile.hpp" +#include "Logging.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + +namespace core { + +using Result::FileType; + +ResultRetriever::ResultRetriever(const Context& ctx, + std::optional result_key) + : m_ctx(ctx), + m_result_key(result_key) +{ +} + +void +ResultRetriever::on_embedded_file(uint8_t file_number, + FileType file_type, + nonstd::span data) +{ + LOG("Reading embedded entry #{} {} ({} bytes)", + file_number, + Result::file_type_to_string(file_type), + data.size()); + + if (file_type == FileType::stdout_output) { + Util::send_to_fd( + m_ctx, + util::to_string_view(MsvcShowIncludesOutput::strip_includes(m_ctx, data)), + STDOUT_FILENO); + } else if (file_type == FileType::stderr_output) { + Util::send_to_fd(m_ctx, util::to_string_view(data), STDERR_FILENO); + } else { + const auto dest_path = get_dest_path(file_type); + if (dest_path.empty()) { + LOG_RAW("Not writing"); + } else if (dest_path == "/dev/null") { + LOG_RAW("Not writing to /dev/null"); + } else { + LOG("Writing to {}", dest_path); + if (file_type == FileType::dependency) { + write_dependency_file(dest_path, data); + } else { + util::throw_on_error( + util::write_file(dest_path, data), + FMT("Failed to write to {}: ", dest_path)); + } + } + } +} + +void +ResultRetriever::on_raw_file(uint8_t file_number, + FileType file_type, + uint64_t file_size) +{ + LOG("Reading raw entry #{} {} ({} bytes)", + file_number, + Result::file_type_to_string(file_type), + file_size); + + if (!m_result_key) { + throw core::Error("Raw entry for non-local result"); + } + const auto raw_file_path = + m_ctx.storage.local.get_raw_file_path(*m_result_key, file_number); + const auto st = Stat::stat(raw_file_path, Stat::OnError::throw_error); + if (st.size() != file_size) { + throw core::Error( + FMT("Bad file size of {} (actual {} bytes, expected {} bytes)", + raw_file_path, + st.size(), + file_size)); + } + + const auto dest_path = get_dest_path(file_type); + if (!dest_path.empty()) { + try { + Util::clone_hard_link_or_copy_file( + m_ctx.config, raw_file_path, dest_path, false); + } catch (core::Error& e) { + throw WriteError(FMT("Failed to clone/link/copy {} to {}: {}", + raw_file_path, + dest_path, + e.what())); + } + + // Update modification timestamp to save the file from LRU cleanup (and, if + // hard-linked, to make the object file newer than the source file). + util::set_timestamps(raw_file_path); + } else { + // Should never happen. + LOG("Did not copy {} since destination path is unknown for type {}", + raw_file_path, + static_cast(file_type)); + } +} + +std::string +ResultRetriever::get_dest_path(FileType file_type) const +{ + switch (file_type) { + case FileType::object: + return m_ctx.args_info.output_obj; + + case FileType::dependency: + if (m_ctx.args_info.generating_dependencies) { + return m_ctx.args_info.output_dep; + } + break; + + case FileType::stdout_output: + case FileType::stderr_output: + // Should never get here. + break; + + case FileType::coverage_unmangled: + if (m_ctx.args_info.generating_coverage) { + return Util::change_extension(m_ctx.args_info.output_obj, ".gcno"); + } + break; + + case FileType::stackusage: + if (m_ctx.args_info.generating_stackusage) { + return m_ctx.args_info.output_su; + } + break; + + case FileType::diagnostic: + if (m_ctx.args_info.generating_diagnostics) { + return m_ctx.args_info.output_dia; + } + break; + + case FileType::dwarf_object: + if (m_ctx.args_info.seen_split_dwarf + && m_ctx.args_info.output_obj != "/dev/null") { + return m_ctx.args_info.output_dwo; + } + break; + + case FileType::coverage_mangled: + if (m_ctx.args_info.generating_coverage) { + return Result::gcno_file_in_mangled_form(m_ctx); + } + break; + + case FileType::assembler_listing: + return m_ctx.args_info.output_al; + } + + return {}; +} + +void +ResultRetriever::write_dependency_file(const std::string& path, + nonstd::span data) +{ + ASSERT(m_ctx.args_info.dependency_target); + + Fd fd(open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666)); + if (!fd) { + throw WriteError(FMT("Failed to open {} for writing", path)); + } + + auto write_data = [&](auto data, auto size) { + util::throw_on_error(util::write_fd(*fd, data, size), + FMT("Failed to write to {}: ", path)); + }; + + std::string_view str_data = util::to_string_view(data); + size_t start_pos = 0; + const size_t colon_pos = str_data.find(": "); + if (colon_pos != std::string::npos) { + const auto obj_in_dep_file = str_data.substr(0, colon_pos); + const auto& dep_target = *m_ctx.args_info.dependency_target; + if (obj_in_dep_file != dep_target) { + write_data(dep_target.data(), dep_target.length()); + start_pos = colon_pos; + } + } + + write_data(str_data.data() + start_pos, str_data.length() - start_pos); +} + +} // namespace core diff --git a/src/core/ResultRetriever.hpp b/src/core/ResultRetriever.hpp new file mode 100644 index 0000000..92a3af8 --- /dev/null +++ b/src/core/ResultRetriever.hpp @@ -0,0 +1,64 @@ +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Fd.hpp" + +#include +#include +#include + +#include + +class Context; + +namespace core { + +// This class retrieves a result entry to the local file system. +class ResultRetriever : public Result::Deserializer::Visitor +{ +public: + class WriteError : public Error + { + using Error::Error; + }; + + //`path` should be the path to the local result entry file if the result comes + // from local storage. + ResultRetriever(const Context& ctx, + std::optional result_key = std::nullopt); + + void on_embedded_file(uint8_t file_number, + Result::FileType file_type, + nonstd::span data) override; + void on_raw_file(uint8_t file_number, + Result::FileType file_type, + uint64_t file_size) override; + +private: + const Context& m_ctx; + std::optional m_result_key; + + std::string get_dest_path(Result::FileType file_type) const; + + void write_dependency_file(const std::string& path, + nonstd::span data); +}; + +} // namespace core diff --git a/src/core/Serializer.hpp b/src/core/Serializer.hpp new file mode 100644 index 0000000..bb5ddd0 --- /dev/null +++ b/src/core/Serializer.hpp @@ -0,0 +1,37 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include + +#include + +namespace core { + +class Serializer +{ +public: + virtual ~Serializer() = default; + virtual uint32_t serialized_size() const = 0; + virtual void serialize(util::Bytes& output) = 0; +}; + +} // namespace core diff --git a/src/core/Sloppiness.hpp b/src/core/Sloppiness.hpp new file mode 100644 index 0000000..ef45907 --- /dev/null +++ b/src/core/Sloppiness.hpp @@ -0,0 +1,100 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +namespace core { + +enum class Sloppy : uint32_t { + none = 0U, + + include_file_mtime = 1U << 0, + include_file_ctime = 1U << 1, + time_macros = 1U << 2, + pch_defines = 1U << 3, + // Allow us to match files based on their stats (size, mtime, ctime), without + // looking at their contents. + file_stat_matches = 1U << 4, + // Allow us to not include any system headers in the manifest include files, + // similar to -MM versus -M for dependencies. + system_headers = 1U << 5, + // Allow us to ignore ctimes when comparing file stats, so we can fake mtimes + // if we want to (it is much harder to fake ctimes, requires changing clock) + file_stat_matches_ctime = 1U << 6, + // Allow us to not include the -index-store-path option in the manifest hash. + clang_index_store = 1U << 7, + // Ignore locale settings. + locale = 1U << 8, + // Allow caching even if -fmodules is used. + modules = 1U << 9, + // Ignore virtual file system (VFS) overlay file. + ivfsoverlay = 1U << 10, + // Allow us to include incorrect working directory in .gcno files. + gcno_cwd = 1U << 11, + // Ignore -frandom-seed=*string*. + random_seed = 1U << 12, +}; + +class Sloppiness +{ +public: + Sloppiness(Sloppy value = Sloppy::none); + explicit Sloppiness(uint32_t value); + + void enable(Sloppy value); + bool is_enabled(Sloppy value) const; + uint32_t to_bitmask() const; + +private: + Sloppy m_sloppiness = Sloppy::none; +}; + +// --- Inline implementations --- + +inline Sloppiness::Sloppiness(Sloppy value) : m_sloppiness(value) +{ +} + +inline Sloppiness::Sloppiness(uint32_t value) + : m_sloppiness(static_cast(value)) +{ +} + +inline void +Sloppiness::enable(Sloppy value) +{ + m_sloppiness = static_cast(static_cast(m_sloppiness) + | static_cast(value)); +} + +inline bool +Sloppiness::is_enabled(Sloppy value) const +{ + return static_cast(m_sloppiness) & static_cast(value); +} + +inline uint32_t +Sloppiness::to_bitmask() const +{ + return static_cast(m_sloppiness); +} + +} // namespace core diff --git a/src/core/Statistic.hpp b/src/core/Statistic.hpp new file mode 100644 index 0000000..1f4fb92 --- /dev/null +++ b/src/core/Statistic.hpp @@ -0,0 +1,78 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +namespace core { + +// Statistics fields in storage order. +enum class Statistic { + none = 0, + compiler_produced_stdout = 1, + compile_failed = 2, + internal_error = 3, + cache_miss = 4, + preprocessor_error = 5, + could_not_find_compiler = 6, + missing_cache_file = 7, + preprocessed_cache_hit = 8, + bad_compiler_arguments = 9, + called_for_link = 10, + files_in_cache = 11, + cache_size_kibibyte = 12, + obsolete_max_files = 13, + obsolete_max_size = 14, + unsupported_source_language = 15, + bad_output_file = 16, + no_input_file = 17, + multiple_source_files = 18, + autoconf_test = 19, + unsupported_compiler_option = 20, + output_to_stdout = 21, + direct_cache_hit = 22, + compiler_produced_no_output = 23, + compiler_produced_empty_output = 24, + error_hashing_extra_file = 25, + compiler_check_failed = 26, + could_not_use_precompiled_header = 27, + called_for_preprocessing = 28, + cleanups_performed = 29, + unsupported_code_directive = 30, + stats_zeroed_timestamp = 31, + could_not_use_modules = 32, + direct_cache_miss = 33, + preprocessed_cache_miss = 34, + local_storage_read_hit = 35, + local_storage_read_miss = 36, + remote_storage_read_hit = 37, + remote_storage_read_miss = 38, + remote_storage_error = 39, + remote_storage_timeout = 40, + recache = 41, + unsupported_environment_variable = 42, + local_storage_write = 43, + local_storage_hit = 44, + local_storage_miss = 45, + remote_storage_write = 46, + remote_storage_hit = 47, + remote_storage_miss = 48, + + END +}; + +} // namespace core diff --git a/src/core/Statistics.cpp b/src/core/Statistics.cpp new file mode 100644 index 0000000..662b59b --- /dev/null +++ b/src/core/Statistics.cpp @@ -0,0 +1,415 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "Statistics.hpp" + +#include +#include +#include +#include +#include +#include + +#include + +namespace core { + +using core::Statistic; + +const unsigned FLAG_NOZERO = 1U << 0; // don't zero with --zero-stats +const unsigned FLAG_NEVER = 1U << 1; // don't include in --print-stats +const unsigned FLAG_ERROR = 1U << 2; // include in error count +const unsigned FLAG_UNCACHEABLE = 1U << 3; // include in uncacheable count + +namespace { + +struct StatisticsField +{ + StatisticsField(const Statistic statistic_, + const char* const id_, + const char* const description_, + const unsigned flags_ = 0) + : statistic(statistic_), + id(id_), + description(description_), + flags(flags_) + { + } + + const Statistic statistic; + const char* const id; // for --print-stats + const char* const description; // for --show-stats --verbose + const unsigned flags; // bitmask of FLAG_* values +}; + +} // namespace + +#define FIELD(id, ...) \ + { \ + Statistic::id, #id, __VA_ARGS__ \ + } + +const StatisticsField k_statistics_fields[] = { + // Field "none" intentionally omitted. + FIELD(autoconf_test, "Autoconf compile/link", FLAG_UNCACHEABLE), + FIELD(bad_compiler_arguments, "Bad compiler arguments", FLAG_UNCACHEABLE), + FIELD(bad_output_file, "Could not write to output file", FLAG_ERROR), + FIELD(cache_miss, nullptr), + FIELD(cache_size_kibibyte, nullptr, FLAG_NOZERO), + FIELD(called_for_link, "Called for linking", FLAG_UNCACHEABLE), + FIELD(called_for_preprocessing, "Called for preprocessing", FLAG_UNCACHEABLE), + FIELD(cleanups_performed, nullptr), + FIELD(compile_failed, "Compilation failed", FLAG_UNCACHEABLE), + FIELD(compiler_check_failed, "Compiler check failed", FLAG_ERROR), + FIELD(compiler_produced_no_output, + "Compiler output file missing", + FLAG_UNCACHEABLE), + FIELD(compiler_produced_empty_output, + "Compiler produced empty output", + FLAG_UNCACHEABLE), + FIELD(compiler_produced_stdout, "Compiler produced stdout", FLAG_UNCACHEABLE), + FIELD(could_not_find_compiler, "Could not find compiler", FLAG_ERROR), + FIELD(could_not_use_modules, "Could not use modules", FLAG_UNCACHEABLE), + FIELD(could_not_use_precompiled_header, + "Could not use precompiled header", + FLAG_UNCACHEABLE), + FIELD(direct_cache_hit, nullptr), + FIELD(direct_cache_miss, nullptr), + FIELD(error_hashing_extra_file, "Error hashing extra file", FLAG_ERROR), + FIELD(files_in_cache, nullptr, FLAG_NOZERO), + FIELD(internal_error, "Internal error", FLAG_ERROR), + FIELD(local_storage_hit, nullptr), + FIELD(local_storage_miss, nullptr), + FIELD(local_storage_read_hit, nullptr), + FIELD(local_storage_read_miss, nullptr), + FIELD(local_storage_write, nullptr), + FIELD(missing_cache_file, "Missing cache file", FLAG_ERROR), + FIELD(multiple_source_files, "Multiple source files", FLAG_UNCACHEABLE), + FIELD(no_input_file, "No input file", FLAG_UNCACHEABLE), + FIELD(obsolete_max_files, nullptr, FLAG_NOZERO | FLAG_NEVER), + FIELD(obsolete_max_size, nullptr, FLAG_NOZERO | FLAG_NEVER), + FIELD(output_to_stdout, "Output to stdout", FLAG_UNCACHEABLE), + FIELD(preprocessed_cache_hit, nullptr), + FIELD(preprocessed_cache_miss, nullptr), + FIELD(preprocessor_error, "Preprocessing failed", FLAG_UNCACHEABLE), + FIELD(recache, "Forced recache", FLAG_UNCACHEABLE), + FIELD(remote_storage_error, nullptr), + FIELD(remote_storage_hit, nullptr), + FIELD(remote_storage_miss, nullptr), + FIELD(remote_storage_read_hit, nullptr), + FIELD(remote_storage_read_miss, nullptr), + FIELD(remote_storage_write, nullptr), + FIELD(remote_storage_timeout, nullptr), + FIELD(stats_zeroed_timestamp, nullptr), + FIELD( + unsupported_code_directive, "Unsupported code directive", FLAG_UNCACHEABLE), + FIELD(unsupported_compiler_option, + "Unsupported compiler option", + FLAG_UNCACHEABLE), + FIELD(unsupported_environment_variable, + "Unsupported environment variable", + FLAG_UNCACHEABLE), + FIELD(unsupported_source_language, + "Unsupported source language", + FLAG_UNCACHEABLE), +}; + +static_assert(sizeof(k_statistics_fields) / sizeof(k_statistics_fields[0]) + == static_cast(Statistic::END) - 1); + +static std::string +format_timestamp(const util::TimePoint& value) +{ + if (value.sec() == 0) { + return "never"; + } else { + const auto tm = Util::localtime(value); + char buffer[100] = "?"; + if (tm) { + strftime(buffer, sizeof(buffer), "%c", &*tm); + } + return buffer; + } +} + +static std::string +percent(const uint64_t nominator, const uint64_t denominator) +{ + if (denominator == 0) { + return ""; + } else if (nominator >= denominator) { + return FMT("({:5.1f}%)", (100.0 * nominator) / denominator); + } else { + return FMT("({:5.2f}%)", (100.0 * nominator) / denominator); + } +} + +Statistics::Statistics(const StatisticsCounters& counters) + : m_counters(counters) +{ +} + +std::vector +Statistics::get_statistics_ids() const +{ + std::vector result; + for (const auto& field : k_statistics_fields) { + if (!(field.flags & FLAG_NOZERO)) { + for (size_t i = 0; i < m_counters.get(field.statistic); ++i) { + result.emplace_back(field.id); + } + } + } + std::sort(result.begin(), result.end()); + return result; +} + +uint64_t +Statistics::count_stats(const unsigned flags) const +{ + uint64_t sum = 0; + for (const auto& field : k_statistics_fields) { + if (field.flags & flags) { + sum += m_counters.get(field.statistic); + } + } + return sum; +} + +std::vector> +Statistics::get_stats(unsigned flags, const bool all) const +{ + std::vector> result; + for (const auto& field : k_statistics_fields) { + const auto count = m_counters.get(field.statistic); + if ((field.flags & flags) && (all || count > 0)) { + result.emplace_back(field.description, count); + } + } + return result; +} + +static void +add_ratio_row(util::TextTable& table, + const std::string& text, + const uint64_t nominator, + const uint64_t denominator) +{ + if (denominator > 0) { + table.add_row({ + text, + nominator, + "/", + denominator, + percent(nominator, denominator), + }); + } else { + table.add_row({text, nominator}); + } +} + +std::string +Statistics::format_human_readable(const Config& config, + const util::TimePoint& last_updated, + const uint8_t verbosity, + const bool from_log) const +{ + util::TextTable table; + using C = util::TextTable::Cell; + +#define S(x_) m_counters.get(Statistic::x_) + + const uint64_t d_hits = S(direct_cache_hit); + const uint64_t d_misses = S(direct_cache_miss); + const uint64_t p_hits = S(preprocessed_cache_hit); + const uint64_t p_misses = S(preprocessed_cache_miss); + const uint64_t hits = d_hits + p_hits; + const uint64_t misses = S(cache_miss); + const uint64_t uncacheable = count_stats(FLAG_UNCACHEABLE); + const uint64_t errors = count_stats(FLAG_ERROR); + const uint64_t total_calls = hits + misses + errors + uncacheable; + + auto cmp_fn = [](const auto& e1, const auto& e2) { + return e1.first.compare(e2.first) < 0; + }; + + if (verbosity > 0 && !from_log) { + table.add_row({"Cache directory:", C(config.cache_dir()).colspan(4)}); + table.add_row({"Config file:", C(config.config_path()).colspan(4)}); + table.add_row( + {"System config file:", C(config.system_config_path()).colspan(4)}); + table.add_row( + {"Stats updated:", C(format_timestamp(last_updated)).colspan(4)}); + if (verbosity > 1) { + const util::TimePoint last_zeroed(S(stats_zeroed_timestamp)); + table.add_row( + {"Stats zeroed:", C(format_timestamp(last_zeroed)).colspan(4)}); + } + } + + if (total_calls > 0 || verbosity > 1) { + add_ratio_row(table, "Cacheable calls:", hits + misses, total_calls); + add_ratio_row(table, " Hits:", hits, hits + misses); + add_ratio_row(table, " Direct:", d_hits, hits); + add_ratio_row(table, " Preprocessed:", p_hits, hits); + add_ratio_row(table, " Misses:", misses, hits + misses); + } + + if (uncacheable > 0 || verbosity > 1) { + add_ratio_row(table, "Uncacheable calls:", uncacheable, total_calls); + if (verbosity > 0) { + auto uncacheable_stats = get_stats(FLAG_UNCACHEABLE, verbosity > 1); + std::sort(uncacheable_stats.begin(), uncacheable_stats.end(), cmp_fn); + for (const auto& [name, value] : uncacheable_stats) { + add_ratio_row(table, FMT(" {}:", name), value, uncacheable); + } + } + } + + if (errors > 0 || verbosity > 1) { + add_ratio_row(table, "Errors:", errors, total_calls); + if (verbosity > 0) { + auto error_stats = get_stats(FLAG_ERROR, verbosity > 1); + std::sort(error_stats.begin(), error_stats.end(), cmp_fn); + for (const auto& [name, value] : error_stats) { + add_ratio_row(table, FMT(" {}:", name), value, errors); + } + } + } + + if (total_calls > 0 && verbosity > 0) { + table.add_heading("Successful lookups:"); + add_ratio_row(table, " Direct:", d_hits, d_hits + d_misses); + add_ratio_row(table, " Preprocessed:", p_hits, p_hits + p_misses); + } + + const uint64_t g = 1'000'000'000; + const uint64_t local_hits = S(local_storage_hit); + const uint64_t local_misses = S(local_storage_miss); + const uint64_t local_reads = + S(local_storage_read_hit) + S(local_storage_read_miss); + const uint64_t local_writes = S(local_storage_write); + const uint64_t local_size = S(cache_size_kibibyte) * 1024; + const uint64_t cleanups = S(cleanups_performed); + const uint64_t remote_hits = S(remote_storage_hit); + const uint64_t remote_misses = S(remote_storage_miss); + const uint64_t remote_reads = + S(remote_storage_read_hit) + S(remote_storage_read_miss); + const uint64_t remote_writes = S(remote_storage_write); + const uint64_t remote_errors = S(remote_storage_error); + const uint64_t remote_timeouts = S(remote_storage_timeout); + + table.add_heading("Local storage:"); + if (!from_log) { + std::vector size_cells{ + " Cache size (GB):", + C(FMT("{:.2f}", static_cast(local_size) / g)).right_align()}; + if (config.max_size() != 0) { + size_cells.emplace_back("/"); + size_cells.emplace_back( + C(FMT("{:.2f}", static_cast(config.max_size()) / g)) + .right_align()); + size_cells.emplace_back(percent(local_size, config.max_size())); + } + table.add_row(size_cells); + + if (verbosity > 0) { + std::vector files_cells{" Files:", S(files_in_cache)}; + if (config.max_files() > 0) { + files_cells.emplace_back("/"); + files_cells.emplace_back(config.max_files()); + files_cells.emplace_back( + percent(S(files_in_cache), config.max_files())); + } + table.add_row(files_cells); + } + if (cleanups > 0 || verbosity > 1) { + table.add_row({" Cleanups:", cleanups}); + } + } + if (verbosity > 0 || (remote_hits + remote_misses) > 0) { + add_ratio_row(table, " Hits:", local_hits, local_hits + local_misses); + add_ratio_row(table, " Misses:", local_misses, local_hits + local_misses); + } + if (verbosity > 0) { + table.add_row({" Reads:", local_reads}); + table.add_row({" Writes:", local_writes}); + } + + if (verbosity > 1 + || remote_hits + remote_misses + remote_errors + remote_timeouts > 0) { + table.add_heading("Remote storage:"); + add_ratio_row(table, " Hits:", remote_hits, remote_hits + remote_misses); + add_ratio_row( + table, " Misses:", remote_misses, remote_hits + remote_misses); + if (verbosity > 0) { + table.add_row({" Reads:", remote_reads}); + table.add_row({" Writes:", remote_writes}); + } + if (verbosity > 1 || remote_errors > 0) { + table.add_row({" Errors:", remote_errors}); + } + if (verbosity > 1 || remote_timeouts > 0) { + table.add_row({" Timeouts:", remote_timeouts}); + } + } + + return table.render(); +} + +std::string +Statistics::format_machine_readable(const util::TimePoint& last_updated) const +{ + std::vector lines; + + lines.push_back(FMT("stats_updated_timestamp\t{}\n", last_updated.sec())); + + for (const auto& field : k_statistics_fields) { + if (!(field.flags & FLAG_NEVER)) { + lines.push_back( + FMT("{}\t{}\n", field.id, m_counters.get(field.statistic))); + } + } + + std::sort(lines.begin(), lines.end()); + return util::join(lines, ""); +} + +std::unordered_map +Statistics::get_id_map() +{ + std::unordered_map result; + for (const auto& field : k_statistics_fields) { + result[field.id] = field.statistic; + } + return result; +} + +std::vector +Statistics::get_zeroable_fields() +{ + std::vector result; + for (const auto& field : k_statistics_fields) { + if (!(field.flags & FLAG_NOZERO)) { + result.push_back(field.statistic); + } + } + return result; +} + +} // namespace core diff --git a/src/core/Statistics.hpp b/src/core/Statistics.hpp new file mode 100644 index 0000000..f74d81e --- /dev/null +++ b/src/core/Statistics.hpp @@ -0,0 +1,74 @@ +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +class Config; + +namespace core { + +class Statistics +{ +public: + Statistics(const StatisticsCounters& counters); + + // Return machine-readable strings representing the statistics counters. + std::vector get_statistics_ids() const; + + // Format cache statistics in human-readable format. + std::string format_human_readable(const Config& config, + const util::TimePoint& last_updated, + uint8_t verbosity, + bool from_log) const; + + // Format cache statistics in machine-readable format. + std::string + format_machine_readable(const util::TimePoint& last_updated) const; + + const StatisticsCounters& counters() const; + + static std::unordered_map get_id_map(); + + static std::vector get_zeroable_fields(); + +private: + const StatisticsCounters m_counters; + + uint64_t count_stats(unsigned flags) const; + std::vector> get_stats(unsigned flags, + bool all) const; +}; + +// --- Inline implementations --- + +inline const StatisticsCounters& +Statistics::counters() const +{ + return m_counters; +} + +} // namespace core diff --git a/src/core/StatisticsCounters.cpp b/src/core/StatisticsCounters.cpp new file mode 100644 index 0000000..3f1f2d6 --- /dev/null +++ b/src/core/StatisticsCounters.cpp @@ -0,0 +1,117 @@ +// Copyright (C) 2010-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "StatisticsCounters.hpp" + +#include + +#include + +namespace core { + +StatisticsCounters::StatisticsCounters() + : m_counters(static_cast(Statistic::END)) +{ +} + +StatisticsCounters::StatisticsCounters(const Statistic statistic) + : StatisticsCounters({statistic}) +{ +} + +StatisticsCounters::StatisticsCounters( + const std::initializer_list statistics) + : StatisticsCounters() +{ + for (auto st : statistics) { + increment(st); + } +} + +uint64_t +StatisticsCounters::get(Statistic statistic) const +{ + const auto index = static_cast(statistic); + ASSERT(index < static_cast(Statistic::END)); + return index < m_counters.size() ? m_counters[index] : 0; +} + +void +StatisticsCounters::set(Statistic statistic, uint64_t value) +{ + const auto index = static_cast(statistic); + ASSERT(index < static_cast(Statistic::END)); + m_counters[index] = value; +} + +uint64_t +StatisticsCounters::get_raw(size_t index) const +{ + ASSERT(index < size()); + return m_counters[index]; +} + +void +StatisticsCounters::set_raw(size_t index, uint64_t value) +{ + if (index >= m_counters.size()) { + m_counters.resize(index + 1); + } + m_counters[index] = value; +} + +void +StatisticsCounters::increment(Statistic statistic, int64_t value) +{ + if (value == 0) { + return; + } + const auto i = static_cast(statistic); + if (i >= m_counters.size()) { + m_counters.resize(i + 1); + } + auto& counter = m_counters[i]; + counter = + std::max(static_cast(0), static_cast(counter + value)); +} + +void +StatisticsCounters::increment(const StatisticsCounters& other) +{ + m_counters.resize(std::max(size(), other.size())); + for (size_t i = 0; i < other.size(); ++i) { + auto& counter = m_counters[i]; + counter = std::max(static_cast(0), + static_cast(counter + other.m_counters[i])); + } +} + +size_t +StatisticsCounters::size() const +{ + return m_counters.size(); +} + +bool +StatisticsCounters::all_zero() const +{ + return !std::any_of( + m_counters.begin(), m_counters.end(), [](unsigned v) { return v != 0; }); +} + +} // namespace core diff --git a/src/core/StatisticsCounters.hpp b/src/core/StatisticsCounters.hpp new file mode 100644 index 0000000..21bb1b3 --- /dev/null +++ b/src/core/StatisticsCounters.hpp @@ -0,0 +1,57 @@ +// Copyright (C) 2010-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "Statistic.hpp" + +#include +#include +#include +#include + +namespace core { + +// A simple wrapper around a vector of integers used for the statistics +// counters. +class StatisticsCounters +{ +public: + StatisticsCounters(); + StatisticsCounters(Statistic statistic); + StatisticsCounters(std::initializer_list statistics); + + uint64_t get(Statistic statistic) const; + void set(Statistic statistic, uint64_t value); + + uint64_t get_raw(size_t index) const; + void set_raw(size_t index, uint64_t value); + + void increment(Statistic statistic, int64_t value = 1); + void increment(const StatisticsCounters& other); + + size_t size() const; + + // Return true if all counters are zero, false otherwise. + bool all_zero() const; + +private: + std::vector m_counters; +}; + +} // namespace core diff --git a/src/core/StatsLog.cpp b/src/core/StatsLog.cpp new file mode 100644 index 0000000..3202d2c --- /dev/null +++ b/src/core/StatsLog.cpp @@ -0,0 +1,71 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "StatsLog.hpp" + +#include +#include +#include +#include + +#include + +namespace core { + +StatisticsCounters +StatsLog::read() const +{ + core::StatisticsCounters counters; + + const auto id_map = Statistics::get_id_map(); + + std::ifstream in(m_path); + std::string line; + while (std::getline(in, line, '\n')) { + if (line[0] == '#') { + continue; + } + const auto entry = id_map.find(line); + if (entry != id_map.end()) { + Statistic statistic = entry->second; + counters.increment(statistic, 1); + } else { + LOG("Unknown statistic: {}", line); + } + } + + return counters; +} + +void +StatsLog::log_result(const std::string& input_file, + const std::vector& result_ids) +{ + File file(m_path, "ab"); + if (!file) { + LOG("Failed to open {}: {}", m_path, strerror(errno)); + return; + } + + PRINT(*file, "# {}\n", input_file); + for (const auto& id : result_ids) { + PRINT(*file, "{}\n", id); + } +} + +} // namespace core diff --git a/src/core/StatsLog.hpp b/src/core/StatsLog.hpp new file mode 100644 index 0000000..19284de --- /dev/null +++ b/src/core/StatsLog.hpp @@ -0,0 +1,47 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include "StatisticsCounters.hpp" + +#include +#include + +namespace core { + +class StatsLog +{ +public: + StatsLog(const std::string& path); + + StatisticsCounters read() const; + void log_result(const std::string& input_file, + const std::vector& result_ids); + +private: + const std::string m_path; +}; + +// --- Inline implementations --- + +inline StatsLog::StatsLog(const std::string& path) : m_path(path) +{ +} + +} // namespace core diff --git a/src/core/exceptions.hpp b/src/core/exceptions.hpp new file mode 100644 index 0000000..7bdf509 --- /dev/null +++ b/src/core/exceptions.hpp @@ -0,0 +1,51 @@ +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include +#include +#include +#include + +namespace core { + +// Don't throw or catch ErrorBase directly, use a subclass. +class ErrorBase : public std::runtime_error +{ + using std::runtime_error::runtime_error; +}; + +// Throw an Error to indicate a potentially non-fatal error that may be caught +// and handled by callers. An uncaught Error that reaches the top level will be +// treated similar to Fatal. +class Error : public ErrorBase +{ + using ErrorBase::ErrorBase; +}; + +// Throw a Fatal to make ccache print the error message to stderr and exit +// with a non-zero exit code. +class Fatal : public ErrorBase +{ + using ErrorBase::ErrorBase; +}; + +} // namespace core diff --git a/src/core/mainoptions.cpp b/src/core/mainoptions.cpp new file mode 100644 index 0000000..2283162 --- /dev/null +++ b/src/core/mainoptions.cpp @@ -0,0 +1,685 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "mainoptions.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + +#ifdef HAVE_GETOPT_LONG +# include +#elif defined(_WIN32) +# include +#else +extern "C" { +# include +} +#endif + +namespace core { + +constexpr const char VERSION_TEXT[] = + R"({0} version {1} +Features: {2} + +Copyright (C) 2002-2007 Andrew Tridgell +Copyright (C) 2009-2022 Joel Rosdahl and other contributors + +See for a complete list of contributors. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 3 of the License, or (at your option) any later +version. +)"; + +constexpr const char USAGE_TEXT[] = + R"(Usage: + {0} [options] + {0} compiler [compiler options] + compiler [compiler options] (ccache masquerading as the compiler) + +Common options: + -c, --cleanup delete old files and recalculate size counters + (normally not needed as this is done + automatically) + -C, --clear clear the cache completely (except configuration) + --config-path PATH operate on configuration file PATH instead of the + default + -d, --dir PATH operate on cache directory PATH instead of the + default + --evict-namespace NAMESPACE + remove files created in namespace NAMESPACE + --evict-older-than AGE remove files older than AGE (unsigned integer + with a d (days) or s (seconds) suffix) + -F, --max-files NUM set maximum number of files in cache to NUM (use + 0 for no limit) + -M, --max-size SIZE set maximum size of cache to SIZE (use 0 for no + limit); available suffixes: k, M, G, T (decimal) + and Ki, Mi, Gi, Ti (binary); default suffix: G + -X, --recompress LEVEL recompress the cache to level LEVEL (integer or + "uncompressed") using the Zstandard algorithm; + see "Cache compression" in the manual for details + -o, --set-config KEY=VAL set configuration item KEY to value VAL + -x, --show-compression show compression statistics + -p, --show-config show current configuration options in + human-readable format + --show-log-stats print statistics counters from the stats log + in human-readable format + -s, --show-stats show summary of configuration and statistics + counters in human-readable format (use + -v/--verbose once or twice for more details) + -v, --verbose increase verbosity + -z, --zero-stats zero statistics counters + + -h, --help print this help text + -V, --version print version and copyright information + +Options for remote file-based storage: + --trim-dir PATH remove old files from directory PATH until it is + at most the size specified by --trim-max-size + (note: don't use this option to trim the local + cache) + --trim-max-size SIZE specify the maximum size for --trim-dir; + available suffixes: k, M, G, T (decimal) and Ki, + Mi, Gi, Ti (binary); default suffix: G + --trim-method METHOD specify the method (atime or mtime) for + --trim-dir; default: atime + +Options for scripting or debugging: + --checksum-file PATH print the checksum (128 bit XXH3) of the file at + PATH + --extract-result PATH extract file data stored in result file at PATH + to the current working directory + -k, --get-config KEY print the value of configuration key KEY + --hash-file PATH print the hash (160 bit BLAKE3) of the file at + PATH + --inspect PATH print result/manifest file at PATH in + human-readable format + --print-stats print statistics counter IDs and corresponding + values in machine-parsable format + +See also the manual on . +)"; + +static void +configuration_printer(const std::string& key, + const std::string& value, + const std::string& origin) +{ + PRINT(stdout, "({}) {} = {}\n", origin, key, value); +} + +static nonstd::expected, std::string> +read_from_path_or_stdin(const std::string& path) +{ + if (path == "-") { + std::vector output; + const auto result = + util::read_fd(STDIN_FILENO, [&](const uint8_t* data, size_t size) { + output.insert(output.end(), data, data + size); + }); + if (!result) { + return nonstd::make_unexpected( + FMT("Failed to read from stdin: {}", result.error())); + } + return output; + } else { + const auto result = util::read_file>(path); + if (!result) { + return nonstd::make_unexpected( + FMT("Failed to read {}: {}", path, result.error())); + } + return *result; + } +} + +static int +inspect_path(const std::string& path) +{ + const auto cache_entry_data = read_from_path_or_stdin(path); + if (!cache_entry_data) { + PRINT(stderr, "Error: {}\n", cache_entry_data.error()); + return EXIT_FAILURE; + } + + core::CacheEntry cache_entry(*cache_entry_data); + fputs(cache_entry.header().inspect().c_str(), stdout); + + const auto payload = cache_entry.payload(); + + switch (cache_entry.header().entry_type) { + case core::CacheEntryType::manifest: { + core::Manifest manifest; + manifest.read(payload); + manifest.inspect(stdout); + break; + } + case core::CacheEntryType::result: + Result::Deserializer result_deserializer(payload); + ResultInspector result_inspector(stdout); + result_deserializer.visit(result_inspector); + break; + } + + cache_entry.verify_checksum(); + + return EXIT_SUCCESS; +} + +static void +print_compression_statistics(const storage::local::CompressionStatistics& cs) +{ + const double ratio = cs.compr_size > 0 + ? static_cast(cs.content_size) / cs.compr_size + : 0.0; + const double savings = ratio > 0.0 ? 100.0 - (100.0 / ratio) : 0.0; + + using C = util::TextTable::Cell; + auto human_readable = Util::format_human_readable_size; + util::TextTable table; + + table.add_row({ + "Total data:", + C(human_readable(cs.compr_size + cs.incompr_size)).right_align(), + FMT("({} disk blocks)", human_readable(cs.on_disk_size)), + }); + table.add_row({ + "Compressed data:", + C(human_readable(cs.compr_size)).right_align(), + FMT("({:.1f}% of original size)", 100.0 - savings), + }); + table.add_row({ + " Original size:", + C(human_readable(cs.content_size)).right_align(), + }); + table.add_row({ + " Compression ratio:", + C(FMT("{:.3f} x ", ratio)).right_align(), + FMT("({:.1f}% space savings)", savings), + }); + table.add_row({ + "Incompressible data:", + C(human_readable(cs.incompr_size)).right_align(), + }); + + PRINT_RAW(stdout, table.render()); +} + +static void +trim_dir(const std::string& dir, + const uint64_t trim_max_size, + const bool trim_lru_mtime) +{ + struct File + { + std::string path; + Stat stat; + }; + std::vector files; + uint64_t size_before = 0; + + Util::traverse(dir, [&](const std::string& path, const bool is_dir) { + const auto stat = Stat::lstat(path); + if (!stat) { + // Probably some race, ignore. + return; + } + size_before += stat.size_on_disk(); + if (!is_dir) { + const auto name = Util::base_name(path); + if (name == "ccache.conf" || name == "stats") { + throw Fatal( + FMT("this looks like a local cache directory (found {})", path)); + } + files.push_back({path, stat}); + } + }); + + std::sort(files.begin(), files.end(), [&](const auto& f1, const auto& f2) { + return trim_lru_mtime ? f1.stat.mtime() < f2.stat.mtime() + : f1.stat.atime() < f2.stat.atime(); + }); + + uint64_t size_after = size_before; + + for (const auto& file : files) { + if (size_after <= trim_max_size) { + break; + } + Util::unlink_tmp(file.path); + size_after -= file.stat.size(); + } + + PRINT(stdout, + "Removed {} ({} -> {})\n", + Util::format_human_readable_size(size_before - size_after), + Util::format_human_readable_size(size_before), + Util::format_human_readable_size(size_after)); +} + +static std::string +get_version_text(const std::string_view ccache_name) +{ + return FMT( + VERSION_TEXT, ccache_name, CCACHE_VERSION, storage::get_features()); +} + +std::string +get_usage_text(const std::string_view ccache_name) +{ + return FMT(USAGE_TEXT, ccache_name); +} + +enum { + CHECKSUM_FILE, + CONFIG_PATH, + DUMP_MANIFEST, + DUMP_RESULT, + EVICT_NAMESPACE, + EVICT_OLDER_THAN, + EXTRACT_RESULT, + HASH_FILE, + INSPECT, + PRINT_STATS, + SHOW_LOG_STATS, + TRIM_DIR, + TRIM_MAX_SIZE, + TRIM_METHOD, +}; + +const char options_string[] = "cCd:k:hF:M:po:svVxX:z"; +const option long_options[] = { + {"checksum-file", required_argument, nullptr, CHECKSUM_FILE}, + {"cleanup", no_argument, nullptr, 'c'}, + {"clear", no_argument, nullptr, 'C'}, + {"config-path", required_argument, nullptr, CONFIG_PATH}, + {"dir", required_argument, nullptr, 'd'}, + {"directory", required_argument, nullptr, 'd'}, // bwd compat + {"dump-manifest", required_argument, nullptr, DUMP_MANIFEST}, // bwd compat + {"dump-result", required_argument, nullptr, DUMP_RESULT}, // bwd compat + {"evict-namespace", required_argument, nullptr, EVICT_NAMESPACE}, + {"evict-older-than", required_argument, nullptr, EVICT_OLDER_THAN}, + {"extract-result", required_argument, nullptr, EXTRACT_RESULT}, + {"get-config", required_argument, nullptr, 'k'}, + {"hash-file", required_argument, nullptr, HASH_FILE}, + {"help", no_argument, nullptr, 'h'}, + {"inspect", required_argument, nullptr, INSPECT}, + {"max-files", required_argument, nullptr, 'F'}, + {"max-size", required_argument, nullptr, 'M'}, + {"print-stats", no_argument, nullptr, PRINT_STATS}, + {"recompress", required_argument, nullptr, 'X'}, + {"set-config", required_argument, nullptr, 'o'}, + {"show-compression", no_argument, nullptr, 'x'}, + {"show-config", no_argument, nullptr, 'p'}, + {"show-log-stats", no_argument, nullptr, SHOW_LOG_STATS}, + {"show-stats", no_argument, nullptr, 's'}, + {"trim-dir", required_argument, nullptr, TRIM_DIR}, + {"trim-max-size", required_argument, nullptr, TRIM_MAX_SIZE}, + {"trim-method", required_argument, nullptr, TRIM_METHOD}, + {"verbose", no_argument, nullptr, 'v'}, + {"version", no_argument, nullptr, 'V'}, + {"zero-stats", no_argument, nullptr, 'z'}, + {nullptr, 0, nullptr, 0}}; + +int +process_main_options(int argc, const char* const* argv) +{ + int c; + std::optional trim_max_size; + bool trim_lru_mtime = false; + uint8_t verbosity = 0; + std::optional evict_namespace; + std::optional evict_max_age; + + // First pass: Handle non-command options that affect command options. + while ((c = getopt_long(argc, + const_cast(argv), + options_string, + long_options, + nullptr)) + != -1) { + const std::string arg = optarg ? optarg : std::string(); + + switch (c) { + case 'd': // --dir + Util::setenv("CCACHE_DIR", arg); + break; + + case CONFIG_PATH: + Util::setenv("CCACHE_CONFIGPATH", arg); + break; + + case TRIM_MAX_SIZE: + trim_max_size = Util::parse_size(arg); + break; + + case TRIM_METHOD: + trim_lru_mtime = (arg == "ctime"); + break; + + case 'v': // --verbose + ++verbosity; + break; + + case '?': // unknown option + return EXIT_FAILURE; + } + } + + // Second pass: Handle command options in order. + optind = 1; + while ((c = getopt_long(argc, + const_cast(argv), + options_string, + long_options, + nullptr)) + != -1) { + Config config; + config.read(); + + UmaskScope umask_scope(config.umask()); + + const std::string arg = optarg ? optarg : std::string(); + + switch (c) { + case CONFIG_PATH: + case 'd': // --dir + case TRIM_MAX_SIZE: + case TRIM_METHOD: + case 'v': // --verbose + // Already handled in the first pass. + break; + + case CHECKSUM_FILE: { + util::XXH3_128 checksum; + Fd fd(arg == "-" ? STDIN_FILENO : open(arg.c_str(), O_RDONLY)); + if (fd) { + util::read_fd(*fd, [&checksum](const uint8_t* data, size_t size) { + checksum.update({data, size}); + }); + const auto digest = checksum.digest(); + PRINT( + stdout, "{}\n", Util::format_base16(digest.data(), digest.size())); + } else { + PRINT(stderr, "Error: Failed to checksum {}\n", arg); + } + break; + } + + case EVICT_NAMESPACE: { + evict_namespace = arg; + break; + } + + case EVICT_OLDER_THAN: { + evict_max_age = Util::parse_duration(arg); + break; + } + + case EXTRACT_RESULT: { + umask_scope.release(); // Use original umask for files outside cache dir + const auto cache_entry_data = read_from_path_or_stdin(arg); + if (!cache_entry_data) { + PRINT(stderr, "Error: \"{}\"", cache_entry_data.error()); + return EXIT_FAILURE; + } + std::optional get_raw_file_path; + if (arg != "-") { + get_raw_file_path = [&](uint8_t file_number) { + return storage::local::LocalStorage::get_raw_file_path(arg, + file_number); + }; + } + ResultExtractor result_extractor(".", get_raw_file_path); + core::CacheEntry cache_entry(*cache_entry_data); + const auto payload = cache_entry.payload(); + + Result::Deserializer result_deserializer(payload); + result_deserializer.visit(result_extractor); + cache_entry.verify_checksum(); + return EXIT_SUCCESS; + } + + case HASH_FILE: { + Hash hash; + const auto result = + arg == "-" ? hash.hash_fd(STDIN_FILENO) : hash.hash_file(arg); + if (result) { + PRINT(stdout, "{}\n", hash.digest().to_string()); + } else { + PRINT(stderr, "Error: Failed to hash {}: {}\n", arg, result.error()); + return EXIT_FAILURE; + } + break; + } + + case INSPECT: + case DUMP_MANIFEST: // Backward compatibility + case DUMP_RESULT: // Backward compatibility + return inspect_path(arg); + + case PRINT_STATS: { + const auto [counters, last_updated] = + storage::local::LocalStorage(config).get_all_statistics(); + Statistics statistics(counters); + PRINT_RAW(stdout, statistics.format_machine_readable(last_updated)); + break; + } + + case 'c': // --cleanup + { + ProgressBar progress_bar("Cleaning..."); + storage::local::LocalStorage(config).clean_all( + [&](double progress) { progress_bar.update(progress); }); + if (isatty(STDOUT_FILENO)) { + PRINT_RAW(stdout, "\n"); + } + break; + } + + case 'C': // --clear + { + ProgressBar progress_bar("Clearing..."); + storage::local::LocalStorage(config).wipe_all( + [&](double progress) { progress_bar.update(progress); }); + if (isatty(STDOUT_FILENO)) { + PRINT_RAW(stdout, "\n"); + } + break; + } + + case 'h': // --help + PRINT(stdout, USAGE_TEXT, Util::base_name(argv[0])); + return EXIT_SUCCESS; + + case 'k': // --get-config + PRINT(stdout, "{}\n", config.get_string_value(arg)); + break; + + case 'F': { // --max-files + auto files = util::value_or_throw(util::parse_unsigned(arg)); + config.set_value_in_file(config.config_path(), "max_files", arg); + if (files == 0) { + PRINT_RAW(stdout, "Unset cache file limit\n"); + } else { + PRINT(stdout, "Set cache file limit to {}\n", files); + } + break; + } + + case 'M': { // --max-size + uint64_t size = Util::parse_size(arg); + config.set_value_in_file(config.config_path(), "max_size", arg); + if (size == 0) { + PRINT_RAW(stdout, "Unset cache size limit\n"); + } else { + PRINT(stdout, + "Set cache size limit to {}\n", + Util::format_human_readable_size(size)); + } + break; + } + + case 'o': { // --set-config + // Start searching for equal sign at position 1 to improve error message + // for the -o=K=V case (key "=K" and value "V"). + size_t eq_pos = arg.find('=', 1); + if (eq_pos == std::string::npos) { + throw Error(FMT("missing equal sign in \"{}\"", arg)); + } + std::string key = arg.substr(0, eq_pos); + std::string value = arg.substr(eq_pos + 1); + config.set_value_in_file(config.config_path(), key, value); + break; + } + + case 'p': // --show-config + config.visit_items(configuration_printer); + break; + + case SHOW_LOG_STATS: { + if (config.stats_log().empty()) { + throw Fatal("No stats log has been configured"); + } + Statistics statistics(StatsLog(config.stats_log()).read()); + const auto timestamp = + Stat::stat(config.stats_log(), Stat::OnError::log).mtime(); + PRINT_RAW( + stdout, + statistics.format_human_readable(config, timestamp, verbosity, true)); + break; + } + + case 's': { // --show-stats + const auto [counters, last_updated] = + storage::local::LocalStorage(config).get_all_statistics(); + Statistics statistics(counters); + PRINT_RAW(stdout, + statistics.format_human_readable( + config, last_updated, verbosity, false)); + break; + } + + case TRIM_DIR: + if (!trim_max_size) { + throw Error("please specify --trim-max-size when using --trim-dir"); + } + trim_dir(arg, *trim_max_size, trim_lru_mtime); + break; + + case 'V': // --version + { + std::string_view name = Util::base_name(argv[0]); +#ifdef _WIN32 + name = Util::remove_extension(name); +#endif + PRINT_RAW(stdout, get_version_text(name)); + break; + } + + case 'x': // --show-compression + { + ProgressBar progress_bar("Scanning..."); + const auto compression_statistics = + storage::local::LocalStorage(config).get_compression_statistics( + [&](double progress) { progress_bar.update(progress); }); + if (isatty(STDOUT_FILENO)) { + PRINT_RAW(stdout, "\n\n"); + } + print_compression_statistics(compression_statistics); + break; + } + + case 'X': // --recompress + { + std::optional wanted_level; + if (arg == "uncompressed") { + wanted_level = std::nullopt; + } else { + wanted_level = util::value_or_throw( + util::parse_signed(arg, INT8_MIN, INT8_MAX, "compression level")); + } + + ProgressBar progress_bar("Recompressing..."); + storage::local::LocalStorage(config).recompress( + wanted_level, [&](double progress) { progress_bar.update(progress); }); + break; + } + + case 'z': // --zero-stats + storage::local::LocalStorage(config).zero_all_statistics(); + PRINT_RAW(stdout, "Statistics zeroed\n"); + break; + + default: + PRINT(stderr, USAGE_TEXT, Util::base_name(argv[0])); + return EXIT_FAILURE; + } + } + + if (evict_max_age || evict_namespace) { + Config config; + config.read(); + + ProgressBar progress_bar("Evicting..."); + storage::local::LocalStorage(config).evict( + [&](double progress) { progress_bar.update(progress); }, + evict_max_age, + evict_namespace); + if (isatty(STDOUT_FILENO)) { + PRINT_RAW(stdout, "\n"); + } + } + + return EXIT_SUCCESS; +} + +} // namespace core diff --git a/src/core/mainoptions.hpp b/src/core/mainoptions.hpp new file mode 100644 index 0000000..2cf84cd --- /dev/null +++ b/src/core/mainoptions.hpp @@ -0,0 +1,31 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +namespace core { + +// The main program when not doing a compile. +int process_main_options(int argc, const char* const* argv); + +std::string get_usage_text(std::string_view ccache_name); + +} // namespace core diff --git a/src/core/types.cpp b/src/core/types.cpp new file mode 100644 index 0000000..ecd05e5 --- /dev/null +++ b/src/core/types.cpp @@ -0,0 +1,83 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "types.hpp" + +#include +#include +#include +#include + +namespace core { + +std::string +to_string(const CacheEntryType type) +{ + switch (type) { + case CacheEntryType::manifest: + return "manifest"; + + case CacheEntryType::result: + return "result"; + + default: + return "unknown"; + } +} + +int8_t +compression_level_from_config(const Config& config) +{ + return config.compression() ? config.compression_level() : 0; +} + +CompressionType +compression_type_from_config(const Config& config) +{ + return config.compression() ? CompressionType::zstd : CompressionType::none; +} + +CompressionType +compression_type_from_int(const uint8_t type) +{ + switch (type) { + case static_cast(CompressionType::none): + return CompressionType::none; + + case static_cast(CompressionType::zstd): + return CompressionType::zstd; + } + + throw core::Error(FMT("Unknown type: {}", type)); +} + +std::string +to_string(const CompressionType type) +{ + switch (type) { + case CompressionType::none: + return "none"; + + case CompressionType::zstd: + return "zstd"; + } + + ASSERT(false); +} + +} // namespace core diff --git a/src/core/types.hpp b/src/core/types.hpp new file mode 100644 index 0000000..592eb4f --- /dev/null +++ b/src/core/types.hpp @@ -0,0 +1,45 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +class Config; + +namespace core { + +enum class CacheEntryType : uint8_t { result = 0, manifest = 1 }; + +std::string to_string(CacheEntryType type); + +enum class CompressionType : uint8_t { + none = 0, + zstd = 1, +}; + +int8_t compression_level_from_config(const Config& config); + +CompressionType compression_type_from_config(const Config& config); + +CompressionType compression_type_from_int(uint8_t type); + +std::string to_string(CompressionType type); + +} // namespace core diff --git a/src/core/wincompat.hpp b/src/core/wincompat.hpp new file mode 100644 index 0000000..2082710 --- /dev/null +++ b/src/core/wincompat.hpp @@ -0,0 +1,113 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#ifdef _WIN32 +# include + +# define NOMINMAX 1 +# define STDIN_FILENO 0 +# define STDOUT_FILENO 1 +# define STDERR_FILENO 2 + +# ifdef _MSC_VER +# define PATH_MAX MAX_PATH +# endif // _MSC_VER + +// From: +// http://mesos.apache.org/api/latest/c++/3rdparty_2stout_2include_2stout_2windows_8hpp_source.html +# ifdef _MSC_VER +const mode_t S_IRUSR = mode_t(_S_IREAD); +const mode_t S_IWUSR = mode_t(_S_IWRITE); +# endif + +# ifndef S_IFIFO +# define S_IFIFO 0x1000 +# endif + +# ifndef S_IFBLK +# define S_IFBLK 0x6000 +# endif + +# ifndef S_IFLNK +# define S_IFLNK 0xA000 +# endif + +# ifndef S_ISREG +# define S_ISREG(m) (((m)&S_IFMT) == S_IFREG) +# endif +# ifndef S_ISDIR +# define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR) +# endif +# ifndef S_ISFIFO +# define S_ISFIFO(m) (((m)&S_IFMT) == S_IFIFO) +# endif +# ifndef S_ISCHR +# define S_ISCHR(m) (((m)&S_IFMT) == S_IFCHR) +# endif +# ifndef S_ISLNK +# define S_ISLNK(m) (((m)&S_IFMT) == S_IFLNK) +# endif +# ifndef S_ISBLK +# define S_ISBLK(m) (((m)&S_IFMT) == S_IFBLK) +# endif + +# include +# include +# include +# include +# define NOMINMAX 1 +# define WIN32_NO_STATUS +// clang-format off +# include // struct timeval +// windows must be included after winsock2 +// https://stackoverflow.com/questions/1372480/c-redefinition-header-files-winsock2-h +# include +// bccrypt must go after windows.h +// https://stackoverflow.com/questions/57472787/compile-errors-when-using-c-and-bcrypt-header +# include // NTSTATUS +// clang-format on +# undef WIN32_NO_STATUS +# include +# define mkdir(a, b) _mkdir(a) + +// Protect against incidental use of MinGW execv. +# define execv(a, b) do_not_call_execv_on_windows + +# ifdef _MSC_VER +# define PATH_MAX MAX_PATH +# endif + +# ifdef _MSC_VER +# define DLLIMPORT __declspec(dllimport) +# else +# define DLLIMPORT +# endif + +# define STDIN_FILENO 0 +# define STDOUT_FILENO 1 +# define STDERR_FILENO 2 + +# ifndef O_BINARY +# define O_BINARY 0 +# endif + +#else +# define DLLIMPORT +#endif // _WIN32 diff --git a/src/exceptions.hpp b/src/exceptions.hpp deleted file mode 100644 index f35f50c..0000000 --- a/src/exceptions.hpp +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#include "system.hpp" - -#include "FormatNonstdStringView.hpp" - -#include "third_party/fmt/core.h" -#include "third_party/nonstd/optional.hpp" - -#include -#include -#include - -// Don't throw or catch ErrorBase directly, use a subclass. -class ErrorBase : public std::runtime_error -{ - using std::runtime_error::runtime_error; -}; - -// Throw an Error to indicate a potentially non-fatal error that may be caught -// and handled by callers. An uncaught Error that reaches the top level will be -// treated similar to Fatal. -class Error : public ErrorBase -{ -public: - // Special case: If given only one string, don't parse it as a format string. - Error(const std::string& message); - - // `args` are forwarded to `fmt::format`. - template inline Error(T&&... args); -}; - -inline Error::Error(const std::string& message) : ErrorBase(message) -{ -} - -template -inline Error::Error(T&&... args) - : ErrorBase(fmt::format(std::forward(args)...)) -{ -} - -// Throw a Fatal to make ccache print the error message to stderr and exit -// with a non-zero exit code. -class Fatal : public ErrorBase -{ -public: - // Special case: If given only one string, don't parse it as a format string. - Fatal(const std::string& message); - - // `args` are forwarded to `fmt::format`. - template inline Fatal(T&&... args); -}; - -inline Fatal::Fatal(const std::string& message) : ErrorBase(message) -{ -} - -template -inline Fatal::Fatal(T&&... args) - : ErrorBase(fmt::format(std::forward(args)...)) -{ -} diff --git a/src/execute.cpp b/src/execute.cpp index ebcf2be..a217d65 100644 --- a/src/execute.cpp +++ b/src/execute.cpp @@ -1,5 +1,5 @@ // Copyright (C) 2002 Andrew Tridgell -// Copyright (C) 2011-2021 Joel Rosdahl and other contributors +// Copyright (C) 2011-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -27,15 +27,26 @@ #include "Stat.hpp" #include "TemporaryFile.hpp" #include "Util.hpp" -#include "fmtmacros.hpp" +#include "Win32Util.hpp" + +#include +#include +#include +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + +#ifdef HAVE_SYS_WAIT_H +# include +#endif #ifdef _WIN32 # include "Finalizer.hpp" -# include "Win32Util.hpp" #endif -using nonstd::string_view; - #ifdef _WIN32 static int win32execute(const char* path, const char* const* argv, @@ -64,10 +75,10 @@ execute_noreturn(const char* const* argv, const std::string& temp_dir) std::string win32getshell(const std::string& path) { - const char* path_env = getenv("PATH"); + const char* path_list = getenv("PATH"); std::string sh; - if (Util::to_lowercase(Util::get_extension(path)) == ".sh" && path_env) { - sh = find_executable_in_path("sh.exe", "", path_env); + if (Util::to_lowercase(Util::get_extension(path)) == ".sh" && path_list) { + sh = find_executable_in_path("sh.exe", path_list); } if (sh.empty() && getenv("CCACHE_DETECT_SHEBANG")) { // Detect shebang. @@ -75,8 +86,8 @@ win32getshell(const std::string& path) if (fp) { char buf[10] = {0}; fgets(buf, sizeof(buf) - 1, fp.get()); - if (std::string(buf) == "#!/bin/sh" && path_env) { - sh = find_executable_in_path("sh.exe", "", path_env); + if (std::string(buf) == "#!/bin/sh" && path_list) { + sh = find_executable_in_path("sh.exe", path_list); } } } @@ -138,8 +149,8 @@ win32execute(const char* path, if (args.length() > 8192) { TemporaryFile tmp_file(FMT("{}/cmd_args", temp_dir)); args = Win32Util::argv_to_string(argv + 1, sh, true); - Util::write_fd(*tmp_file.fd, args.data(), args.length()); - args = FMT("{} @{}", full_path, tmp_file.path); + util::write_fd(*tmp_file.fd, args.data(), args.length()); + args = FMT(R"("{}" "@{}")", full_path, tmp_file.path); tmp_file_path = tmp_file.path; LOG("Arguments from {}", tmp_file.path); } @@ -192,7 +203,7 @@ execute(Context& ctx, const char* const* argv, Fd&& fd_out, Fd&& fd_err) } if (ctx.compiler_pid == -1) { - throw Fatal("Failed to fork: {}", strerror(errno)); + throw core::Fatal(FMT("Failed to fork: {}", strerror(errno))); } if (ctx.compiler_pid == 0) { @@ -214,7 +225,7 @@ execute(Context& ctx, const char* const* argv, Fd&& fd_out, Fd&& fd_err) if (result == -1 && errno == EINTR) { continue; } - throw Fatal("waitpid failed: {}", strerror(errno)); + throw core::Fatal(FMT("waitpid failed: {}", strerror(errno))); } { @@ -239,72 +250,70 @@ execute_noreturn(const char* const* argv, const std::string& /*temp_dir*/) std::string find_executable(const Context& ctx, const std::string& name, - const std::string& exclude_name) + const std::string& exclude_path) { - if (Util::is_absolute_path(name)) { + if (util::is_absolute_path(name)) { return name; } - std::string path = ctx.config.path(); - if (path.empty()) { - path = getenv("PATH"); + std::string path_list = ctx.config.path(); + if (path_list.empty()) { + path_list = getenv("PATH"); } - if (path.empty()) { + if (path_list.empty()) { LOG_RAW("No PATH variable"); return {}; } - return find_executable_in_path(name, exclude_name, path); + return find_executable_in_path(name, path_list, exclude_path); } std::string find_executable_in_path(const std::string& name, - const std::string& exclude_name, - const std::string& path) + const std::string& path_list, + std::optional exclude_path) { - if (path.empty()) { + if (path_list.empty()) { return {}; } - // Search the path looking for the first compiler of the right name that isn't - // us. - for (const std::string& dir : Util::split_into_strings(path, PATH_DELIM)) { + const auto real_exclude_path = + exclude_path ? Util::real_path(*exclude_path) : ""; + + // Search the path list looking for the first compiler of the right name that + // isn't us. + for (const std::string& dir : util::split_path_list(path_list)) { + const std::vector candidates = { + FMT("{}/{}", dir, name), #ifdef _WIN32 - char namebuf[MAX_PATH]; - int ret = SearchPath( - dir.c_str(), name.c_str(), nullptr, sizeof(namebuf), namebuf, nullptr); - if (!ret) { - std::string exename = FMT("{}.exe", name); - ret = SearchPath(dir.c_str(), - exename.c_str(), - nullptr, - sizeof(namebuf), - namebuf, - nullptr); - } - (void)exclude_name; - if (ret) { - return namebuf; - } + FMT("{}/{}.exe", dir, name), +#endif + }; + for (const auto& candidate : candidates) { + // A valid candidate: + // + // 1. Must exist (e.g., should not be a broken symlink) and be an + // executable. + // 2. Must not resolve to the same program as argv[0] (i.e., + // exclude_path). This can happen if ccache is masquerading as the + // compiler (with or without using a symlink). + // 3. As an extra safety measure: must not be a ccache executable after + // resolving symlinks. This can happen if the candidate compiler is a + // symlink to another ccache executable. + const bool candidate_exists = +#ifdef _WIN32 + Stat::stat(candidate); #else - ASSERT(!exclude_name.empty()); - std::string fname = FMT("{}/{}", dir, name); - auto st1 = Stat::lstat(fname); - auto st2 = Stat::stat(fname); - // Look for a normal executable file. - if (st1 && st2 && st2.is_regular() && access(fname.c_str(), X_OK) == 0) { - if (st1.is_symlink()) { - std::string real_path = Util::real_path(fname, true); - if (Util::base_name(real_path) == exclude_name) { - // It's a link to "ccache"! - continue; + access(candidate.c_str(), X_OK) == 0; +#endif + if (candidate_exists) { + const auto real_candidate = Util::real_path(candidate); + if ((real_exclude_path.empty() || real_candidate != real_exclude_path) + && !Util::is_ccache_executable(real_candidate)) { + return candidate; } } - - // Found it! - return fname; } -#endif } return {}; diff --git a/src/execute.hpp b/src/execute.hpp index 628ceb3..c44a465 100644 --- a/src/execute.hpp +++ b/src/execute.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,10 +18,9 @@ #pragma once -#include "system.hpp" - #include "Fd.hpp" +#include #include class Context; @@ -31,14 +30,15 @@ int execute(Context& ctx, const char* const* argv, Fd&& fd_out, Fd&& fd_err); void execute_noreturn(const char* const* argv, const std::string& temp_dir); // Find an executable named `name` in `$PATH`. Exclude any executables that are -// links to `exclude_name`. +// links to `exclude_path`. std::string find_executable(const Context& ctx, const std::string& name, - const std::string& exclude_name); + const std::string& exclude_path); -std::string find_executable_in_path(const std::string& name, - const std::string& exclude_name, - const std::string& path); +std::string +find_executable_in_path(const std::string& name, + const std::string& path_list, + std::optional exclude_path = std::nullopt); #ifdef _WIN32 std::string win32getshell(const std::string& path); diff --git a/src/fmtmacros.hpp b/src/fmtmacros.hpp index 9a017fa..6f396fd 100644 --- a/src/fmtmacros.hpp +++ b/src/fmtmacros.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,8 +18,8 @@ #pragma once -#include "third_party/fmt/core.h" -#include "third_party/fmt/format.h" +#include +#include // Convenience macro for calling `fmt::format` with `FMT_STRING` around the // format string literal. diff --git a/src/hashutil.cpp b/src/hashutil.cpp index 7378c02..8d7151d 100644 --- a/src/hashutil.cpp +++ b/src/hashutil.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2009-2021 Joel Rosdahl and other contributors +// Copyright (C) 2009-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -23,28 +23,36 @@ #include "Context.hpp" #include "Hash.hpp" #include "Logging.hpp" -#include "Sloppiness.hpp" #include "Stat.hpp" +#include "Util.hpp" +#include "Win32Util.hpp" #include "execute.hpp" -#include "fmtmacros.hpp" #include "macroskip.hpp" -#include "third_party/blake3/blake3_cpu_supports_avx2.h" +#include +#include +#include +#include +#include #ifdef INODE_CACHE_SUPPORTED # include "InodeCache.hpp" #endif -#ifdef _WIN32 -# include "Win32Util.hpp" +#include "third_party/blake3/blake3_cpu_supports_avx2.h" + +#ifdef HAVE_UNISTD_H +# include +#endif + +#ifdef HAVE_SYS_WAIT_H +# include #endif #ifdef HAVE_AVX2 # include #endif -using nonstd::string_view; - namespace { // Returns one of HASH_SOURCE_CODE_FOUND_DATE, HASH_SOURCE_CODE_FOUND_TIME or @@ -53,7 +61,7 @@ namespace { // // Pre-condition: str[pos - 1] == '_' int -check_for_temporal_macros_helper(string_view str, size_t pos) +check_for_temporal_macros_helper(std::string_view str, size_t pos) { if (pos + 7 > str.length()) { return 0; @@ -85,7 +93,7 @@ check_for_temporal_macros_helper(string_view str, size_t pos) } int -check_for_temporal_macros_bmh(string_view str) +check_for_temporal_macros_bmh(std::string_view str) { int result = 0; @@ -111,14 +119,16 @@ check_for_temporal_macros_bmh(string_view str) } #ifdef HAVE_AVX2 -int check_for_temporal_macros_avx2(string_view str) +# ifndef _MSC_VER // MSVC does not need explicit enabling of AVX2. +int check_for_temporal_macros_avx2(std::string_view str) __attribute__((target("avx2"))); +# endif // The following algorithm, which uses AVX2 instructions to find __DATE__, // __TIME__ and __TIMESTAMP__, is heavily inspired by // . int -check_for_temporal_macros_avx2(string_view str) +check_for_temporal_macros_avx2(std::string_view str) { int result = 0; @@ -148,7 +158,13 @@ check_for_temporal_macros_avx2(string_view str) // A bit set in mask now indicates a possible location for a temporal macro. while (mask != 0) { // The start position + 1 (as we know the first char is _). +# ifndef _MSC_VER const auto start = pos + __builtin_ctz(mask) + 1; +# else + unsigned long index; + _BitScanForward(&index, mask); + const auto start = pos + index + 1; +# endif // Clear the least significant bit set. mask = mask & (mask - 1); @@ -164,48 +180,52 @@ check_for_temporal_macros_avx2(string_view str) #endif int -hash_source_code_file_nocache(const Context& ctx, - Hash& hash, - const std::string& path, - size_t size_hint, - bool is_precompiled) +do_hash_file(const Context& ctx, + Digest& digest, + const std::string& path, + size_t size_hint, + bool check_temporal_macros) { - if (is_precompiled) { - if (hash.hash_file(path)) { - return HASH_SOURCE_CODE_OK; - } else { - return HASH_SOURCE_CODE_ERROR; - } - } else { - std::string data; - try { - data = Util::read_file(path, size_hint); - } catch (Error&) { - return HASH_SOURCE_CODE_ERROR; +#ifdef INODE_CACHE_SUPPORTED + const InodeCache::ContentType content_type = + check_temporal_macros ? InodeCache::ContentType::checked_for_temporal_macros + : InodeCache::ContentType::raw; + if (ctx.config.inode_cache()) { + int result; + if (ctx.inode_cache.get(path, content_type, digest, &result)) { + return result; } - int result = hash_source_code_string(ctx, hash, data, path); - return result; } -} +#else + (void)ctx; +#endif -#ifdef INODE_CACHE_SUPPORTED -InodeCache::ContentType -get_content_type(const Config& config, const std::string& path) -{ - if (Util::is_precompiled_header(path)) { - return InodeCache::ContentType::precompiled_header; + const auto data = util::read_file(path, size_hint); + if (!data) { + LOG("Failed to read {}: {}", path, data.error()); + return HASH_SOURCE_CODE_ERROR; } - if (config.sloppiness() & SLOPPY_TIME_MACROS) { - return InodeCache::ContentType::code_with_sloppy_time_macros; + + int result = HASH_SOURCE_CODE_OK; + if (check_temporal_macros) { + result |= check_for_temporal_macros(*data); } - return InodeCache::ContentType::code; -} + + Hash hash; + hash.hash(*data); + digest = hash.digest(); + +#ifdef INODE_CACHE_SUPPORTED + ctx.inode_cache.put(path, content_type, digest, result); #endif + return result; +} + } // namespace int -check_for_temporal_macros(string_view str) +check_for_temporal_macros(std::string_view str) { #ifdef HAVE_AVX2 if (blake3_cpu_supports_avx2()) { @@ -216,27 +236,42 @@ check_for_temporal_macros(string_view str) } int -hash_source_code_string(const Context& ctx, - Hash& hash, - string_view str, - const std::string& path) +hash_source_code_file(const Context& ctx, + Digest& digest, + const std::string& path, + size_t size_hint) { - int result = HASH_SOURCE_CODE_OK; + const bool check_temporal_macros = + !ctx.config.sloppiness().is_enabled(core::Sloppy::time_macros); + int result = + do_hash_file(ctx, digest, path, size_hint, check_temporal_macros); - // Check for __DATE__, __TIME__ and __TIMESTAMP__if the sloppiness - // configuration tells us we should. - if (!(ctx.config.sloppiness() & SLOPPY_TIME_MACROS)) { - result |= check_for_temporal_macros(str); + if (!check_temporal_macros || result == HASH_SOURCE_CODE_OK + || (result & HASH_SOURCE_CODE_ERROR)) { + return result; + } + + if (result & HASH_SOURCE_CODE_FOUND_TIME) { + // We don't know for sure that the program actually uses the __TIME__ macro, + // but we have to assume it anyway and hash the time stamp. However, that's + // not very useful since the chance that we get a cache hit later the same + // second should be quite slim... So, just signal back to the caller that + // __TIME__ has been found so that the direct mode can be disabled. + LOG("Found __TIME__ in {}", path); + return result; } - // Hash the source string. - hash.hash(str); + // __DATE__ or __TIMESTAMP__ found. We now make sure that the digest changes + // if the (potential) expansion of those macros changes by computing a new + // digest comprising the file digest and time information that represents the + // macro expansions. + + Hash hash; + hash.hash(digest.to_string()); if (result & HASH_SOURCE_CODE_FOUND_DATE) { LOG("Found __DATE__ in {}", path); - // Make sure that the hash sum changes if the (potential) expansion of - // __DATE__ changes. hash.hash_delimiter("date"); auto now = Util::localtime(); if (!now) { @@ -255,20 +290,10 @@ hash_source_code_string(const Context& ctx, hash.hash(source_date_epoch); } } - if (result & HASH_SOURCE_CODE_FOUND_TIME) { - // We don't know for sure that the program actually uses the __TIME__ macro, - // but we have to assume it anyway and hash the time stamp. However, that's - // not very useful since the chance that we get a cache hit later the same - // second should be quite slim... So, just signal back to the caller that - // __TIME__ has been found so that the direct mode can be disabled. - LOG("Found __TIME__ in {}", path); - } if (result & HASH_SOURCE_CODE_FOUND_TIMESTAMP) { LOG("Found __TIMESTAMP__ in {}", path); - // Make sure that the hash sum changes if the (potential) expansion of - // __TIMESTAMP__ changes. const auto stat = Stat::stat(path); if (!stat) { return HASH_SOURCE_CODE_ERROR; @@ -291,74 +316,29 @@ hash_source_code_string(const Context& ctx, hash.hash(timestamp); } + digest = hash.digest(); return result; } -int -hash_source_code_file(const Context& ctx, - Hash& hash, - const std::string& path, - size_t size_hint) +bool +hash_binary_file(const Context& ctx, + Digest& digest, + const std::string& path, + size_t size_hint) { -#ifdef INODE_CACHE_SUPPORTED - if (!ctx.config.inode_cache()) { -#endif - return hash_source_code_file_nocache( - ctx, hash, path, size_hint, Util::is_precompiled_header(path)); - -#ifdef INODE_CACHE_SUPPORTED - } - - // Reusable file hashes must be independent of the outer context. Thus hash - // files separately so that digests based on file contents can be reused. Then - // add the digest into the outer hash instead. - InodeCache::ContentType content_type = get_content_type(ctx.config, path); - Digest digest; - int return_value; - if (!ctx.inode_cache.get(path, content_type, digest, &return_value)) { - Hash file_hash; - return_value = hash_source_code_file_nocache( - ctx, - file_hash, - path, - size_hint, - content_type == InodeCache::ContentType::precompiled_header); - if (return_value == HASH_SOURCE_CODE_ERROR) { - return HASH_SOURCE_CODE_ERROR; - } - digest = file_hash.digest(); - ctx.inode_cache.put(path, content_type, digest, return_value); - } - hash.hash(digest.bytes(), Digest::size(), Hash::HashType::binary); - return return_value; -#endif + return do_hash_file(ctx, digest, path, size_hint, false) + == HASH_SOURCE_CODE_OK; } bool hash_binary_file(const Context& ctx, Hash& hash, const std::string& path) { - if (!ctx.config.inode_cache()) { - return hash.hash_file(path); - } - -#ifdef INODE_CACHE_SUPPORTED - // Reusable file hashes must be independent of the outer context. Thus hash - // files separately so that digests based on file contents can be reused. Then - // add the digest into the outer hash instead. Digest digest; - if (!ctx.inode_cache.get(path, InodeCache::ContentType::binary, digest)) { - Hash file_hash; - if (!file_hash.hash_file(path)) { - return false; - } - digest = file_hash.digest(); - ctx.inode_cache.put(path, InodeCache::ContentType::binary, digest); + const bool success = hash_binary_file(ctx, digest, path); + if (success) { + hash.hash(digest.to_string()); } - hash.hash(digest.bytes(), Digest::size(), Hash::HashType::binary); - return true; -#else - return hash.hash_file(path); -#endif + return success; } bool @@ -367,14 +347,14 @@ hash_command_output(Hash& hash, const std::string& compiler) { #ifdef _WIN32 - std::string adjusted_command = Util::strip_whitespace(command); + std::string adjusted_command = util::strip_whitespace(command); // Add "echo" command. bool using_cmd_exe; - if (Util::starts_with(adjusted_command, "echo")) { + if (util::starts_with(adjusted_command, "echo")) { adjusted_command = FMT("cmd.exe /c \"{}\"", adjusted_command); using_cmd_exe = true; - } else if (Util::starts_with(adjusted_command, "%compiler%") + } else if (util::starts_with(adjusted_command, "%compiler%") && compiler == "echo") { adjusted_command = FMT("cmd.exe /c \"{}{}\"", compiler, adjusted_command.substr(10)); @@ -403,7 +383,7 @@ hash_command_output(Hash& hash, STARTUPINFO si; memset(&si, 0x00, sizeof(si)); - std::string path = find_executable_in_path(args[0], "", getenv("PATH")); + std::string path = find_executable_in_path(args[0], getenv("PATH")); if (path.empty()) { path = args[0]; } @@ -444,9 +424,10 @@ hash_command_output(Hash& hash, return false; } int fd = _open_osfhandle((intptr_t)pipe_out[0], O_BINARY); - bool ok = hash.hash_fd(fd); - if (!ok) { - LOG("Error hashing compiler check command output: {}", strerror(errno)); + const auto compiler_check_result = hash.hash_fd(fd); + if (!compiler_check_result) { + LOG("Error hashing compiler check command output: {}", + compiler_check_result.error()); } WaitForSingleObject(pi.hProcess, INFINITE); DWORD exitcode; @@ -458,16 +439,16 @@ hash_command_output(Hash& hash, LOG("Compiler check command returned {}", exitcode); return false; } - return ok; + return bool(compiler_check_result); #else int pipefd[2]; if (pipe(pipefd) == -1) { - throw Fatal("pipe failed: {}", strerror(errno)); + throw core::Fatal(FMT("pipe failed: {}", strerror(errno))); } pid_t pid = fork(); if (pid == -1) { - throw Fatal("fork failed: {}", strerror(errno)); + throw core::Fatal(FMT("fork failed: {}", strerror(errno))); } if (pid == 0) { @@ -481,9 +462,10 @@ hash_command_output(Hash& hash, } else { // Parent. close(pipefd[1]); - bool ok = hash.hash_fd(pipefd[0]); - if (!ok) { - LOG("Error hashing compiler check command output: {}", strerror(errno)); + const auto hash_result = hash.hash_fd(pipefd[0]); + if (!hash_result) { + LOG("Error hashing compiler check command output: {}", + hash_result.error()); } close(pipefd[0]); @@ -500,7 +482,7 @@ hash_command_output(Hash& hash, LOG("Compiler check command returned {}", WEXITSTATUS(status)); return false; } - return ok; + return bool(hash_result); } #endif } diff --git a/src/hashutil.hpp b/src/hashutil.hpp index ae7bcb0..d0fd142 100644 --- a/src/hashutil.hpp +++ b/src/hashutil.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2009-2020 Joel Rosdahl and other contributors +// Copyright (C) 2009-2022 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,14 +18,13 @@ #pragma once -#include "system.hpp" - -#include "third_party/nonstd/string_view.hpp" - +#include #include +#include class Config; class Context; +class Digest; class Hash; const int HASH_SOURCE_CODE_OK = 0; @@ -40,22 +39,26 @@ const int HASH_SOURCE_CODE_FOUND_TIMESTAMP = (1 << 3); // Returns a bitmask with HASH_SOURCE_CODE_FOUND_DATE, // HASH_SOURCE_CODE_FOUND_TIME and HASH_SOURCE_CODE_FOUND_TIMESTAMP set // appropriately. -int check_for_temporal_macros(nonstd::string_view str); - -// Hash a string. Returns a bitmask of HASH_SOURCE_CODE_* results. -int hash_source_code_string(const Context& ctx, - Hash& hash, - nonstd::string_view str, - const std::string& path); +int check_for_temporal_macros(std::string_view str); -// Hash a file ignoring comments. Returns a bitmask of HASH_SOURCE_CODE_* -// results. +// Hash a source code file using the inode cache if enabled. Returns a bitmask +// of HASH_SOURCE_CODE_* results. int hash_source_code_file(const Context& ctx, - Hash& hash, + Digest& digest, const std::string& path, size_t size_hint = 0); -// Hash a binary file using the inode cache if enabled. +// Hash a binary file (using the inode cache if enabled) and put its digest in +// `digest` +// +// Returns true on success, otherwise false. +bool hash_binary_file(const Context& ctx, + Digest& digest, + const std::string& path, + size_t size_hint = 0); + +// Hash a binary file (using the inode cache if enabled) and hash the digest to +// `hash`. // // Returns true on success, otherwise false. bool hash_binary_file(const Context& ctx, Hash& hash, const std::string& path); diff --git a/src/language.hpp b/src/language.hpp index 99bf386..74be7aa 100644 --- a/src/language.hpp +++ b/src/language.hpp @@ -18,8 +18,6 @@ #pragma once -#include "system.hpp" - #include "Config.hpp" #include diff --git a/src/macroskip.hpp b/src/macroskip.hpp index f159323..5cacba8 100644 --- a/src/macroskip.hpp +++ b/src/macroskip.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2010-2020 Joel Rosdahl and other contributors +// Copyright (C) 2010-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,7 +18,7 @@ #pragma once -#include "system.hpp" +#include // A Boyer-Moore-Horspool skip table used for searching for the strings // "__TIME__", "__DATE__" and "__TIMESTAMP__". diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt new file mode 100644 index 0000000..907d16f --- /dev/null +++ b/src/storage/CMakeLists.txt @@ -0,0 +1,12 @@ +add_subdirectory(local) +add_subdirectory(remote) + +set( + sources + Storage.cpp +) + +file(GLOB headers *.hpp) +list(APPEND sources ${headers}) + +target_sources(ccache_framework PRIVATE ${sources}) diff --git a/src/storage/Storage.cpp b/src/storage/Storage.cpp new file mode 100644 index 0000000..5c843db --- /dev/null +++ b/src/storage/Storage.cpp @@ -0,0 +1,545 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "Storage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_REDIS_STORAGE_BACKEND +# include +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace storage { + +const std::unordered_map> + k_remote_storage_implementations = { + {"file", std::make_shared()}, + {"http", std::make_shared()}, +#ifdef HAVE_REDIS_STORAGE_BACKEND + {"redis", std::make_shared()}, + {"redis+unix", std::make_shared()}, +#endif +}; + +std::string +get_features() +{ + std::vector features; + features.reserve(k_remote_storage_implementations.size()); + std::transform(k_remote_storage_implementations.begin(), + k_remote_storage_implementations.end(), + std::back_inserter(features), + [](auto& entry) { return FMT("{}-storage", entry.first); }); + std::sort(features.begin(), features.end()); + return util::join(features, " "); +} + +struct RemoteStorageShardConfig +{ + std::string name; + double weight; +}; + +struct RemoteStorageConfig +{ + std::vector shards; + remote::RemoteStorage::Backend::Params params; + bool read_only = false; +}; + +struct RemoteStorageBackendEntry +{ + Url url; // With expanded "*". + std::string url_for_logging; // With expanded "*". + std::unique_ptr impl; + bool failed = false; +}; + +struct RemoteStorageEntry +{ + RemoteStorageConfig config; + std::string url_for_logging; // With unexpanded "*". + std::shared_ptr storage; + std::vector backends; +}; + +static std::string +to_string(const RemoteStorageConfig& entry) +{ + std::string result = entry.params.url.str(); + for (const auto& attr : entry.params.attributes) { + result += FMT("|{}={}", attr.key, attr.raw_value); + } + return result; +} + +static RemoteStorageConfig +parse_storage_config(const std::string_view entry) +{ + const auto parts = + Util::split_into_views(entry, "|", util::Tokenizer::Mode::include_empty); + + if (parts.empty() || parts.front().empty()) { + throw core::Error( + FMT("remote storage config must provide a URL: {}", entry)); + } + + RemoteStorageConfig result; + const auto url_str = std::string(parts[0]); + result.params.url = url_str; + + // The Url class is parsing the URL object lazily. Check if the URL is valid + // now to avoid exceptions later. + try { + std::ignore = result.params.url.str(); + } catch (const std::exception& e) { + throw core::Error(FMT("Cannot parse URL {}: {}", url_str, e.what())); + } + + if (result.params.url.scheme().empty()) { + throw core::Error(FMT("URL scheme must not be empty: {}", entry)); + } + + for (size_t i = 1; i < parts.size(); ++i) { + if (parts[i].empty()) { + continue; + } + const auto [key, right_hand_side] = util::split_once(parts[i], '='); + const auto& raw_value = right_hand_side.value_or("true"); + const auto value = + util::value_or_throw(util::percent_decode(raw_value)); + if (key == "read-only") { + result.read_only = (value == "true"); + } else if (key == "shards") { + if (url_str.find('*') == std::string::npos) { + throw core::Error( + FMT(R"(Missing "*" in URL when using shards: "{}")", url_str)); + } + for (const auto& shard : util::Tokenizer(value, ",")) { + double weight = 1.0; + std::string_view name; + const auto lp_pos = shard.find('('); + if (lp_pos != std::string_view::npos) { + if (shard.back() != ')') { + throw core::Error(FMT("Invalid shard name: \"{}\"", shard)); + } + weight = + util::value_or_throw(util::parse_double(std::string( + shard.substr(lp_pos + 1, shard.length() - lp_pos - 2)))); + if (weight < 0.0) { + throw core::Error(FMT("Invalid shard weight: \"{}\"", weight)); + } + name = shard.substr(0, lp_pos); + } else { + name = shard; + } + + result.shards.push_back({std::string(name), weight}); + } + } + + result.params.attributes.push_back( + {std::string(key), value, std::string(raw_value)}); + } + + return result; +} + +static std::vector +parse_storage_configs(const std::string_view& configs) +{ + std::vector result; + for (const auto& config : util::Tokenizer(configs, " ")) { + result.push_back(parse_storage_config(config)); + } + return result; +} + +static std::shared_ptr +get_storage(const Url& url) +{ + const auto it = k_remote_storage_implementations.find(url.scheme()); + if (it != k_remote_storage_implementations.end()) { + return it->second; + } else { + return {}; + } +} + +Storage::Storage(const Config& config) : local(config), m_config(config) +{ +} + +// Define the destructor in the implementation file to avoid having to declare +// RemoteStorageEntry and its constituents in the header file. +// NOLINTNEXTLINE(modernize-use-equals-default) +Storage::~Storage() +{ +} + +void +Storage::initialize() +{ + add_remote_storages(); +} + +void +Storage::finalize() +{ + local.finalize(); +} + +void +Storage::get(const Digest& key, + const core::CacheEntryType type, + const EntryReceiver& entry_receiver) +{ + MTR_SCOPE("storage", "get"); + + if (!m_config.remote_only()) { + auto value = local.get(key, type); + if (value) { + if (m_config.reshare()) { + put_in_remote_storage(key, *value, true); + } + if (entry_receiver(std::move(*value))) { + return; + } + } + } + + get_from_remote_storage(key, type, [&](util::Bytes&& data) { + if (!m_config.remote_only()) { + local.put(key, type, data, true); + } + return entry_receiver(std::move(data)); + }); +} + +void +Storage::put(const Digest& key, + const core::CacheEntryType type, + nonstd::span value) +{ + MTR_SCOPE("storage", "put"); + + if (!m_config.remote_only()) { + local.put(key, type, value); + } + put_in_remote_storage(key, value, false); +} + +void +Storage::remove(const Digest& key, const core::CacheEntryType type) +{ + MTR_SCOPE("storage", "remove"); + + if (!m_config.remote_only()) { + local.remove(key, type); + } + remove_from_remote_storage(key); +} + +bool +Storage::has_remote_storage() const +{ + return !m_remote_storages.empty(); +} + +std::string +Storage::get_remote_storage_config_for_logging() const +{ + auto configs = parse_storage_configs(m_config.remote_storage()); + for (auto& config : configs) { + const auto storage = get_storage(config.params.url); + if (storage) { + storage->redact_secrets(config.params); + } + } + return util::join(configs, " "); +} + +static void +redact_url_for_logging(Url& url_for_logging) +{ + url_for_logging.user_info(""); +} + +void +Storage::add_remote_storages() +{ + const auto configs = parse_storage_configs(m_config.remote_storage()); + for (const auto& config : configs) { + auto url_for_logging = config.params.url; + redact_url_for_logging(url_for_logging); + const auto storage = get_storage(config.params.url); + if (!storage) { + throw core::Error( + FMT("unknown remote storage URL: {}", url_for_logging.str())); + } + m_remote_storages.push_back(std::make_unique( + RemoteStorageEntry{config, url_for_logging.str(), storage, {}})); + } +} + +void +Storage::mark_backend_as_failed( + RemoteStorageBackendEntry& backend_entry, + const remote::RemoteStorage::Backend::Failure failure) +{ + // The backend is expected to log details about the error. + backend_entry.failed = true; + local.increment_statistic( + failure == remote::RemoteStorage::Backend::Failure::timeout + ? core::Statistic::remote_storage_timeout + : core::Statistic::remote_storage_error); +} + +static double +to_half_open_unit_interval(uint64_t value) +{ + constexpr uint8_t double_significand_bits = 53; + constexpr uint64_t denominator = 1ULL << double_significand_bits; + constexpr uint64_t mask = denominator - 1; + return static_cast(value & mask) / denominator; +} + +static Url +get_shard_url(const Digest& key, + const std::string& url, + const std::vector& shards) +{ + ASSERT(!shards.empty()); + + // This is the "weighted rendezvous hashing" algorithm. + double highest_score = -1.0; + std::string best_shard; + for (const auto& shard_config : shards) { + util::XXH3_64 hash; + hash.update(key.bytes(), key.size()); + hash.update(shard_config.name.data(), shard_config.name.length()); + const double score = to_half_open_unit_interval(hash.digest()); + ASSERT(score >= 0.0 && score < 1.0); + const double weighted_score = + score == 0.0 ? 0.0 : shard_config.weight / -std::log(score); + if (weighted_score > highest_score) { + best_shard = shard_config.name; + highest_score = weighted_score; + } + } + + return util::replace_first(url, "*", best_shard); +} + +RemoteStorageBackendEntry* +Storage::get_backend(RemoteStorageEntry& entry, + const Digest& key, + const std::string_view operation_description, + const bool for_writing) +{ + if (for_writing && entry.config.read_only) { + LOG("Not {} {} since it is read-only", + operation_description, + entry.url_for_logging); + return nullptr; + } + + const auto shard_url = + entry.config.shards.empty() + ? entry.config.params.url + : get_shard_url(key, entry.config.params.url.str(), entry.config.shards); + auto backend = + std::find_if(entry.backends.begin(), + entry.backends.end(), + [&](const auto& x) { return x.url.str() == shard_url.str(); }); + + if (backend == entry.backends.end()) { + auto shard_url_for_logging = shard_url; + redact_url_for_logging(shard_url_for_logging); + entry.backends.push_back( + {shard_url, shard_url_for_logging.str(), {}, false}); + auto shard_params = entry.config.params; + shard_params.url = shard_url; + try { + entry.backends.back().impl = entry.storage->create_backend(shard_params); + } catch (const remote::RemoteStorage::Backend::Failed& e) { + LOG("Failed to construct backend for {}{}", + entry.url_for_logging, + std::string_view(e.what()).empty() ? "" : FMT(": {}", e.what())); + mark_backend_as_failed(entry.backends.back(), e.failure()); + return nullptr; + } + return &entry.backends.back(); + } else if (backend->failed) { + LOG("Not {} {} since it failed earlier", + operation_description, + entry.url_for_logging); + return nullptr; + } else { + return &*backend; + } +} + +void +Storage::get_from_remote_storage(const Digest& key, + const core::CacheEntryType type, + const EntryReceiver& entry_receiver) +{ + MTR_SCOPE("remote_storage", "get"); + + for (const auto& entry : m_remote_storages) { + auto backend = get_backend(*entry, key, "getting from", false); + if (!backend) { + continue; + } + + Timer timer; + auto result = backend->impl->get(key); + const auto ms = timer.measure_ms(); + if (!result) { + mark_backend_as_failed(*backend, result.error()); + continue; + } + + auto& value = *result; + if (value) { + LOG("Retrieved {} from {} ({:.2f} ms)", + key.to_string(), + backend->url_for_logging, + ms); + local.increment_statistic(core::Statistic::remote_storage_read_hit); + if (type == core::CacheEntryType::result) { + local.increment_statistic(core::Statistic::remote_storage_hit); + } + if (entry_receiver(std::move(*value))) { + return; + } + } else { + LOG("No {} in {} ({:.2f} ms)", + key.to_string(), + backend->url_for_logging, + ms); + local.increment_statistic(core::Statistic::remote_storage_read_miss); + if (type == core::CacheEntryType::result) { + local.increment_statistic(core::Statistic::remote_storage_miss); + } + } + } +} + +void +Storage::put_in_remote_storage(const Digest& key, + nonstd::span value, + bool only_if_missing) +{ + MTR_SCOPE("remote_storage", "put"); + + if (!core::CacheEntry::Header(value).self_contained) { + LOG("Not putting {} in remote storage since it's not self-contained", + key.to_string()); + return; + } + + for (const auto& entry : m_remote_storages) { + auto backend = get_backend(*entry, key, "putting in", true); + if (!backend) { + continue; + } + + Timer timer; + const auto result = backend->impl->put(key, value, only_if_missing); + const auto ms = timer.measure_ms(); + if (!result) { + // The backend is expected to log details about the error. + mark_backend_as_failed(*backend, result.error()); + continue; + } + + const bool stored = *result; + LOG("{} {} in {} ({:.2f} ms)", + stored ? "Stored" : "Did not have to store", + key.to_string(), + backend->url_for_logging, + ms); + local.increment_statistic(core::Statistic::remote_storage_write); + } +} + +void +Storage::remove_from_remote_storage(const Digest& key) +{ + MTR_SCOPE("remote_storage", "remove"); + + for (const auto& entry : m_remote_storages) { + auto backend = get_backend(*entry, key, "removing from", true); + if (!backend) { + continue; + } + + Timer timer; + const auto result = backend->impl->remove(key); + const auto ms = timer.measure_ms(); + if (!result) { + mark_backend_as_failed(*backend, result.error()); + continue; + } + + const bool removed = *result; + if (removed) { + LOG("Removed {} from {} ({:.2f} ms)", + key.to_string(), + backend->url_for_logging, + ms); + } else { + LOG("No {} to remove from {} ({:.2f} ms)", + key.to_string(), + backend->url_for_logging, + ms); + } + + local.increment_statistic(core::Statistic::remote_storage_write); + } +} + +} // namespace storage diff --git a/src/storage/Storage.hpp b/src/storage/Storage.hpp new file mode 100644 index 0000000..1b72106 --- /dev/null +++ b/src/storage/Storage.hpp @@ -0,0 +1,95 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +class Digest; + +namespace storage { + +std::string get_features(); + +struct RemoteStorageBackendEntry; +struct RemoteStorageEntry; + +class Storage +{ +public: + Storage(const Config& config); + ~Storage(); + + void initialize(); + void finalize(); + + local::LocalStorage local; + + using EntryReceiver = std::function; + + void get(const Digest& key, + core::CacheEntryType type, + const EntryReceiver& entry_receiver); + + void put(const Digest& key, + core::CacheEntryType type, + nonstd::span value); + + void remove(const Digest& key, core::CacheEntryType type); + + bool has_remote_storage() const; + std::string get_remote_storage_config_for_logging() const; + +private: + const Config& m_config; + std::vector> m_remote_storages; + + void add_remote_storages(); + + void mark_backend_as_failed(RemoteStorageBackendEntry& backend_entry, + remote::RemoteStorage::Backend::Failure failure); + + RemoteStorageBackendEntry* get_backend(RemoteStorageEntry& entry, + const Digest& key, + std::string_view operation_description, + const bool for_writing); + + void get_from_remote_storage(const Digest& key, + core::CacheEntryType type, + const EntryReceiver& entry_receiver); + + void put_in_remote_storage(const Digest& key, + nonstd::span value, + bool only_if_missing); + + void remove_from_remote_storage(const Digest& key); +}; + +} // namespace storage diff --git a/src/storage/local/CMakeLists.txt b/src/storage/local/CMakeLists.txt new file mode 100644 index 0000000..7b0d4f0 --- /dev/null +++ b/src/storage/local/CMakeLists.txt @@ -0,0 +1,15 @@ +set( + sources + CacheFile.cpp + LocalStorage.cpp + LocalStorage_cleanup.cpp + LocalStorage_compress.cpp + LocalStorage_statistics.cpp + StatsFile.cpp + util.cpp +) + +file(GLOB headers *.hpp) +list(APPEND sources ${headers}) + +target_sources(ccache_framework PRIVATE ${sources}) diff --git a/src/storage/local/CacheFile.cpp b/src/storage/local/CacheFile.cpp new file mode 100644 index 0000000..a2716e8 --- /dev/null +++ b/src/storage/local/CacheFile.cpp @@ -0,0 +1,47 @@ +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "CacheFile.hpp" + +#include +#include +#include + +const Stat& +CacheFile::lstat() const +{ + if (!m_stat) { + m_stat = Stat::lstat(m_path); + } + + return *m_stat; +} + +CacheFile::Type +CacheFile::type() const +{ + if (util::ends_with(m_path, "M")) { + return Type::manifest; + } else if (util::ends_with(m_path, "R")) { + return Type::result; + } else if (util::ends_with(m_path, "W")) { + return Type::raw; + } else { + return Type::unknown; + } +} diff --git a/src/storage/local/CacheFile.hpp b/src/storage/local/CacheFile.hpp new file mode 100644 index 0000000..e653363 --- /dev/null +++ b/src/storage/local/CacheFile.hpp @@ -0,0 +1,50 @@ +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include +#include + +class CacheFile +{ +public: + enum class Type { result, manifest, raw, unknown }; + + explicit CacheFile(const std::string& path); + + const Stat& lstat() const; + const std::string& path() const; + Type type() const; + +private: + std::string m_path; + mutable std::optional m_stat; +}; + +inline CacheFile::CacheFile(const std::string& path) : m_path(path) +{ +} + +inline const std::string& +CacheFile::path() const +{ + return m_path; +} diff --git a/src/storage/local/LocalStorage.cpp b/src/storage/local/LocalStorage.cpp new file mode 100644 index 0000000..001e2a9 --- /dev/null +++ b/src/storage/local/LocalStorage.cpp @@ -0,0 +1,496 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "LocalStorage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + +using core::Statistic; + +namespace storage::local { + +// How often (in seconds) to scan $CCACHE_DIR/tmp for left-over temporary +// files. +const util::Duration k_tempdir_cleanup_interval(2 * 24 * 60 * 60); // 2 days + +// Maximum files per cache directory. This constant is somewhat arbitrarily +// chosen to be large enough to avoid unnecessary cache levels but small enough +// not to make it too slow for legacy file systems with bad performance for +// large directories. It could be made configurable, but hopefully there will be +// no need to do that. +const uint64_t k_max_cache_files_per_directory = 2000; + +// Minimum number of cache levels ($CCACHE_DIR/1/2/stored_file). +const uint8_t k_min_cache_levels = 2; + +// Maximum number of cache levels ($CCACHE_DIR/1/2/3/stored_file). +// +// On a cache miss, (k_max_cache_levels - k_min_cache_levels + 1) cache lookups +// (i.e. stat system calls) will be performed for a cache entry. +// +// An assumption made here is that if a cache is so large that it holds more +// than 16^4 * k_max_cache_files_per_directory files then we can assume that the +// file system is sane enough to handle more than +// k_max_cache_files_per_directory. +const uint8_t k_max_cache_levels = 4; + +static std::string +suffix_from_type(const core::CacheEntryType type) +{ + switch (type) { + case core::CacheEntryType::manifest: + return "M"; + + case core::CacheEntryType::result: + return "R"; + } + + ASSERT(false); +} + +static uint8_t +calculate_wanted_cache_level(const uint64_t files_in_level_1) +{ + uint64_t files_per_directory = files_in_level_1 / 16; + for (uint8_t i = k_min_cache_levels; i <= k_max_cache_levels; ++i) { + if (files_per_directory < k_max_cache_files_per_directory) { + return i; + } + files_per_directory /= 16; + } + return k_max_cache_levels; +} + +LocalStorage::LocalStorage(const Config& config) : m_config(config) +{ +} + +void +LocalStorage::finalize() +{ + if (m_config.temporary_dir() == m_config.default_temporary_dir()) { + clean_internal_tempdir(); + } + + if (!m_config.stats()) { + return; + } + + if (m_manifest_key) { + // A manifest entry was written. + ASSERT(!m_manifest_path.empty()); + update_stats_and_maybe_move_cache_file(*m_manifest_key, + m_manifest_path, + m_manifest_counter_updates, + core::CacheEntryType::manifest); + } + + if (!m_result_key) { + // No result entry was written, so we just choose one of the stats files in + // the 256 level 2 directories. + + ASSERT(m_result_counter_updates.get(Statistic::cache_size_kibibyte) == 0); + ASSERT(m_result_counter_updates.get(Statistic::files_in_cache) == 0); + + const auto bucket = getpid() % 256; + const auto stats_file = + FMT("{}/{:x}/{:x}/stats", m_config.cache_dir(), bucket / 16, bucket % 16); + StatsFile(stats_file).update([&](auto& cs) { + cs.increment(m_result_counter_updates); + }); + return; + } + + ASSERT(!m_result_path.empty()); + + const auto counters = + update_stats_and_maybe_move_cache_file(*m_result_key, + m_result_path, + m_result_counter_updates, + core::CacheEntryType::result); + if (!counters) { + return; + } + + const auto subdir = + FMT("{}/{:x}", m_config.cache_dir(), m_result_key->bytes()[0] >> 4); + bool need_cleanup = false; + + if (m_config.max_files() != 0 + && counters->get(Statistic::files_in_cache) > m_config.max_files() / 16) { + LOG("Need to clean up {} since it holds {} files (limit: {} files)", + subdir, + counters->get(Statistic::files_in_cache), + m_config.max_files() / 16); + need_cleanup = true; + } + if (m_config.max_size() != 0 + && counters->get(Statistic::cache_size_kibibyte) + > m_config.max_size() / 1024 / 16) { + LOG("Need to clean up {} since it holds {} KiB (limit: {} KiB)", + subdir, + counters->get(Statistic::cache_size_kibibyte), + m_config.max_size() / 1024 / 16); + need_cleanup = true; + } + + if (need_cleanup) { + const double factor = m_config.limit_multiple() / 16; + const uint64_t max_size = round(m_config.max_size() * factor); + const uint32_t max_files = round(m_config.max_files() * factor); + clean_dir(subdir, + max_size, + max_files, + std::nullopt, + std::nullopt, + [](double /*progress*/) {}); + } +} + +std::optional +LocalStorage::get(const Digest& key, const core::CacheEntryType type) +{ + MTR_SCOPE("local_storage", "get"); + + std::optional return_value; + + const auto cache_file = look_up_cache_file(key, type); + if (cache_file.stat) { + const auto value = util::read_file(cache_file.path); + if (value) { + LOG("Retrieved {} from local storage ({})", + key.to_string(), + cache_file.path); + + // Update modification timestamp to save file from LRU cleanup. + util::set_timestamps(cache_file.path); + + return_value = *value; + } else { + LOG("Failed to read {}: {}", cache_file.path, value.error()); + } + } else { + LOG("No {} in local storage", key.to_string()); + } + + increment_statistic(return_value ? core::Statistic::local_storage_read_hit + : core::Statistic::local_storage_read_miss); + if (type == core::CacheEntryType::result) { + increment_statistic(return_value ? core::Statistic::local_storage_hit + : core::Statistic::local_storage_miss); + } + + return return_value; +} + +void +LocalStorage::put(const Digest& key, + const core::CacheEntryType type, + nonstd::span value, + bool only_if_missing) +{ + MTR_SCOPE("local_storage", "put"); + + const auto cache_file = look_up_cache_file(key, type); + if (only_if_missing && cache_file.stat) { + LOG("Not storing {} in local storage since it already exists", + cache_file.path); + return; + } + + switch (type) { + case core::CacheEntryType::manifest: + m_manifest_key = key; + m_manifest_path = cache_file.path; + break; + + case core::CacheEntryType::result: + m_result_key = key; + m_result_path = cache_file.path; + break; + } + + try { + increment_statistic(core::Statistic::local_storage_write); + AtomicFile result_file(cache_file.path, AtomicFile::Mode::binary); + result_file.write(value); + result_file.commit(); + } catch (core::Error& e) { + LOG("Failed to write to {}: {}", cache_file.path, e.what()); + return; + } + + const auto new_stat = Stat::stat(cache_file.path, Stat::OnError::log); + if (!new_stat) { + LOG("Failed to stat {}: {}", cache_file.path, strerror(errno)); + return; + } + + LOG("Stored {} in local storage ({})", key.to_string(), cache_file.path); + + auto& counter_updates = (type == core::CacheEntryType::manifest) + ? m_manifest_counter_updates + : m_result_counter_updates; + counter_updates.increment( + Statistic::cache_size_kibibyte, + Util::size_change_kibibyte(cache_file.stat, new_stat)); + counter_updates.increment(Statistic::files_in_cache, cache_file.stat ? 0 : 1); + + // Make sure we have a CACHEDIR.TAG in the cache part of cache_dir. This can + // be done almost anywhere, but we might as well do it near the end as we save + // the stat call if we exit early. + util::create_cachedir_tag( + FMT("{}/{}", m_config.cache_dir(), key.to_string()[0])); +} + +void +LocalStorage::remove(const Digest& key, const core::CacheEntryType type) +{ + MTR_SCOPE("local_storage", "remove"); + + const auto cache_file = look_up_cache_file(key, type); + if (cache_file.stat) { + increment_statistic(core::Statistic::local_storage_write); + Util::unlink_safe(cache_file.path); + LOG("Removed {} from local storage ({})", key.to_string(), cache_file.path); + } else { + LOG("No {} to remove from local storage", key.to_string()); + } +} + +std::string +LocalStorage::get_raw_file_path(std::string_view result_path, + uint8_t file_number) +{ + if (file_number >= 10) { + // To support more entries in the future, encode to [0-9a-z]. Note that + // LocalStorage::evict currently assumes that the entry number is + // represented as one character. + throw core::Error(FMT("Too high raw file entry number: {}", file_number)); + } + + const auto prefix = result_path.substr(0, result_path.length() - 1); + return FMT("{}{}W", prefix, file_number); +} + +std::string +LocalStorage::get_raw_file_path(const Digest& result_key, + uint8_t file_number) const +{ + const auto cache_file = + look_up_cache_file(result_key, core::CacheEntryType::result); + return get_raw_file_path(cache_file.path, file_number); +} + +void +LocalStorage::put_raw_files( + const Digest& key, + const std::vector raw_files) +{ + const auto cache_file = look_up_cache_file(key, core::CacheEntryType::result); + Util::ensure_dir_exists(Util::dir_name(cache_file.path)); + + for (auto [file_number, source_path] : raw_files) { + const auto dest_path = get_raw_file_path(cache_file.path, file_number); + const auto old_stat = Stat::stat(dest_path); + try { + Util::clone_hard_link_or_copy_file( + m_config, source_path, dest_path, true); + m_added_raw_files.push_back(dest_path); + } catch (core::Error& e) { + LOG("Failed to store {} as raw file {}: {}", + source_path, + dest_path, + e.what()); + throw; + } + const auto new_stat = Stat::stat(dest_path); + increment_statistic(Statistic::cache_size_kibibyte, + Util::size_change_kibibyte(old_stat, new_stat)); + increment_statistic(Statistic::files_in_cache, + (new_stat ? 1 : 0) - (old_stat ? 1 : 0)); + } +} + +void +LocalStorage::increment_statistic(const Statistic statistic, + const int64_t value) +{ + m_result_counter_updates.increment(statistic, value); +} + +void +LocalStorage::increment_statistics(const core::StatisticsCounters& statistics) +{ + m_result_counter_updates.increment(statistics); +} + +// Private methods + +LocalStorage::LookUpCacheFileResult +LocalStorage::look_up_cache_file(const Digest& key, + const core::CacheEntryType type) const +{ + const auto key_string = FMT("{}{}", key.to_string(), suffix_from_type(type)); + + for (uint8_t level = k_min_cache_levels; level <= k_max_cache_levels; + ++level) { + const auto path = get_path_in_cache(level, key_string); + const auto stat = Stat::stat(path); + if (stat) { + return {path, stat, level}; + } + } + + const auto shallowest_path = + get_path_in_cache(k_min_cache_levels, key_string); + return {shallowest_path, Stat(), k_min_cache_levels}; +} + +void +LocalStorage::clean_internal_tempdir() +{ + MTR_SCOPE("local_storage", "clean_internal_tempdir"); + + const auto now = util::TimePoint::now(); + const auto cleaned_stamp = FMT("{}/.cleaned", m_config.temporary_dir()); + const auto cleaned_stat = Stat::stat(cleaned_stamp); + if (cleaned_stat + && cleaned_stat.mtime() + k_tempdir_cleanup_interval >= now) { + // No cleanup needed. + return; + } + + LOG("Cleaning up {}", m_config.temporary_dir()); + Util::ensure_dir_exists(m_config.temporary_dir()); + Util::traverse(m_config.temporary_dir(), + [now](const std::string& path, bool is_dir) { + if (is_dir) { + return; + } + const auto st = Stat::lstat(path, Stat::OnError::log); + if (st && st.mtime() + k_tempdir_cleanup_interval < now) { + Util::unlink_tmp(path); + } + }); + + util::write_file(cleaned_stamp, ""); +} + +std::optional +LocalStorage::update_stats_and_maybe_move_cache_file( + const Digest& key, + const std::string& current_path, + const core::StatisticsCounters& counter_updates, + const core::CacheEntryType type) +{ + if (counter_updates.all_zero()) { + return std::nullopt; + } + + // Use stats file in the level one subdirectory for cache bookkeeping counters + // since cleanup is performed on level one. Use stats file in the level two + // subdirectory for other counters to reduce lock contention. + const bool use_stats_on_level_1 = + counter_updates.get(Statistic::cache_size_kibibyte) != 0 + || counter_updates.get(Statistic::files_in_cache) != 0; + std::string level_string = FMT("{:x}", key.bytes()[0] >> 4); + if (!use_stats_on_level_1) { + level_string += FMT("/{:x}", key.bytes()[0] & 0xF); + } + + const auto stats_file = + FMT("{}/{}/stats", m_config.cache_dir(), level_string); + auto counters = StatsFile(stats_file).update([&counter_updates](auto& cs) { + cs.increment(counter_updates); + }); + if (!counters) { + return std::nullopt; + } + + if (use_stats_on_level_1) { + // Only consider moving the cache file to another level when we have read + // the level 1 stats file since it's only then we know the proper + // files_in_cache value. + const auto wanted_level = + calculate_wanted_cache_level(counters->get(Statistic::files_in_cache)); + const auto wanted_path = + get_path_in_cache(wanted_level, key.to_string() + suffix_from_type(type)); + if (current_path != wanted_path) { + Util::ensure_dir_exists(Util::dir_name(wanted_path)); + LOG("Moving {} to {}", current_path, wanted_path); + try { + Util::rename(current_path, wanted_path); + } catch (const core::Error&) { + // Two ccache processes may move the file at the same time, so failure + // to rename is OK. + } + for (const auto& raw_file : m_added_raw_files) { + try { + Util::rename(raw_file, + FMT("{}/{}", + Util::dir_name(wanted_path), + Util::base_name(raw_file))); + } catch (const core::Error&) { + // Two ccache processes may move the file at the same time, so failure + // to rename is OK. + } + } + } + } + return counters; +} + +std::string +LocalStorage::get_path_in_cache(const uint8_t level, + const std::string_view name) const +{ + ASSERT(level >= 1 && level <= 8); + ASSERT(name.length() >= level); + + std::string path(m_config.cache_dir()); + path.reserve(path.size() + level * 2 + 1 + name.length() - level); + + for (uint8_t i = 0; i < level; ++i) { + path.push_back('/'); + path.push_back(name.at(i)); + } + + path.push_back('/'); + const std::string_view name_remaining = name.substr(level); + path.append(name_remaining.data(), name_remaining.length()); + + return path; +} + +} // namespace storage::local diff --git a/src/storage/local/LocalStorage.hpp b/src/storage/local/LocalStorage.hpp new file mode 100644 index 0000000..6251b3e --- /dev/null +++ b/src/storage/local/LocalStorage.hpp @@ -0,0 +1,171 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +class Config; + +namespace storage { +namespace local { + +struct CompressionStatistics +{ + uint64_t compr_size; + uint64_t content_size; + uint64_t incompr_size; + uint64_t on_disk_size; +}; + +class LocalStorage +{ +public: + LocalStorage(const Config& config); + + void finalize(); + + // --- Cache entry handling --- + + std::optional get(const Digest& key, core::CacheEntryType type); + + void put(const Digest& key, + core::CacheEntryType type, + nonstd::span value, + bool only_if_missing = false); + + void remove(const Digest& key, core::CacheEntryType type); + + static std::string get_raw_file_path(std::string_view result_path, + uint8_t file_number); + std::string get_raw_file_path(const Digest& result_key, + uint8_t file_number) const; + + void + put_raw_files(const Digest& key, + const std::vector raw_files); + + // --- Statistics --- + + void increment_statistic(core::Statistic statistic, int64_t value = 1); + void increment_statistics(const core::StatisticsCounters& statistics); + + const core::StatisticsCounters& get_statistics_updates() const; + + // Zero all statistics counters except those tracking cache size and number of + // files in the cache. + void zero_all_statistics(); + + // Get statistics and last time of update for the whole local storage cache. + std::pair + get_all_statistics() const; + + // --- Cleanup --- + + void evict(const ProgressReceiver& progress_receiver, + std::optional max_age, + std::optional namespace_); + + void clean_all(const ProgressReceiver& progress_receiver); + + void wipe_all(const ProgressReceiver& progress_receiver); + + // --- Compression --- + + CompressionStatistics + get_compression_statistics(const ProgressReceiver& progress_receiver) const; + + void recompress(std::optional level, + const ProgressReceiver& progress_receiver); + +private: + const Config& m_config; + + // Main statistics updates (result statistics and size/count change for result + // file) which get written into the statistics file belonging to the result + // file. + core::StatisticsCounters m_result_counter_updates; + + // Statistics updates (only for manifest size/count change) which get written + // into the statistics file belonging to the manifest. + core::StatisticsCounters m_manifest_counter_updates; + + // The manifest and result keys and paths are stored by put() so that + // finalize() can use them to move the files in place. + std::optional m_manifest_key; + std::optional m_result_key; + std::string m_manifest_path; + std::string m_result_path; + + std::vector m_added_raw_files; + + struct LookUpCacheFileResult + { + std::string path; + Stat stat; + uint8_t level; + }; + + LookUpCacheFileResult look_up_cache_file(const Digest& key, + core::CacheEntryType type) const; + + void clean_internal_tempdir(); + + std::optional + update_stats_and_maybe_move_cache_file( + const Digest& key, + const std::string& current_path, + const core::StatisticsCounters& counter_updates, + core::CacheEntryType type); + + // Join the cache directory, a '/' and `name` into a single path and return + // it. Additionally, `level` single-character, '/'-separated subpaths are + // split from the beginning of `name` before joining them all. + std::string get_path_in_cache(uint8_t level, std::string_view name) const; + + static void clean_dir(const std::string& subdir, + uint64_t max_size, + uint64_t max_files, + std::optional max_age, + std::optional namespace_, + const ProgressReceiver& progress_receiver); +}; + +// --- Inline implementations --- + +inline const core::StatisticsCounters& +LocalStorage::get_statistics_updates() const +{ + return m_result_counter_updates; +} + +} // namespace local +} // namespace storage diff --git a/src/storage/local/LocalStorage_cleanup.cpp b/src/storage/local/LocalStorage_cleanup.cpp new file mode 100644 index 0000000..3039e83 --- /dev/null +++ b/src/storage/local/LocalStorage_cleanup.cpp @@ -0,0 +1,275 @@ +// Copyright (C) 2002-2006 Andrew Tridgell +// Copyright (C) 2009-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "LocalStorage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef INODE_CACHE_SUPPORTED +# include +#endif + +#include + +using core::Statistic; + +namespace storage::local { + +static void +delete_file(const std::string& path, + const uint64_t size, + uint64_t* cache_size, + uint64_t* files_in_cache) +{ + const bool deleted = Util::unlink_safe(path, Util::UnlinkLog::ignore_failure); + if (!deleted && errno != ENOENT && errno != ESTALE) { + LOG("Failed to unlink {} ({})", path, strerror(errno)); + } else if (cache_size && files_in_cache) { + // The counters are intentionally subtracted even if there was no file to + // delete since the final cache size calculation will be incorrect if they + // aren't. (This can happen when there are several parallel ongoing + // cleanups of the same directory.) + *cache_size -= size; + --*files_in_cache; + } +} + +static void +update_counters(const std::string& dir, + const uint64_t files_in_cache, + const uint64_t cache_size, + const bool cleanup_performed) +{ + const std::string stats_file = dir + "/stats"; + StatsFile(stats_file).update([=](auto& cs) { + if (cleanup_performed) { + cs.increment(Statistic::cleanups_performed); + } + cs.set(Statistic::files_in_cache, files_in_cache); + cs.set(Statistic::cache_size_kibibyte, cache_size / 1024); + }); +} + +void +LocalStorage::evict(const ProgressReceiver& progress_receiver, + std::optional max_age, + std::optional namespace_) +{ + for_each_level_1_subdir( + m_config.cache_dir(), + [&](const std::string& subdir, + const ProgressReceiver& sub_progress_receiver) { + clean_dir(subdir, 0, 0, max_age, namespace_, sub_progress_receiver); + }, + progress_receiver); +} + +// Clean up one cache subdirectory. +void +LocalStorage::clean_dir(const std::string& subdir, + const uint64_t max_size, + const uint64_t max_files, + const std::optional max_age, + const std::optional namespace_, + const ProgressReceiver& progress_receiver) +{ + LOG("Cleaning up cache directory {}", subdir); + + std::vector files = get_level_1_files( + subdir, [&](double progress) { progress_receiver(progress / 3); }); + + uint64_t cache_size = 0; + uint64_t files_in_cache = 0; + auto current_time = util::TimePoint::now(); + std::unordered_map /*associated_raw_files*/> + raw_files_map; + + for (size_t i = 0; i < files.size(); + ++i, progress_receiver(1.0 / 3 + 1.0 * i / files.size() / 3)) { + const auto& file = files[i]; + + if (!file.lstat().is_regular()) { + // Not a file or missing file. + continue; + } + + // Delete any tmp files older than 1 hour right away. + if (file.lstat().mtime() + util::Duration(3600) < current_time + && TemporaryFile::is_tmp_file(file.path())) { + Util::unlink_tmp(file.path()); + continue; + } + + if (namespace_ && file.type() == CacheFile::Type::raw) { + const auto result_filename = + FMT("{}R", file.path().substr(0, file.path().length() - 2)); + raw_files_map[result_filename].push_back(file.path()); + } + + cache_size += file.lstat().size_on_disk(); + files_in_cache += 1; + } + + // Sort according to modification time, oldest first. + std::sort(files.begin(), files.end(), [](const auto& f1, const auto& f2) { + return f1.lstat().mtime() < f2.lstat().mtime(); + }); + + LOG("Before cleanup: {:.0f} KiB, {:.0f} files", + static_cast(cache_size) / 1024, + static_cast(files_in_cache)); + + bool cleaned = false; + for (size_t i = 0; i < files.size(); + ++i, progress_receiver(2.0 / 3 + 1.0 * i / files.size() / 3)) { + const auto& file = files[i]; + + if (!file.lstat() || file.lstat().is_directory()) { + continue; + } + + if ((max_size == 0 || cache_size <= max_size) + && (max_files == 0 || files_in_cache <= max_files) + && (!max_age + || file.lstat().mtime() > (current_time - util::Duration(*max_age))) + && (!namespace_ || max_age)) { + break; + } + + if (namespace_) { + try { + core::CacheEntry::Header header(file.path()); + if (header.namespace_ != *namespace_) { + continue; + } + } catch (core::Error&) { + // Failed to read header: ignore. + continue; + } + + // For namespace eviction we need to remove raw files based on result + // filename since they don't have a header. + if (file.type() == CacheFile::Type::result) { + const auto entry = raw_files_map.find(file.path()); + if (entry != raw_files_map.end()) { + for (const auto& raw_file : entry->second) { + delete_file(raw_file, + Stat::lstat(raw_file).size_on_disk(), + &cache_size, + &files_in_cache); + } + } + } + } + + if (util::ends_with(file.path(), ".stderr")) { + // In order to be nice to legacy ccache versions, make sure that the .o + // file is deleted before .stderr, because if the ccache process gets + // killed after deleting the .stderr but before deleting the .o, the + // cached result will be inconsistent. (.stderr is the only file that is + // optional for legacy ccache versions; any other file missing from the + // cache will be detected.) + std::string o_file = file.path().substr(0, file.path().size() - 6) + "o"; + + // Don't subtract this extra deletion from the cache size; that + // bookkeeping will be done when the loop reaches the .o file. If the + // loop doesn't reach the .o file since the target limits have been + // reached, the bookkeeping won't happen, but that small counter + // discrepancy won't do much harm and it will correct itself in the next + // cleanup. + delete_file(o_file, 0, nullptr, nullptr); + } + + delete_file( + file.path(), file.lstat().size_on_disk(), &cache_size, &files_in_cache); + cleaned = true; + } + + LOG("After cleanup: {:.0f} KiB, {:.0f} files", + static_cast(cache_size) / 1024, + static_cast(files_in_cache)); + + if (cleaned) { + LOG("Cleaned up cache directory {}", subdir); + } + + update_counters(subdir, files_in_cache, cache_size, cleaned); +} + +// Clean up all cache subdirectories. +void +LocalStorage::clean_all(const ProgressReceiver& progress_receiver) +{ + for_each_level_1_subdir( + m_config.cache_dir(), + [&](const std::string& subdir, + const ProgressReceiver& sub_progress_receiver) { + clean_dir(subdir, + m_config.max_size() / 16, + m_config.max_files() / 16, + std::nullopt, + std::nullopt, + sub_progress_receiver); + }, + progress_receiver); +} + +// Wipe one cache subdirectory. +static void +wipe_dir(const std::string& subdir, const ProgressReceiver& progress_receiver) +{ + LOG("Clearing out cache directory {}", subdir); + + const std::vector files = get_level_1_files( + subdir, [&](double progress) { progress_receiver(progress / 2); }); + + for (size_t i = 0; i < files.size(); ++i) { + Util::unlink_safe(files[i].path()); + progress_receiver(0.5 + 0.5 * i / files.size()); + } + + const bool cleared = !files.empty(); + if (cleared) { + LOG("Cleared out cache directory {}", subdir); + } + update_counters(subdir, 0, 0, cleared); +} + +// Wipe all cached files in all subdirectories. +void +LocalStorage::wipe_all(const ProgressReceiver& progress_receiver) +{ + for_each_level_1_subdir(m_config.cache_dir(), wipe_dir, progress_receiver); +} + +} // namespace storage::local diff --git a/src/storage/local/LocalStorage_compress.cpp b/src/storage/local/LocalStorage_compress.cpp new file mode 100644 index 0000000..d983b5c --- /dev/null +++ b/src/storage/local/LocalStorage_compress.cpp @@ -0,0 +1,294 @@ +// Copyright (C) 2019-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "LocalStorage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifdef HAVE_UNISTD_H +# include +#endif + +#include +#include +#include + +namespace storage::local { + +namespace { + +class RecompressionStatistics +{ +public: + void update(uint64_t content_size, + uint64_t old_size, + uint64_t new_size, + uint64_t incompressible_size); + uint64_t content_size() const; + uint64_t old_size() const; + uint64_t new_size() const; + uint64_t incompressible_size() const; + +private: + mutable std::mutex m_mutex; + uint64_t m_content_size = 0; + uint64_t m_old_size = 0; + uint64_t m_new_size = 0; + uint64_t m_incompressible_size = 0; +}; + +void +RecompressionStatistics::update(const uint64_t content_size, + const uint64_t old_size, + const uint64_t new_size, + const uint64_t incompressible_size) +{ + std::unique_lock lock(m_mutex); + m_incompressible_size += incompressible_size; + m_content_size += content_size; + m_old_size += old_size; + m_new_size += new_size; +} + +uint64_t +RecompressionStatistics::content_size() const +{ + std::unique_lock lock(m_mutex); + return m_content_size; +} + +uint64_t +RecompressionStatistics::old_size() const +{ + std::unique_lock lock(m_mutex); + return m_old_size; +} + +uint64_t +RecompressionStatistics::new_size() const +{ + std::unique_lock lock(m_mutex); + return m_new_size; +} + +uint64_t +RecompressionStatistics::incompressible_size() const +{ + std::unique_lock lock(m_mutex); + return m_incompressible_size; +} + +} // namespace + +static void +recompress_file(RecompressionStatistics& statistics, + const std::string& stats_file, + const CacheFile& cache_file, + const std::optional level) +{ + core::CacheEntry::Header header(cache_file.path()); + + const int8_t wanted_level = + level ? (*level == 0 ? core::CacheEntry::default_compression_level : *level) + : 0; + const auto old_stat = Stat::stat(cache_file.path(), Stat::OnError::log); + + if (header.compression_level == wanted_level) { + statistics.update(header.entry_size, old_stat.size(), old_stat.size(), 0); + return; + } + + const auto cache_file_data = util::value_or_throw( + util::read_file(cache_file.path()), + FMT("Failed to read {}: ", cache_file.path())); + core::CacheEntry cache_entry(cache_file_data); + cache_entry.verify_checksum(); + + header.entry_format_version = core::CacheEntry::k_format_version; + header.compression_type = + level ? core::CompressionType::zstd : core::CompressionType::none; + header.compression_level = wanted_level; + + AtomicFile new_cache_file(cache_file.path(), AtomicFile::Mode::binary); + new_cache_file.write( + core::CacheEntry::serialize(header, cache_entry.payload())); + new_cache_file.commit(); + + // Restore mtime/atime to keep cache LRU cleanup working as expected: + util::set_timestamps(cache_file.path(), old_stat.mtime(), old_stat.atime()); + + const auto new_stat = Stat::stat(cache_file.path(), Stat::OnError::log); + StatsFile(stats_file).update([=](auto& cs) { + cs.increment(core::Statistic::cache_size_kibibyte, + Util::size_change_kibibyte(old_stat, new_stat)); + }); + statistics.update(header.entry_size, old_stat.size(), new_stat.size(), 0); +} + +CompressionStatistics +LocalStorage::get_compression_statistics( + const ProgressReceiver& progress_receiver) const +{ + CompressionStatistics cs{}; + + for_each_level_1_subdir( + m_config.cache_dir(), + [&](const auto& subdir, const auto& sub_progress_receiver) { + const std::vector files = get_level_1_files( + subdir, [&](double progress) { sub_progress_receiver(progress / 2); }); + + for (size_t i = 0; i < files.size(); ++i) { + const auto& cache_file = files[i]; + cs.on_disk_size += cache_file.lstat().size_on_disk(); + + try { + core::CacheEntry::Header header(cache_file.path()); + cs.compr_size += cache_file.lstat().size(); + cs.content_size += header.entry_size; + } catch (core::Error&) { + cs.incompr_size += cache_file.lstat().size(); + } + + sub_progress_receiver(1.0 / 2 + 1.0 * i / files.size() / 2); + } + }, + progress_receiver); + + return cs; +} + +void +LocalStorage::recompress(const std::optional level, + const ProgressReceiver& progress_receiver) +{ + const size_t threads = std::thread::hardware_concurrency(); + const size_t read_ahead = 2 * threads; + ThreadPool thread_pool(threads, read_ahead); + RecompressionStatistics statistics; + + for_each_level_1_subdir( + m_config.cache_dir(), + [&](const auto& subdir, const auto& sub_progress_receiver) { + std::vector files = + get_level_1_files(subdir, [&](double progress) { + sub_progress_receiver(0.1 * progress); + }); + + auto stats_file = subdir + "/stats"; + + for (size_t i = 0; i < files.size(); ++i) { + const auto& file = files[i]; + + if (file.type() != CacheFile::Type::unknown) { + thread_pool.enqueue([&statistics, stats_file, file, level] { + try { + recompress_file(statistics, stats_file, file, level); + } catch (core::Error&) { + // Ignore for now. + } + }); + } else if (!TemporaryFile::is_tmp_file(file.path())) { + statistics.update(0, 0, 0, file.lstat().size()); + } + + sub_progress_receiver(0.1 + 0.9 * i / files.size()); + } + + if (util::ends_with(subdir, "f")) { + // Wait here instead of after for_each_level_1_subdir to avoid + // updating the progress bar to 100% before all work is done. + thread_pool.shut_down(); + } + }, + progress_receiver); + + // In case there was no f subdir, shut down the thread pool now. + thread_pool.shut_down(); + + if (isatty(STDOUT_FILENO)) { + PRINT_RAW(stdout, "\n\n"); + } + + const double old_ratio = + statistics.old_size() > 0 + ? static_cast(statistics.content_size()) / statistics.old_size() + : 0.0; + const double old_savings = + old_ratio > 0.0 ? 100.0 - (100.0 / old_ratio) : 0.0; + const double new_ratio = + statistics.new_size() > 0 + ? static_cast(statistics.content_size()) / statistics.new_size() + : 0.0; + const double new_savings = + new_ratio > 0.0 ? 100.0 - (100.0 / new_ratio) : 0.0; + const int64_t size_difference = static_cast(statistics.new_size()) + - static_cast(statistics.old_size()); + + const std::string old_compr_size_str = + Util::format_human_readable_size(statistics.old_size()); + const std::string new_compr_size_str = + Util::format_human_readable_size(statistics.new_size()); + const std::string content_size_str = + Util::format_human_readable_size(statistics.content_size()); + const std::string incompr_size_str = + Util::format_human_readable_size(statistics.incompressible_size()); + const std::string size_difference_str = + FMT("{}{}", + size_difference < 0 ? "-" : (size_difference > 0 ? "+" : " "), + Util::format_human_readable_size( + size_difference < 0 ? -size_difference : size_difference)); + + PRINT(stdout, "Original data: {:>8s}\n", content_size_str); + PRINT(stdout, + "Old compressed data: {:>8s} ({:.1f}% of original size)\n", + old_compr_size_str, + 100.0 - old_savings); + PRINT(stdout, + " - Compression ratio: {:>5.3f} x ({:.1f}% space savings)\n", + old_ratio, + old_savings); + PRINT(stdout, + "New compressed data: {:>8s} ({:.1f}% of original size)\n", + new_compr_size_str, + 100.0 - new_savings); + PRINT(stdout, + " - Compression ratio: {:>5.3f} x ({:.1f}% space savings)\n", + new_ratio, + new_savings); + PRINT(stdout, "Size change: {:>9s}\n", size_difference_str); +} + +} // namespace storage::local diff --git a/src/storage/local/LocalStorage_statistics.cpp b/src/storage/local/LocalStorage_statistics.cpp new file mode 100644 index 0000000..b801313 --- /dev/null +++ b/src/storage/local/LocalStorage_statistics.cpp @@ -0,0 +1,84 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "LocalStorage.hpp" + +#include +#include +#include +#include + +#include + +namespace storage::local { + +static void +for_each_level_1_and_2_stats_file( + const std::string& cache_dir, + const std::function function) +{ + for (size_t level_1 = 0; level_1 <= 0xF; ++level_1) { + function(FMT("{}/{:x}/stats", cache_dir, level_1)); + for (size_t level_2 = 0; level_2 <= 0xF; ++level_2) { + function(FMT("{}/{:x}/{:x}/stats", cache_dir, level_1, level_2)); + } + } +} + +// Zero all statistics counters except those tracking cache size and number of +// files in the cache. +void +LocalStorage::zero_all_statistics() +{ + const auto now = util::TimePoint::now(); + const auto zeroable_fields = core::Statistics::get_zeroable_fields(); + + for_each_level_1_and_2_stats_file( + m_config.cache_dir(), [=](const std::string& path) { + StatsFile(path).update([=](auto& cs) { + for (const auto statistic : zeroable_fields) { + cs.set(statistic, 0); + } + cs.set(core::Statistic::stats_zeroed_timestamp, now.sec()); + }); + }); +} + +// Get statistics and last time of update for the whole local storage cache. +std::pair +LocalStorage::get_all_statistics() const +{ + core::StatisticsCounters counters; + uint64_t zero_timestamp = 0; + util::TimePoint last_updated; + + // Add up the stats in each directory. + for_each_level_1_and_2_stats_file( + m_config.cache_dir(), [&](const auto& path) { + counters.set(core::Statistic::stats_zeroed_timestamp, 0); // Don't add + counters.increment(StatsFile(path).read()); + zero_timestamp = std::max( + counters.get(core::Statistic::stats_zeroed_timestamp), zero_timestamp); + last_updated = std::max(last_updated, Stat::stat(path).mtime()); + }); + + counters.set(core::Statistic::stats_zeroed_timestamp, zero_timestamp); + return std::make_pair(counters, last_updated); +} + +} // namespace storage::local diff --git a/src/storage/local/StatsFile.cpp b/src/storage/local/StatsFile.cpp new file mode 100644 index 0000000..4332d80 --- /dev/null +++ b/src/storage/local/StatsFile.cpp @@ -0,0 +1,91 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "StatsFile.hpp" + +#include +#include +#include +#include +#include +#include + +namespace storage::local { + +StatsFile::StatsFile(const std::string& path) : m_path(path) +{ +} + +core::StatisticsCounters +StatsFile::read() const +{ + core::StatisticsCounters counters; + + const auto data = util::read_file(m_path); + if (!data) { + // A nonexistent stats file is OK. + return counters; + } + + size_t i = 0; + const char* str = data->c_str(); + while (true) { + char* end; + const uint64_t value = std::strtoull(str, &end, 10); + if (end == str) { + break; + } + counters.set_raw(i, value); + ++i; + str = end; + } + + return counters; +} + +std::optional +StatsFile::update( + std::function function) const +{ + util::ShortLivedLockFile lock_file(m_path); + util::LockFileGuard lock(lock_file); + if (!lock.acquired()) { + LOG("Failed to acquire lock for {}", m_path); + return std::nullopt; + } + + auto counters = read(); + function(counters); + + AtomicFile file(m_path, AtomicFile::Mode::text); + for (size_t i = 0; i < counters.size(); ++i) { + file.write(FMT("{}\n", counters.get_raw(i))); + } + try { + file.commit(); + } catch (const core::Error& e) { + // Make failure to write a stats file a soft error since it's not important + // enough to fail whole the process and also because it is called in the + // Context destructor. + LOG("Error: {}", e.what()); + } + + return counters; +} + +} // namespace storage::local diff --git a/src/storage/local/StatsFile.hpp b/src/storage/local/StatsFile.hpp new file mode 100644 index 0000000..2ac4026 --- /dev/null +++ b/src/storage/local/StatsFile.hpp @@ -0,0 +1,48 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include +#include +#include + +namespace storage::local { + +class StatsFile +{ +public: + StatsFile(const std::string& path); + + // Read counters. No lock is acquired. If the file doesn't exist all returned + // counters will be zero. + core::StatisticsCounters read() const; + + // Acquire a lock, read counters, call `function` with the counters, write the + // counters and release the lock. Returns the resulting counters or nullopt on + // error (e.g. if the lock could not be acquired). + std::optional + update(std::function) const; + +private: + const std::string m_path; +}; + +} // namespace storage::local diff --git a/src/storage/local/util.cpp b/src/storage/local/util.cpp new file mode 100644 index 0000000..1035925 --- /dev/null +++ b/src/storage/local/util.cpp @@ -0,0 +1,75 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "util.hpp" + +#include +#include +#include + +namespace storage::local { + +void +for_each_level_1_subdir(const std::string& cache_dir, + const SubdirVisitor& visitor, + const ProgressReceiver& progress_receiver) +{ + for (int i = 0; i <= 0xF; i++) { + double progress = 1.0 * i / 16; + progress_receiver(progress); + std::string subdir_path = FMT("{}/{:x}", cache_dir, i); + visitor(subdir_path, [&](double inner_progress) { + progress_receiver(progress + inner_progress / 16); + }); + } + progress_receiver(1.0); +} + +std::vector +get_level_1_files(const std::string& dir, + const ProgressReceiver& progress_receiver) +{ + std::vector files; + + if (!Stat::stat(dir)) { + return files; + } + + size_t level_2_directories = 0; + + Util::traverse(dir, [&](const std::string& path, bool is_dir) { + auto name = Util::base_name(path); + if (name == "CACHEDIR.TAG" || name == "stats" + || util::starts_with(name, ".nfs")) { + return; + } + + if (!is_dir) { + files.emplace_back(path); + } else if (path != dir + && path.find('/', dir.size() + 1) == std::string::npos) { + ++level_2_directories; + progress_receiver(level_2_directories / 16.0); + } + }); + + progress_receiver(1.0); + return files; +} + +} // namespace storage::local diff --git a/src/storage/local/util.hpp b/src/storage/local/util.hpp new file mode 100644 index 0000000..8aa9b87 --- /dev/null +++ b/src/storage/local/util.hpp @@ -0,0 +1,62 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +#include +#include +#include + +namespace storage::local { + +using ProgressReceiver = std::function; +using SubdirVisitor = std::function; + +// Call a function for each subdir (0-9a-f) in the cache. +// +// Parameters: +// - cache_dir: Path to the cache directory. +// - visitor: Function to call with directory path and progress_receiver as +// arguments. +// - progress_receiver: Function that will be called for progress updates. +void for_each_level_1_subdir(const std::string& cache_dir, + const SubdirVisitor& visitor, + const ProgressReceiver& progress_receiver); + +// Get a list of files in a level 1 subdirectory of the cache. +// +// The function works under the assumption that directory entries with one +// character names (except ".") are subdirectories and that there are no other +// subdirectories. +// +// Files ignored: +// - CACHEDIR.TAG +// - stats +// - .nfs* (temporary NFS files that may be left for open but deleted files). +// +// Parameters: +// - dir: The directory to traverse recursively. +// - progress_receiver: Function that will be called for progress updates. +std::vector +get_level_1_files(const std::string& dir, + const ProgressReceiver& progress_receiver); + +} // namespace storage::local diff --git a/src/storage/remote/CMakeLists.txt b/src/storage/remote/CMakeLists.txt new file mode 100644 index 0000000..96dbebb --- /dev/null +++ b/src/storage/remote/CMakeLists.txt @@ -0,0 +1,15 @@ +set( + sources + FileStorage.cpp + HttpStorage.cpp + RemoteStorage.cpp +) + +if(REDIS_STORAGE_BACKEND) + list(APPEND sources RedisStorage.cpp) +endif() + +file(GLOB headers *.hpp) +list(APPEND sources ${headers}) + +target_sources(ccache_framework PRIVATE ${sources}) diff --git a/src/storage/remote/FileStorage.cpp b/src/storage/remote/FileStorage.cpp new file mode 100644 index 0000000..b1c8a33 --- /dev/null +++ b/src/storage/remote/FileStorage.cpp @@ -0,0 +1,200 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "FileStorage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include // for mode_t + +#include + +namespace storage::remote { + +namespace { + +class FileStorageBackend : public RemoteStorage::Backend +{ +public: + FileStorageBackend(const Params& params); + + nonstd::expected, Failure> + get(const Digest& key) override; + + nonstd::expected put(const Digest& key, + nonstd::span value, + bool only_if_missing) override; + + nonstd::expected remove(const Digest& key) override; + +private: + enum class Layout { flat, subdirs }; + + std::string m_dir; + std::optional m_umask; + bool m_update_mtime = false; + Layout m_layout = Layout::subdirs; + + std::string get_entry_path(const Digest& key) const; +}; + +FileStorageBackend::FileStorageBackend(const Params& params) +{ + ASSERT(params.url.scheme() == "file"); + + const auto& host = params.url.host(); +#ifdef _WIN32 + m_dir = util::replace_all(params.url.path(), "/", "\\"); + if (!host.empty()) { + m_dir = FMT("\\\\{}{}", host, m_dir); + } +#else + if (!host.empty() && host != "localhost") { + throw core::Fatal( + FMT("invalid file URL \"{}\": specifying a host other than localhost is" + " not supported", + params.url.str())); + } + m_dir = params.url.path(); +#endif + + for (const auto& attr : params.attributes) { + if (attr.key == "layout") { + if (attr.value == "flat") { + m_layout = Layout::flat; + } else if (attr.value == "subdirs") { + m_layout = Layout::subdirs; + } else { + LOG("Unknown layout: {}", attr.value); + } + } else if (attr.key == "umask") { + m_umask = + util::value_or_throw(util::parse_umask(attr.value)); + } else if (attr.key == "update-mtime") { + m_update_mtime = attr.value == "true"; + } else if (!is_framework_attribute(attr.key)) { + LOG("Unknown attribute: {}", attr.key); + } + } +} + +nonstd::expected, RemoteStorage::Backend::Failure> +FileStorageBackend::get(const Digest& key) +{ + const auto path = get_entry_path(key); + const bool exists = Stat::stat(path); + + if (!exists) { + // Don't log failure if the entry doesn't exist. + return std::nullopt; + } + + if (m_update_mtime) { + // Update modification timestamp for potential LRU cleanup by some external + // mechanism. + util::set_timestamps(path); + } + + auto value = util::read_file(path); + if (!value) { + LOG("Failed to read {}: {}", path, value.error()); + return nonstd::make_unexpected(Failure::error); + } + return std::move(*value); +} + +nonstd::expected +FileStorageBackend::put(const Digest& key, + const nonstd::span value, + const bool only_if_missing) +{ + const auto path = get_entry_path(key); + + if (only_if_missing && Stat::stat(path)) { + LOG("{} already in cache", path); + return false; + } + + { + UmaskScope umask_scope(m_umask); + + const auto dir = Util::dir_name(path); + if (!Util::create_dir(dir)) { + LOG("Failed to create directory {}: {}", dir, strerror(errno)); + return nonstd::make_unexpected(Failure::error); + } + + util::create_cachedir_tag(m_dir); + + LOG("Writing {}", path); + try { + AtomicFile file(path, AtomicFile::Mode::binary); + file.write(value); + file.commit(); + return true; + } catch (const core::Error& e) { + LOG("Failed to write {}: {}", path, e.what()); + return nonstd::make_unexpected(Failure::error); + } + } +} + +nonstd::expected +FileStorageBackend::remove(const Digest& key) +{ + return Util::unlink_safe(get_entry_path(key)); +} + +std::string +FileStorageBackend::get_entry_path(const Digest& key) const +{ + switch (m_layout) { + case Layout::flat: + return FMT("{}/{}", m_dir, key.to_string()); + + case Layout::subdirs: { + const auto key_str = key.to_string(); + const uint8_t digits = 2; + ASSERT(key_str.length() > digits); + return FMT("{}/{:.{}}/{}", m_dir, key_str, digits, &key_str[digits]); + } + } + + ASSERT(false); +} + +} // namespace + +std::unique_ptr +FileStorage::create_backend(const Backend::Params& params) const +{ + return std::make_unique(params); +} + +} // namespace storage::remote diff --git a/src/storage/remote/FileStorage.hpp b/src/storage/remote/FileStorage.hpp new file mode 100644 index 0000000..ae72cfd --- /dev/null +++ b/src/storage/remote/FileStorage.hpp @@ -0,0 +1,32 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +namespace storage::remote { + +class FileStorage : public RemoteStorage +{ +public: + std::unique_ptr + create_backend(const Backend::Params& params) const override; +}; + +} // namespace storage::remote diff --git a/src/storage/remote/HttpStorage.cpp b/src/storage/remote/HttpStorage.cpp new file mode 100644 index 0000000..26c8918 --- /dev/null +++ b/src/storage/remote/HttpStorage.cpp @@ -0,0 +1,308 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "HttpStorage.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace storage::remote { + +namespace { + +class HttpStorageBackend : public RemoteStorage::Backend +{ +public: + HttpStorageBackend(const Params& params); + + nonstd::expected, Failure> + get(const Digest& key) override; + + nonstd::expected put(const Digest& key, + nonstd::span value, + bool only_if_missing) override; + + nonstd::expected remove(const Digest& key) override; + +private: + enum class Layout { bazel, flat, subdirs }; + + const std::string m_url_path; + httplib::Client m_http_client; + Layout m_layout = Layout::subdirs; + + std::string get_entry_path(const Digest& key) const; +}; + +std::string +get_url_path(const Url& url) +{ + auto path = url.path(); + if (path.empty() || path.back() != '/') { + path += '/'; + } + return path; +} + +Url +get_partial_url(const Url& from_url) +{ + Url url; + url.scheme(from_url.scheme()); + url.host(from_url.host(), from_url.ip_version()); + if (!from_url.port().empty()) { + url.port(from_url.port()); + } + return url; +} + +std::string +get_url(const Url& url) +{ + if (url.host().empty()) { + throw core::Fatal( + FMT("A host is required in HTTP storage URL \"{}\"", url.str())); + } + + // httplib requires a partial URL with just scheme, host and port. + return get_partial_url(url).str(); +} + +RemoteStorage::Backend::Failure +failure_from_httplib_error(httplib::Error error) +{ + return error == httplib::Error::ConnectionTimeout + ? RemoteStorage::Backend::Failure::timeout + : RemoteStorage::Backend::Failure::error; +} + +HttpStorageBackend::HttpStorageBackend(const Params& params) + : m_url_path(get_url_path(params.url)), + m_http_client(get_url(params.url)) +{ + if (!params.url.user_info().empty()) { + const auto [user, password] = util::split_once(params.url.user_info(), ':'); + if (!password) { + throw core::Fatal(FMT("Expected username:password in URL but got \"{}\"", + params.url.user_info())); + } + m_http_client.set_basic_auth(std::string(user), std::string(*password)); + } + + m_http_client.set_default_headers({ + {"User-Agent", FMT("ccache/{}", CCACHE_VERSION)}, + }); + m_http_client.set_keep_alive(true); + + auto connect_timeout = k_default_connect_timeout; + auto operation_timeout = k_default_operation_timeout; + + for (const auto& attr : params.attributes) { + if (attr.key == "bearer-token") { + m_http_client.set_bearer_token_auth(attr.value); + } else if (attr.key == "connect-timeout") { + connect_timeout = parse_timeout_attribute(attr.value); + } else if (attr.key == "keep-alive") { + m_http_client.set_keep_alive(attr.value == "true"); + } else if (attr.key == "layout") { + if (attr.value == "bazel") { + m_layout = Layout::bazel; + } else if (attr.value == "flat") { + m_layout = Layout::flat; + } else if (attr.value == "subdirs") { + m_layout = Layout::subdirs; + } else { + LOG("Unknown layout: {}", attr.value); + } + } else if (attr.key == "operation-timeout") { + operation_timeout = parse_timeout_attribute(attr.value); + } else if (!is_framework_attribute(attr.key)) { + LOG("Unknown attribute: {}", attr.key); + } + } + + m_http_client.set_connection_timeout(connect_timeout); + m_http_client.set_read_timeout(operation_timeout); + m_http_client.set_write_timeout(operation_timeout); +} + +nonstd::expected, RemoteStorage::Backend::Failure> +HttpStorageBackend::get(const Digest& key) +{ + const auto url_path = get_entry_path(key); + const auto result = m_http_client.Get(url_path); + + if (result.error() != httplib::Error::Success || !result) { + LOG("Failed to get {} from http storage: {} ({})", + url_path, + to_string(result.error()), + static_cast(result.error())); + return nonstd::make_unexpected(failure_from_httplib_error(result.error())); + } + + if (result->status < 200 || result->status >= 300) { + // Don't log failure if the entry doesn't exist. + return std::nullopt; + } + + return util::Bytes(result->body.data(), result->body.size()); +} + +nonstd::expected +HttpStorageBackend::put(const Digest& key, + const nonstd::span value, + const bool only_if_missing) +{ + const auto url_path = get_entry_path(key); + + if (only_if_missing) { + const auto result = m_http_client.Head(url_path); + + if (result.error() != httplib::Error::Success || !result) { + LOG("Failed to check for {} in http storage: {} ({})", + url_path, + to_string(result.error()), + static_cast(result.error())); + return nonstd::make_unexpected( + failure_from_httplib_error(result.error())); + } + + if (result->status >= 200 && result->status < 300) { + LOG("Found entry {} already within http storage: status code: {}", + url_path, + result->status); + return false; + } + } + + static const auto content_type = "application/octet-stream"; + const auto result = + m_http_client.Put(url_path, + reinterpret_cast(value.data()), + value.size(), + content_type); + + if (result.error() != httplib::Error::Success || !result) { + LOG("Failed to put {} to http storage: {} ({})", + url_path, + to_string(result.error()), + static_cast(result.error())); + return nonstd::make_unexpected(failure_from_httplib_error(result.error())); + } + + if (result->status < 200 || result->status >= 300) { + LOG("Failed to put {} to http storage: status code: {}", + url_path, + result->status); + return nonstd::make_unexpected(failure_from_httplib_error(result.error())); + } + + return true; +} + +nonstd::expected +HttpStorageBackend::remove(const Digest& key) +{ + const auto url_path = get_entry_path(key); + const auto result = m_http_client.Delete(url_path); + + if (result.error() != httplib::Error::Success || !result) { + LOG("Failed to delete {} from http storage: {} ({})", + url_path, + to_string(result.error()), + static_cast(result.error())); + return nonstd::make_unexpected(failure_from_httplib_error(result.error())); + } + + if (result->status < 200 || result->status >= 300) { + LOG("Failed to delete {} from http storage: status code: {}", + url_path, + result->status); + return nonstd::make_unexpected(failure_from_httplib_error(result.error())); + } + + return true; +} + +std::string +HttpStorageBackend::get_entry_path(const Digest& key) const +{ + switch (m_layout) { + case Layout::bazel: { + // Mimic hex representation of a SHA256 hash value. + const auto sha256_hex_size = 64; + static_assert(Digest::size() == 20, "Update below if digest size changes"); + std::string hex_digits = Util::format_base16(key.bytes(), key.size()); + hex_digits.append(hex_digits.data(), sha256_hex_size - hex_digits.size()); + LOG("Translated key {} to Bazel layout ac/{}", key.to_string(), hex_digits); + return FMT("{}ac/{}", m_url_path, hex_digits); + } + + case Layout::flat: + return m_url_path + key.to_string(); + + case Layout::subdirs: { + const auto key_str = key.to_string(); + const uint8_t digits = 2; + ASSERT(key_str.length() > digits); + return FMT("{}{:.{}}/{}", m_url_path, key_str, digits, &key_str[digits]); + } + } + + ASSERT(false); +} + +} // namespace + +std::unique_ptr +HttpStorage::create_backend(const Backend::Params& params) const +{ + return std::make_unique(params); +} + +void +HttpStorage::redact_secrets(Backend::Params& params) const +{ + auto& url = params.url; + const auto [user, password] = util::split_once(url.user_info(), ':'); + if (password) { + url.user_info(FMT("{}:{}", user, k_redacted_password)); + } + + auto bearer_token_attribute = + std::find_if(params.attributes.begin(), + params.attributes.end(), + [&](const auto& attr) { return attr.key == "bearer-token"; }); + if (bearer_token_attribute != params.attributes.end()) { + bearer_token_attribute->value = k_redacted_password; + bearer_token_attribute->raw_value = k_redacted_password; + } +} + +} // namespace storage::remote diff --git a/src/storage/remote/HttpStorage.hpp b/src/storage/remote/HttpStorage.hpp new file mode 100644 index 0000000..dbe45c9 --- /dev/null +++ b/src/storage/remote/HttpStorage.hpp @@ -0,0 +1,34 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +namespace storage::remote { + +class HttpStorage : public RemoteStorage +{ +public: + std::unique_ptr + create_backend(const Backend::Params& params) const override; + + void redact_secrets(Backend::Params& params) const override; +}; + +} // namespace storage::remote diff --git a/src/storage/remote/RedisStorage.cpp b/src/storage/remote/RedisStorage.cpp new file mode 100644 index 0000000..e6c8ce3 --- /dev/null +++ b/src/storage/remote/RedisStorage.cpp @@ -0,0 +1,367 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "RedisStorage.hpp" + +#include +#include +#include +#include +#include +#include + +// Ignore "ISO C++ forbids flexible array member ‘buf’" warning from -Wpedantic. +#ifdef __GNUC__ +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wpedantic" +#endif +#ifdef _MSC_VER +# pragma warning(push) +# pragma warning(disable : 4200) +#endif +#include +#ifdef _MSC_VER +# pragma warning(pop) +#endif +#ifdef __GNUC__ +# pragma GCC diagnostic pop +#endif + +#include +#include +#include + +namespace storage::remote { + +namespace { + +using RedisContext = std::unique_ptr; +using RedisReply = std::unique_ptr; + +const uint32_t DEFAULT_PORT = 6379; + +class RedisStorageBackend : public RemoteStorage::Backend +{ +public: + RedisStorageBackend(const RemoteStorage::Backend::Params& params); + + nonstd::expected, Failure> + get(const Digest& key) override; + + nonstd::expected put(const Digest& key, + nonstd::span value, + bool only_if_missing) override; + + nonstd::expected remove(const Digest& key) override; + +private: + const std::string m_prefix; + RedisContext m_context; + + void + connect(const Url& url, uint32_t connect_timeout, uint32_t operation_timeout); + void select_database(const Url& url); + void authenticate(const Url& url); + nonstd::expected redis_command(const char* format, ...); + std::string get_key_string(const Digest& digest) const; +}; + +timeval +to_timeval(const uint32_t ms) +{ + timeval tv; + tv.tv_sec = ms / 1000; + tv.tv_usec = (ms % 1000) * 1000; + return tv; +} + +std::pair, std::optional> +split_user_info(const std::string& user_info) +{ + const auto [left, right] = util::split_once(user_info, ':'); + if (left.empty()) { + // redis://HOST + return {std::nullopt, std::nullopt}; + } else if (right) { + // redis://USERNAME:PASSWORD@HOST + return {std::string(left), std::string(*right)}; + } else { + // redis://PASSWORD@HOST + return {std::nullopt, std::string(left)}; + } +} + +RedisStorageBackend::RedisStorageBackend(const Params& params) + : m_prefix("ccache"), // TODO: attribute + m_context(nullptr, redisFree) +{ + const auto& url = params.url; + ASSERT(url.scheme() == "redis" || url.scheme() == "redis+unix"); + if (url.scheme() == "redis+unix" && !params.url.host().empty() + && params.url.host() != "localhost") { + throw core::Fatal( + FMT("invalid file path \"{}\": specifying a host other than localhost is" + " not supported", + params.url.str(), + params.url.host())); + } + + auto connect_timeout = k_default_connect_timeout; + auto operation_timeout = k_default_operation_timeout; + + for (const auto& attr : params.attributes) { + if (attr.key == "connect-timeout") { + connect_timeout = parse_timeout_attribute(attr.value); + } else if (attr.key == "operation-timeout") { + operation_timeout = parse_timeout_attribute(attr.value); + } else if (!is_framework_attribute(attr.key)) { + LOG("Unknown attribute: {}", attr.key); + } + } + + connect(url, connect_timeout.count(), operation_timeout.count()); + authenticate(url); + select_database(url); +} + +inline bool +is_error(int err) +{ + return err != REDIS_OK; +} + +inline bool +is_timeout(int err) +{ +#ifdef REDIS_ERR_TIMEOUT + // Only returned for hiredis version 1.0.0 and above + return err == REDIS_ERR_TIMEOUT; +#else + (void)err; + return false; +#endif +} + +nonstd::expected, RemoteStorage::Backend::Failure> +RedisStorageBackend::get(const Digest& key) +{ + const auto key_string = get_key_string(key); + LOG("Redis GET {}", key_string); + const auto reply = redis_command("GET %s", key_string.c_str()); + if (!reply) { + return nonstd::make_unexpected(reply.error()); + } else if ((*reply)->type == REDIS_REPLY_STRING) { + return util::Bytes((*reply)->str, (*reply)->len); + } else if ((*reply)->type == REDIS_REPLY_NIL) { + return std::nullopt; + } else { + LOG("Unknown reply type: {}", (*reply)->type); + return nonstd::make_unexpected(Failure::error); + } +} + +nonstd::expected +RedisStorageBackend::put(const Digest& key, + nonstd::span value, + bool only_if_missing) +{ + const auto key_string = get_key_string(key); + + if (only_if_missing) { + LOG("Redis EXISTS {}", key_string); + const auto reply = redis_command("EXISTS %s", key_string.c_str()); + if (!reply) { + return nonstd::make_unexpected(reply.error()); + } else if ((*reply)->type != REDIS_REPLY_INTEGER) { + LOG("Unknown reply type: {}", (*reply)->type); + } else if ((*reply)->integer > 0) { + LOG("Entry {} already in Redis", key_string); + return false; + } + } + + LOG("Redis SET {} [{} bytes]", key_string, value.size()); + const auto reply = + redis_command("SET %s %b", key_string.c_str(), value.data(), value.size()); + if (!reply) { + return nonstd::make_unexpected(reply.error()); + } else if ((*reply)->type == REDIS_REPLY_STATUS) { + return true; + } else { + LOG("Unknown reply type: {}", (*reply)->type); + return nonstd::make_unexpected(Failure::error); + } +} + +nonstd::expected +RedisStorageBackend::remove(const Digest& key) +{ + const auto key_string = get_key_string(key); + LOG("Redis DEL {}", key_string); + const auto reply = redis_command("DEL %s", key_string.c_str()); + if (!reply) { + return nonstd::make_unexpected(reply.error()); + } else if ((*reply)->type == REDIS_REPLY_INTEGER) { + return (*reply)->integer > 0; + } else { + LOG("Unknown reply type: {}", (*reply)->type); + return nonstd::make_unexpected(Failure::error); + } +} + +void +RedisStorageBackend::connect(const Url& url, + const uint32_t connect_timeout, + const uint32_t operation_timeout) +{ + if (url.scheme() == "redis+unix") { + LOG("Redis connecting to {} (connect timeout {} ms)", + url.path(), + connect_timeout); + m_context.reset(redisConnectUnixWithTimeout(url.path().c_str(), + to_timeval(connect_timeout))); + } else { + const std::string host = url.host().empty() ? "localhost" : url.host(); + const uint32_t port = + url.port().empty() ? DEFAULT_PORT + : util::value_or_throw( + util::parse_unsigned(url.port(), 1, 65535, "port")); + ASSERT(url.path().empty() || url.path()[0] == '/'); + + LOG("Redis connecting to {}:{} (connect timeout {} ms)", + host, + port, + connect_timeout); + m_context.reset( + redisConnectWithTimeout(host.c_str(), port, to_timeval(connect_timeout))); + } + + if (!m_context) { + throw Failed("Redis context construction error"); + } + if (is_timeout(m_context->err)) { + throw Failed(FMT("Redis connection timeout: {}", m_context->errstr), + Failure::timeout); + } + if (is_error(m_context->err)) { + throw Failed(FMT("Redis connection error: {}", m_context->errstr)); + } + + LOG("Redis operation timeout set to {} ms", operation_timeout); + if (redisSetTimeout(m_context.get(), to_timeval(operation_timeout)) + != REDIS_OK) { + throw Failed("Failed to set operation timeout"); + } + + LOG_RAW("Redis connection OK"); +} + +void +RedisStorageBackend::select_database(const Url& url) +{ + std::optional db; + if (url.scheme() == "redis+unix") { + for (const auto& param : url.query()) { + if (param.key() == "db") { + db = param.val(); + break; + } + } + } else if (!url.path().empty()) { + db = url.path().substr(1); + } + const uint32_t db_number = + !db ? 0 + : util::value_or_throw(util::parse_unsigned( + *db, 0, std::numeric_limits::max(), "db number")); + + if (db_number != 0) { + LOG("Redis SELECT {}", db_number); + util::value_or_throw(redis_command("SELECT %d", db_number)); + } +} + +void +RedisStorageBackend::authenticate(const Url& url) +{ + const auto [user, password] = split_user_info(url.user_info()); + if (password) { + if (user) { + // redis://user:password@host + LOG("Redis AUTH {} {}", *user, k_redacted_password); + util::value_or_throw( + redis_command("AUTH %s %s", user->c_str(), password->c_str())); + } else { + // redis://password@host + LOG("Redis AUTH {}", k_redacted_password); + util::value_or_throw(redis_command("AUTH %s", password->c_str())); + } + } +} + +nonstd::expected +RedisStorageBackend::redis_command(const char* format, ...) +{ + va_list ap; + va_start(ap, format); + auto reply = + static_cast(redisvCommand(m_context.get(), format, ap)); + va_end(ap); + if (!reply) { + LOG("Redis command failed: {}", m_context->errstr); + return nonstd::make_unexpected(is_timeout(m_context->err) ? Failure::timeout + : Failure::error); + } else if (reply->type == REDIS_REPLY_ERROR) { + LOG("Redis command failed: {}", reply->str); + return nonstd::make_unexpected(Failure::error); + } else { + return RedisReply(reply, freeReplyObject); + } +} + +std::string +RedisStorageBackend::get_key_string(const Digest& digest) const +{ + return FMT("{}:{}", m_prefix, digest.to_string()); +} + +} // namespace + +std::unique_ptr +RedisStorage::create_backend(const Backend::Params& params) const +{ + return std::make_unique(params); +} + +void +RedisStorage::redact_secrets(Backend::Params& params) const +{ + auto& url = params.url; + const auto [user, password] = split_user_info(url.user_info()); + if (password) { + if (user) { + // redis://user:password@host + url.user_info(FMT("{}:{}", *user, k_redacted_password)); + } else { + // redis://password@host + url.user_info(k_redacted_password); + } + } +} + +} // namespace storage::remote diff --git a/src/storage/remote/RedisStorage.hpp b/src/storage/remote/RedisStorage.hpp new file mode 100644 index 0000000..530cb43 --- /dev/null +++ b/src/storage/remote/RedisStorage.hpp @@ -0,0 +1,34 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include + +namespace storage::remote { + +class RedisStorage : public RemoteStorage +{ +public: + std::unique_ptr + create_backend(const Backend::Params& params) const override; + + void redact_secrets(Backend::Params& params) const override; +}; + +} // namespace storage::remote diff --git a/src/storage/remote/RemoteStorage.cpp b/src/storage/remote/RemoteStorage.cpp new file mode 100644 index 0000000..e310390 --- /dev/null +++ b/src/storage/remote/RemoteStorage.cpp @@ -0,0 +1,39 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "RemoteStorage.hpp" + +#include +#include + +namespace storage::remote { + +bool +RemoteStorage::Backend::is_framework_attribute(const std::string& name) +{ + return name == "read-only" || name == "shards"; +} + +std::chrono::milliseconds +RemoteStorage::Backend::parse_timeout_attribute(const std::string& value) +{ + return std::chrono::milliseconds(util::value_or_throw( + util::parse_unsigned(value, 1, 60 * 1000, "timeout"))); +} + +} // namespace storage::remote diff --git a/src/storage/remote/RemoteStorage.hpp b/src/storage/remote/RemoteStorage.hpp new file mode 100644 index 0000000..a80237b --- /dev/null +++ b/src/storage/remote/RemoteStorage.hpp @@ -0,0 +1,146 @@ +// Copyright (C) 2021-2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +class Digest; + +namespace storage::remote { + +constexpr auto k_redacted_password = "********"; +const auto k_default_connect_timeout = std::chrono::milliseconds{100}; +const auto k_default_operation_timeout = std::chrono::milliseconds{10000}; + +// This class defines the API that a remote storage must implement. +class RemoteStorage +{ +public: + class Backend + { + public: + struct Attribute + { + std::string key; // Key part. + std::string value; // Value part, percent-decoded. + std::string raw_value; // Value part, not percent-decoded. + }; + + struct Params + { + Url url; + std::vector attributes; + }; + + enum class Failure { + error, // Operation error, e.g. bad parameters or failed connection. + timeout, // Timeout, e.g. due to slow network or server. + }; + + class Failed : public std::runtime_error + { + public: + Failed(Failure failure); + Failed(const std::string& message, Failure failure = Failure::error); + + Failure failure() const; + + private: + Failure m_failure; + }; + + virtual ~Backend() = default; + + // Get the value associated with `key`. Returns the value on success or + // std::nullopt if the entry is not present. + virtual nonstd::expected, Failure> + get(const Digest& key) = 0; + + // Put `value` associated to `key` in the storage. A true `only_if_missing` + // is a hint that the value does not have to be set if already present. + // Returns true if the entry was stored, otherwise false. + virtual nonstd::expected + put(const Digest& key, + nonstd::span value, + bool only_if_missing = false) = 0; + + // Remove `key` and its associated value. Returns true if the entry was + // removed, otherwise false. + virtual nonstd::expected remove(const Digest& key) = 0; + + // Determine whether an attribute is handled by the remote storage + // framework itself. + static bool is_framework_attribute(const std::string& name); + + // Parse a timeout `value`, throwing `Failed` on error. + static std::chrono::milliseconds + parse_timeout_attribute(const std::string& value); + }; + + virtual ~RemoteStorage() = default; + + // Create an instance of the backend. The instance is created just before the + // first call to a backend method, so the backend constructor can open a + // connection or similar right away if wanted. The method should throw + // `core::Fatal` on fatal configuration error or `Backend::Failed` on + // connection error or timeout. + virtual std::unique_ptr + create_backend(const Backend::Params& parameters) const = 0; + + // Redact secrets in backend parameters, if any. + virtual void redact_secrets(Backend::Params& parameters) const; +}; + +// --- Inline implementations --- + +inline void +RemoteStorage::redact_secrets(RemoteStorage::Backend::Params& /*config*/) const +{ +} + +inline RemoteStorage::Backend::Failed::Failed(Failure failure) + : Failed("", failure) +{ +} + +inline RemoteStorage::Backend::Failed::Failed(const std::string& message, + Failure failure) + : std::runtime_error::runtime_error(message), + m_failure(failure) +{ +} + +inline RemoteStorage::Backend::Failure +RemoteStorage::Backend::Failed::failure() const +{ + return m_failure; +} + +} // namespace storage::remote diff --git a/src/storage/types.hpp b/src/storage/types.hpp new file mode 100644 index 0000000..16e70cd --- /dev/null +++ b/src/storage/types.hpp @@ -0,0 +1,28 @@ +// Copyright (C) 2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +namespace storage { + +using EntryWriter = std::function; + +} // namespace storage diff --git a/src/system.hpp b/src/system.hpp deleted file mode 100644 index fa4bbf3..0000000 --- a/src/system.hpp +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright (C) 2010-2020 Joel Rosdahl and other contributors -// -// See doc/AUTHORS.adoc for a complete list of contributors. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation; either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along with -// this program; if not, write to the Free Software Foundation, Inc., 51 -// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -#pragma once - -#ifdef __MINGW32__ -# define __USE_MINGW_ANSI_STDIO 1 -# define __STDC_FORMAT_MACROS 1 -#endif - -#include "config.h" - -#ifdef HAVE_SYS_FILE_H -# include -#endif - -#ifdef HAVE_SYS_MMAN_H -# include -#endif -#include -#include -#ifdef HAVE_SYS_WAIT_H -# include -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef HAVE_DIRENT_H -# include -#endif - -#include - -#ifdef HAVE_STRINGS_H -# include -#endif - -#ifdef HAVE_UNISTD_H -# include -#endif - -#ifdef HAVE_UTIME_H -# include -#elif defined(HAVE_SYS_UTIME_H) -# include -#endif - -#ifdef HAVE_VARARGS_H -# include -#endif - -// AIX/PASE does not properly define usleep within its headers. However, the -// function is available in libc.a. This extern define ensures that it is -// usable within the ccache code base. -#ifdef _AIX -extern "C" int usleep(useconds_t); -#endif - -#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0])) - -// Buffer size for I/O operations. Should be a multiple of 4 KiB. -const size_t READ_BUFFER_SIZE = 65536; - -#ifndef ESTALE -# define ESTALE -1 -#endif - -#ifdef _WIN32 -# ifndef _WIN32_WINNT -// _WIN32_WINNT is set in the generated header config.h -# error _WIN32_WINNT is undefined -# endif - -# ifdef _MSC_VER -typedef int mode_t; -typedef int pid_t; -# endif - -# ifndef __MINGW32__ -typedef int64_t ssize_t; -# endif - -// Defined in Win32Util.cpp -void usleep(int64_t usec); -struct tm* localtime_r(time_t* _clock, struct tm* _result); - -# ifdef _MSC_VER -int gettimeofday(struct timeval* tp, struct timezone* tzp); -int asprintf(char** strp, const char* fmt, ...); -# endif - -// From: -// http://mesos.apache.org/api/latest/c++/3rdparty_2stout_2include_2stout_2windows_8hpp_source.html -# ifdef _MSC_VER -const mode_t S_IRUSR = mode_t(_S_IREAD); -const mode_t S_IWUSR = mode_t(_S_IWRITE); -# endif - -# ifndef S_IFIFO -# define S_IFIFO 0x1000 -# endif - -# ifndef S_IFBLK -# define S_IFBLK 0x6000 -# endif - -# ifndef S_IFLNK -# define S_IFLNK 0xA000 -# endif - -# ifndef S_ISREG -# define S_ISREG(m) (((m)&S_IFMT) == S_IFREG) -# endif -# ifndef S_ISDIR -# define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR) -# endif -# ifndef S_ISFIFO -# define S_ISFIFO(m) (((m)&S_IFMT) == S_IFIFO) -# endif -# ifndef S_ISCHR -# define S_ISCHR(m) (((m)&S_IFMT) == S_IFCHR) -# endif -# ifndef S_ISLNK -# define S_ISLNK(m) (((m)&S_IFMT) == S_IFLNK) -# endif -# ifndef S_ISBLK -# define S_ISBLK(m) (((m)&S_IFMT) == S_IFBLK) -# endif - -# include -# include -# include -# define NOMINMAX 1 -# define WIN32_NO_STATUS -# include -# undef WIN32_NO_STATUS -# include -# define mkdir(a, b) _mkdir(a) -# define execv(a, b) \ - do_not_call_execv_on_windows // to protect against incidental use of MinGW - // execv -# define strncasecmp _strnicmp -# define strcasecmp _stricmp - -# ifdef _MSC_VER -# define PATH_MAX MAX_PATH -# endif - -# ifdef _MSC_VER -# define DLLIMPORT __declspec(dllimport) -# else -# define DLLIMPORT -# endif - -# define STDIN_FILENO 0 -# define STDOUT_FILENO 1 -# define STDERR_FILENO 2 -# define DIR_DELIM_CH '\\' -# define PATH_DELIM ";" -#else -# define DLLIMPORT -# define DIR_DELIM_CH '/' -# define PATH_DELIM ":" -#endif - -DLLIMPORT extern char** environ; - -// Work with silly DOS binary open. -#ifndef O_BINARY -# define O_BINARY 0 -#endif - -#if defined(HAVE_SYS_MMAN_H) && defined(HAVE_PTHREAD_MUTEXATTR_SETPSHARED) -# define INODE_CACHE_SUPPORTED -#endif - -// Workaround for missing std::is_trivially_copyable in GCC < 5. -#if __GNUG__ && __GNUC__ < 5 -# define IS_TRIVIALLY_COPYABLE(T) __has_trivial_copy(T) -#else -# define IS_TRIVIALLY_COPYABLE(T) std::is_trivially_copyable::value -#endif - -// GCC version of a couple of standard C++ attributes -#ifdef __GNUC__ -# define nodiscard gnu::warn_unused_result -# define maybe_unused gnu::unused -#endif diff --git a/src/test_lockfile.cpp b/src/test_lockfile.cpp new file mode 100644 index 0000000..3240654 --- /dev/null +++ b/src/test_lockfile.cpp @@ -0,0 +1,79 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include +#include +#include +#include +#include + +#include +#include +#include + +int +main(int argc, char** argv) +{ + if (argc != 5) { + PRINT_RAW(stderr, + "Usage: test-lockfile PATH SECONDS " + " \n"); + return 1; + } + Config config; + config.update_from_environment(); + Logging::init(config); + + const std::string path(argv[1]); + const auto seconds = util::parse_signed(argv[2]); + const bool long_lived = std::string(argv[3]) == "long"; + const bool blocking = std::string(argv[4]) == "blocking"; + if (!seconds) { + PRINT_RAW(stderr, "Error: Failed to parse seconds\n"); + return 1; + } + + using LockFilePtr = std::unique_ptr; + LockFilePtr lock_file; + lock_file = long_lived + ? LockFilePtr{std::make_unique(path)} + : LockFilePtr{std::make_unique(path)}; + const auto mode = blocking ? util::LockFileGuard::Mode::blocking + : util::LockFileGuard::Mode::non_blocking; + + PRINT(stdout, "{}\n", blocking ? "Acquiring" : "Trying to acquire"); + bool acquired = false; + { + util::LockFileGuard lock(*lock_file, mode); + acquired = lock.acquired(); + if (acquired) { + PRINT_RAW(stdout, "Acquired\n"); + PRINT( + stdout, "Sleeping {} second{}\n", *seconds, *seconds == 1 ? "" : "s"); + std::this_thread::sleep_for(std::chrono::seconds{*seconds}); + } else { + PRINT(stdout, "{} acquire\n", blocking ? "Failed to" : "Did not"); + } + if (acquired) { + PRINT_RAW(stdout, "Releasing\n"); + } + } + if (acquired) { + PRINT_RAW(stdout, "Released\n"); + } +} diff --git a/src/third_party/.clang-tidy b/src/third_party/.clang-tidy index d2f78e1..f291965 100644 --- a/src/third_party/.clang-tidy +++ b/src/third_party/.clang-tidy @@ -2,8 +2,7 @@ # config file... So just pick a fast check that will never fail. --- Checks: '-*,readability-function-size' -CheckOptions: +CheckOptions: - key: readability-function-size.LineThreshold value: 99999999 ... - diff --git a/src/third_party/CMakeLists.txt b/src/third_party/CMakeLists.txt index ed0ff9e..0d6ffb3 100644 --- a/src/third_party/CMakeLists.txt +++ b/src/third_party/CMakeLists.txt @@ -1,19 +1,22 @@ -add_library(third_party_lib STATIC base32hex.c format.cpp xxhash.c) +add_library(third_party STATIC base32hex.c format.cpp httplib.cpp url.cpp xxhash.c) if(NOT MSVC) - target_sources(third_party_lib PRIVATE getopt_long.c) + target_sources(third_party PRIVATE getopt_long.c) else() - target_sources(third_party_lib PRIVATE win32/getopt.c) - target_compile_definitions(third_party_lib PUBLIC -DSTATIC_GETOPT) + target_sources(third_party PRIVATE win32/getopt.c) + target_compile_definitions(third_party PUBLIC -DSTATIC_GETOPT) endif() if(WIN32) - target_sources(third_party_lib PRIVATE win32/mktemp.c) + target_sources(third_party PRIVATE win32/mktemp.c) endif () if(ENABLE_TRACING) - target_sources(third_party_lib PRIVATE minitrace.c) + target_sources(third_party PRIVATE minitrace.c) endif() +file(GLOB headers *.h fmt/*.h nonstd/*.hpp win32/*.h) +target_sources(third_party PRIVATE ${headers}) + set(xxhdispatchtest [=[ #include "xxh_x86dispatch.c" @@ -31,47 +34,28 @@ try_compile(USE_XXH_DISPATCH ${CMAKE_CURRENT_BINARY_DIR} CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${CMAKE_CURRENT_SOURCE_DIR}" COMPILE_DEFINITIONS "-DXXH_STATIC_LINKING_ONLY") -target_compile_definitions(third_party_lib INTERFACE "-DXXH_STATIC_LINKING_ONLY") +target_compile_definitions(third_party INTERFACE "-DXXH_STATIC_LINKING_ONLY") if(USE_XXH_DISPATCH) - target_sources(third_party_lib PRIVATE xxh_x86dispatch.c) - target_compile_definitions(third_party_lib INTERFACE "-DUSE_XXH_DISPATCH") + target_sources(third_party PRIVATE xxh_x86dispatch.c) + target_compile_definitions(third_party INTERFACE "-DUSE_XXH_DISPATCH") endif() # Treat third party headers as system files (no warning for those headers). target_include_directories( - third_party_lib - PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} SYSTEM) - -target_link_libraries(third_party_lib PRIVATE standard_settings) -target_link_libraries(third_party_lib INTERFACE blake3) + third_party + PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} SYSTEM +) -# These warnings are enabled by default even without e.g. -Wall, but we don't -# want them in third_party. -if(CMAKE_CXX_COMPILER_ID MATCHES "^GNU|Clang$") - target_compile_options( - third_party_lib - PRIVATE - $<$:-Wno-implicit-function-declaration - -Wno-int-conversion>) -endif() +target_link_libraries(third_party PRIVATE standard_settings) +target_link_libraries(third_party INTERFACE blake3) -if(CMAKE_C_COMPILER_ID STREQUAL "GNU") - target_compile_options( - third_party_lib - PRIVATE $<$:-Wno-attributes>) +if(WIN32) + target_link_libraries(third_party PRIVATE ws2_32) endif() # Silence warning from winbase.h due to /Zc:preprocessor. if(MSVC) - target_compile_options( - third_party_lib - PRIVATE /wd5105) -endif() - -# The headers are included from the rest of the project, so turn off warnings as -# required. -if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - target_compile_options(third_party_lib INTERFACE -Wno-shadow) + target_compile_options(third_party PRIVATE /wd5105) endif() add_subdirectory(blake3) diff --git a/src/third_party/blake3/CMakeLists.txt b/src/third_party/blake3/CMakeLists.txt index 581ee81..a30342d 100644 --- a/src/third_party/blake3/CMakeLists.txt +++ b/src/third_party/blake3/CMakeLists.txt @@ -17,17 +17,17 @@ function(add_source_if_enabled feature msvc_flags others_flags intrinsic) set(compile_flags "${others_flags}") endif() + if(MSVC) + set(suffix "_x86-64_windows_msvc.asm") + elseif(WIN32) + set(suffix "_x86-64_windows_gnu.S") + else() + set(suffix "_x86-64_unix.S") + endif() + # First check if it's possible to use the assembler variant for the feature. string(TOUPPER "have_asm_${feature}" have_feature) if(NOT DEFINED "${have_feature}" AND CMAKE_SIZEOF_VOID_P EQUAL 8) - if(MSVC) - set(suffix "_x86-64_windows_msvc.asm") - elseif(WIN32) - set(suffix "_x86-64_windows_gnu.S") - else() - set(suffix "_x86-64_unix.S") - endif() - if(NOT CMAKE_REQUIRED_QUIET) message(STATUS "Performing Test ${have_feature}") endif() @@ -115,3 +115,6 @@ if(CMAKE_SIZEOF_VOID_P EQUAL 8) target_compile_definitions(blake3 PRIVATE BLAKE3_USE_NEON) endif() endif() + +file(GLOB headers *.h) +target_sources(blake3 PRIVATE ${headers}) diff --git a/src/third_party/blake3/blake3.c b/src/third_party/blake3/blake3.c index 7abf532..1239433 100644 --- a/src/third_party/blake3/blake3.c +++ b/src/third_party/blake3/blake3.c @@ -5,9 +5,7 @@ #include "blake3.h" #include "blake3_impl.h" -const char * blake3_version(void) { - return BLAKE3_VERSION_STRING; -} +const char *blake3_version(void) { return BLAKE3_VERSION_STRING; } INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8], uint8_t flags) { @@ -342,12 +340,18 @@ INLINE void compress_subtree_to_parent_node( uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; size_t num_cvs = blake3_compress_subtree_wide(input, input_len, key, chunk_counter, flags, cv_array); + assert(num_cvs <= MAX_SIMD_DEGREE_OR_2); // If MAX_SIMD_DEGREE is greater than 2 and there's enough input, // compress_subtree_wide() returns more than 2 chaining values. Condense // them into 2 by forming parent nodes repeatedly. uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; - while (num_cvs > 2) { + // The second half of this loop condition is always true, and we just + // asserted it above. But GCC can't tell that it's always true, and if NDEBUG + // is set on platforms where MAX_SIMD_DEGREE_OR_2 == 2, GCC emits spurious + // warnings here. GCC 8.5 is particularly sensitive, so if you're changing + // this code, test it against that version. + while (num_cvs > 2 && num_cvs <= MAX_SIMD_DEGREE_OR_2) { num_cvs = compress_parents_parallel(cv_array, num_cvs, key, flags, out_array); memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); @@ -605,3 +609,8 @@ void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, } output_root_bytes(&output, seek, out, out_len); } + +void blake3_hasher_reset(blake3_hasher *self) { + chunk_state_reset(&self->chunk, self->key, 0); + self->cv_stack_len = 0; +} diff --git a/src/third_party/blake3/blake3.h b/src/third_party/blake3/blake3.h index 57ebd5a..7caf9b4 100644 --- a/src/third_party/blake3/blake3.h +++ b/src/third_party/blake3/blake3.h @@ -8,13 +8,12 @@ extern "C" { #endif -#define BLAKE3_VERSION_STRING "0.3.7" +#define BLAKE3_VERSION_STRING "1.3.1" #define BLAKE3_KEY_LEN 32 #define BLAKE3_OUT_LEN 32 #define BLAKE3_BLOCK_LEN 64 #define BLAKE3_CHUNK_LEN 1024 #define BLAKE3_MAX_DEPTH 54 -#define BLAKE3_MAX_SIMD_DEGREE 16 // This struct is a private implementation detail. It has to be here because // it's part of blake3_hasher below. @@ -39,12 +38,12 @@ typedef struct { uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN]; } blake3_hasher; -const char * blake3_version(void); +const char *blake3_version(void); void blake3_hasher_init(blake3_hasher *self); void blake3_hasher_init_keyed(blake3_hasher *self, const uint8_t key[BLAKE3_KEY_LEN]); void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context); -void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, +void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, size_t context_len); void blake3_hasher_update(blake3_hasher *self, const void *input, size_t input_len); @@ -52,6 +51,7 @@ void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, size_t out_len); void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, uint8_t *out, size_t out_len); +void blake3_hasher_reset(blake3_hasher *self); #ifdef __cplusplus } diff --git a/src/third_party/blake3/blake3_avx2.c b/src/third_party/blake3/blake3_avx2.c index c5a2ce9..e76aa1a 100644 --- a/src/third_party/blake3/blake3_avx2.c +++ b/src/third_party/blake3/blake3_avx2.c @@ -208,7 +208,7 @@ INLINE void transpose_msg_vecs(const uint8_t *const *inputs, out[14] = loadu(&inputs[6][block_offset + 1 * sizeof(__m256i)]); out[15] = loadu(&inputs[7][block_offset + 1 * sizeof(__m256i)]); for (size_t i = 0; i < 8; ++i) { - _mm_prefetch(&inputs[i][block_offset + 256], _MM_HINT_T0); + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); } transpose_vecs(&out[0]); transpose_vecs(&out[8]); @@ -219,14 +219,15 @@ INLINE void load_counters(uint64_t counter, bool increment_counter, const __m256i mask = _mm256_set1_epi32(-(int32_t)increment_counter); const __m256i add0 = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); const __m256i add1 = _mm256_and_si256(mask, add0); - __m256i l = _mm256_add_epi32(_mm256_set1_epi32(counter), add1); + __m256i l = _mm256_add_epi32(_mm256_set1_epi32((int32_t)counter), add1); __m256i carry = _mm256_cmpgt_epi32(_mm256_xor_si256(add1, _mm256_set1_epi32(0x80000000)), _mm256_xor_si256( l, _mm256_set1_epi32(0x80000000))); - __m256i h = _mm256_sub_epi32(_mm256_set1_epi32(counter >> 32), carry); + __m256i h = _mm256_sub_epi32(_mm256_set1_epi32((int32_t)(counter >> 32)), carry); *out_lo = l; *out_hi = h; } +static void blake3_hash8_avx2(const uint8_t *const *inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, diff --git a/src/third_party/blake3/blake3_avx512.c b/src/third_party/blake3/blake3_avx512.c index 77a5c38..9c35b08 100644 --- a/src/third_party/blake3/blake3_avx512.c +++ b/src/third_party/blake3/blake3_avx512.c @@ -468,7 +468,7 @@ INLINE void transpose_msg_vecs4(const uint8_t *const *inputs, out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(__m128i)]); out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(__m128i)]); for (size_t i = 0; i < 4; ++i) { - _mm_prefetch(&inputs[i][block_offset + 256], _MM_HINT_T0); + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); } transpose_vecs_128(&out[0]); transpose_vecs_128(&out[4]); @@ -488,6 +488,7 @@ INLINE void load_counters4(uint64_t counter, bool increment_counter, *out_hi = _mm256_cvtepi64_epi32(_mm256_srli_epi64(counters, 32)); } +static void blake3_hash4_avx512(const uint8_t *const *inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, @@ -724,7 +725,7 @@ INLINE void transpose_msg_vecs8(const uint8_t *const *inputs, out[14] = loadu_256(&inputs[6][block_offset + 1 * sizeof(__m256i)]); out[15] = loadu_256(&inputs[7][block_offset + 1 * sizeof(__m256i)]); for (size_t i = 0; i < 8; ++i) { - _mm_prefetch(&inputs[i][block_offset + 256], _MM_HINT_T0); + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); } transpose_vecs_256(&out[0]); transpose_vecs_256(&out[8]); @@ -742,6 +743,7 @@ INLINE void load_counters8(uint64_t counter, bool increment_counter, *out_hi = _mm512_cvtepi64_epi32(_mm512_srli_epi64(counters, 32)); } +static void blake3_hash8_avx512(const uint8_t *const *inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, @@ -1037,7 +1039,7 @@ INLINE void transpose_msg_vecs16(const uint8_t *const *inputs, out[14] = loadu_512(&inputs[14][block_offset]); out[15] = loadu_512(&inputs[15][block_offset]); for (size_t i = 0; i < 16; ++i) { - _mm_prefetch(&inputs[i][block_offset + 256], _MM_HINT_T0); + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); } transpose_vecs_512(out); } @@ -1047,13 +1049,14 @@ INLINE void load_counters16(uint64_t counter, bool increment_counter, const __m512i mask = _mm512_set1_epi32(-(int32_t)increment_counter); const __m512i add0 = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); const __m512i add1 = _mm512_and_si512(mask, add0); - __m512i l = _mm512_add_epi32(_mm512_set1_epi32(counter), add1); + __m512i l = _mm512_add_epi32(_mm512_set1_epi32((int32_t)counter), add1); __mmask16 carry = _mm512_cmp_epu32_mask(l, add1, _MM_CMPINT_LT); - __m512i h = _mm512_mask_add_epi32(_mm512_set1_epi32(counter >> 32), carry, _mm512_set1_epi32(counter >> 32), _mm512_set1_epi32(1)); + __m512i h = _mm512_mask_add_epi32(_mm512_set1_epi32((int32_t)(counter >> 32)), carry, _mm512_set1_epi32((int32_t)(counter >> 32)), _mm512_set1_epi32(1)); *out_lo = l; *out_hi = h; } +static void blake3_hash16_avx512(const uint8_t *const *inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, diff --git a/src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm index 97a7268..b19efba 100644 --- a/src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm +++ b/src/third_party/blake3/blake3_avx512_x86-64_windows_msvc.asm @@ -2421,8 +2421,8 @@ _blake3_compress_in_place_avx512 PROC movzx r8d, r8b shl rax, 32 add r8, rax - vmovd xmm3, r9 - vmovd xmm4, r8 + vmovq xmm3, r9 + vmovq xmm4, r8 vpunpcklqdq xmm3, xmm3, xmm4 vmovaps xmm2, xmmword ptr [BLAKE3_IV] vmovups xmm8, xmmword ptr [rdx] @@ -2516,8 +2516,8 @@ _blake3_compress_xof_avx512 PROC mov r10, qword ptr [rsp+78H] shl rax, 32 add r8, rax - vmovd xmm3, r9 - vmovd xmm4, r8 + vmovq xmm3, r9 + vmovq xmm4, r8 vpunpcklqdq xmm3, xmm3, xmm4 vmovaps xmm2, xmmword ptr [BLAKE3_IV] vmovups xmm8, xmmword ptr [rdx] diff --git a/src/third_party/blake3/blake3_dispatch.c b/src/third_party/blake3/blake3_dispatch.c index 6518478..b498058 100644 --- a/src/third_party/blake3/blake3_dispatch.c +++ b/src/third_party/blake3/blake3_dispatch.c @@ -232,7 +232,7 @@ void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, #endif #endif -#if defined(BLAKE3_USE_NEON) +#if BLAKE3_USE_NEON == 1 blake3_hash_many_neon(inputs, num_inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out); return; @@ -269,7 +269,7 @@ size_t blake3_simd_degree(void) { } #endif #endif -#if defined(BLAKE3_USE_NEON) +#if BLAKE3_USE_NEON == 1 return 4; #endif return 1; diff --git a/src/third_party/blake3/blake3_impl.h b/src/third_party/blake3/blake3_impl.h index 86ab6aa..cc5672f 100644 --- a/src/third_party/blake3/blake3_impl.h +++ b/src/third_party/blake3/blake3_impl.h @@ -38,6 +38,10 @@ enum blake3_flags { #define IS_X86_32 #endif +#if defined(__aarch64__) || defined(_M_ARM64) +#define IS_AARCH64 +#endif + #if defined(IS_X86) #if defined(_MSC_VER) #include @@ -45,9 +49,18 @@ enum blake3_flags { #include #endif +#if !defined(BLAKE3_USE_NEON) + // If BLAKE3_USE_NEON not manually set, autodetect based on AArch64ness + #if defined(IS_AARCH64) + #define BLAKE3_USE_NEON 1 + #else + #define BLAKE3_USE_NEON 0 + #endif +#endif + #if defined(IS_X86) #define MAX_SIMD_DEGREE 16 -#elif defined(BLAKE3_USE_NEON) +#elif BLAKE3_USE_NEON == 1 #define MAX_SIMD_DEGREE 4 #else #define MAX_SIMD_DEGREE 1 @@ -83,11 +96,11 @@ static unsigned int highest_one(uint64_t x) { #elif defined(_MSC_VER) && defined(IS_X86_32) if(x >> 32) { unsigned long index; - _BitScanReverse(&index, x >> 32); + _BitScanReverse(&index, (unsigned long)(x >> 32)); return 32 + index; } else { unsigned long index; - _BitScanReverse(&index, x); + _BitScanReverse(&index, (unsigned long)x); return index; } #else @@ -257,7 +270,7 @@ void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, #endif #endif -#if defined(BLAKE3_USE_NEON) +#if BLAKE3_USE_NEON == 1 void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, diff --git a/src/third_party/blake3/blake3_neon.c b/src/third_party/blake3/blake3_neon.c index 46691f5..a6f6da9 100644 --- a/src/third_party/blake3/blake3_neon.c +++ b/src/third_party/blake3/blake3_neon.c @@ -2,7 +2,12 @@ #include -// TODO: This is probably incorrect for big-endian ARM. How should that work? +#ifdef __ARM_BIG_ENDIAN +#error "This implementation only supports little-endian ARM." +// It might be that all we need for big-endian support here is to get the loads +// and stores right, but step zero would be finding a way to test it in CI. +#endif + INLINE uint32x4_t loadu_128(const uint8_t src[16]) { // vld1q_u32 has alignment requirements. Don't use it. uint32x4_t x; diff --git a/src/third_party/blake3/blake3_sse2.c b/src/third_party/blake3/blake3_sse2.c index 1592966..f4449ac 100644 --- a/src/third_party/blake3/blake3_sse2.c +++ b/src/third_party/blake3/blake3_sse2.c @@ -78,7 +78,7 @@ INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); } -INLINE __m128i blend_epi16(__m128i a, __m128i b, const int imm8) { +INLINE __m128i blend_epi16(__m128i a, __m128i b, const int16_t imm8) { const __m128i bits = _mm_set_epi16(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01); __m128i mask = _mm_set1_epi16(imm8); mask = _mm_and_si128(mask, bits); @@ -435,7 +435,7 @@ INLINE void transpose_msg_vecs(const uint8_t *const *inputs, out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); for (size_t i = 0; i < 4; ++i) { - _mm_prefetch(&inputs[i][block_offset + 256], _MM_HINT_T0); + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); } transpose_vecs(&out[0]); transpose_vecs(&out[4]); @@ -448,14 +448,15 @@ INLINE void load_counters(uint64_t counter, bool increment_counter, const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); const __m128i add1 = _mm_and_si128(mask, add0); - __m128i l = _mm_add_epi32(_mm_set1_epi32(counter), add1); + __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); - __m128i h = _mm_sub_epi32(_mm_set1_epi32(counter >> 32), carry); + __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); *out_lo = l; *out_hi = h; } +static void blake3_hash4_sse2(const uint8_t *const *inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, diff --git a/src/third_party/blake3/blake3_sse2_x86-64_unix.S b/src/third_party/blake3/blake3_sse2_x86-64_unix.S index d144046..99f033f 100644 --- a/src/third_party/blake3/blake3_sse2_x86-64_unix.S +++ b/src/third_party/blake3/blake3_sse2_x86-64_unix.S @@ -1704,7 +1704,7 @@ blake3_hash_many_sse2: pshufd xmm15, xmm11, 0x93 shl rax, 0x20 or rax, 0x40 - movd xmm3, rax + movq xmm3, rax movdqa xmmword ptr [rsp+0x20], xmm3 movaps xmm3, xmmword ptr [rsp] movaps xmm11, xmmword ptr [rsp+0x10] @@ -1917,7 +1917,7 @@ blake3_hash_many_sse2: movaps xmm2, xmmword ptr [BLAKE3_IV+rip] shl rax, 32 or rax, 64 - movd xmm12, rax + movq xmm12, rax movdqa xmm3, xmm13 punpcklqdq xmm3, xmm12 movups xmm4, xmmword ptr [r8+rdx-0x40] diff --git a/src/third_party/blake3/blake3_sse2_x86-64_windows_gnu.S b/src/third_party/blake3/blake3_sse2_x86-64_windows_gnu.S index 494c0c6..8852ba5 100644 --- a/src/third_party/blake3/blake3_sse2_x86-64_windows_gnu.S +++ b/src/third_party/blake3/blake3_sse2_x86-64_windows_gnu.S @@ -1715,7 +1715,7 @@ blake3_hash_many_sse2: pshufd xmm15, xmm11, 0x93 shl rax, 0x20 or rax, 0x40 - movd xmm3, rax + movq xmm3, rax movdqa xmmword ptr [rsp+0x20], xmm3 movaps xmm3, xmmword ptr [rsp] movaps xmm11, xmmword ptr [rsp+0x10] @@ -1928,7 +1928,7 @@ blake3_hash_many_sse2: movaps xmm2, xmmword ptr [BLAKE3_IV+rip] shl rax, 32 or rax, 64 - movd xmm12, rax + movq xmm12, rax movdqa xmm3, xmm13 punpcklqdq xmm3, xmm12 movups xmm4, xmmword ptr [r8+rdx-0x40] @@ -2137,10 +2137,10 @@ _blake3_compress_in_place_sse2: por xmm9, xmm8 movdqa xmm8, xmm7 punpcklqdq xmm8, xmm5 - movdqa xmm10, xmm6 + movdqa xmm14, xmm6 pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK+rip] - pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK+rip] - por xmm8, xmm10 + pand xmm14, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm8, xmm14 pshufd xmm8, xmm8, 0x78 punpckhdq xmm5, xmm7 punpckldq xmm6, xmm5 @@ -2268,10 +2268,10 @@ blake3_compress_xof_sse2: por xmm9, xmm8 movdqa xmm8, xmm7 punpcklqdq xmm8, xmm5 - movdqa xmm10, xmm6 + movdqa xmm14, xmm6 pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK+rip] - pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK+rip] - por xmm8, xmm10 + pand xmm14, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm8, xmm14 pshufd xmm8, xmm8, 0x78 punpckhdq xmm5, xmm7 punpckldq xmm6, xmm5 diff --git a/src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm b/src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm index ff9bb4d..507502f 100644 --- a/src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm +++ b/src/third_party/blake3/blake3_sse2_x86-64_windows_msvc.asm @@ -2139,10 +2139,10 @@ _blake3_compress_in_place_sse2 PROC por xmm9, xmm8 movdqa xmm8, xmm7 punpcklqdq xmm8, xmm5 - movdqa xmm10, xmm6 + movdqa xmm14, xmm6 pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK] - pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK] - por xmm8, xmm10 + pand xmm14, xmmword ptr [PBLENDW_0xC0_MASK] + por xmm8, xmm14 pshufd xmm8, xmm8, 78H punpckhdq xmm5, xmm7 punpckldq xmm6, xmm5 @@ -2271,10 +2271,10 @@ _blake3_compress_xof_sse2 PROC por xmm9, xmm8 movdqa xmm8, xmm7 punpcklqdq xmm8, xmm5 - movdqa xmm10, xmm6 + movdqa xmm14, xmm6 pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK] - pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK] - por xmm8, xmm10 + pand xmm14, xmmword ptr [PBLENDW_0xC0_MASK] + por xmm8, xmm14 pshufd xmm8, xmm8, 78H punpckhdq xmm5, xmm7 punpckldq xmm6, xmm5 diff --git a/src/third_party/blake3/blake3_sse41.c b/src/third_party/blake3/blake3_sse41.c index b311225..87a8dae 100644 --- a/src/third_party/blake3/blake3_sse41.c +++ b/src/third_party/blake3/blake3_sse41.c @@ -429,7 +429,7 @@ INLINE void transpose_msg_vecs(const uint8_t *const *inputs, out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); for (size_t i = 0; i < 4; ++i) { - _mm_prefetch(&inputs[i][block_offset + 256], _MM_HINT_T0); + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); } transpose_vecs(&out[0]); transpose_vecs(&out[4]); @@ -442,14 +442,15 @@ INLINE void load_counters(uint64_t counter, bool increment_counter, const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); const __m128i add1 = _mm_and_si128(mask, add0); - __m128i l = _mm_add_epi32(_mm_set1_epi32(counter), add1); + __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); - __m128i h = _mm_sub_epi32(_mm_set1_epi32(counter >> 32), carry); + __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); *out_lo = l; *out_hi = h; } +static void blake3_hash4_sse41(const uint8_t *const *inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, diff --git a/src/third_party/doctest.h b/src/third_party/doctest.h index 42eb039..aa2724c 100644 --- a/src/third_party/doctest.h +++ b/src/third_party/doctest.h @@ -11,7 +11,7 @@ // https://opensource.org/licenses/MIT // // The documentation can be found at the library's page: -// https://github.com/onqtam/doctest/blob/master/doc/markdown/readme.md +// https://github.com/doctest/doctest/blob/master/doc/markdown/readme.md // // ================================================================================================= // ================================================================================================= @@ -48,8 +48,16 @@ #define DOCTEST_VERSION_MAJOR 2 #define DOCTEST_VERSION_MINOR 4 -#define DOCTEST_VERSION_PATCH 6 -#define DOCTEST_VERSION_STR "2.4.6" +#define DOCTEST_VERSION_PATCH 9 + +// util we need here +#define DOCTEST_TOSTR_IMPL(x) #x +#define DOCTEST_TOSTR(x) DOCTEST_TOSTR_IMPL(x) + +#define DOCTEST_VERSION_STR \ + DOCTEST_TOSTR(DOCTEST_VERSION_MAJOR) "." \ + DOCTEST_TOSTR(DOCTEST_VERSION_MINOR) "." \ + DOCTEST_TOSTR(DOCTEST_VERSION_PATCH) #define DOCTEST_VERSION \ (DOCTEST_VERSION_MAJOR * 10000 + DOCTEST_VERSION_MINOR * 100 + DOCTEST_VERSION_PATCH) @@ -60,6 +68,12 @@ // ideas for the version stuff are taken from here: https://github.com/cxxstuff/cxx_detect +#ifdef _MSC_VER +#define DOCTEST_CPLUSPLUS _MSVC_LANG +#else +#define DOCTEST_CPLUSPLUS __cplusplus +#endif + #define DOCTEST_COMPILER(MAJOR, MINOR, PATCH) ((MAJOR)*10000000 + (MINOR)*100000 + (PATCH)) // GCC/Clang and GCC/MSVC are mutually exclusive, but Clang/MSVC are not because of clang-cl... @@ -137,85 +151,92 @@ // == COMPILER WARNINGS ============================================================================ // ================================================================================================= +// both the header and the implementation suppress all of these, +// so it only makes sense to aggregrate them like so +#define DOCTEST_SUPPRESS_COMMON_WARNINGS_PUSH \ + DOCTEST_CLANG_SUPPRESS_WARNING_PUSH \ + DOCTEST_CLANG_SUPPRESS_WARNING("-Wunknown-pragmas") \ + DOCTEST_CLANG_SUPPRESS_WARNING("-Wweak-vtables") \ + DOCTEST_CLANG_SUPPRESS_WARNING("-Wpadded") \ + DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-prototypes") \ + DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat") \ + DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat-pedantic") \ + \ + DOCTEST_GCC_SUPPRESS_WARNING_PUSH \ + DOCTEST_GCC_SUPPRESS_WARNING("-Wunknown-pragmas") \ + DOCTEST_GCC_SUPPRESS_WARNING("-Wpragmas") \ + DOCTEST_GCC_SUPPRESS_WARNING("-Weffc++") \ + DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-overflow") \ + DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-aliasing") \ + DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-declarations") \ + DOCTEST_GCC_SUPPRESS_WARNING("-Wuseless-cast") \ + DOCTEST_GCC_SUPPRESS_WARNING("-Wnoexcept") \ + \ + DOCTEST_MSVC_SUPPRESS_WARNING_PUSH \ + /* these 4 also disabled globally via cmake: */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4514) /* unreferenced inline function has been removed */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4571) /* SEH related */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4710) /* function not inlined */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4711) /* function selected for inline expansion*/ \ + /* */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4616) /* invalid compiler warning */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4619) /* invalid compiler warning */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4996) /* The compiler encountered a deprecated declaration */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4706) /* assignment within conditional expression */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4512) /* 'class' : assignment operator could not be generated */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4127) /* conditional expression is constant */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4820) /* padding */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4625) /* copy constructor was implicitly deleted */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4626) /* assignment operator was implicitly deleted */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(5027) /* move assignment operator implicitly deleted */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(5026) /* move constructor was implicitly deleted */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4640) /* construction of local static object not thread-safe */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(5045) /* Spectre mitigation for memory load */ \ + /* static analysis */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(26439) /* Function may not throw. Declare it 'noexcept' */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(26495) /* Always initialize a member variable */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(26451) /* Arithmetic overflow ... */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(26444) /* Avoid unnamed objects with custom ctor and dtor... */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(26812) /* Prefer 'enum class' over 'enum' */ + +#define DOCTEST_SUPPRESS_COMMON_WARNINGS_POP \ + DOCTEST_CLANG_SUPPRESS_WARNING_POP \ + DOCTEST_GCC_SUPPRESS_WARNING_POP \ + DOCTEST_MSVC_SUPPRESS_WARNING_POP + +DOCTEST_SUPPRESS_COMMON_WARNINGS_PUSH + DOCTEST_CLANG_SUPPRESS_WARNING_PUSH -DOCTEST_CLANG_SUPPRESS_WARNING("-Wunknown-pragmas") DOCTEST_CLANG_SUPPRESS_WARNING("-Wnon-virtual-dtor") -DOCTEST_CLANG_SUPPRESS_WARNING("-Wweak-vtables") -DOCTEST_CLANG_SUPPRESS_WARNING("-Wpadded") DOCTEST_CLANG_SUPPRESS_WARNING("-Wdeprecated") -DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-prototypes") -DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-local-typedef") -DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat") -DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat-pedantic") DOCTEST_GCC_SUPPRESS_WARNING_PUSH -DOCTEST_GCC_SUPPRESS_WARNING("-Wunknown-pragmas") -DOCTEST_GCC_SUPPRESS_WARNING("-Wpragmas") -DOCTEST_GCC_SUPPRESS_WARNING("-Weffc++") -DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-overflow") -DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-aliasing") DOCTEST_GCC_SUPPRESS_WARNING("-Wctor-dtor-privacy") -DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-declarations") DOCTEST_GCC_SUPPRESS_WARNING("-Wnon-virtual-dtor") -DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-local-typedefs") -DOCTEST_GCC_SUPPRESS_WARNING("-Wuseless-cast") -DOCTEST_GCC_SUPPRESS_WARNING("-Wnoexcept") DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-promo") DOCTEST_MSVC_SUPPRESS_WARNING_PUSH -DOCTEST_MSVC_SUPPRESS_WARNING(4616) // invalid compiler warning -DOCTEST_MSVC_SUPPRESS_WARNING(4619) // invalid compiler warning -DOCTEST_MSVC_SUPPRESS_WARNING(4996) // The compiler encountered a deprecated declaration -DOCTEST_MSVC_SUPPRESS_WARNING(4706) // assignment within conditional expression -DOCTEST_MSVC_SUPPRESS_WARNING(4512) // 'class' : assignment operator could not be generated -DOCTEST_MSVC_SUPPRESS_WARNING(4127) // conditional expression is constant -DOCTEST_MSVC_SUPPRESS_WARNING(4820) // padding -DOCTEST_MSVC_SUPPRESS_WARNING(4625) // copy constructor was implicitly defined as deleted -DOCTEST_MSVC_SUPPRESS_WARNING(4626) // assignment operator was implicitly defined as deleted -DOCTEST_MSVC_SUPPRESS_WARNING(5027) // move assignment operator was implicitly defined as deleted -DOCTEST_MSVC_SUPPRESS_WARNING(5026) // move constructor was implicitly defined as deleted DOCTEST_MSVC_SUPPRESS_WARNING(4623) // default constructor was implicitly defined as deleted -DOCTEST_MSVC_SUPPRESS_WARNING(4640) // construction of local static object is not thread-safe -// static analysis -DOCTEST_MSVC_SUPPRESS_WARNING(26439) // This kind of function may not throw. Declare it 'noexcept' -DOCTEST_MSVC_SUPPRESS_WARNING(26495) // Always initialize a member variable -DOCTEST_MSVC_SUPPRESS_WARNING(26451) // Arithmetic overflow ... -DOCTEST_MSVC_SUPPRESS_WARNING(26444) // Avoid unnamed objects with custom construction and dtr... -DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum' - -// 4548 - expression before comma has no effect; expected expression with side - effect -// 4265 - class has virtual functions, but destructor is not virtual -// 4986 - exception specification does not match previous declaration -// 4350 - behavior change: 'member1' called instead of 'member2' -// 4668 - 'x' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif' -// 4365 - conversion from 'int' to 'unsigned long', signed/unsigned mismatch -// 4774 - format string expected in argument 'x' is not a string literal -// 4820 - padding in structs - -// only 4 should be disabled globally: -// - 4514 # unreferenced inline function has been removed -// - 4571 # SEH related -// - 4710 # function not inlined -// - 4711 # function 'x' selected for automatic inline expansion #define DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN \ DOCTEST_MSVC_SUPPRESS_WARNING_PUSH \ - DOCTEST_MSVC_SUPPRESS_WARNING(4548) \ - DOCTEST_MSVC_SUPPRESS_WARNING(4265) \ - DOCTEST_MSVC_SUPPRESS_WARNING(4986) \ - DOCTEST_MSVC_SUPPRESS_WARNING(4350) \ - DOCTEST_MSVC_SUPPRESS_WARNING(4668) \ - DOCTEST_MSVC_SUPPRESS_WARNING(4365) \ - DOCTEST_MSVC_SUPPRESS_WARNING(4774) \ - DOCTEST_MSVC_SUPPRESS_WARNING(4820) \ - DOCTEST_MSVC_SUPPRESS_WARNING(4625) \ - DOCTEST_MSVC_SUPPRESS_WARNING(4626) \ - DOCTEST_MSVC_SUPPRESS_WARNING(5027) \ - DOCTEST_MSVC_SUPPRESS_WARNING(5026) \ - DOCTEST_MSVC_SUPPRESS_WARNING(4623) \ - DOCTEST_MSVC_SUPPRESS_WARNING(5039) \ - DOCTEST_MSVC_SUPPRESS_WARNING(5045) \ - DOCTEST_MSVC_SUPPRESS_WARNING(5105) + DOCTEST_MSVC_SUPPRESS_WARNING(4548) /* before comma no effect; expected side - effect */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4265) /* virtual functions, but destructor is not virtual */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4986) /* exception specification does not match previous */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4350) /* 'member1' called instead of 'member2' */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4668) /* not defined as a preprocessor macro */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4365) /* signed/unsigned mismatch */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4774) /* format string not a string literal */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4820) /* padding */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4625) /* copy constructor was implicitly deleted */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4626) /* assignment operator was implicitly deleted */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(5027) /* move assignment operator implicitly deleted */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(5026) /* move constructor was implicitly deleted */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4623) /* default constructor was implicitly deleted */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(5039) /* pointer to pot. throwing function passed to extern C */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(5045) /* Spectre mitigation for memory load */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(5105) /* macro producing 'defined' has undefined behavior */ \ + DOCTEST_MSVC_SUPPRESS_WARNING(4738) /* storing float result in memory, loss of performance */ #define DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_END DOCTEST_MSVC_SUPPRESS_WARNING_POP @@ -228,6 +249,7 @@ DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum' // GCC C++11 feature support table: https://gcc.gnu.org/projects/cxx-status.html // MSVC version table: // https://en.wikipedia.org/wiki/Microsoft_Visual_C%2B%2B#Internal_version_numbering +// MSVC++ 14.3 (17) _MSC_VER == 1930 (Visual Studio 2022) // MSVC++ 14.2 (16) _MSC_VER == 1920 (Visual Studio 2019) // MSVC++ 14.1 (15) _MSC_VER == 1910 (Visual Studio 2017) // MSVC++ 14.0 _MSC_VER == 1900 (Visual Studio 2015) @@ -237,6 +259,10 @@ DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum' // MSVC++ 9.0 _MSC_VER == 1500 (Visual Studio 2008) // MSVC++ 8.0 _MSC_VER == 1400 (Visual Studio 2005) +// Universal Windows Platform support +#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP) +#define DOCTEST_CONFIG_NO_WINDOWS_SEH +#endif // WINAPI_FAMILY #if DOCTEST_MSVC && !defined(DOCTEST_CONFIG_WINDOWS_SEH) #define DOCTEST_CONFIG_WINDOWS_SEH #endif // MSVC @@ -245,7 +271,7 @@ DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum' #endif // DOCTEST_CONFIG_NO_WINDOWS_SEH #if !defined(_WIN32) && !defined(__QNX__) && !defined(DOCTEST_CONFIG_POSIX_SIGNALS) && \ - !defined(__EMSCRIPTEN__) + !defined(__EMSCRIPTEN__) && !defined(__wasi__) #define DOCTEST_CONFIG_POSIX_SIGNALS #endif // _WIN32 #if defined(DOCTEST_CONFIG_NO_POSIX_SIGNALS) && defined(DOCTEST_CONFIG_POSIX_SIGNALS) @@ -253,7 +279,8 @@ DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum' #endif // DOCTEST_CONFIG_NO_POSIX_SIGNALS #ifndef DOCTEST_CONFIG_NO_EXCEPTIONS -#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND) +#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND) \ + || defined(__wasi__) #define DOCTEST_CONFIG_NO_EXCEPTIONS #endif // no exceptions #endif // DOCTEST_CONFIG_NO_EXCEPTIONS @@ -268,6 +295,10 @@ DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum' #define DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS #endif // DOCTEST_CONFIG_NO_EXCEPTIONS && !DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS +#ifdef __wasi__ +#define DOCTEST_CONFIG_NO_MULTITHREADING +#endif + #if defined(DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN) && !defined(DOCTEST_CONFIG_IMPLEMENT) #define DOCTEST_CONFIG_IMPLEMENT #endif // DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN @@ -295,6 +326,16 @@ DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum' #define DOCTEST_INTERFACE #endif // DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL +// needed for extern template instantiations +// see https://github.com/fmtlib/fmt/issues/2228 +#if DOCTEST_MSVC +#define DOCTEST_INTERFACE_DECL +#define DOCTEST_INTERFACE_DEF DOCTEST_INTERFACE +#else // DOCTEST_MSVC +#define DOCTEST_INTERFACE_DECL DOCTEST_INTERFACE +#define DOCTEST_INTERFACE_DEF +#endif // DOCTEST_MSVC + #define DOCTEST_EMPTY #if DOCTEST_MSVC @@ -312,17 +353,46 @@ DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum' #endif #ifndef DOCTEST_NORETURN +#if DOCTEST_MSVC && (DOCTEST_MSVC < DOCTEST_COMPILER(19, 0, 0)) +#define DOCTEST_NORETURN +#else // DOCTEST_MSVC #define DOCTEST_NORETURN [[noreturn]] +#endif // DOCTEST_MSVC #endif // DOCTEST_NORETURN #ifndef DOCTEST_NOEXCEPT +#if DOCTEST_MSVC && (DOCTEST_MSVC < DOCTEST_COMPILER(19, 0, 0)) +#define DOCTEST_NOEXCEPT +#else // DOCTEST_MSVC #define DOCTEST_NOEXCEPT noexcept +#endif // DOCTEST_MSVC #endif // DOCTEST_NOEXCEPT +#ifndef DOCTEST_CONSTEXPR +#if DOCTEST_MSVC && (DOCTEST_MSVC < DOCTEST_COMPILER(19, 0, 0)) +#define DOCTEST_CONSTEXPR const +#define DOCTEST_CONSTEXPR_FUNC inline +#else // DOCTEST_MSVC +#define DOCTEST_CONSTEXPR constexpr +#define DOCTEST_CONSTEXPR_FUNC constexpr +#endif // DOCTEST_MSVC +#endif // DOCTEST_CONSTEXPR + // ================================================================================================= // == FEATURE DETECTION END ======================================================================== // ================================================================================================= +#define DOCTEST_DECLARE_INTERFACE(name) \ + virtual ~name(); \ + name() = default; \ + name(const name&) = delete; \ + name(name&&) = delete; \ + name& operator=(const name&) = delete; \ + name& operator=(name&&) = delete; + +#define DOCTEST_DEFINE_INTERFACE(name) \ + name::~name() = default; + // internal macros for string concatenation and anonymous variable name generation #define DOCTEST_CAT_IMPL(s1, s2) s1##s2 #define DOCTEST_CAT(s1, s2) DOCTEST_CAT_IMPL(s1, s2) @@ -332,8 +402,6 @@ DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum' #define DOCTEST_ANONYMOUS(x) DOCTEST_CAT(x, __LINE__) #endif // __COUNTER__ -#define DOCTEST_TOSTR(x) #x - #ifndef DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE #define DOCTEST_REF_WRAP(x) x& #else // DOCTEST_CONFIG_ASSERTION_PARAMETERS_BY_VALUE @@ -347,31 +415,39 @@ DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum' #define DOCTEST_PLATFORM_IPHONE #elif defined(_WIN32) #define DOCTEST_PLATFORM_WINDOWS +#elif defined(__wasi__) +#define DOCTEST_PLATFORM_WASI #else // DOCTEST_PLATFORM #define DOCTEST_PLATFORM_LINUX #endif // DOCTEST_PLATFORM -#define DOCTEST_GLOBAL_NO_WARNINGS(var) \ - DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wglobal-constructors") \ - DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-variable") \ - static const int var DOCTEST_UNUSED // NOLINT(fuchsia-statically-constructed-objects,cert-err58-cpp) -#define DOCTEST_GLOBAL_NO_WARNINGS_END() DOCTEST_CLANG_SUPPRESS_WARNING_POP +namespace doctest { namespace detail { + static DOCTEST_CONSTEXPR int consume(const int*, int) noexcept { return 0; } +}} + +#define DOCTEST_GLOBAL_NO_WARNINGS(var, ...) \ + DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wglobal-constructors") \ + static const int var = doctest::detail::consume(&var, __VA_ARGS__); \ + DOCTEST_CLANG_SUPPRESS_WARNING_POP #ifndef DOCTEST_BREAK_INTO_DEBUGGER // should probably take a look at https://github.com/scottt/debugbreak #ifdef DOCTEST_PLATFORM_LINUX #if defined(__GNUC__) && (defined(__i386) || defined(__x86_64)) // Break at the location of the failing check if possible -#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("int $3\n" : :) // NOLINT (hicpp-no-assembler) +#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("int $3\n" : :) // NOLINT(hicpp-no-assembler) #else #include #define DOCTEST_BREAK_INTO_DEBUGGER() raise(SIGTRAP) #endif #elif defined(DOCTEST_PLATFORM_MAC) #if defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__i386) -#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("int $3\n" : :) // NOLINT (hicpp-no-assembler) +#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("int $3\n" : :) // NOLINT(hicpp-no-assembler) +#elif defined(__ppc__) || defined(__ppc64__) +// https://www.cocoawithlove.com/2008/03/break-into-debugger.html +#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("li r0, 20\nsc\nnop\nli r0, 37\nli r4, 2\nsc\nnop\n": : : "memory","r0","r3","r4") // NOLINT(hicpp-no-assembler) #else -#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("brk #0"); // NOLINT (hicpp-no-assembler) +#define DOCTEST_BREAK_INTO_DEBUGGER() __asm__("brk #0"); // NOLINT(hicpp-no-assembler) #endif #elif DOCTEST_MSVC #define DOCTEST_BREAK_INTO_DEBUGGER() __debugbreak() @@ -387,54 +463,66 @@ DOCTEST_GCC_SUPPRESS_WARNING_POP // this is kept here for backwards compatibility since the config option was changed #ifdef DOCTEST_CONFIG_USE_IOSFWD +#ifndef DOCTEST_CONFIG_USE_STD_HEADERS #define DOCTEST_CONFIG_USE_STD_HEADERS +#endif #endif // DOCTEST_CONFIG_USE_IOSFWD +// for clang - always include ciso646 (which drags some std stuff) because +// we want to check if we are using libc++ with the _LIBCPP_VERSION macro in +// which case we don't want to forward declare stuff from std - for reference: +// https://github.com/doctest/doctest/issues/126 +// https://github.com/doctest/doctest/issues/356 +#if DOCTEST_CLANG +#include +#ifdef _LIBCPP_VERSION +#ifndef DOCTEST_CONFIG_USE_STD_HEADERS +#define DOCTEST_CONFIG_USE_STD_HEADERS +#endif +#endif // _LIBCPP_VERSION +#endif // clang + #ifdef DOCTEST_CONFIG_USE_STD_HEADERS #ifndef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS #define DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS #endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS -#include +DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN #include #include +#include +DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_END #else // DOCTEST_CONFIG_USE_STD_HEADERS -#if DOCTEST_CLANG -// to detect if libc++ is being used with clang (the _LIBCPP_VERSION identifier) -#include -#endif // clang - -#ifdef _LIBCPP_VERSION -#define DOCTEST_STD_NAMESPACE_BEGIN _LIBCPP_BEGIN_NAMESPACE_STD -#define DOCTEST_STD_NAMESPACE_END _LIBCPP_END_NAMESPACE_STD -#else // _LIBCPP_VERSION -#define DOCTEST_STD_NAMESPACE_BEGIN namespace std { -#define DOCTEST_STD_NAMESPACE_END } -#endif // _LIBCPP_VERSION - // Forward declaring 'X' in namespace std is not permitted by the C++ Standard. DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4643) -DOCTEST_STD_NAMESPACE_BEGIN // NOLINT (cert-dcl58-cpp) -typedef decltype(nullptr) nullptr_t; +namespace std { // NOLINT(cert-dcl58-cpp) +typedef decltype(nullptr) nullptr_t; // NOLINT(modernize-use-using) +typedef decltype(sizeof(void*)) size_t; // NOLINT(modernize-use-using) template struct char_traits; template <> struct char_traits; template -class basic_ostream; -typedef basic_ostream> ostream; +class basic_ostream; // NOLINT(fuchsia-virtual-inheritance) +typedef basic_ostream> ostream; // NOLINT(modernize-use-using) +template +// NOLINTNEXTLINE +basic_ostream& operator<<(basic_ostream&, const char*); +template +class basic_istream; +typedef basic_istream> istream; // NOLINT(modernize-use-using) template class tuple; #if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0) -// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183 -template +// see this issue on why this is needed: https://github.com/doctest/doctest/issues/183 +template class allocator; -template +template class basic_string; using string = basic_string, allocator>; #endif // VS 2019 -DOCTEST_STD_NAMESPACE_END +} // namespace std DOCTEST_MSVC_SUPPRESS_WARNING_POP @@ -446,8 +534,14 @@ DOCTEST_MSVC_SUPPRESS_WARNING_POP namespace doctest { +using std::size_t; + DOCTEST_INTERFACE extern bool is_running_in_test; +#ifndef DOCTEST_CONFIG_STRING_SIZE_TYPE +#define DOCTEST_CONFIG_STRING_SIZE_TYPE unsigned +#endif + // A 24 byte string class (can be as small as 17 for x64 and 13 for x86) that can hold strings with length // of up to 23 chars on the stack before going on the heap - the last byte of the buffer is used for: // - "is small" bit - the highest bit - if "0" then it is small - otherwise its "1" (128) @@ -460,7 +554,6 @@ DOCTEST_INTERFACE extern bool is_running_in_test; // TODO: // - optimizations - like not deleting memory unnecessarily in operator= and etc. // - resize/reserve/clear -// - substr // - replace // - back/front // - iterator stuff @@ -470,63 +563,84 @@ DOCTEST_INTERFACE extern bool is_running_in_test; // - relational operators as free functions - taking const char* as one of the params class DOCTEST_INTERFACE String { - static const unsigned len = 24; //!OCLINT avoid private static members - static const unsigned last = len - 1; //!OCLINT avoid private static members +public: + using size_type = DOCTEST_CONFIG_STRING_SIZE_TYPE; + +private: + static DOCTEST_CONSTEXPR size_type len = 24; //!OCLINT avoid private static members + static DOCTEST_CONSTEXPR size_type last = len - 1; //!OCLINT avoid private static members struct view // len should be more than sizeof(view) - because of the final byte for flags { char* ptr; - unsigned size; - unsigned capacity; + size_type size; + size_type capacity; }; union { - char buf[len]; + char buf[len]; // NOLINT(*-avoid-c-arrays) view data; }; - bool isOnStack() const { return (buf[last] & 128) == 0; } - void setOnHeap(); - void setLast(unsigned in = last); + char* allocate(size_type sz); + + bool isOnStack() const noexcept { return (buf[last] & 128) == 0; } + void setOnHeap() noexcept; + void setLast(size_type in = last) noexcept; + void setSize(size_type sz) noexcept; void copy(const String& other); public: - String(); + static DOCTEST_CONSTEXPR size_type npos = static_cast(-1); + + String() noexcept; ~String(); // cppcheck-suppress noExplicitConstructor String(const char* in); - String(const char* in, unsigned in_size); + String(const char* in, size_type in_size); + + String(std::istream& in, size_type in_size); String(const String& other); String& operator=(const String& other); String& operator+=(const String& other); - String operator+(const String& other) const; - String(String&& other); - String& operator=(String&& other); + String(String&& other) noexcept; + String& operator=(String&& other) noexcept; - char operator[](unsigned i) const; - char& operator[](unsigned i); + char operator[](size_type i) const; + char& operator[](size_type i); // the only functions I'm willing to leave in the interface - available for inlining const char* c_str() const { return const_cast(this)->c_str(); } // NOLINT char* c_str() { - if(isOnStack()) + if (isOnStack()) { return reinterpret_cast(buf); + } return data.ptr; } - unsigned size() const; - unsigned capacity() const; + size_type size() const; + size_type capacity() const; + + String substr(size_type pos, size_type cnt = npos) &&; + String substr(size_type pos, size_type cnt = npos) const &; + + size_type find(char ch, size_type pos = 0) const; + size_type rfind(char ch, size_type pos = npos) const; int compare(const char* other, bool no_case = false) const; int compare(const String& other, bool no_case = false) const; + +friend DOCTEST_INTERFACE std::ostream& operator<<(std::ostream& s, const String& in); }; +DOCTEST_INTERFACE String operator+(const String& lhs, const String& rhs); + DOCTEST_INTERFACE bool operator==(const String& lhs, const String& rhs); DOCTEST_INTERFACE bool operator!=(const String& lhs, const String& rhs); DOCTEST_INTERFACE bool operator<(const String& lhs, const String& rhs); @@ -534,7 +648,21 @@ DOCTEST_INTERFACE bool operator>(const String& lhs, const String& rhs); DOCTEST_INTERFACE bool operator<=(const String& lhs, const String& rhs); DOCTEST_INTERFACE bool operator>=(const String& lhs, const String& rhs); -DOCTEST_INTERFACE std::ostream& operator<<(std::ostream& s, const String& in); +class DOCTEST_INTERFACE Contains { +public: + explicit Contains(const String& string); + + bool checkWith(const String& other) const; + + String string; +}; + +DOCTEST_INTERFACE String toString(const Contains& in); + +DOCTEST_INTERFACE bool operator==(const String& lhs, const Contains& rhs); +DOCTEST_INTERFACE bool operator==(const Contains& lhs, const String& rhs); +DOCTEST_INTERFACE bool operator!=(const String& lhs, const Contains& rhs); +DOCTEST_INTERFACE bool operator!=(const Contains& lhs, const String& rhs); namespace Color { enum Enum @@ -607,7 +735,7 @@ namespace assertType { DT_WARN_THROWS_WITH = is_throws_with | is_warn, DT_CHECK_THROWS_WITH = is_throws_with | is_check, DT_REQUIRE_THROWS_WITH = is_throws_with | is_require, - + DT_WARN_THROWS_WITH_AS = is_throws_with | is_throws_as | is_warn, DT_CHECK_THROWS_WITH_AS = is_throws_with | is_throws_as | is_check, DT_REQUIRE_THROWS_WITH_AS = is_throws_with | is_throws_as | is_require, @@ -688,9 +816,27 @@ struct DOCTEST_INTERFACE AssertData String m_decomp; // for specific exception-related asserts - bool m_threw_as; - const char* m_exception_type; - const char* m_exception_string; + bool m_threw_as; + const char* m_exception_type; + + class DOCTEST_INTERFACE StringContains { + private: + Contains content; + bool isContains; + + public: + StringContains(const String& str) : content(str), isContains(false) { } + StringContains(Contains cntn) : content(static_cast(cntn)), isContains(true) { } + + bool check(const String& str) { return isContains ? (content == str) : (content.string == str); } + + operator const String&() const { return content.string; } + + const char* c_str() const { return content.string.c_str(); } + } m_exception_string; + + AssertData(assertType::Enum at, const char* file, int line, const char* expr, + const char* exception_type, const StringContains& exception_string); }; struct DOCTEST_INTERFACE MessageData @@ -707,13 +853,13 @@ struct DOCTEST_INTERFACE SubcaseSignature const char* m_file; int m_line; + bool operator==(const SubcaseSignature& other) const; bool operator<(const SubcaseSignature& other) const; }; struct DOCTEST_INTERFACE IContextScope { - IContextScope(); - virtual ~IContextScope(); + DOCTEST_DECLARE_INTERFACE(IContextScope) virtual void stringify(std::ostream*) const = 0; }; @@ -723,9 +869,8 @@ namespace detail { struct ContextOptions //!OCLINT too many fields { - std::ostream* cout; // stdout stream - std::cout by default - std::ostream* cerr; // stderr stream - std::cerr by default - String binary_name; // the test binary name + std::ostream* cout = nullptr; // stdout stream + String binary_name; // the test binary name const detail::TestCase* currentTest = nullptr; @@ -744,9 +889,12 @@ struct ContextOptions //!OCLINT too many fields bool case_sensitive; // if filtering should be case sensitive bool exit; // if the program should be exited after the tests are ran/whatever bool duration; // print the time duration of each test case + bool minimal; // minimal console output (only test failures) + bool quiet; // no console output bool no_throw; // to skip exceptions-related assertion macros bool no_exitcode; // if the framework should return 0 as the exitcode bool no_run; // to not run the tests at all (can be done with an "*" exclude) + bool no_intro; // to not print the intro of the framework bool no_version; // to not print the version of the framework bool no_colors; // if output to the console should be colorized bool force_colors; // forces the use of colors even when a tty cannot be detected @@ -768,150 +916,184 @@ struct ContextOptions //!OCLINT too many fields }; namespace detail { - template - struct enable_if - {}; + namespace types { +#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS + using namespace std; +#else + template + struct enable_if { }; - template - struct enable_if - { typedef TYPE type; }; + template + struct enable_if { using type = T; }; - // clang-format off - template struct remove_reference { typedef T type; }; - template struct remove_reference { typedef T type; }; - template struct remove_reference { typedef T type; }; + struct true_type { static DOCTEST_CONSTEXPR bool value = true; }; + struct false_type { static DOCTEST_CONSTEXPR bool value = false; }; + + template struct remove_reference { using type = T; }; + template struct remove_reference { using type = T; }; + template struct remove_reference { using type = T; }; + + template struct is_rvalue_reference : false_type { }; + template struct is_rvalue_reference : true_type { }; - template U declval(int); + template struct remove_const { using type = T; }; + template struct remove_const { using type = T; }; - template T declval(long); + // Compiler intrinsics + template struct is_enum { static DOCTEST_CONSTEXPR bool value = __is_enum(T); }; + template struct underlying_type { using type = __underlying_type(T); }; - template auto declval() DOCTEST_NOEXCEPT -> decltype(declval(0)) ; + template struct is_pointer : false_type { }; + template struct is_pointer : true_type { }; - template struct is_lvalue_reference { const static bool value=false; }; - template struct is_lvalue_reference { const static bool value=true; }; + template struct is_array : false_type { }; + // NOLINTNEXTLINE(*-avoid-c-arrays) + template struct is_array : true_type { }; +#endif + } + + // + template + T&& declval(); template - inline T&& forward(typename remove_reference::type& t) DOCTEST_NOEXCEPT - { + DOCTEST_CONSTEXPR_FUNC T&& forward(typename types::remove_reference::type& t) DOCTEST_NOEXCEPT { return static_cast(t); } template - inline T&& forward(typename remove_reference::type&& t) DOCTEST_NOEXCEPT - { - static_assert(!is_lvalue_reference::value, - "Can not forward an rvalue as an lvalue."); + DOCTEST_CONSTEXPR_FUNC T&& forward(typename types::remove_reference::type&& t) DOCTEST_NOEXCEPT { return static_cast(t); } - template struct remove_const { typedef T type; }; - template struct remove_const { typedef T type; }; -#ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS - template struct is_enum : public std::is_enum {}; - template struct underlying_type : public std::underlying_type {}; -#else - // Use compiler intrinsics - template struct is_enum { constexpr static bool value = __is_enum(T); }; - template struct underlying_type { typedef __underlying_type(T) type; }; -#endif - // clang-format on + template + struct deferred_false : types::false_type { }; + +// MSVS 2015 :( +#if defined(_MSC_VER) && _MSC_VER <= 1900 + template + struct has_global_insertion_operator : types::false_type { }; template - struct deferred_false - // cppcheck-suppress unusedStructMember - { static const bool value = false; }; - - namespace has_insertion_operator_impl { - std::ostream &os(); - template - DOCTEST_REF_WRAP(T) val(); - - template - struct check { - static constexpr bool value = false; - }; + struct has_global_insertion_operator(), declval()), void())> : types::true_type { }; - template - struct check(), void())> { - static constexpr bool value = true; - }; - } // namespace has_insertion_operator_impl + template + struct has_insertion_operator { static DOCTEST_CONSTEXPR bool value = has_global_insertion_operator::value; }; - template - using has_insertion_operator = has_insertion_operator_impl::check; + template + struct insert_hack; - DOCTEST_INTERFACE void my_memcpy(void* dest, const void* src, unsigned num); + template + struct insert_hack { + static void insert(std::ostream& os, const T& t) { ::operator<<(os, t); } + }; + + template + struct insert_hack { + static void insert(std::ostream& os, const T& t) { operator<<(os, t); } + }; + + template + using insert_hack_t = insert_hack::value>; +#else + template + struct has_insertion_operator : types::false_type { }; +#endif + +template +struct has_insertion_operator(), declval()), void())> : types::true_type { }; - DOCTEST_INTERFACE std::ostream* getTlsOss(); // returns a thread-local ostringstream - DOCTEST_INTERFACE String getTlsOssResult(); + DOCTEST_INTERFACE std::ostream* tlssPush(); + DOCTEST_INTERFACE String tlssPop(); template - struct StringMakerBase - { + struct StringMakerBase { template static String convert(const DOCTEST_REF_WRAP(T)) { +#ifdef DOCTEST_CONFIG_REQUIRE_STRINGIFICATION_FOR_ALL_USED_TYPES + static_assert(deferred_false::value, "No stringification detected for type T. See string conversion manual"); +#endif return "{?}"; } }; - template <> - struct StringMakerBase - { - template - static String convert(const DOCTEST_REF_WRAP(T) in) { - *getTlsOss() << in; - return getTlsOssResult(); - } - }; - - DOCTEST_INTERFACE String rawMemoryToString(const void* object, unsigned size); + template + struct filldata; template - String rawMemoryToString(const DOCTEST_REF_WRAP(T) object) { - return rawMemoryToString(&object, sizeof(object)); + void filloss(std::ostream* stream, const T& in) { + filldata::fill(stream, in); + } + + template + void filloss(std::ostream* stream, const T (&in)[N]) { // NOLINT(*-avoid-c-arrays) + // T[N], T(&)[N], T(&&)[N] have same behaviour. + // Hence remove reference. + filloss::type>(stream, in); } template - const char* type_to_string() { - return "<>"; + String toStream(const T& in) { + std::ostream* stream = tlssPush(); + filloss(stream, in); + return tlssPop(); } + + template <> + struct StringMakerBase { + template + static String convert(const DOCTEST_REF_WRAP(T) in) { + return toStream(in); + } + }; } // namespace detail template -struct StringMaker : public detail::StringMakerBase::value> +struct StringMaker : public detail::StringMakerBase< + detail::has_insertion_operator::value || detail::types::is_pointer::value || detail::types::is_array::value> {}; -template -struct StringMaker -{ - template - static String convert(U* p) { - if(p) - return detail::rawMemoryToString(p); - return "NULL"; - } -}; +#ifndef DOCTEST_STRINGIFY +#ifdef DOCTEST_CONFIG_DOUBLE_STRINGIFY +#define DOCTEST_STRINGIFY(...) toString(toString(__VA_ARGS__)) +#else +#define DOCTEST_STRINGIFY(...) toString(__VA_ARGS__) +#endif +#endif -template -struct StringMaker -{ - static String convert(R C::*p) { - if(p) - return detail::rawMemoryToString(p); - return "NULL"; - } -}; +template +String toString() { +#if DOCTEST_MSVC >= 0 && DOCTEST_CLANG == 0 && DOCTEST_GCC == 0 + String ret = __FUNCSIG__; // class doctest::String __cdecl doctest::toString(void) + String::size_type beginPos = ret.find('<'); + return ret.substr(beginPos + 1, ret.size() - beginPos - static_cast(sizeof(">(void)"))); +#else + String ret = __PRETTY_FUNCTION__; // doctest::String toString() [with T = TYPE] + String::size_type begin = ret.find('=') + 2; + return ret.substr(begin, ret.size() - begin - 1); +#endif +} -template ::value, bool>::type = true> +template ::value, bool>::type = true> String toString(const DOCTEST_REF_WRAP(T) value) { return StringMaker::convert(value); } #ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING -DOCTEST_INTERFACE String toString(char* in); DOCTEST_INTERFACE String toString(const char* in); #endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING + +#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0) +// see this issue on why this is needed: https://github.com/doctest/doctest/issues/183 +DOCTEST_INTERFACE String toString(const std::string& in); +#endif // VS 2019 + +DOCTEST_INTERFACE String toString(String in); + +DOCTEST_INTERFACE String toString(std::nullptr_t); + DOCTEST_INTERFACE String toString(bool in); + DOCTEST_INTERFACE String toString(float in); DOCTEST_INTERFACE String toString(double in); DOCTEST_INTERFACE String toString(double long in); @@ -919,40 +1101,85 @@ DOCTEST_INTERFACE String toString(double long in); DOCTEST_INTERFACE String toString(char in); DOCTEST_INTERFACE String toString(char signed in); DOCTEST_INTERFACE String toString(char unsigned in); -DOCTEST_INTERFACE String toString(int short in); -DOCTEST_INTERFACE String toString(int short unsigned in); -DOCTEST_INTERFACE String toString(int in); -DOCTEST_INTERFACE String toString(int unsigned in); -DOCTEST_INTERFACE String toString(int long in); -DOCTEST_INTERFACE String toString(int long unsigned in); -DOCTEST_INTERFACE String toString(int long long in); -DOCTEST_INTERFACE String toString(int long long unsigned in); -DOCTEST_INTERFACE String toString(std::nullptr_t in); - -template ::value, bool>::type = true> +DOCTEST_INTERFACE String toString(short in); +DOCTEST_INTERFACE String toString(short unsigned in); +DOCTEST_INTERFACE String toString(signed in); +DOCTEST_INTERFACE String toString(unsigned in); +DOCTEST_INTERFACE String toString(long in); +DOCTEST_INTERFACE String toString(long unsigned in); +DOCTEST_INTERFACE String toString(long long in); +DOCTEST_INTERFACE String toString(long long unsigned in); + +template ::value, bool>::type = true> String toString(const DOCTEST_REF_WRAP(T) value) { - typedef typename detail::underlying_type::type UT; - return toString(static_cast(value)); + using UT = typename detail::types::underlying_type::type; + return (DOCTEST_STRINGIFY(static_cast(value))); } -#if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0) -// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183 -DOCTEST_INTERFACE String toString(const std::string& in); -#endif // VS 2019 +namespace detail { + template + struct filldata + { + static void fill(std::ostream* stream, const T& in) { +#if defined(_MSC_VER) && _MSC_VER <= 1900 + insert_hack_t::insert(*stream, in); +#else + operator<<(*stream, in); +#endif + } + }; + +DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4866) +// NOLINTBEGIN(*-avoid-c-arrays) + template + struct filldata { + static void fill(std::ostream* stream, const T(&in)[N]) { + *stream << "["; + for (size_t i = 0; i < N; i++) { + if (i != 0) { *stream << ", "; } + *stream << (DOCTEST_STRINGIFY(in[i])); + } + *stream << "]"; + } + }; +// NOLINTEND(*-avoid-c-arrays) +DOCTEST_MSVC_SUPPRESS_WARNING_POP -class DOCTEST_INTERFACE Approx + // Specialized since we don't want the terminating null byte! +// NOLINTBEGIN(*-avoid-c-arrays) + template + struct filldata { + static void fill(std::ostream* stream, const char (&in)[N]) { + *stream << String(in, in[N - 1] ? N : N - 1); + } // NOLINT(clang-analyzer-cplusplus.NewDeleteLeaks) + }; +// NOLINTEND(*-avoid-c-arrays) + + template <> + struct filldata { + static void fill(std::ostream* stream, const void* in); + }; + + template + struct filldata { + static void fill(std::ostream* stream, const T* in) { + filldata::fill(stream, in); + } + }; +} + +struct DOCTEST_INTERFACE Approx { -public: - explicit Approx(double value); + Approx(double value); Approx operator()(double value) const; #ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS template explicit Approx(const T& value, - typename detail::enable_if::value>::type* = + typename detail::types::enable_if::value>::type* = static_cast(nullptr)) { - *this = Approx(static_cast(value)); + *this = static_cast(value); } #endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS @@ -960,7 +1187,7 @@ public: #ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS template - typename detail::enable_if::value, Approx&>::type epsilon( + typename std::enable_if::value, Approx&>::type epsilon( const T& newEpsilon) { m_epsilon = static_cast(newEpsilon); return *this; @@ -971,7 +1198,7 @@ public: #ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS template - typename detail::enable_if::value, Approx&>::type scale( + typename std::enable_if::value, Approx&>::type scale( const T& newScale) { m_scale = static_cast(newScale); return *this; @@ -992,30 +1219,27 @@ public: DOCTEST_INTERFACE friend bool operator> (double lhs, const Approx & rhs); DOCTEST_INTERFACE friend bool operator> (const Approx & lhs, double rhs); - DOCTEST_INTERFACE friend String toString(const Approx& in); - #ifdef DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS #define DOCTEST_APPROX_PREFIX \ - template friend typename detail::enable_if::value, bool>::type + template friend typename std::enable_if::value, bool>::type - DOCTEST_APPROX_PREFIX operator==(const T& lhs, const Approx& rhs) { return operator==(double(lhs), rhs); } + DOCTEST_APPROX_PREFIX operator==(const T& lhs, const Approx& rhs) { return operator==(static_cast(lhs), rhs); } DOCTEST_APPROX_PREFIX operator==(const Approx& lhs, const T& rhs) { return operator==(rhs, lhs); } DOCTEST_APPROX_PREFIX operator!=(const T& lhs, const Approx& rhs) { return !operator==(lhs, rhs); } DOCTEST_APPROX_PREFIX operator!=(const Approx& lhs, const T& rhs) { return !operator==(rhs, lhs); } - DOCTEST_APPROX_PREFIX operator<=(const T& lhs, const Approx& rhs) { return double(lhs) < rhs.m_value || lhs == rhs; } - DOCTEST_APPROX_PREFIX operator<=(const Approx& lhs, const T& rhs) { return lhs.m_value < double(rhs) || lhs == rhs; } - DOCTEST_APPROX_PREFIX operator>=(const T& lhs, const Approx& rhs) { return double(lhs) > rhs.m_value || lhs == rhs; } - DOCTEST_APPROX_PREFIX operator>=(const Approx& lhs, const T& rhs) { return lhs.m_value > double(rhs) || lhs == rhs; } - DOCTEST_APPROX_PREFIX operator< (const T& lhs, const Approx& rhs) { return double(lhs) < rhs.m_value && lhs != rhs; } - DOCTEST_APPROX_PREFIX operator< (const Approx& lhs, const T& rhs) { return lhs.m_value < double(rhs) && lhs != rhs; } - DOCTEST_APPROX_PREFIX operator> (const T& lhs, const Approx& rhs) { return double(lhs) > rhs.m_value && lhs != rhs; } - DOCTEST_APPROX_PREFIX operator> (const Approx& lhs, const T& rhs) { return lhs.m_value > double(rhs) && lhs != rhs; } + DOCTEST_APPROX_PREFIX operator<=(const T& lhs, const Approx& rhs) { return static_cast(lhs) < rhs.m_value || lhs == rhs; } + DOCTEST_APPROX_PREFIX operator<=(const Approx& lhs, const T& rhs) { return lhs.m_value < static_cast(rhs) || lhs == rhs; } + DOCTEST_APPROX_PREFIX operator>=(const T& lhs, const Approx& rhs) { return static_cast(lhs) > rhs.m_value || lhs == rhs; } + DOCTEST_APPROX_PREFIX operator>=(const Approx& lhs, const T& rhs) { return lhs.m_value > static_cast(rhs) || lhs == rhs; } + DOCTEST_APPROX_PREFIX operator< (const T& lhs, const Approx& rhs) { return static_cast(lhs) < rhs.m_value && lhs != rhs; } + DOCTEST_APPROX_PREFIX operator< (const Approx& lhs, const T& rhs) { return lhs.m_value < static_cast(rhs) && lhs != rhs; } + DOCTEST_APPROX_PREFIX operator> (const T& lhs, const Approx& rhs) { return static_cast(lhs) > rhs.m_value && lhs != rhs; } + DOCTEST_APPROX_PREFIX operator> (const Approx& lhs, const T& rhs) { return lhs.m_value > static_cast(rhs) && lhs != rhs; } #undef DOCTEST_APPROX_PREFIX #endif // DOCTEST_CONFIG_INCLUDE_TYPE_TRAITS // clang-format on -private: double m_epsilon; double m_scale; double m_value; @@ -1025,18 +1249,35 @@ DOCTEST_INTERFACE String toString(const Approx& in); DOCTEST_INTERFACE const ContextOptions* getContextOptions(); -#if !defined(DOCTEST_CONFIG_DISABLE) +template +struct DOCTEST_INTERFACE_DECL IsNaN +{ + F value; bool flipped; + IsNaN(F f, bool flip = false) : value(f), flipped(flip) { } + IsNaN operator!() const { return { value, !flipped }; } + operator bool() const; +}; +#ifndef __MINGW32__ +extern template struct DOCTEST_INTERFACE_DECL IsNaN; +extern template struct DOCTEST_INTERFACE_DECL IsNaN; +extern template struct DOCTEST_INTERFACE_DECL IsNaN; +#endif +DOCTEST_INTERFACE String toString(IsNaN in); +DOCTEST_INTERFACE String toString(IsNaN in); +DOCTEST_INTERFACE String toString(IsNaN in); + +#ifndef DOCTEST_CONFIG_DISABLE namespace detail { // clang-format off #ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING - template struct decay_array { typedef T type; }; - template struct decay_array { typedef T* type; }; - template struct decay_array { typedef T* type; }; + template struct decay_array { using type = T; }; + template struct decay_array { using type = T*; }; + template struct decay_array { using type = T*; }; - template struct not_char_pointer { enum { value = 1 }; }; - template<> struct not_char_pointer { enum { value = 0 }; }; - template<> struct not_char_pointer { enum { value = 0 }; }; + template struct not_char_pointer { static DOCTEST_CONSTEXPR value = 1; }; + template<> struct not_char_pointer { static DOCTEST_CONSTEXPR value = 0; }; + template<> struct not_char_pointer { static DOCTEST_CONSTEXPR value = 0; }; template struct can_use_op : public not_char_pointer::type> {}; #endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING @@ -1059,16 +1300,22 @@ namespace detail { bool m_entered = false; Subcase(const String& name, const char* file, int line); + Subcase(const Subcase&) = delete; + Subcase(Subcase&&) = delete; + Subcase& operator=(const Subcase&) = delete; + Subcase& operator=(Subcase&&) = delete; ~Subcase(); operator bool() const; + + private: + bool checkFilters(); }; template String stringifyBinaryExpr(const DOCTEST_REF_WRAP(L) lhs, const char* op, const DOCTEST_REF_WRAP(R) rhs) { - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - return toString(lhs) + op + toString(rhs); + return (DOCTEST_STRINGIFY(lhs)) + op + (DOCTEST_STRINGIFY(rhs)); } #if DOCTEST_CLANG && DOCTEST_CLANG < DOCTEST_COMPILER(3, 6, 0) @@ -1079,12 +1326,12 @@ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wunused-comparison") // If not it doesn't find the operator or if the operator at global scope is defined after // this template, the template won't be instantiated due to SFINAE. Once the template is not // instantiated it can look for global operator using normal conversions. -#define SFINAE_OP(ret,op) decltype(doctest::detail::declval() op doctest::detail::declval(),static_cast(0)) +#define SFINAE_OP(ret,op) decltype((void)(doctest::detail::declval() op doctest::detail::declval()),ret{}) #define DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(op, op_str, op_macro) \ template \ - DOCTEST_NOINLINE SFINAE_OP(Result,op) operator op(R&& rhs) { \ - bool res = op_macro(doctest::detail::forward(lhs), doctest::detail::forward(rhs)); \ + DOCTEST_NOINLINE SFINAE_OP(Result,op) operator op(R&& rhs) { \ + bool res = op_macro(doctest::detail::forward(lhs), doctest::detail::forward(rhs)); \ if(m_at & assertType::is_false) \ res = !res; \ if(!res || doctest::getContextOptions()->success) \ @@ -1103,11 +1350,12 @@ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wunused-comparison") return *this; \ } - struct DOCTEST_INTERFACE Result + struct DOCTEST_INTERFACE Result // NOLINT(*-member-init) { bool m_passed; String m_decomp; + Result() = default; // TODO: Why do we need this? (To remove NOLINT) Result(bool passed, const String& decomposition = String()); // forbidding some expressions based on this table: https://en.cppreference.com/w/cpp/language/operator_precedence @@ -1164,8 +1412,7 @@ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wunused-comparison") #ifndef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING #define DOCTEST_COMPARISON_RETURN_TYPE bool #else // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING -#define DOCTEST_COMPARISON_RETURN_TYPE typename enable_if::value || can_use_op::value, bool>::type - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) +#define DOCTEST_COMPARISON_RETURN_TYPE typename types::enable_if::value || can_use_op::value, bool>::type inline bool eq(const char* lhs, const char* rhs) { return String(lhs) == String(rhs); } inline bool ne(const char* lhs, const char* rhs) { return String(lhs) != String(rhs); } inline bool lt(const char* lhs, const char* rhs) { return String(lhs) < String(rhs); } @@ -1213,26 +1460,26 @@ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wunused-comparison") assertType::Enum m_at; explicit Expression_lhs(L&& in, assertType::Enum at) - : lhs(doctest::detail::forward(in)) + : lhs(static_cast(in)) , m_at(at) {} DOCTEST_NOINLINE operator Result() { -// this is needed only foc MSVC 2015: -// https://ci.appveyor.com/project/onqtam/doctest/builds/38181202 +// this is needed only for MSVC 2015 DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4800) // 'int': forcing value to bool bool res = static_cast(lhs); DOCTEST_MSVC_SUPPRESS_WARNING_POP - if(m_at & assertType::is_false) //!OCLINT bitwise operator in conditional + if(m_at & assertType::is_false) { //!OCLINT bitwise operator in conditional res = !res; + } - if(!res || getContextOptions()->success) - return Result(res, toString(lhs)); - return Result(res); + if(!res || getContextOptions()->success) { + return { res, (DOCTEST_STRINGIFY(lhs)) }; + } + return { res }; } - /* This is required for user-defined conversions from Expression_lhs to L */ - //operator L() const { return lhs; } - operator L() const { return lhs; } + /* This is required for user-defined conversions from Expression_lhs to L */ + operator L() const { return lhs; } // clang-format off DOCTEST_DO_BINARY_EXPRESSION_COMPARISON(==, " == ", DOCTEST_CMP_EQ) //!OCLINT bitwise operator in conditional @@ -1289,22 +1536,27 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP // https://github.com/catchorg/Catch2/issues/870 // https://github.com/catchorg/Catch2/issues/565 template - Expression_lhs operator<<(L &&operand) { - return Expression_lhs(doctest::detail::forward(operand), m_at); + Expression_lhs operator<<(L&& operand) { + return Expression_lhs(static_cast(operand), m_at); + } + + template ::value,void >::type* = nullptr> + Expression_lhs operator<<(const L &operand) { + return Expression_lhs(operand, m_at); } }; struct DOCTEST_INTERFACE TestSuite { - const char* m_test_suite; - const char* m_description; - bool m_skip; - bool m_no_breaks; - bool m_no_output; - bool m_may_fail; - bool m_should_fail; - int m_expected_failures; - double m_timeout; + const char* m_test_suite = nullptr; + const char* m_description = nullptr; + bool m_skip = false; + bool m_no_breaks = false; + bool m_no_output = false; + bool m_may_fail = false; + bool m_should_fail = false; + int m_expected_failures = 0; + double m_timeout = 0; TestSuite& operator*(const char* in); @@ -1315,25 +1567,28 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP } }; - typedef void (*funcType)(); + using funcType = void (*)(); struct DOCTEST_INTERFACE TestCase : public TestCaseData { funcType m_test; // a function pointer to the test case - const char* m_type; // for templated test cases - gets appended to the real name + String m_type; // for templated test cases - gets appended to the real name int m_template_id; // an ID used to distinguish between the different versions of a templated test case String m_full_name; // contains the name (only for templated test cases!) + the template type TestCase(funcType test, const char* file, unsigned line, const TestSuite& test_suite, - const char* type = "", int template_id = -1); + const String& type = String(), int template_id = -1); TestCase(const TestCase& other); + TestCase(TestCase&&) = delete; DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(26434) // hides a non-virtual function TestCase& operator=(const TestCase& other); DOCTEST_MSVC_SUPPRESS_WARNING_POP + TestCase& operator=(TestCase&&) = delete; + TestCase& operator*(const char* in); template @@ -1343,6 +1598,8 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP } bool operator<(const TestCase& other) const; + + ~TestCase() = default; }; // forward declarations of functions used by the macros @@ -1382,27 +1639,36 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP struct DOCTEST_INTERFACE ResultBuilder : public AssertData { ResultBuilder(assertType::Enum at, const char* file, int line, const char* expr, - const char* exception_type = "", const char* exception_string = ""); + const char* exception_type = "", const String& exception_string = ""); + + ResultBuilder(assertType::Enum at, const char* file, int line, const char* expr, + const char* exception_type, const Contains& exception_string); void setResult(const Result& res); template - DOCTEST_NOINLINE void binary_assert(const DOCTEST_REF_WRAP(L) lhs, + DOCTEST_NOINLINE bool binary_assert(const DOCTEST_REF_WRAP(L) lhs, const DOCTEST_REF_WRAP(R) rhs) { m_failed = !RelationalComparator()(lhs, rhs); - if(m_failed || getContextOptions()->success) + if (m_failed || getContextOptions()->success) { m_decomp = stringifyBinaryExpr(lhs, ", ", rhs); + } + return !m_failed; } template - DOCTEST_NOINLINE void unary_assert(const DOCTEST_REF_WRAP(L) val) { + DOCTEST_NOINLINE bool unary_assert(const DOCTEST_REF_WRAP(L) val) { m_failed = !val; - if(m_at & assertType::is_false) //!OCLINT bitwise operator in conditional + if (m_at & assertType::is_false) { //!OCLINT bitwise operator in conditional m_failed = !m_failed; + } + + if (m_failed || getContextOptions()->success) { + m_decomp = (DOCTEST_STRINGIFY(val)); + } - if(m_failed || getContextOptions()->success) - m_decomp = toString(val); + return !m_failed; } void translateException(); @@ -1422,8 +1688,8 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP DOCTEST_INTERFACE void failed_out_of_a_testing_context(const AssertData& ad); - DOCTEST_INTERFACE void decomp_assert(assertType::Enum at, const char* file, int line, - const char* expr, Result result); + DOCTEST_INTERFACE bool decomp_assert(assertType::Enum at, const char* file, int line, + const char* expr, const Result& result); #define DOCTEST_ASSERT_OUT_OF_TESTS(decomp) \ do { \ @@ -1438,7 +1704,7 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP if(checkIfShouldThrow(at)) \ throwException(); \ } \ - return; \ + return !failed; \ } \ } while(false) @@ -1453,7 +1719,7 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP throwException() template - DOCTEST_NOINLINE void binary_assert(assertType::Enum at, const char* file, int line, + DOCTEST_NOINLINE bool binary_assert(assertType::Enum at, const char* file, int line, const char* expr, const DOCTEST_REF_WRAP(L) lhs, const DOCTEST_REF_WRAP(R) rhs) { bool failed = !RelationalComparator()(lhs, rhs); @@ -1464,10 +1730,11 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP // ################################################################################### DOCTEST_ASSERT_OUT_OF_TESTS(stringifyBinaryExpr(lhs, ", ", rhs)); DOCTEST_ASSERT_IN_TESTS(stringifyBinaryExpr(lhs, ", ", rhs)); + return !failed; } template - DOCTEST_NOINLINE void unary_assert(assertType::Enum at, const char* file, int line, + DOCTEST_NOINLINE bool unary_assert(assertType::Enum at, const char* file, int line, const char* expr, const DOCTEST_REF_WRAP(L) val) { bool failed = !val; @@ -1478,14 +1745,14 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP // IF THE DEBUGGER BREAKS HERE - GO 1 LEVEL UP IN THE CALLSTACK FOR THE FAILING ASSERT // THIS IS THE EFFECT OF HAVING 'DOCTEST_CONFIG_SUPER_FAST_ASSERTS' DEFINED // ################################################################################### - DOCTEST_ASSERT_OUT_OF_TESTS(toString(val)); - DOCTEST_ASSERT_IN_TESTS(toString(val)); + DOCTEST_ASSERT_OUT_OF_TESTS((DOCTEST_STRINGIFY(val))); + DOCTEST_ASSERT_IN_TESTS((DOCTEST_STRINGIFY(val))); + return !failed; } struct DOCTEST_INTERFACE IExceptionTranslator { - IExceptionTranslator(); - virtual ~IExceptionTranslator(); + DOCTEST_DECLARE_INTERFACE(IExceptionTranslator) virtual bool translate(String&) const = 0; }; @@ -1501,7 +1768,7 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP try { throw; // lgtm [cpp/rethrow-no-exception] // cppcheck-suppress catchExceptionByValue - } catch(T ex) { // NOLINT + } catch(const T& ex) { res = m_translateFunction(ex); //!OCLINT parameter reassignment return true; } catch(...) {} //!OCLINT - empty catch statement @@ -1516,95 +1783,70 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP DOCTEST_INTERFACE void registerExceptionTranslatorImpl(const IExceptionTranslator* et); - template - struct StringStreamBase - { - template - static void convert(std::ostream* s, const T& in) { - *s << toString(in); - } - - // always treat char* as a string in this context - no matter - // if DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING is defined - static void convert(std::ostream* s, const char* in) { *s << String(in); } - }; + // ContextScope base class used to allow implementing methods of ContextScope + // that don't depend on the template parameter in doctest.cpp. + struct DOCTEST_INTERFACE ContextScopeBase : public IContextScope { + ContextScopeBase(const ContextScopeBase&) = delete; - template <> - struct StringStreamBase - { - template - static void convert(std::ostream* s, const T& in) { - *s << in; - } - }; + ContextScopeBase& operator=(const ContextScopeBase&) = delete; + ContextScopeBase& operator=(ContextScopeBase&&) = delete; - template - struct StringStream : public StringStreamBase::value> - {}; + ~ContextScopeBase() override = default; - template - void toStream(std::ostream* s, const T& value) { - StringStream::convert(s, value); - } - -#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING - DOCTEST_INTERFACE void toStream(std::ostream* s, char* in); - DOCTEST_INTERFACE void toStream(std::ostream* s, const char* in); -#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING - DOCTEST_INTERFACE void toStream(std::ostream* s, bool in); - DOCTEST_INTERFACE void toStream(std::ostream* s, float in); - DOCTEST_INTERFACE void toStream(std::ostream* s, double in); - DOCTEST_INTERFACE void toStream(std::ostream* s, double long in); - - DOCTEST_INTERFACE void toStream(std::ostream* s, char in); - DOCTEST_INTERFACE void toStream(std::ostream* s, char signed in); - DOCTEST_INTERFACE void toStream(std::ostream* s, char unsigned in); - DOCTEST_INTERFACE void toStream(std::ostream* s, int short in); - DOCTEST_INTERFACE void toStream(std::ostream* s, int short unsigned in); - DOCTEST_INTERFACE void toStream(std::ostream* s, int in); - DOCTEST_INTERFACE void toStream(std::ostream* s, int unsigned in); - DOCTEST_INTERFACE void toStream(std::ostream* s, int long in); - DOCTEST_INTERFACE void toStream(std::ostream* s, int long unsigned in); - DOCTEST_INTERFACE void toStream(std::ostream* s, int long long in); - DOCTEST_INTERFACE void toStream(std::ostream* s, int long long unsigned in); - - // ContextScope base class used to allow implementing methods of ContextScope - // that don't depend on the template parameter in doctest.cpp. - class DOCTEST_INTERFACE ContextScopeBase : public IContextScope { protected: ContextScopeBase(); + ContextScopeBase(ContextScopeBase&& other) noexcept; void destroy(); + bool need_to_destroy{true}; }; template class ContextScope : public ContextScopeBase { - const L lambda_; + L lambda_; public: explicit ContextScope(const L &lambda) : lambda_(lambda) {} + explicit ContextScope(L&& lambda) : lambda_(static_cast(lambda)) { } + + ContextScope(const ContextScope&) = delete; + ContextScope(ContextScope&&) noexcept = default; - ContextScope(ContextScope &&other) : lambda_(other.lambda_) {} + ContextScope& operator=(const ContextScope&) = delete; + ContextScope& operator=(ContextScope&&) = delete; void stringify(std::ostream* s) const override { lambda_(s); } - ~ContextScope() override { destroy(); } + ~ContextScope() override { + if (need_to_destroy) { + destroy(); + } + } }; struct DOCTEST_INTERFACE MessageBuilder : public MessageData { std::ostream* m_stream; + bool logged = false; MessageBuilder(const char* file, int line, assertType::Enum severity); - MessageBuilder() = delete; + + MessageBuilder(const MessageBuilder&) = delete; + MessageBuilder(MessageBuilder&&) = delete; + + MessageBuilder& operator=(const MessageBuilder&) = delete; + MessageBuilder& operator=(MessageBuilder&&) = delete; + ~MessageBuilder(); // the preferred way of chaining parameters for stringification +DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4866) template MessageBuilder& operator,(const T& in) { - toStream(m_stream, in); + *m_stream << (DOCTEST_STRINGIFY(in)); return *this; } +DOCTEST_MSVC_SUPPRESS_WARNING_POP // kept here just for backwards-compatibility - the comma operator should be preferred now template @@ -1620,7 +1862,7 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP bool log(); void react(); }; - + template ContextScope MakeContextScope(const L &lambda) { return ContextScope(lambda); @@ -1673,7 +1915,7 @@ int registerExceptionTranslator(String (*)(T)) { #endif // DOCTEST_CONFIG_DISABLE namespace detail { - typedef void (*assert_handler)(const AssertData&); + using assert_handler = void (*)(const AssertData&); struct ContextState; } // namespace detail @@ -1686,12 +1928,19 @@ class DOCTEST_INTERFACE Context public: explicit Context(int argc = 0, const char* const* argv = nullptr); - ~Context(); + Context(const Context&) = delete; + Context(Context&&) = delete; + + Context& operator=(const Context&) = delete; + Context& operator=(Context&&) = delete; + + ~Context(); // NOLINT(performance-trivially-destructible) void applyCommandLine(int argc, const char* const* argv); void addFilter(const char* filter, const char* value); void clearFilters(); + void setOption(const char* option, bool value); void setOption(const char* option, int value); void setOption(const char* option, const char* value); @@ -1701,6 +1950,8 @@ public: void setAssertHandler(detail::assert_handler ah); + void setCout(std::ostream* out); + int run(); }; @@ -1727,6 +1978,7 @@ struct DOCTEST_INTERFACE CurrentTestCaseStats int numAssertsFailedCurrentTest; double seconds; int failure_flags; // use TestCaseFailureReason::Enum + bool testCaseSuccess; }; struct DOCTEST_INTERFACE TestCaseException @@ -1790,8 +2042,7 @@ struct DOCTEST_INTERFACE IReporter // or isn't in the execution range (between first and last) (safe to cache a pointer to the input) virtual void test_case_skipped(const TestCaseData&) = 0; - // doctest will not be managing the lifetimes of reporters given to it but this would still be nice to have - virtual ~IReporter(); + DOCTEST_DECLARE_INTERFACE(IReporter) // can obtain all currently active contexts and stringify them if one wishes to do so static int get_num_active_contexts(); @@ -1803,7 +2054,7 @@ struct DOCTEST_INTERFACE IReporter }; namespace detail { - typedef IReporter* (*reporterCreatorFunc)(const ContextOptions&); + using reporterCreatorFunc = IReporter* (*)(const ContextOptions&); DOCTEST_INTERFACE void registerReporterImpl(const char* name, int prio, reporterCreatorFunc c, bool isReporter); @@ -1820,14 +2071,30 @@ int registerReporter(const char* name, int priority, bool isReporter) { } } // namespace doctest +#ifdef DOCTEST_CONFIG_ASSERTS_RETURN_VALUES +#define DOCTEST_FUNC_EMPTY [] { return false; }() +#else +#define DOCTEST_FUNC_EMPTY (void)0 +#endif + // if registering is not disabled -#if !defined(DOCTEST_CONFIG_DISABLE) +#ifndef DOCTEST_CONFIG_DISABLE + +#ifdef DOCTEST_CONFIG_ASSERTS_RETURN_VALUES +#define DOCTEST_FUNC_SCOPE_BEGIN [&] +#define DOCTEST_FUNC_SCOPE_END () +#define DOCTEST_FUNC_SCOPE_RET(v) return v +#else +#define DOCTEST_FUNC_SCOPE_BEGIN do +#define DOCTEST_FUNC_SCOPE_END while(false) +#define DOCTEST_FUNC_SCOPE_RET(v) (void)0 +#endif // common code in asserts - for convenience -#define DOCTEST_ASSERT_LOG_AND_REACT(b) \ - if(b.log()) \ - DOCTEST_BREAK_INTO_DEBUGGER(); \ - b.react() +#define DOCTEST_ASSERT_LOG_REACT_RETURN(b) \ + if(b.log()) DOCTEST_BREAK_INTO_DEBUGGER(); \ + b.react(); \ + DOCTEST_FUNC_SCOPE_RET(!b.m_failed) #ifdef DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS #define DOCTEST_WRAP_IN_TRY(x) x; @@ -1835,7 +2102,7 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_WRAP_IN_TRY(x) \ try { \ x; \ - } catch(...) { _DOCTEST_RB.translateException(); } + } catch(...) { DOCTEST_RB.translateException(); } #endif // DOCTEST_CONFIG_NO_TRY_CATCH_IN_ASSERTS #ifdef DOCTEST_CONFIG_VOID_CAST_EXPRESSIONS @@ -1849,27 +2116,26 @@ int registerReporter(const char* name, int priority, bool isReporter) { // registers the test by initializing a dummy var with a function #define DOCTEST_REGISTER_FUNCTION(global_prefix, f, decorators) \ - global_prefix DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \ + global_prefix DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(DOCTEST_ANON_VAR_), /* NOLINT */ \ doctest::detail::regTest( \ doctest::detail::TestCase( \ f, __FILE__, __LINE__, \ doctest_detail_test_suite_ns::getCurrentTestSuite()) * \ - decorators); \ - DOCTEST_GLOBAL_NO_WARNINGS_END() + decorators)) #define DOCTEST_IMPLEMENT_FIXTURE(der, base, func, decorators) \ - namespace { \ + namespace { /* NOLINT */ \ struct der : public base \ { \ void f(); \ }; \ - static void func() { \ + static inline DOCTEST_NOINLINE void func() { \ der v; \ v.f(); \ } \ DOCTEST_REGISTER_FUNCTION(DOCTEST_EMPTY, func, decorators) \ } \ - inline DOCTEST_NOINLINE void der::f() + inline DOCTEST_NOINLINE void der::f() // NOLINT(misc-definitions-in-headers) #define DOCTEST_CREATE_AND_REGISTER_FUNCTION(f, decorators) \ static void f(); \ @@ -1878,18 +2144,18 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_CREATE_AND_REGISTER_FUNCTION_IN_CLASS(f, proxy, decorators) \ static doctest::detail::funcType proxy() { return f; } \ - DOCTEST_REGISTER_FUNCTION(inline const, proxy(), decorators) \ + DOCTEST_REGISTER_FUNCTION(inline, proxy(), decorators) \ static void f() // for registering tests #define DOCTEST_TEST_CASE(decorators) \ - DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), decorators) + DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(DOCTEST_ANON_FUNC_), decorators) // for registering tests in classes - requires C++17 for inline variables! -#if __cplusplus >= 201703L || (DOCTEST_MSVC >= DOCTEST_COMPILER(19, 12, 0) && _MSVC_LANG >= 201703L) +#if DOCTEST_CPLUSPLUS >= 201703L #define DOCTEST_TEST_CASE_CLASS(decorators) \ - DOCTEST_CREATE_AND_REGISTER_FUNCTION_IN_CLASS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), \ - DOCTEST_ANONYMOUS(_DOCTEST_ANON_PROXY_), \ + DOCTEST_CREATE_AND_REGISTER_FUNCTION_IN_CLASS(DOCTEST_ANONYMOUS(DOCTEST_ANON_FUNC_), \ + DOCTEST_ANONYMOUS(DOCTEST_ANON_PROXY_), \ decorators) #else // DOCTEST_TEST_CASE_CLASS #define DOCTEST_TEST_CASE_CLASS(...) \ @@ -1898,26 +2164,25 @@ int registerReporter(const char* name, int priority, bool isReporter) { // for registering tests with a fixture #define DOCTEST_TEST_CASE_FIXTURE(c, decorators) \ - DOCTEST_IMPLEMENT_FIXTURE(DOCTEST_ANONYMOUS(_DOCTEST_ANON_CLASS_), c, \ - DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), decorators) + DOCTEST_IMPLEMENT_FIXTURE(DOCTEST_ANONYMOUS(DOCTEST_ANON_CLASS_), c, \ + DOCTEST_ANONYMOUS(DOCTEST_ANON_FUNC_), decorators) // for converting types to strings without the header and demangling -#define DOCTEST_TYPE_TO_STRING_IMPL(...) \ - template <> \ - inline const char* type_to_string<__VA_ARGS__>() { \ - return "<" #__VA_ARGS__ ">"; \ - } -#define DOCTEST_TYPE_TO_STRING(...) \ - namespace doctest { namespace detail { \ - DOCTEST_TYPE_TO_STRING_IMPL(__VA_ARGS__) \ +#define DOCTEST_TYPE_TO_STRING_AS(str, ...) \ + namespace doctest { \ + template <> \ + inline String toString<__VA_ARGS__>() { \ + return str; \ } \ } \ - typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_) + static_assert(true, "") + +#define DOCTEST_TYPE_TO_STRING(...) DOCTEST_TYPE_TO_STRING_AS(#__VA_ARGS__, __VA_ARGS__) #define DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, iter, func) \ template \ static void func(); \ - namespace { \ + namespace { /* NOLINT */ \ template \ struct iter; \ template \ @@ -1926,7 +2191,7 @@ int registerReporter(const char* name, int priority, bool isReporter) { iter(const char* file, unsigned line, int index) { \ doctest::detail::regTest(doctest::detail::TestCase(func, file, line, \ doctest_detail_test_suite_ns::getCurrentTestSuite(), \ - doctest::detail::type_to_string(), \ + doctest::toString(), \ int(line) * 1000 + index) \ * dec); \ iter>(file, line, index + 1); \ @@ -1943,20 +2208,20 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_TEST_CASE_TEMPLATE_DEFINE(dec, T, id) \ DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, DOCTEST_CAT(id, ITERATOR), \ - DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_)) + DOCTEST_ANONYMOUS(DOCTEST_ANON_TMP_)) #define DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, anon, ...) \ - DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_CAT(anon, DUMMY)) = \ - doctest::detail::instantiationHelper(DOCTEST_CAT(id, ITERATOR)<__VA_ARGS__>(__FILE__, __LINE__, 0));\ - DOCTEST_GLOBAL_NO_WARNINGS_END() + DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_CAT(anon, DUMMY), /* NOLINT(cert-err58-cpp, fuchsia-statically-constructed-objects) */ \ + doctest::detail::instantiationHelper( \ + DOCTEST_CAT(id, ITERATOR)<__VA_ARGS__>(__FILE__, __LINE__, 0))) #define DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, ...) \ - DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), std::tuple<__VA_ARGS__>) \ - typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_) + DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, DOCTEST_ANONYMOUS(DOCTEST_ANON_TMP_), std::tuple<__VA_ARGS__>) \ + static_assert(true, "") #define DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, ...) \ - DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), __VA_ARGS__) \ - typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_) + DOCTEST_TEST_CASE_TEMPLATE_INSTANTIATE_IMPL(id, DOCTEST_ANONYMOUS(DOCTEST_ANON_TMP_), __VA_ARGS__) \ + static_assert(true, "") #define DOCTEST_TEST_CASE_TEMPLATE_IMPL(dec, T, anon, ...) \ DOCTEST_TEST_CASE_TEMPLATE_DEFINE_IMPL(dec, T, DOCTEST_CAT(anon, ITERATOR), anon); \ @@ -1965,17 +2230,17 @@ int registerReporter(const char* name, int priority, bool isReporter) { static void anon() #define DOCTEST_TEST_CASE_TEMPLATE(dec, T, ...) \ - DOCTEST_TEST_CASE_TEMPLATE_IMPL(dec, T, DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_), __VA_ARGS__) + DOCTEST_TEST_CASE_TEMPLATE_IMPL(dec, T, DOCTEST_ANONYMOUS(DOCTEST_ANON_TMP_), __VA_ARGS__) // for subcases #define DOCTEST_SUBCASE(name) \ - if(const doctest::detail::Subcase & DOCTEST_ANONYMOUS(_DOCTEST_ANON_SUBCASE_) DOCTEST_UNUSED = \ + if(const doctest::detail::Subcase & DOCTEST_ANONYMOUS(DOCTEST_ANON_SUBCASE_) DOCTEST_UNUSED = \ doctest::detail::Subcase(name, __FILE__, __LINE__)) // for grouping tests in test suites by using code blocks #define DOCTEST_TEST_SUITE_IMPL(decorators, ns_name) \ namespace ns_name { namespace doctest_detail_test_suite_ns { \ - static DOCTEST_NOINLINE doctest::detail::TestSuite& getCurrentTestSuite() { \ + static DOCTEST_NOINLINE doctest::detail::TestSuite& getCurrentTestSuite() noexcept { \ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4640) \ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wexit-time-destructors") \ DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wmissing-field-initializers") \ @@ -1995,53 +2260,53 @@ int registerReporter(const char* name, int priority, bool isReporter) { namespace ns_name #define DOCTEST_TEST_SUITE(decorators) \ - DOCTEST_TEST_SUITE_IMPL(decorators, DOCTEST_ANONYMOUS(_DOCTEST_ANON_SUITE_)) + DOCTEST_TEST_SUITE_IMPL(decorators, DOCTEST_ANONYMOUS(DOCTEST_ANON_SUITE_)) // for starting a testsuite block #define DOCTEST_TEST_SUITE_BEGIN(decorators) \ - DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \ - doctest::detail::setTestSuite(doctest::detail::TestSuite() * decorators); \ - DOCTEST_GLOBAL_NO_WARNINGS_END() \ - typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_) + DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(DOCTEST_ANON_VAR_), /* NOLINT(cert-err58-cpp) */ \ + doctest::detail::setTestSuite(doctest::detail::TestSuite() * decorators)) \ + static_assert(true, "") // for ending a testsuite block #define DOCTEST_TEST_SUITE_END \ - DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_VAR_)) = \ - doctest::detail::setTestSuite(doctest::detail::TestSuite() * ""); \ - DOCTEST_GLOBAL_NO_WARNINGS_END() \ - typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_) + DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(DOCTEST_ANON_VAR_), /* NOLINT(cert-err58-cpp) */ \ + doctest::detail::setTestSuite(doctest::detail::TestSuite() * "")) \ + using DOCTEST_ANONYMOUS(DOCTEST_ANON_FOR_SEMICOLON_) = int // for registering exception translators #define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR_IMPL(translatorName, signature) \ inline doctest::String translatorName(signature); \ - DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_)) = \ - doctest::registerExceptionTranslator(translatorName); \ - DOCTEST_GLOBAL_NO_WARNINGS_END() \ + DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(DOCTEST_ANON_TRANSLATOR_), /* NOLINT(cert-err58-cpp) */ \ + doctest::registerExceptionTranslator(translatorName)) \ doctest::String translatorName(signature) #define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature) \ - DOCTEST_REGISTER_EXCEPTION_TRANSLATOR_IMPL(DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_), \ + DOCTEST_REGISTER_EXCEPTION_TRANSLATOR_IMPL(DOCTEST_ANONYMOUS(DOCTEST_ANON_TRANSLATOR_), \ signature) // for registering reporters #define DOCTEST_REGISTER_REPORTER(name, priority, reporter) \ - DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_REPORTER_)) = \ - doctest::registerReporter(name, priority, true); \ - DOCTEST_GLOBAL_NO_WARNINGS_END() typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_) + DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(DOCTEST_ANON_REPORTER_), /* NOLINT(cert-err58-cpp) */ \ + doctest::registerReporter(name, priority, true)) \ + static_assert(true, "") // for registering listeners #define DOCTEST_REGISTER_LISTENER(name, priority, reporter) \ - DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(_DOCTEST_ANON_REPORTER_)) = \ - doctest::registerReporter(name, priority, false); \ - DOCTEST_GLOBAL_NO_WARNINGS_END() typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_) + DOCTEST_GLOBAL_NO_WARNINGS(DOCTEST_ANONYMOUS(DOCTEST_ANON_REPORTER_), /* NOLINT(cert-err58-cpp) */ \ + doctest::registerReporter(name, priority, false)) \ + static_assert(true, "") -// for logging +// clang-format off +// for logging - disabling formatting because it's important to have these on 2 separate lines - see PR #557 #define DOCTEST_INFO(...) \ - DOCTEST_INFO_IMPL(DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), \ + DOCTEST_INFO_IMPL(DOCTEST_ANONYMOUS(DOCTEST_CAPTURE_), \ + DOCTEST_ANONYMOUS(DOCTEST_CAPTURE_OTHER_), \ __VA_ARGS__) +// clang-format on #define DOCTEST_INFO_IMPL(mb_name, s_name, ...) \ - auto DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_) = doctest::detail::MakeContextScope( \ + auto DOCTEST_ANONYMOUS(DOCTEST_CAPTURE_) = doctest::detail::MakeContextScope( \ [&](std::ostream* s_name) { \ doctest::detail::MessageBuilder mb_name(__FILE__, __LINE__, doctest::assertType::is_warn); \ mb_name.m_stream = s_name; \ @@ -2051,16 +2316,18 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_CAPTURE(x) DOCTEST_INFO(#x " := ", x) #define DOCTEST_ADD_AT_IMPL(type, file, line, mb, ...) \ - do { \ + DOCTEST_FUNC_SCOPE_BEGIN { \ doctest::detail::MessageBuilder mb(file, line, doctest::assertType::type); \ mb * __VA_ARGS__; \ - DOCTEST_ASSERT_LOG_AND_REACT(mb); \ - } while(false) + if(mb.log()) \ + DOCTEST_BREAK_INTO_DEBUGGER(); \ + mb.react(); \ + } DOCTEST_FUNC_SCOPE_END // clang-format off -#define DOCTEST_ADD_MESSAGE_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_warn, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__) -#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_check, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__) -#define DOCTEST_ADD_FAIL_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_require, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__) +#define DOCTEST_ADD_MESSAGE_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_warn, file, line, DOCTEST_ANONYMOUS(DOCTEST_MESSAGE_), __VA_ARGS__) +#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_check, file, line, DOCTEST_ANONYMOUS(DOCTEST_MESSAGE_), __VA_ARGS__) +#define DOCTEST_ADD_FAIL_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_require, file, line, DOCTEST_ANONYMOUS(DOCTEST_MESSAGE_), __VA_ARGS__) // clang-format on #define DOCTEST_MESSAGE(...) DOCTEST_ADD_MESSAGE_AT(__FILE__, __LINE__, __VA_ARGS__) @@ -2073,18 +2340,37 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_ASSERT_IMPLEMENT_2(assert_type, ...) \ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Woverloaded-shift-op-parentheses") \ - doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \ + /* NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) */ \ + doctest::detail::ResultBuilder DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \ __LINE__, #__VA_ARGS__); \ - DOCTEST_WRAP_IN_TRY(_DOCTEST_RB.setResult( \ + DOCTEST_WRAP_IN_TRY(DOCTEST_RB.setResult( \ doctest::detail::ExpressionDecomposer(doctest::assertType::assert_type) \ - << __VA_ARGS__)) \ - DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB) \ + << __VA_ARGS__)) /* NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) */ \ + DOCTEST_ASSERT_LOG_REACT_RETURN(DOCTEST_RB) \ DOCTEST_CLANG_SUPPRESS_WARNING_POP #define DOCTEST_ASSERT_IMPLEMENT_1(assert_type, ...) \ - do { \ + DOCTEST_FUNC_SCOPE_BEGIN { \ DOCTEST_ASSERT_IMPLEMENT_2(assert_type, __VA_ARGS__); \ - } while(false) + } DOCTEST_FUNC_SCOPE_END // NOLINT(clang-analyzer-cplusplus.NewDeleteLeaks) + +#define DOCTEST_BINARY_ASSERT(assert_type, comp, ...) \ + DOCTEST_FUNC_SCOPE_BEGIN { \ + doctest::detail::ResultBuilder DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \ + __LINE__, #__VA_ARGS__); \ + DOCTEST_WRAP_IN_TRY( \ + DOCTEST_RB.binary_assert( \ + __VA_ARGS__)) \ + DOCTEST_ASSERT_LOG_REACT_RETURN(DOCTEST_RB); \ + } DOCTEST_FUNC_SCOPE_END + +#define DOCTEST_UNARY_ASSERT(assert_type, ...) \ + DOCTEST_FUNC_SCOPE_BEGIN { \ + doctest::detail::ResultBuilder DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \ + __LINE__, #__VA_ARGS__); \ + DOCTEST_WRAP_IN_TRY(DOCTEST_RB.unary_assert(__VA_ARGS__)) \ + DOCTEST_ASSERT_LOG_REACT_RETURN(DOCTEST_RB); \ + } DOCTEST_FUNC_SCOPE_END #else // DOCTEST_CONFIG_SUPER_FAST_ASSERTS @@ -2098,6 +2384,14 @@ int registerReporter(const char* name, int priority, bool isReporter) { doctest::detail::ExpressionDecomposer(doctest::assertType::assert_type) \ << __VA_ARGS__) DOCTEST_CLANG_SUPPRESS_WARNING_POP +#define DOCTEST_BINARY_ASSERT(assert_type, comparison, ...) \ + doctest::detail::binary_assert( \ + doctest::assertType::assert_type, __FILE__, __LINE__, #__VA_ARGS__, __VA_ARGS__) + +#define DOCTEST_UNARY_ASSERT(assert_type, ...) \ + doctest::detail::unary_assert(doctest::assertType::assert_type, __FILE__, __LINE__, \ + #__VA_ARGS__, __VA_ARGS__) + #endif // DOCTEST_CONFIG_SUPER_FAST_ASSERTS #define DOCTEST_WARN(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_WARN, __VA_ARGS__) @@ -2108,51 +2402,83 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_REQUIRE_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_REQUIRE_FALSE, __VA_ARGS__) // clang-format off -#define DOCTEST_WARN_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN, cond); } while(false) -#define DOCTEST_CHECK_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK, cond); } while(false) -#define DOCTEST_REQUIRE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE, cond); } while(false) -#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN_FALSE, cond); } while(false) -#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK_FALSE, cond); } while(false) -#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE_FALSE, cond); } while(false) +#define DOCTEST_WARN_MESSAGE(cond, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN, cond); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_CHECK_MESSAGE(cond, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK, cond); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_REQUIRE_MESSAGE(cond, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE, cond); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN_FALSE, cond); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK_FALSE, cond); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE_FALSE, cond); } DOCTEST_FUNC_SCOPE_END // clang-format on +#define DOCTEST_WARN_EQ(...) DOCTEST_BINARY_ASSERT(DT_WARN_EQ, eq, __VA_ARGS__) +#define DOCTEST_CHECK_EQ(...) DOCTEST_BINARY_ASSERT(DT_CHECK_EQ, eq, __VA_ARGS__) +#define DOCTEST_REQUIRE_EQ(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_EQ, eq, __VA_ARGS__) +#define DOCTEST_WARN_NE(...) DOCTEST_BINARY_ASSERT(DT_WARN_NE, ne, __VA_ARGS__) +#define DOCTEST_CHECK_NE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_NE, ne, __VA_ARGS__) +#define DOCTEST_REQUIRE_NE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_NE, ne, __VA_ARGS__) +#define DOCTEST_WARN_GT(...) DOCTEST_BINARY_ASSERT(DT_WARN_GT, gt, __VA_ARGS__) +#define DOCTEST_CHECK_GT(...) DOCTEST_BINARY_ASSERT(DT_CHECK_GT, gt, __VA_ARGS__) +#define DOCTEST_REQUIRE_GT(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_GT, gt, __VA_ARGS__) +#define DOCTEST_WARN_LT(...) DOCTEST_BINARY_ASSERT(DT_WARN_LT, lt, __VA_ARGS__) +#define DOCTEST_CHECK_LT(...) DOCTEST_BINARY_ASSERT(DT_CHECK_LT, lt, __VA_ARGS__) +#define DOCTEST_REQUIRE_LT(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_LT, lt, __VA_ARGS__) +#define DOCTEST_WARN_GE(...) DOCTEST_BINARY_ASSERT(DT_WARN_GE, ge, __VA_ARGS__) +#define DOCTEST_CHECK_GE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_GE, ge, __VA_ARGS__) +#define DOCTEST_REQUIRE_GE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_GE, ge, __VA_ARGS__) +#define DOCTEST_WARN_LE(...) DOCTEST_BINARY_ASSERT(DT_WARN_LE, le, __VA_ARGS__) +#define DOCTEST_CHECK_LE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_LE, le, __VA_ARGS__) +#define DOCTEST_REQUIRE_LE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_LE, le, __VA_ARGS__) + +#define DOCTEST_WARN_UNARY(...) DOCTEST_UNARY_ASSERT(DT_WARN_UNARY, __VA_ARGS__) +#define DOCTEST_CHECK_UNARY(...) DOCTEST_UNARY_ASSERT(DT_CHECK_UNARY, __VA_ARGS__) +#define DOCTEST_REQUIRE_UNARY(...) DOCTEST_UNARY_ASSERT(DT_REQUIRE_UNARY, __VA_ARGS__) +#define DOCTEST_WARN_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_WARN_UNARY_FALSE, __VA_ARGS__) +#define DOCTEST_CHECK_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_CHECK_UNARY_FALSE, __VA_ARGS__) +#define DOCTEST_REQUIRE_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_REQUIRE_UNARY_FALSE, __VA_ARGS__) + +#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS + #define DOCTEST_ASSERT_THROWS_AS(expr, assert_type, message, ...) \ - do { \ + DOCTEST_FUNC_SCOPE_BEGIN { \ if(!doctest::getContextOptions()->no_throw) { \ - doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \ + doctest::detail::ResultBuilder DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \ __LINE__, #expr, #__VA_ARGS__, message); \ try { \ DOCTEST_CAST_TO_VOID(expr) \ - } catch(const typename doctest::detail::remove_const< \ - typename doctest::detail::remove_reference<__VA_ARGS__>::type>::type&) { \ - _DOCTEST_RB.translateException(); \ - _DOCTEST_RB.m_threw_as = true; \ - } catch(...) { _DOCTEST_RB.translateException(); } \ - DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \ + } catch(const typename doctest::detail::types::remove_const< \ + typename doctest::detail::types::remove_reference<__VA_ARGS__>::type>::type&) {\ + DOCTEST_RB.translateException(); \ + DOCTEST_RB.m_threw_as = true; \ + } catch(...) { DOCTEST_RB.translateException(); } \ + DOCTEST_ASSERT_LOG_REACT_RETURN(DOCTEST_RB); \ + } else { /* NOLINT(*-else-after-return) */ \ + DOCTEST_FUNC_SCOPE_RET(false); \ } \ - } while(false) + } DOCTEST_FUNC_SCOPE_END #define DOCTEST_ASSERT_THROWS_WITH(expr, expr_str, assert_type, ...) \ - do { \ + DOCTEST_FUNC_SCOPE_BEGIN { \ if(!doctest::getContextOptions()->no_throw) { \ - doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \ + doctest::detail::ResultBuilder DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \ __LINE__, expr_str, "", __VA_ARGS__); \ try { \ DOCTEST_CAST_TO_VOID(expr) \ - } catch(...) { _DOCTEST_RB.translateException(); } \ - DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \ + } catch(...) { DOCTEST_RB.translateException(); } \ + DOCTEST_ASSERT_LOG_REACT_RETURN(DOCTEST_RB); \ + } else { /* NOLINT(*-else-after-return) */ \ + DOCTEST_FUNC_SCOPE_RET(false); \ } \ - } while(false) + } DOCTEST_FUNC_SCOPE_END #define DOCTEST_ASSERT_NOTHROW(assert_type, ...) \ - do { \ - doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \ + DOCTEST_FUNC_SCOPE_BEGIN { \ + doctest::detail::ResultBuilder DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \ __LINE__, #__VA_ARGS__); \ try { \ DOCTEST_CAST_TO_VOID(__VA_ARGS__) \ - } catch(...) { _DOCTEST_RB.translateException(); } \ - DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \ - } while(false) + } catch(...) { DOCTEST_RB.translateException(); } \ + DOCTEST_ASSERT_LOG_REACT_RETURN(DOCTEST_RB); \ + } DOCTEST_FUNC_SCOPE_END // clang-format off #define DOCTEST_WARN_THROWS(...) DOCTEST_ASSERT_THROWS_WITH((__VA_ARGS__), #__VA_ARGS__, DT_WARN_THROWS, "") @@ -2175,166 +2501,23 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_CHECK_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_CHECK_NOTHROW, __VA_ARGS__) #define DOCTEST_REQUIRE_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_REQUIRE_NOTHROW, __VA_ARGS__) -#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS(expr); } while(false) -#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS(expr); } while(false) -#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS(expr); } while(false) -#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_AS(expr, ex); } while(false) -#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_AS(expr, ex); } while(false) -#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_AS(expr, ex); } while(false) -#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH(expr, with); } while(false) -#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH(expr, with); } while(false) -#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH(expr, with); } while(false) -#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH_AS(expr, with, ex); } while(false) -#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ex); } while(false) -#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ex); } while(false) -#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_NOTHROW(expr); } while(false) -#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_NOTHROW(expr); } while(false) -#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_NOTHROW(expr); } while(false) +#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS(expr); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS(expr); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS(expr); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_AS(expr, ex); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_AS(expr, ex); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_AS(expr, ex); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH(expr, with); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH(expr, with); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH(expr, with); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH_AS(expr, with, ex); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ex); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ex); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_NOTHROW(expr); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_NOTHROW(expr); } DOCTEST_FUNC_SCOPE_END +#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) DOCTEST_FUNC_SCOPE_BEGIN { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_NOTHROW(expr); } DOCTEST_FUNC_SCOPE_END // clang-format on -#ifndef DOCTEST_CONFIG_SUPER_FAST_ASSERTS - -#define DOCTEST_BINARY_ASSERT(assert_type, comp, ...) \ - do { \ - doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \ - __LINE__, #__VA_ARGS__); \ - DOCTEST_WRAP_IN_TRY( \ - _DOCTEST_RB.binary_assert( \ - __VA_ARGS__)) \ - DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \ - } while(false) - -#define DOCTEST_UNARY_ASSERT(assert_type, ...) \ - do { \ - doctest::detail::ResultBuilder _DOCTEST_RB(doctest::assertType::assert_type, __FILE__, \ - __LINE__, #__VA_ARGS__); \ - DOCTEST_WRAP_IN_TRY(_DOCTEST_RB.unary_assert(__VA_ARGS__)) \ - DOCTEST_ASSERT_LOG_AND_REACT(_DOCTEST_RB); \ - } while(false) - -#else // DOCTEST_CONFIG_SUPER_FAST_ASSERTS - -#define DOCTEST_BINARY_ASSERT(assert_type, comparison, ...) \ - doctest::detail::binary_assert( \ - doctest::assertType::assert_type, __FILE__, __LINE__, #__VA_ARGS__, __VA_ARGS__) - -#define DOCTEST_UNARY_ASSERT(assert_type, ...) \ - doctest::detail::unary_assert(doctest::assertType::assert_type, __FILE__, __LINE__, \ - #__VA_ARGS__, __VA_ARGS__) - -#endif // DOCTEST_CONFIG_SUPER_FAST_ASSERTS - -#define DOCTEST_WARN_EQ(...) DOCTEST_BINARY_ASSERT(DT_WARN_EQ, eq, __VA_ARGS__) -#define DOCTEST_CHECK_EQ(...) DOCTEST_BINARY_ASSERT(DT_CHECK_EQ, eq, __VA_ARGS__) -#define DOCTEST_REQUIRE_EQ(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_EQ, eq, __VA_ARGS__) -#define DOCTEST_WARN_NE(...) DOCTEST_BINARY_ASSERT(DT_WARN_NE, ne, __VA_ARGS__) -#define DOCTEST_CHECK_NE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_NE, ne, __VA_ARGS__) -#define DOCTEST_REQUIRE_NE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_NE, ne, __VA_ARGS__) -#define DOCTEST_WARN_GT(...) DOCTEST_BINARY_ASSERT(DT_WARN_GT, gt, __VA_ARGS__) -#define DOCTEST_CHECK_GT(...) DOCTEST_BINARY_ASSERT(DT_CHECK_GT, gt, __VA_ARGS__) -#define DOCTEST_REQUIRE_GT(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_GT, gt, __VA_ARGS__) -#define DOCTEST_WARN_LT(...) DOCTEST_BINARY_ASSERT(DT_WARN_LT, lt, __VA_ARGS__) -#define DOCTEST_CHECK_LT(...) DOCTEST_BINARY_ASSERT(DT_CHECK_LT, lt, __VA_ARGS__) -#define DOCTEST_REQUIRE_LT(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_LT, lt, __VA_ARGS__) -#define DOCTEST_WARN_GE(...) DOCTEST_BINARY_ASSERT(DT_WARN_GE, ge, __VA_ARGS__) -#define DOCTEST_CHECK_GE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_GE, ge, __VA_ARGS__) -#define DOCTEST_REQUIRE_GE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_GE, ge, __VA_ARGS__) -#define DOCTEST_WARN_LE(...) DOCTEST_BINARY_ASSERT(DT_WARN_LE, le, __VA_ARGS__) -#define DOCTEST_CHECK_LE(...) DOCTEST_BINARY_ASSERT(DT_CHECK_LE, le, __VA_ARGS__) -#define DOCTEST_REQUIRE_LE(...) DOCTEST_BINARY_ASSERT(DT_REQUIRE_LE, le, __VA_ARGS__) - -#define DOCTEST_WARN_UNARY(...) DOCTEST_UNARY_ASSERT(DT_WARN_UNARY, __VA_ARGS__) -#define DOCTEST_CHECK_UNARY(...) DOCTEST_UNARY_ASSERT(DT_CHECK_UNARY, __VA_ARGS__) -#define DOCTEST_REQUIRE_UNARY(...) DOCTEST_UNARY_ASSERT(DT_REQUIRE_UNARY, __VA_ARGS__) -#define DOCTEST_WARN_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_WARN_UNARY_FALSE, __VA_ARGS__) -#define DOCTEST_CHECK_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_CHECK_UNARY_FALSE, __VA_ARGS__) -#define DOCTEST_REQUIRE_UNARY_FALSE(...) DOCTEST_UNARY_ASSERT(DT_REQUIRE_UNARY_FALSE, __VA_ARGS__) - -#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS - -#undef DOCTEST_WARN_THROWS -#undef DOCTEST_CHECK_THROWS -#undef DOCTEST_REQUIRE_THROWS -#undef DOCTEST_WARN_THROWS_AS -#undef DOCTEST_CHECK_THROWS_AS -#undef DOCTEST_REQUIRE_THROWS_AS -#undef DOCTEST_WARN_THROWS_WITH -#undef DOCTEST_CHECK_THROWS_WITH -#undef DOCTEST_REQUIRE_THROWS_WITH -#undef DOCTEST_WARN_THROWS_WITH_AS -#undef DOCTEST_CHECK_THROWS_WITH_AS -#undef DOCTEST_REQUIRE_THROWS_WITH_AS -#undef DOCTEST_WARN_NOTHROW -#undef DOCTEST_CHECK_NOTHROW -#undef DOCTEST_REQUIRE_NOTHROW - -#undef DOCTEST_WARN_THROWS_MESSAGE -#undef DOCTEST_CHECK_THROWS_MESSAGE -#undef DOCTEST_REQUIRE_THROWS_MESSAGE -#undef DOCTEST_WARN_THROWS_AS_MESSAGE -#undef DOCTEST_CHECK_THROWS_AS_MESSAGE -#undef DOCTEST_REQUIRE_THROWS_AS_MESSAGE -#undef DOCTEST_WARN_THROWS_WITH_MESSAGE -#undef DOCTEST_CHECK_THROWS_WITH_MESSAGE -#undef DOCTEST_REQUIRE_THROWS_WITH_MESSAGE -#undef DOCTEST_WARN_THROWS_WITH_AS_MESSAGE -#undef DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE -#undef DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE -#undef DOCTEST_WARN_NOTHROW_MESSAGE -#undef DOCTEST_CHECK_NOTHROW_MESSAGE -#undef DOCTEST_REQUIRE_NOTHROW_MESSAGE - -#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS - -#define DOCTEST_WARN_THROWS(...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS(...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS(...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_AS(expr, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_AS(expr, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_WITH(expr, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_WITH(expr, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_WITH_AS(expr, with, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ...) (static_cast(0)) -#define DOCTEST_WARN_NOTHROW(...) (static_cast(0)) -#define DOCTEST_CHECK_NOTHROW(...) (static_cast(0)) -#define DOCTEST_REQUIRE_NOTHROW(...) (static_cast(0)) - -#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast(0)) -#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) (static_cast(0)) -#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) (static_cast(0)) - -#else // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS - -#undef DOCTEST_REQUIRE -#undef DOCTEST_REQUIRE_FALSE -#undef DOCTEST_REQUIRE_MESSAGE -#undef DOCTEST_REQUIRE_FALSE_MESSAGE -#undef DOCTEST_REQUIRE_EQ -#undef DOCTEST_REQUIRE_NE -#undef DOCTEST_REQUIRE_GT -#undef DOCTEST_REQUIRE_LT -#undef DOCTEST_REQUIRE_GE -#undef DOCTEST_REQUIRE_LE -#undef DOCTEST_REQUIRE_UNARY -#undef DOCTEST_REQUIRE_UNARY_FALSE - -#endif // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS - #endif // DOCTEST_CONFIG_NO_EXCEPTIONS // ================================================================================================= @@ -2344,7 +2527,7 @@ int registerReporter(const char* name, int priority, bool isReporter) { #else // DOCTEST_CONFIG_DISABLE #define DOCTEST_IMPLEMENT_FIXTURE(der, base, func, name) \ - namespace { \ + namespace /* NOLINT */ { \ template \ struct der : public base \ { void f(); }; \ @@ -2358,51 +2541,48 @@ int registerReporter(const char* name, int priority, bool isReporter) { // for registering tests #define DOCTEST_TEST_CASE(name) \ - DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name) + DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(DOCTEST_ANON_FUNC_), name) // for registering tests in classes #define DOCTEST_TEST_CASE_CLASS(name) \ - DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name) + DOCTEST_CREATE_AND_REGISTER_FUNCTION(DOCTEST_ANONYMOUS(DOCTEST_ANON_FUNC_), name) // for registering tests with a fixture #define DOCTEST_TEST_CASE_FIXTURE(x, name) \ - DOCTEST_IMPLEMENT_FIXTURE(DOCTEST_ANONYMOUS(_DOCTEST_ANON_CLASS_), x, \ - DOCTEST_ANONYMOUS(_DOCTEST_ANON_FUNC_), name) + DOCTEST_IMPLEMENT_FIXTURE(DOCTEST_ANONYMOUS(DOCTEST_ANON_CLASS_), x, \ + DOCTEST_ANONYMOUS(DOCTEST_ANON_FUNC_), name) // for converting types to strings without the header and demangling -#define DOCTEST_TYPE_TO_STRING(...) typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_) -#define DOCTEST_TYPE_TO_STRING_IMPL(...) +#define DOCTEST_TYPE_TO_STRING_AS(str, ...) static_assert(true, "") +#define DOCTEST_TYPE_TO_STRING(...) static_assert(true, "") // for typed tests #define DOCTEST_TEST_CASE_TEMPLATE(name, type, ...) \ template \ - inline void DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_)() + inline void DOCTEST_ANONYMOUS(DOCTEST_ANON_TMP_)() #define DOCTEST_TEST_CASE_TEMPLATE_DEFINE(name, type, id) \ template \ - inline void DOCTEST_ANONYMOUS(_DOCTEST_ANON_TMP_)() - -#define DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, ...) \ - typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_) + inline void DOCTEST_ANONYMOUS(DOCTEST_ANON_TMP_)() -#define DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, ...) \ - typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_) +#define DOCTEST_TEST_CASE_TEMPLATE_INVOKE(id, ...) static_assert(true, "") +#define DOCTEST_TEST_CASE_TEMPLATE_APPLY(id, ...) static_assert(true, "") // for subcases #define DOCTEST_SUBCASE(name) // for a testsuite block -#define DOCTEST_TEST_SUITE(name) namespace +#define DOCTEST_TEST_SUITE(name) namespace // NOLINT // for starting a testsuite block -#define DOCTEST_TEST_SUITE_BEGIN(name) typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_) +#define DOCTEST_TEST_SUITE_BEGIN(name) static_assert(true, "") // for ending a testsuite block -#define DOCTEST_TEST_SUITE_END typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_) +#define DOCTEST_TEST_SUITE_END using DOCTEST_ANONYMOUS(DOCTEST_ANON_FOR_SEMICOLON_) = int #define DOCTEST_REGISTER_EXCEPTION_TRANSLATOR(signature) \ template \ - static inline doctest::String DOCTEST_ANONYMOUS(_DOCTEST_ANON_TRANSLATOR_)(signature) + static inline doctest::String DOCTEST_ANONYMOUS(DOCTEST_ANON_TRANSLATOR_)(signature) #define DOCTEST_REGISTER_REPORTER(name, priority, reporter) #define DOCTEST_REGISTER_LISTENER(name, priority, reporter) @@ -2416,80 +2596,253 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_FAIL_CHECK(...) (static_cast(0)) #define DOCTEST_FAIL(...) (static_cast(0)) -#define DOCTEST_WARN(...) (static_cast(0)) -#define DOCTEST_CHECK(...) (static_cast(0)) -#define DOCTEST_REQUIRE(...) (static_cast(0)) -#define DOCTEST_WARN_FALSE(...) (static_cast(0)) -#define DOCTEST_CHECK_FALSE(...) (static_cast(0)) -#define DOCTEST_REQUIRE_FALSE(...) (static_cast(0)) - -#define DOCTEST_WARN_MESSAGE(cond, ...) (static_cast(0)) -#define DOCTEST_CHECK_MESSAGE(cond, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_MESSAGE(cond, ...) (static_cast(0)) -#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) (static_cast(0)) -#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) (static_cast(0)) - -#define DOCTEST_WARN_THROWS(...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS(...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS(...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_AS(expr, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_AS(expr, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_WITH(expr, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_WITH(expr, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_WITH_AS(expr, with, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ...) (static_cast(0)) -#define DOCTEST_WARN_NOTHROW(...) (static_cast(0)) -#define DOCTEST_CHECK_NOTHROW(...) (static_cast(0)) -#define DOCTEST_REQUIRE_NOTHROW(...) (static_cast(0)) - -#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast(0)) -#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) (static_cast(0)) -#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) (static_cast(0)) -#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) (static_cast(0)) - -#define DOCTEST_WARN_EQ(...) (static_cast(0)) -#define DOCTEST_CHECK_EQ(...) (static_cast(0)) -#define DOCTEST_REQUIRE_EQ(...) (static_cast(0)) -#define DOCTEST_WARN_NE(...) (static_cast(0)) -#define DOCTEST_CHECK_NE(...) (static_cast(0)) -#define DOCTEST_REQUIRE_NE(...) (static_cast(0)) -#define DOCTEST_WARN_GT(...) (static_cast(0)) -#define DOCTEST_CHECK_GT(...) (static_cast(0)) -#define DOCTEST_REQUIRE_GT(...) (static_cast(0)) -#define DOCTEST_WARN_LT(...) (static_cast(0)) -#define DOCTEST_CHECK_LT(...) (static_cast(0)) -#define DOCTEST_REQUIRE_LT(...) (static_cast(0)) -#define DOCTEST_WARN_GE(...) (static_cast(0)) -#define DOCTEST_CHECK_GE(...) (static_cast(0)) -#define DOCTEST_REQUIRE_GE(...) (static_cast(0)) -#define DOCTEST_WARN_LE(...) (static_cast(0)) -#define DOCTEST_CHECK_LE(...) (static_cast(0)) -#define DOCTEST_REQUIRE_LE(...) (static_cast(0)) - -#define DOCTEST_WARN_UNARY(...) (static_cast(0)) -#define DOCTEST_CHECK_UNARY(...) (static_cast(0)) -#define DOCTEST_REQUIRE_UNARY(...) (static_cast(0)) -#define DOCTEST_WARN_UNARY_FALSE(...) (static_cast(0)) -#define DOCTEST_CHECK_UNARY_FALSE(...) (static_cast(0)) -#define DOCTEST_REQUIRE_UNARY_FALSE(...) (static_cast(0)) +#if defined(DOCTEST_CONFIG_EVALUATE_ASSERTS_EVEN_WHEN_DISABLED) \ + && defined(DOCTEST_CONFIG_ASSERTS_RETURN_VALUES) + +#define DOCTEST_WARN(...) [&] { return __VA_ARGS__; }() +#define DOCTEST_CHECK(...) [&] { return __VA_ARGS__; }() +#define DOCTEST_REQUIRE(...) [&] { return __VA_ARGS__; }() +#define DOCTEST_WARN_FALSE(...) [&] { return !(__VA_ARGS__); }() +#define DOCTEST_CHECK_FALSE(...) [&] { return !(__VA_ARGS__); }() +#define DOCTEST_REQUIRE_FALSE(...) [&] { return !(__VA_ARGS__); }() + +#define DOCTEST_WARN_MESSAGE(cond, ...) [&] { return cond; }() +#define DOCTEST_CHECK_MESSAGE(cond, ...) [&] { return cond; }() +#define DOCTEST_REQUIRE_MESSAGE(cond, ...) [&] { return cond; }() +#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) [&] { return !(cond); }() +#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) [&] { return !(cond); }() +#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) [&] { return !(cond); }() + +namespace doctest { +namespace detail { +#define DOCTEST_RELATIONAL_OP(name, op) \ + template \ + bool name(const DOCTEST_REF_WRAP(L) lhs, const DOCTEST_REF_WRAP(R) rhs) { return lhs op rhs; } + + DOCTEST_RELATIONAL_OP(eq, ==) + DOCTEST_RELATIONAL_OP(ne, !=) + DOCTEST_RELATIONAL_OP(lt, <) + DOCTEST_RELATIONAL_OP(gt, >) + DOCTEST_RELATIONAL_OP(le, <=) + DOCTEST_RELATIONAL_OP(ge, >=) +} // namespace detail +} // namespace doctest + +#define DOCTEST_WARN_EQ(...) [&] { return doctest::detail::eq(__VA_ARGS__); }() +#define DOCTEST_CHECK_EQ(...) [&] { return doctest::detail::eq(__VA_ARGS__); }() +#define DOCTEST_REQUIRE_EQ(...) [&] { return doctest::detail::eq(__VA_ARGS__); }() +#define DOCTEST_WARN_NE(...) [&] { return doctest::detail::ne(__VA_ARGS__); }() +#define DOCTEST_CHECK_NE(...) [&] { return doctest::detail::ne(__VA_ARGS__); }() +#define DOCTEST_REQUIRE_NE(...) [&] { return doctest::detail::ne(__VA_ARGS__); }() +#define DOCTEST_WARN_LT(...) [&] { return doctest::detail::lt(__VA_ARGS__); }() +#define DOCTEST_CHECK_LT(...) [&] { return doctest::detail::lt(__VA_ARGS__); }() +#define DOCTEST_REQUIRE_LT(...) [&] { return doctest::detail::lt(__VA_ARGS__); }() +#define DOCTEST_WARN_GT(...) [&] { return doctest::detail::gt(__VA_ARGS__); }() +#define DOCTEST_CHECK_GT(...) [&] { return doctest::detail::gt(__VA_ARGS__); }() +#define DOCTEST_REQUIRE_GT(...) [&] { return doctest::detail::gt(__VA_ARGS__); }() +#define DOCTEST_WARN_LE(...) [&] { return doctest::detail::le(__VA_ARGS__); }() +#define DOCTEST_CHECK_LE(...) [&] { return doctest::detail::le(__VA_ARGS__); }() +#define DOCTEST_REQUIRE_LE(...) [&] { return doctest::detail::le(__VA_ARGS__); }() +#define DOCTEST_WARN_GE(...) [&] { return doctest::detail::ge(__VA_ARGS__); }() +#define DOCTEST_CHECK_GE(...) [&] { return doctest::detail::ge(__VA_ARGS__); }() +#define DOCTEST_REQUIRE_GE(...) [&] { return doctest::detail::ge(__VA_ARGS__); }() +#define DOCTEST_WARN_UNARY(...) [&] { return __VA_ARGS__; }() +#define DOCTEST_CHECK_UNARY(...) [&] { return __VA_ARGS__; }() +#define DOCTEST_REQUIRE_UNARY(...) [&] { return __VA_ARGS__; }() +#define DOCTEST_WARN_UNARY_FALSE(...) [&] { return !(__VA_ARGS__); }() +#define DOCTEST_CHECK_UNARY_FALSE(...) [&] { return !(__VA_ARGS__); }() +#define DOCTEST_REQUIRE_UNARY_FALSE(...) [&] { return !(__VA_ARGS__); }() + +#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS + +#define DOCTEST_WARN_THROWS_WITH(expr, with, ...) [] { static_assert(false, "Exception translation is not available when doctest is disabled."); return false; }() +#define DOCTEST_CHECK_THROWS_WITH(expr, with, ...) DOCTEST_WARN_THROWS_WITH(,,) +#define DOCTEST_REQUIRE_THROWS_WITH(expr, with, ...) DOCTEST_WARN_THROWS_WITH(,,) +#define DOCTEST_WARN_THROWS_WITH_AS(expr, with, ex, ...) DOCTEST_WARN_THROWS_WITH(,,) +#define DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ex, ...) DOCTEST_WARN_THROWS_WITH(,,) +#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ex, ...) DOCTEST_WARN_THROWS_WITH(,,) + +#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_WARN_THROWS_WITH(,,) +#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_WARN_THROWS_WITH(,,) +#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_WARN_THROWS_WITH(,,) +#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_WARN_THROWS_WITH(,,) +#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_WARN_THROWS_WITH(,,) +#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_WARN_THROWS_WITH(,,) + +#define DOCTEST_WARN_THROWS(...) [&] { try { __VA_ARGS__; return false; } catch (...) { return true; } }() +#define DOCTEST_CHECK_THROWS(...) [&] { try { __VA_ARGS__; return false; } catch (...) { return true; } }() +#define DOCTEST_REQUIRE_THROWS(...) [&] { try { __VA_ARGS__; return false; } catch (...) { return true; } }() +#define DOCTEST_WARN_THROWS_AS(expr, ...) [&] { try { expr; } catch (__VA_ARGS__) { return true; } catch (...) { } return false; }() +#define DOCTEST_CHECK_THROWS_AS(expr, ...) [&] { try { expr; } catch (__VA_ARGS__) { return true; } catch (...) { } return false; }() +#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) [&] { try { expr; } catch (__VA_ARGS__) { return true; } catch (...) { } return false; }() +#define DOCTEST_WARN_NOTHROW(...) [&] { try { __VA_ARGS__; return true; } catch (...) { return false; } }() +#define DOCTEST_CHECK_NOTHROW(...) [&] { try { __VA_ARGS__; return true; } catch (...) { return false; } }() +#define DOCTEST_REQUIRE_NOTHROW(...) [&] { try { __VA_ARGS__; return true; } catch (...) { return false; } }() + +#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) [&] { try { __VA_ARGS__; return false; } catch (...) { return true; } }() +#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) [&] { try { __VA_ARGS__; return false; } catch (...) { return true; } }() +#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) [&] { try { __VA_ARGS__; return false; } catch (...) { return true; } }() +#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) [&] { try { expr; } catch (__VA_ARGS__) { return true; } catch (...) { } return false; }() +#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) [&] { try { expr; } catch (__VA_ARGS__) { return true; } catch (...) { } return false; }() +#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) [&] { try { expr; } catch (__VA_ARGS__) { return true; } catch (...) { } return false; }() +#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) [&] { try { __VA_ARGS__; return true; } catch (...) { return false; } }() +#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) [&] { try { __VA_ARGS__; return true; } catch (...) { return false; } }() +#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) [&] { try { __VA_ARGS__; return true; } catch (...) { return false; } }() + +#endif // DOCTEST_CONFIG_NO_EXCEPTIONS + +#else // DOCTEST_CONFIG_EVALUATE_ASSERTS_EVEN_WHEN_DISABLED + +#define DOCTEST_WARN(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_FALSE(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_FALSE(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_FALSE(...) DOCTEST_FUNC_EMPTY + +#define DOCTEST_WARN_MESSAGE(cond, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_MESSAGE(cond, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_MESSAGE(cond, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) DOCTEST_FUNC_EMPTY + +#define DOCTEST_WARN_EQ(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_EQ(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_EQ(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_NE(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_NE(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_NE(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_GT(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_GT(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_GT(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_LT(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_LT(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_LT(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_GE(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_GE(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_GE(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_LE(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_LE(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_LE(...) DOCTEST_FUNC_EMPTY + +#define DOCTEST_WARN_UNARY(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_UNARY(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_UNARY(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_UNARY_FALSE(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_UNARY_FALSE(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_UNARY_FALSE(...) DOCTEST_FUNC_EMPTY + +#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS + +#define DOCTEST_WARN_THROWS(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_THROWS(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_THROWS(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_THROWS_AS(expr, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_THROWS_AS(expr, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_THROWS_WITH(expr, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_THROWS_WITH(expr, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_THROWS_WITH_AS(expr, with, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_NOTHROW(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_NOTHROW(...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_NOTHROW(...) DOCTEST_FUNC_EMPTY + +#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) DOCTEST_FUNC_EMPTY +#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) DOCTEST_FUNC_EMPTY + +#endif // DOCTEST_CONFIG_NO_EXCEPTIONS + +#endif // DOCTEST_CONFIG_EVALUATE_ASSERTS_EVEN_WHEN_DISABLED #endif // DOCTEST_CONFIG_DISABLE +#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS + +#ifdef DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS +#define DOCTEST_EXCEPTION_EMPTY_FUNC DOCTEST_FUNC_EMPTY +#else // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS +#define DOCTEST_EXCEPTION_EMPTY_FUNC [] { static_assert(false, "Exceptions are disabled! " \ + "Use DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS if you want to compile with exceptions disabled."); return false; }() + +#undef DOCTEST_REQUIRE +#undef DOCTEST_REQUIRE_FALSE +#undef DOCTEST_REQUIRE_MESSAGE +#undef DOCTEST_REQUIRE_FALSE_MESSAGE +#undef DOCTEST_REQUIRE_EQ +#undef DOCTEST_REQUIRE_NE +#undef DOCTEST_REQUIRE_GT +#undef DOCTEST_REQUIRE_LT +#undef DOCTEST_REQUIRE_GE +#undef DOCTEST_REQUIRE_LE +#undef DOCTEST_REQUIRE_UNARY +#undef DOCTEST_REQUIRE_UNARY_FALSE + +#define DOCTEST_REQUIRE DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_FALSE DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_MESSAGE DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_FALSE_MESSAGE DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_EQ DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_NE DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_GT DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_LT DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_GE DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_LE DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_UNARY DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_UNARY_FALSE DOCTEST_EXCEPTION_EMPTY_FUNC + +#endif // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS + +#define DOCTEST_WARN_THROWS(...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_CHECK_THROWS(...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_THROWS(...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_WARN_THROWS_AS(expr, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_CHECK_THROWS_AS(expr, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_THROWS_AS(expr, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_WARN_THROWS_WITH(expr, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_CHECK_THROWS_WITH(expr, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_THROWS_WITH(expr, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_WARN_THROWS_WITH_AS(expr, with, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_WARN_NOTHROW(...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_CHECK_NOTHROW(...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_NOTHROW(...) DOCTEST_EXCEPTION_EMPTY_FUNC + +#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) DOCTEST_EXCEPTION_EMPTY_FUNC +#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) DOCTEST_EXCEPTION_EMPTY_FUNC + +#endif // DOCTEST_CONFIG_NO_EXCEPTIONS + // clang-format off // KEPT FOR BACKWARDS COMPATIBILITY - FORWARDING TO THE RIGHT MACROS #define DOCTEST_FAST_WARN_EQ DOCTEST_WARN_EQ @@ -2536,11 +2889,12 @@ int registerReporter(const char* name, int priority, bool isReporter) { // clang-format on // == SHORT VERSIONS OF THE MACROS -#if !defined(DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES) +#ifndef DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES #define TEST_CASE(name) DOCTEST_TEST_CASE(name) #define TEST_CASE_CLASS(name) DOCTEST_TEST_CASE_CLASS(name) #define TEST_CASE_FIXTURE(x, name) DOCTEST_TEST_CASE_FIXTURE(x, name) +#define TYPE_TO_STRING_AS(str, ...) DOCTEST_TYPE_TO_STRING_AS(str, __VA_ARGS__) #define TYPE_TO_STRING(...) DOCTEST_TYPE_TO_STRING(__VA_ARGS__) #define TEST_CASE_TEMPLATE(name, T, ...) DOCTEST_TEST_CASE_TEMPLATE(name, T, __VA_ARGS__) #define TEST_CASE_TEMPLATE_DEFINE(name, T, id) DOCTEST_TEST_CASE_TEMPLATE_DEFINE(name, T, id) @@ -2673,39 +3027,19 @@ int registerReporter(const char* name, int priority, bool isReporter) { #endif // DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES -#if !defined(DOCTEST_CONFIG_DISABLE) +#ifndef DOCTEST_CONFIG_DISABLE // this is here to clear the 'current test suite' for the current translation unit - at the top DOCTEST_TEST_SUITE_END(); -// add stringification for primitive/fundamental types -namespace doctest { namespace detail { - DOCTEST_TYPE_TO_STRING_IMPL(bool) - DOCTEST_TYPE_TO_STRING_IMPL(float) - DOCTEST_TYPE_TO_STRING_IMPL(double) - DOCTEST_TYPE_TO_STRING_IMPL(long double) - DOCTEST_TYPE_TO_STRING_IMPL(char) - DOCTEST_TYPE_TO_STRING_IMPL(signed char) - DOCTEST_TYPE_TO_STRING_IMPL(unsigned char) -#if !DOCTEST_MSVC || defined(_NATIVE_WCHAR_T_DEFINED) - DOCTEST_TYPE_TO_STRING_IMPL(wchar_t) -#endif // not MSVC or wchar_t support enabled - DOCTEST_TYPE_TO_STRING_IMPL(short int) - DOCTEST_TYPE_TO_STRING_IMPL(unsigned short int) - DOCTEST_TYPE_TO_STRING_IMPL(int) - DOCTEST_TYPE_TO_STRING_IMPL(unsigned int) - DOCTEST_TYPE_TO_STRING_IMPL(long int) - DOCTEST_TYPE_TO_STRING_IMPL(unsigned long int) - DOCTEST_TYPE_TO_STRING_IMPL(long long int) - DOCTEST_TYPE_TO_STRING_IMPL(unsigned long long int) -}} // namespace doctest::detail - #endif // DOCTEST_CONFIG_DISABLE DOCTEST_CLANG_SUPPRESS_WARNING_POP DOCTEST_MSVC_SUPPRESS_WARNING_POP DOCTEST_GCC_SUPPRESS_WARNING_POP +DOCTEST_SUPPRESS_COMMON_WARNINGS_POP + #endif // DOCTEST_LIBRARY_INCLUDED #ifndef DOCTEST_SINGLE_HEADER @@ -2725,13 +3059,11 @@ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wunused-macros") DOCTEST_CLANG_SUPPRESS_WARNING_POP +DOCTEST_SUPPRESS_COMMON_WARNINGS_PUSH + DOCTEST_CLANG_SUPPRESS_WARNING_PUSH -DOCTEST_CLANG_SUPPRESS_WARNING("-Wunknown-pragmas") -DOCTEST_CLANG_SUPPRESS_WARNING("-Wpadded") -DOCTEST_CLANG_SUPPRESS_WARNING("-Wweak-vtables") DOCTEST_CLANG_SUPPRESS_WARNING("-Wglobal-constructors") DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors") -DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-prototypes") DOCTEST_CLANG_SUPPRESS_WARNING("-Wsign-conversion") DOCTEST_CLANG_SUPPRESS_WARNING("-Wshorten-64-to-32") DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-variable-declarations") @@ -2739,65 +3071,35 @@ DOCTEST_CLANG_SUPPRESS_WARNING("-Wswitch") DOCTEST_CLANG_SUPPRESS_WARNING("-Wswitch-enum") DOCTEST_CLANG_SUPPRESS_WARNING("-Wcovered-switch-default") DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-noreturn") -DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-local-typedef") DOCTEST_CLANG_SUPPRESS_WARNING("-Wdisabled-macro-expansion") DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-braces") DOCTEST_CLANG_SUPPRESS_WARNING("-Wmissing-field-initializers") -DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat") -DOCTEST_CLANG_SUPPRESS_WARNING("-Wc++98-compat-pedantic") DOCTEST_CLANG_SUPPRESS_WARNING("-Wunused-member-function") DOCTEST_CLANG_SUPPRESS_WARNING("-Wnonportable-system-include-path") DOCTEST_GCC_SUPPRESS_WARNING_PUSH -DOCTEST_GCC_SUPPRESS_WARNING("-Wunknown-pragmas") -DOCTEST_GCC_SUPPRESS_WARNING("-Wpragmas") DOCTEST_GCC_SUPPRESS_WARNING("-Wconversion") -DOCTEST_GCC_SUPPRESS_WARNING("-Weffc++") DOCTEST_GCC_SUPPRESS_WARNING("-Wsign-conversion") -DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-overflow") -DOCTEST_GCC_SUPPRESS_WARNING("-Wstrict-aliasing") DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-field-initializers") DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-braces") -DOCTEST_GCC_SUPPRESS_WARNING("-Wmissing-declarations") DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch") DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch-enum") DOCTEST_GCC_SUPPRESS_WARNING("-Wswitch-default") DOCTEST_GCC_SUPPRESS_WARNING("-Wunsafe-loop-optimizations") DOCTEST_GCC_SUPPRESS_WARNING("-Wold-style-cast") -DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-local-typedefs") -DOCTEST_GCC_SUPPRESS_WARNING("-Wuseless-cast") DOCTEST_GCC_SUPPRESS_WARNING("-Wunused-function") DOCTEST_GCC_SUPPRESS_WARNING("-Wmultiple-inheritance") -DOCTEST_GCC_SUPPRESS_WARNING("-Wnoexcept") DOCTEST_GCC_SUPPRESS_WARNING("-Wsuggest-attribute") DOCTEST_MSVC_SUPPRESS_WARNING_PUSH -DOCTEST_MSVC_SUPPRESS_WARNING(4616) // invalid compiler warning -DOCTEST_MSVC_SUPPRESS_WARNING(4619) // invalid compiler warning -DOCTEST_MSVC_SUPPRESS_WARNING(4996) // The compiler encountered a deprecated declaration DOCTEST_MSVC_SUPPRESS_WARNING(4267) // 'var' : conversion from 'x' to 'y', possible loss of data -DOCTEST_MSVC_SUPPRESS_WARNING(4706) // assignment within conditional expression -DOCTEST_MSVC_SUPPRESS_WARNING(4512) // 'class' : assignment operator could not be generated -DOCTEST_MSVC_SUPPRESS_WARNING(4127) // conditional expression is constant DOCTEST_MSVC_SUPPRESS_WARNING(4530) // C++ exception handler used, but unwind semantics not enabled DOCTEST_MSVC_SUPPRESS_WARNING(4577) // 'noexcept' used with no exception handling mode specified DOCTEST_MSVC_SUPPRESS_WARNING(4774) // format string expected in argument is not a string literal DOCTEST_MSVC_SUPPRESS_WARNING(4365) // conversion from 'int' to 'unsigned', signed/unsigned mismatch -DOCTEST_MSVC_SUPPRESS_WARNING(4820) // padding in structs -DOCTEST_MSVC_SUPPRESS_WARNING(4640) // construction of local static object is not thread-safe DOCTEST_MSVC_SUPPRESS_WARNING(5039) // pointer to potentially throwing function passed to extern C -DOCTEST_MSVC_SUPPRESS_WARNING(5045) // Spectre mitigation stuff -DOCTEST_MSVC_SUPPRESS_WARNING(4626) // assignment operator was implicitly defined as deleted -DOCTEST_MSVC_SUPPRESS_WARNING(5027) // move assignment operator was implicitly defined as deleted -DOCTEST_MSVC_SUPPRESS_WARNING(5026) // move constructor was implicitly defined as deleted -DOCTEST_MSVC_SUPPRESS_WARNING(4625) // copy constructor was implicitly defined as deleted DOCTEST_MSVC_SUPPRESS_WARNING(4800) // forcing value to bool 'true' or 'false' (performance warning) -// static analysis -DOCTEST_MSVC_SUPPRESS_WARNING(26439) // This kind of function may not throw. Declare it 'noexcept' -DOCTEST_MSVC_SUPPRESS_WARNING(26495) // Always initialize a member variable -DOCTEST_MSVC_SUPPRESS_WARNING(26451) // Arithmetic overflow ... -DOCTEST_MSVC_SUPPRESS_WARNING(26444) // Avoid unnamed objects with custom construction and dtor... -DOCTEST_MSVC_SUPPRESS_WARNING(26812) // Prefer 'enum class' over 'enum' +DOCTEST_MSVC_SUPPRESS_WARNING(5245) // unreferenced function with internal linkage has been removed DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN @@ -2805,7 +3107,7 @@ DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN #include #include #include -// borland (Embarcadero) compiler requires math.h and not cmath - https://github.com/onqtam/doctest/pull/37 +// borland (Embarcadero) compiler requires math.h and not cmath - https://github.com/doctest/doctest/pull/37 #ifdef __BORLANDC__ #include #endif // __BORLANDC__ @@ -2821,16 +3123,27 @@ DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN #include #include #include +#ifndef DOCTEST_CONFIG_NO_MULTITHREADING #include #include +#define DOCTEST_DECLARE_MUTEX(name) std::mutex name; +#define DOCTEST_DECLARE_STATIC_MUTEX(name) static DOCTEST_DECLARE_MUTEX(name) +#define DOCTEST_LOCK_MUTEX(name) std::lock_guard DOCTEST_ANONYMOUS(DOCTEST_ANON_LOCK_)(name); +#else // DOCTEST_CONFIG_NO_MULTITHREADING +#define DOCTEST_DECLARE_MUTEX(name) +#define DOCTEST_DECLARE_STATIC_MUTEX(name) +#define DOCTEST_LOCK_MUTEX(name) +#endif // DOCTEST_CONFIG_NO_MULTITHREADING #include #include +#include #include #include #include #include #include #include +#include #ifdef DOCTEST_PLATFORM_MAC #include @@ -2863,7 +3176,7 @@ DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN #endif // DOCTEST_PLATFORM_WINDOWS -// this is a fix for https://github.com/onqtam/doctest/issues/348 +// this is a fix for https://github.com/doctest/doctest/issues/348 // https://mail.gnome.org/archives/xml/2012-January/msg00000.html #if !defined(HAVE_UNISTD_H) && !defined(STDOUT_FILENO) #define STDOUT_FILENO fileno(stdout) @@ -2885,8 +3198,12 @@ DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_END #endif #ifndef DOCTEST_THREAD_LOCAL +#if defined(DOCTEST_CONFIG_NO_MULTITHREADING) || DOCTEST_MSVC && (DOCTEST_MSVC < DOCTEST_COMPILER(19, 0, 0)) +#define DOCTEST_THREAD_LOCAL +#else // DOCTEST_MSVC #define DOCTEST_THREAD_LOCAL thread_local -#endif +#endif // DOCTEST_MSVC +#endif // DOCTEST_THREAD_LOCAL #ifndef DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES #define DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES 32 @@ -2906,12 +3223,34 @@ DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_END #define DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS #endif +#ifndef DOCTEST_CDECL +#define DOCTEST_CDECL __cdecl +#endif + namespace doctest { -bool is_running_in_test = false; +bool is_running_in_test = false; + +namespace { + using namespace detail; + + template + DOCTEST_NORETURN void throw_exception(Ex const& e) { +#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS + throw e; +#else // DOCTEST_CONFIG_NO_EXCEPTIONS + std::cerr << "doctest will terminate because it needed to throw an exception.\n" + << "The message was: " << e.what() << '\n'; + std::terminate(); +#endif // DOCTEST_CONFIG_NO_EXCEPTIONS + } + +#ifndef DOCTEST_INTERNAL_ERROR +#define DOCTEST_INTERNAL_ERROR(msg) \ + throw_exception(std::logic_error( \ + __FILE__ ":" DOCTEST_TOSTR(__LINE__) ": Internal doctest error: " msg)) +#endif // DOCTEST_INTERNAL_ERROR -namespace { - using namespace detail; // case insensitive strcmp int stricmp(const char* a, const char* b) { for(;; a++, b++) { @@ -2921,20 +3260,6 @@ namespace { } } - template - String fpToString(T value, int precision) { - std::ostringstream oss; - oss << std::setprecision(precision) << std::fixed << value; - std::string d = oss.str(); - size_t i = d.find_last_not_of('0'); - if(i != std::string::npos && i != d.size() - 1) { - if(d[i] == '.') - i++; - d = d.substr(0, i + 1); - } - return d.c_str(); - } - struct Endianness { enum Arch @@ -2955,36 +3280,35 @@ namespace { } // namespace namespace detail { - void my_memcpy(void* dest, const void* src, unsigned num) { memcpy(dest, src, num); } - - String rawMemoryToString(const void* object, unsigned size) { - // Reverse order for little endian architectures - int i = 0, end = static_cast(size), inc = 1; - if(Endianness::which() == Endianness::Little) { - i = end - 1; - end = inc = -1; - } - - unsigned const char* bytes = static_cast(object); - std::ostringstream oss; - oss << "0x" << std::setfill('0') << std::hex; - for(; i != end; i += inc) - oss << std::setw(2) << static_cast(bytes[i]); - return oss.str().c_str(); - } + DOCTEST_THREAD_LOCAL class + { + std::vector stack; + std::stringstream ss; + + public: + std::ostream* push() { + stack.push_back(ss.tellp()); + return &ss; + } + + String pop() { + if (stack.empty()) + DOCTEST_INTERNAL_ERROR("TLSS was empty when trying to pop!"); - DOCTEST_THREAD_LOCAL std::ostringstream g_oss; // NOLINT(cert-err58-cpp) + std::streampos pos = stack.back(); + stack.pop_back(); + unsigned sz = static_cast(ss.tellp() - pos); + ss.rdbuf()->pubseekpos(pos, std::ios::in | std::ios::out); + return String(ss, sz); + } + } g_oss; - std::ostream* getTlsOss() { - g_oss.clear(); // there shouldn't be anything worth clearing in the flags - g_oss.str(""); // the slow way of resetting a string stream - //g_oss.seekp(0); // optimal reset - as seen here: https://stackoverflow.com/a/624291/3162383 - return &g_oss; + std::ostream* tlssPush() { + return g_oss.push(); } - String getTlsOssResult() { - //g_oss << std::ends; // needed - as shown here: https://stackoverflow.com/a/624291/3162383 - return g_oss.str().c_str(); + String tlssPop() { + return g_oss.pop(); } #ifndef DOCTEST_CONFIG_DISABLE @@ -2993,20 +3317,19 @@ namespace timer_large_integer { #if defined(DOCTEST_PLATFORM_WINDOWS) - typedef ULONGLONG type; + using type = ULONGLONG; #else // DOCTEST_PLATFORM_WINDOWS - using namespace std; - typedef uint64_t type; + using type = std::uint64_t; #endif // DOCTEST_PLATFORM_WINDOWS } -typedef timer_large_integer::type ticks_t; +using ticks_t = timer_large_integer::type; #ifdef DOCTEST_CONFIG_GETCURRENTTICKS ticks_t getCurrentTicks() { return DOCTEST_CONFIG_GETCURRENTTICKS(); } #elif defined(DOCTEST_PLATFORM_WINDOWS) ticks_t getCurrentTicks() { - static LARGE_INTEGER hz = {0}, hzo = {0}; + static LARGE_INTEGER hz = { {0} }, hzo = { {0} }; if(!hz.QuadPart) { QueryPerformanceFrequency(&hz); QueryPerformanceCounter(&hzo); @@ -3038,9 +3361,17 @@ typedef timer_large_integer::type ticks_t; ticks_t m_ticks = 0; }; -#ifdef DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS +#ifdef DOCTEST_CONFIG_NO_MULTITHREADING + template + using Atomic = T; +#else // DOCTEST_CONFIG_NO_MULTITHREADING + template + using Atomic = std::atomic; +#endif // DOCTEST_CONFIG_NO_MULTITHREADING + +#if defined(DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS) || defined(DOCTEST_CONFIG_NO_MULTITHREADING) template - using AtomicOrMultiLaneAtomic = std::atomic; + using MultiLaneAtomic = Atomic; #else // DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS // Provides a multilane implementation of an atomic variable that supports add, sub, load, // store. Instead of using a single atomic variable, this splits up into multiple ones, @@ -3057,8 +3388,8 @@ typedef timer_large_integer::type ticks_t; { struct CacheLineAlignedAtomic { - std::atomic atomic{}; - char padding[DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE - sizeof(std::atomic)]; + Atomic atomic{}; + char padding[DOCTEST_MULTI_LANE_ATOMICS_CACHE_LINE_SIZE - sizeof(Atomic)]; }; CacheLineAlignedAtomic m_atomics[DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES]; @@ -3088,7 +3419,7 @@ typedef timer_large_integer::type ticks_t; return result; } - T operator=(T desired) DOCTEST_NOEXCEPT { + T operator=(T desired) DOCTEST_NOEXCEPT { // lgtm [cpp/assignment-does-not-return-this] store(desired); return desired; } @@ -3103,7 +3434,7 @@ typedef timer_large_integer::type ticks_t; private: // Each thread has a different atomic that it operates on. If more than NumLanes threads - // use this, some will use the same atomic. So performance will degrate a bit, but still + // use this, some will use the same atomic. So performance will degrade a bit, but still // everything will work. // // The logic here is a bit tricky. The call should be as fast as possible, so that there @@ -3114,24 +3445,21 @@ typedef timer_large_integer::type ticks_t; // assigned in a round-robin fashion. // 3. This tlsLaneIdx is stored in the thread local data, so it is directly available with // little overhead. - std::atomic& myAtomic() DOCTEST_NOEXCEPT { - static std::atomic laneCounter; + Atomic& myAtomic() DOCTEST_NOEXCEPT { + static Atomic laneCounter; DOCTEST_THREAD_LOCAL size_t tlsLaneIdx = laneCounter++ % DOCTEST_MULTI_LANE_ATOMICS_THREAD_LANES; return m_atomics[tlsLaneIdx].atomic; } }; - - template - using AtomicOrMultiLaneAtomic = MultiLaneAtomic; #endif // DOCTEST_CONFIG_NO_MULTI_LANE_ATOMICS // this holds both parameters from the command line and runtime data for tests struct ContextState : ContextOptions, TestRunStats, CurrentTestCaseStats { - AtomicOrMultiLaneAtomic numAssertsCurrentTest_atomic; - AtomicOrMultiLaneAtomic numAssertsFailedCurrentTest_atomic; + MultiLaneAtomic numAssertsCurrentTest_atomic; + MultiLaneAtomic numAssertsFailedCurrentTest_atomic; std::vector> filters = decltype(filters)(9); // 9 different filters @@ -3144,11 +3472,12 @@ typedef timer_large_integer::type ticks_t; std::vector stringifiedContexts; // logging from INFO() due to an exception // stuff for subcases - std::vector subcasesStack; - std::set subcasesPassed; - int subcasesCurrentMaxLevel; - bool should_reenter; - std::atomic shouldLogCurrentException; + bool reachedLeaf; + std::vector subcaseStack; + std::vector nextSubcaseStack; + std::unordered_set fullyTraversedSubcases; + size_t currentSubcaseDepth; + Atomic shouldLogCurrentException; void resetRunData() { numTestCases = 0; @@ -3198,7 +3527,8 @@ typedef timer_large_integer::type ticks_t; (TestCaseFailureReason::FailedExactlyNumTimes & failure_flags); // if any subcase has failed - the whole test case has failed - if(failure_flags && !ok_to_fail) + testCaseSuccess = !(failure_flags && !ok_to_fail); + if(!testCaseSuccess) numTestCasesFailed++; } }; @@ -3213,23 +3543,37 @@ typedef timer_large_integer::type ticks_t; #endif // DOCTEST_CONFIG_DISABLE } // namespace detail -void String::setOnHeap() { *reinterpret_cast(&buf[last]) = 128; } -void String::setLast(unsigned in) { buf[last] = char(in); } +char* String::allocate(size_type sz) { + if (sz <= last) { + buf[sz] = '\0'; + setLast(last - sz); + return buf; + } else { + setOnHeap(); + data.size = sz; + data.capacity = data.size + 1; + data.ptr = new char[data.capacity]; + data.ptr[sz] = '\0'; + return data.ptr; + } +} + +void String::setOnHeap() noexcept { *reinterpret_cast(&buf[last]) = 128; } +void String::setLast(size_type in) noexcept { buf[last] = char(in); } +void String::setSize(size_type sz) noexcept { + if (isOnStack()) { buf[sz] = '\0'; setLast(last - sz); } + else { data.ptr[sz] = '\0'; data.size = sz; } +} void String::copy(const String& other) { - using namespace std; if(other.isOnStack()) { memcpy(buf, other.buf, len); } else { - setOnHeap(); - data.size = other.data.size; - data.capacity = data.size + 1; - data.ptr = new char[data.capacity]; - memcpy(data.ptr, other.data.ptr, data.size + 1); + memcpy(allocate(other.data.size), other.data.ptr, other.data.size); } } -String::String() { +String::String() noexcept { buf[0] = '\0'; setLast(); } @@ -3237,26 +3581,17 @@ String::String() { String::~String() { if(!isOnStack()) delete[] data.ptr; - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) -} +} // NOLINT(clang-analyzer-cplusplus.NewDeleteLeaks) String::String(const char* in) : String(in, strlen(in)) {} -String::String(const char* in, unsigned in_size) { - using namespace std; - if(in_size <= last) { - memcpy(buf, in, in_size); - buf[in_size] = '\0'; - setLast(last - in_size); - } else { - setOnHeap(); - data.size = in_size; - data.capacity = data.size + 1; - data.ptr = new char[data.capacity]; - memcpy(data.ptr, in, in_size); - data.ptr[in_size] = '\0'; - } +String::String(const char* in, size_type in_size) { + memcpy(allocate(in_size), in, in_size); +} + +String::String(std::istream& in, size_type in_size) { + in.read(allocate(in_size), in_size); } String::String(const String& other) { copy(other); } @@ -3273,10 +3608,9 @@ String& String::operator=(const String& other) { } String& String::operator+=(const String& other) { - const unsigned my_old_size = size(); - const unsigned other_size = other.size(); - const unsigned total_size = my_old_size + other_size; - using namespace std; + const size_type my_old_size = size(); + const size_type other_size = other.size(); + const size_type total_size = my_old_size + other_size; if(isOnStack()) { if(total_size < len) { // append to the current stack space @@ -3323,18 +3657,13 @@ String& String::operator+=(const String& other) { return *this; } -// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) -String String::operator+(const String& other) const { return String(*this) += other; } - -String::String(String&& other) { - using namespace std; +String::String(String&& other) noexcept { memcpy(buf, other.buf, len); other.buf[0] = '\0'; other.setLast(); } -String& String::operator=(String&& other) { - using namespace std; +String& String::operator=(String&& other) noexcept { if(this != &other) { if(!isOnStack()) delete[] data.ptr; @@ -3345,30 +3674,60 @@ String& String::operator=(String&& other) { return *this; } -char String::operator[](unsigned i) const { - return const_cast(this)->operator[](i); // NOLINT +char String::operator[](size_type i) const { + return const_cast(this)->operator[](i); } -char& String::operator[](unsigned i) { +char& String::operator[](size_type i) { if(isOnStack()) return reinterpret_cast(buf)[i]; return data.ptr[i]; } DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wmaybe-uninitialized") -unsigned String::size() const { +String::size_type String::size() const { if(isOnStack()) - return last - (unsigned(buf[last]) & 31); // using "last" would work only if "len" is 32 + return last - (size_type(buf[last]) & 31); // using "last" would work only if "len" is 32 return data.size; } DOCTEST_GCC_SUPPRESS_WARNING_POP -unsigned String::capacity() const { +String::size_type String::capacity() const { if(isOnStack()) return len; return data.capacity; } +String String::substr(size_type pos, size_type cnt) && { + cnt = std::min(cnt, size() - 1 - pos); + char* cptr = c_str(); + memmove(cptr, cptr + pos, cnt); + setSize(cnt); + return std::move(*this); +} + +String String::substr(size_type pos, size_type cnt) const & { + cnt = std::min(cnt, size() - 1 - pos); + return String{ c_str() + pos, cnt }; +} + +String::size_type String::find(char ch, size_type pos) const { + const char* begin = c_str(); + const char* end = begin + size(); + const char* it = begin + pos; + for (; it < end && *it != ch; it++); + if (it < end) { return static_cast(it - begin); } + else { return npos; } +} + +String::size_type String::rfind(char ch, size_type pos) const { + const char* begin = c_str(); + const char* it = begin + std::min(pos, size() - 1); + for (; it >= begin && *it != ch; it--); + if (it >= begin) { return static_cast(it - begin); } + else { return npos; } +} + int String::compare(const char* other, bool no_case) const { if(no_case) return doctest::stricmp(c_str(), other); @@ -3379,17 +3738,32 @@ int String::compare(const String& other, bool no_case) const { return compare(other.c_str(), no_case); } -// clang-format off +String operator+(const String& lhs, const String& rhs) { return String(lhs) += rhs; } + bool operator==(const String& lhs, const String& rhs) { return lhs.compare(rhs) == 0; } bool operator!=(const String& lhs, const String& rhs) { return lhs.compare(rhs) != 0; } bool operator< (const String& lhs, const String& rhs) { return lhs.compare(rhs) < 0; } bool operator> (const String& lhs, const String& rhs) { return lhs.compare(rhs) > 0; } bool operator<=(const String& lhs, const String& rhs) { return (lhs != rhs) ? lhs.compare(rhs) < 0 : true; } bool operator>=(const String& lhs, const String& rhs) { return (lhs != rhs) ? lhs.compare(rhs) > 0 : true; } -// clang-format on std::ostream& operator<<(std::ostream& s, const String& in) { return s << in.c_str(); } +Contains::Contains(const String& str) : string(str) { } + +bool Contains::checkWith(const String& other) const { + return strstr(other.c_str(), string.c_str()) != nullptr; +} + +String toString(const Contains& in) { + return "Contains( " + in.string + " )"; +} + +bool operator==(const String& lhs, const Contains& rhs) { return rhs.checkWith(lhs); } +bool operator==(const Contains& lhs, const String& rhs) { return lhs.checkWith(rhs); } +bool operator!=(const String& lhs, const Contains& rhs) { return !rhs.checkWith(lhs); } +bool operator!=(const Contains& lhs, const String& rhs) { return !lhs.checkWith(rhs); } + namespace { void color_to_stream(std::ostream&, Color::Enum) DOCTEST_BRANCH_ON_DISABLED({}, ;) } // namespace @@ -3403,64 +3777,42 @@ namespace Color { // clang-format off const char* assertString(assertType::Enum at) { - DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4062) // enum 'x' in switch of enum 'y' is not handled - switch(at) { //!OCLINT missing default in switch statements - case assertType::DT_WARN : return "WARN"; - case assertType::DT_CHECK : return "CHECK"; - case assertType::DT_REQUIRE : return "REQUIRE"; - - case assertType::DT_WARN_FALSE : return "WARN_FALSE"; - case assertType::DT_CHECK_FALSE : return "CHECK_FALSE"; - case assertType::DT_REQUIRE_FALSE : return "REQUIRE_FALSE"; - - case assertType::DT_WARN_THROWS : return "WARN_THROWS"; - case assertType::DT_CHECK_THROWS : return "CHECK_THROWS"; - case assertType::DT_REQUIRE_THROWS : return "REQUIRE_THROWS"; - - case assertType::DT_WARN_THROWS_AS : return "WARN_THROWS_AS"; - case assertType::DT_CHECK_THROWS_AS : return "CHECK_THROWS_AS"; - case assertType::DT_REQUIRE_THROWS_AS : return "REQUIRE_THROWS_AS"; - - case assertType::DT_WARN_THROWS_WITH : return "WARN_THROWS_WITH"; - case assertType::DT_CHECK_THROWS_WITH : return "CHECK_THROWS_WITH"; - case assertType::DT_REQUIRE_THROWS_WITH : return "REQUIRE_THROWS_WITH"; - - case assertType::DT_WARN_THROWS_WITH_AS : return "WARN_THROWS_WITH_AS"; - case assertType::DT_CHECK_THROWS_WITH_AS : return "CHECK_THROWS_WITH_AS"; - case assertType::DT_REQUIRE_THROWS_WITH_AS : return "REQUIRE_THROWS_WITH_AS"; - - case assertType::DT_WARN_NOTHROW : return "WARN_NOTHROW"; - case assertType::DT_CHECK_NOTHROW : return "CHECK_NOTHROW"; - case assertType::DT_REQUIRE_NOTHROW : return "REQUIRE_NOTHROW"; - - case assertType::DT_WARN_EQ : return "WARN_EQ"; - case assertType::DT_CHECK_EQ : return "CHECK_EQ"; - case assertType::DT_REQUIRE_EQ : return "REQUIRE_EQ"; - case assertType::DT_WARN_NE : return "WARN_NE"; - case assertType::DT_CHECK_NE : return "CHECK_NE"; - case assertType::DT_REQUIRE_NE : return "REQUIRE_NE"; - case assertType::DT_WARN_GT : return "WARN_GT"; - case assertType::DT_CHECK_GT : return "CHECK_GT"; - case assertType::DT_REQUIRE_GT : return "REQUIRE_GT"; - case assertType::DT_WARN_LT : return "WARN_LT"; - case assertType::DT_CHECK_LT : return "CHECK_LT"; - case assertType::DT_REQUIRE_LT : return "REQUIRE_LT"; - case assertType::DT_WARN_GE : return "WARN_GE"; - case assertType::DT_CHECK_GE : return "CHECK_GE"; - case assertType::DT_REQUIRE_GE : return "REQUIRE_GE"; - case assertType::DT_WARN_LE : return "WARN_LE"; - case assertType::DT_CHECK_LE : return "CHECK_LE"; - case assertType::DT_REQUIRE_LE : return "REQUIRE_LE"; - - case assertType::DT_WARN_UNARY : return "WARN_UNARY"; - case assertType::DT_CHECK_UNARY : return "CHECK_UNARY"; - case assertType::DT_REQUIRE_UNARY : return "REQUIRE_UNARY"; - case assertType::DT_WARN_UNARY_FALSE : return "WARN_UNARY_FALSE"; - case assertType::DT_CHECK_UNARY_FALSE : return "CHECK_UNARY_FALSE"; - case assertType::DT_REQUIRE_UNARY_FALSE : return "REQUIRE_UNARY_FALSE"; + DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4061) // enum 'x' in switch of enum 'y' is not explicitely handled + #define DOCTEST_GENERATE_ASSERT_TYPE_CASE(assert_type) case assertType::DT_ ## assert_type: return #assert_type + #define DOCTEST_GENERATE_ASSERT_TYPE_CASES(assert_type) \ + DOCTEST_GENERATE_ASSERT_TYPE_CASE(WARN_ ## assert_type); \ + DOCTEST_GENERATE_ASSERT_TYPE_CASE(CHECK_ ## assert_type); \ + DOCTEST_GENERATE_ASSERT_TYPE_CASE(REQUIRE_ ## assert_type) + switch(at) { + DOCTEST_GENERATE_ASSERT_TYPE_CASE(WARN); + DOCTEST_GENERATE_ASSERT_TYPE_CASE(CHECK); + DOCTEST_GENERATE_ASSERT_TYPE_CASE(REQUIRE); + + DOCTEST_GENERATE_ASSERT_TYPE_CASES(FALSE); + + DOCTEST_GENERATE_ASSERT_TYPE_CASES(THROWS); + + DOCTEST_GENERATE_ASSERT_TYPE_CASES(THROWS_AS); + + DOCTEST_GENERATE_ASSERT_TYPE_CASES(THROWS_WITH); + + DOCTEST_GENERATE_ASSERT_TYPE_CASES(THROWS_WITH_AS); + + DOCTEST_GENERATE_ASSERT_TYPE_CASES(NOTHROW); + + DOCTEST_GENERATE_ASSERT_TYPE_CASES(EQ); + DOCTEST_GENERATE_ASSERT_TYPE_CASES(NE); + DOCTEST_GENERATE_ASSERT_TYPE_CASES(GT); + DOCTEST_GENERATE_ASSERT_TYPE_CASES(LT); + DOCTEST_GENERATE_ASSERT_TYPE_CASES(GE); + DOCTEST_GENERATE_ASSERT_TYPE_CASES(LE); + + DOCTEST_GENERATE_ASSERT_TYPE_CASES(UNARY); + DOCTEST_GENERATE_ASSERT_TYPE_CASES(UNARY_FALSE); + + default: DOCTEST_INTERNAL_ERROR("Tried stringifying invalid assert type!"); } DOCTEST_MSVC_SUPPRESS_WARNING_POP - return ""; } // clang-format on @@ -3494,6 +3846,12 @@ const char* skipPathFromFilename(const char* file) { DOCTEST_CLANG_SUPPRESS_WARNING_POP DOCTEST_GCC_SUPPRESS_WARNING_POP +bool SubcaseSignature::operator==(const SubcaseSignature& other) const { + return m_line == other.m_line + && std::strcmp(m_file, other.m_file) == 0 + && m_name == other.m_name; +} + bool SubcaseSignature::operator<(const SubcaseSignature& other) const { if(m_line != other.m_line) return m_line < other.m_line; @@ -3502,45 +3860,53 @@ bool SubcaseSignature::operator<(const SubcaseSignature& other) const { return m_name.compare(other.m_name) < 0; } -IContextScope::IContextScope() = default; -IContextScope::~IContextScope() = default; +DOCTEST_DEFINE_INTERFACE(IContextScope) -#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING -String toString(char* in) { return toString(static_cast(in)); } -// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) -String toString(const char* in) { return String("\"") + (in ? in : "{null string}") + "\""; } -#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING -String toString(bool in) { return in ? "true" : "false"; } -String toString(float in) { return fpToString(in, 5) + "f"; } -String toString(double in) { return fpToString(in, 10); } -String toString(double long in) { return fpToString(in, 15); } - -#define DOCTEST_TO_STRING_OVERLOAD(type, fmt) \ - String toString(type in) { \ - char buf[64]; \ - std::sprintf(buf, fmt, in); \ - return buf; \ +namespace detail { + void filldata::fill(std::ostream* stream, const void* in) { + if (in) { *stream << in; } + else { *stream << "nullptr"; } } -DOCTEST_TO_STRING_OVERLOAD(char, "%d") -DOCTEST_TO_STRING_OVERLOAD(char signed, "%d") -DOCTEST_TO_STRING_OVERLOAD(char unsigned, "%u") -DOCTEST_TO_STRING_OVERLOAD(int short, "%d") -DOCTEST_TO_STRING_OVERLOAD(int short unsigned, "%u") -DOCTEST_TO_STRING_OVERLOAD(int, "%d") -DOCTEST_TO_STRING_OVERLOAD(unsigned, "%u") -DOCTEST_TO_STRING_OVERLOAD(int long, "%ld") -DOCTEST_TO_STRING_OVERLOAD(int long unsigned, "%lu") -DOCTEST_TO_STRING_OVERLOAD(int long long, "%lld") -DOCTEST_TO_STRING_OVERLOAD(int long long unsigned, "%llu") + template + String toStreamLit(T t) { + std::ostream* os = tlssPush(); + os->operator<<(t); + return tlssPop(); + } +} -String toString(std::nullptr_t) { return "NULL"; } +#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING +String toString(const char* in) { return String("\"") + (in ? in : "{null string}") + "\""; } +#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING #if DOCTEST_MSVC >= DOCTEST_COMPILER(19, 20, 0) -// see this issue on why this is needed: https://github.com/onqtam/doctest/issues/183 +// see this issue on why this is needed: https://github.com/doctest/doctest/issues/183 String toString(const std::string& in) { return in.c_str(); } #endif // VS 2019 +String toString(String in) { return in; } + +String toString(std::nullptr_t) { return "nullptr"; } + +String toString(bool in) { return in ? "true" : "false"; } + +String toString(float in) { return toStreamLit(in); } +String toString(double in) { return toStreamLit(in); } +String toString(double long in) { return toStreamLit(in); } + +String toString(char in) { return toStreamLit(static_cast(in)); } +String toString(char signed in) { return toStreamLit(static_cast(in)); } +String toString(char unsigned in) { return toStreamLit(static_cast(in)); } +String toString(short in) { return toStreamLit(in); } +String toString(short unsigned in) { return toStreamLit(in); } +String toString(signed in) { return toStreamLit(in); } +String toString(unsigned in) { return toStreamLit(in); } +String toString(long in) { return toStreamLit(in); } +String toString(long unsigned in) { return toStreamLit(in); } +String toString(long long in) { return toStreamLit(in); } +String toString(long long unsigned in) { return toStreamLit(in); } + Approx::Approx(double value) : m_epsilon(static_cast(std::numeric_limits::epsilon()) * 100) , m_scale(1.0) @@ -3580,11 +3946,25 @@ bool operator>(double lhs, const Approx& rhs) { return lhs > rhs.m_value && lhs bool operator>(const Approx& lhs, double rhs) { return lhs.m_value > rhs && lhs != rhs; } String toString(const Approx& in) { - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - return String("Approx( ") + doctest::toString(in.m_value) + " )"; + return "Approx( " + doctest::toString(in.m_value) + " )"; } const ContextOptions* getContextOptions() { return DOCTEST_BRANCH_ON_DISABLED(nullptr, g_cs); } +DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4738) +template +IsNaN::operator bool() const { + return std::isnan(value) ^ flipped; +} +DOCTEST_MSVC_SUPPRESS_WARNING_POP +template struct DOCTEST_INTERFACE_DEF IsNaN; +template struct DOCTEST_INTERFACE_DEF IsNaN; +template struct DOCTEST_INTERFACE_DEF IsNaN; +template +String toString(IsNaN in) { return String(in.flipped ? "! " : "") + "IsNaN( " + doctest::toString(in.value) + " )"; } +String toString(IsNaN in) { return toString(in); } +String toString(IsNaN in) { return toString(in); } +String toString(IsNaN in) { return toString(in); } + } // namespace doctest #ifdef DOCTEST_CONFIG_DISABLE @@ -3594,15 +3974,15 @@ Context::~Context() = default; void Context::applyCommandLine(int, const char* const*) {} void Context::addFilter(const char*, const char*) {} void Context::clearFilters() {} +void Context::setOption(const char*, bool) {} void Context::setOption(const char*, int) {} void Context::setOption(const char*, const char*) {} bool Context::shouldExit() { return false; } void Context::setAsDefaultForAssertsOutOfTestCases() {} void Context::setAssertHandler(detail::assert_handler) {} +void Context::setCout(std::ostream*) {} int Context::run() { return 0; } -IReporter::~IReporter() = default; - int IReporter::get_num_active_contexts() { return 0; } const IContextScope* const* IReporter::get_active_contexts() { return nullptr; } int IReporter::get_num_stringified_contexts() { return 0; } @@ -3635,7 +4015,7 @@ namespace doctest { namespace { // the int (priority) is part of the key for automatic sorting - sadly one can register a // reporter with a duplicate name and a different priority but hopefully that won't happen often :| - typedef std::map, reporterCreatorFunc> reporterMap; + using reporterMap = std::map, reporterCreatorFunc>; reporterMap& getReporters() { static reporterMap data; @@ -3667,8 +4047,8 @@ namespace detail { #ifndef DOCTEST_CONFIG_NO_EXCEPTIONS DOCTEST_NORETURN void throwException() { g_cs->shouldLogCurrentException = false; - throw TestFailureException(); - } // NOLINT(cert-err60-cpp) + throw TestFailureException(); // NOLINT(hicpp-exception-baseclass) + } #else // DOCTEST_CONFIG_NO_EXCEPTIONS void throwException() {} #endif // DOCTEST_CONFIG_NO_EXCEPTIONS @@ -3714,91 +4094,132 @@ namespace { return !*wild; } - //// C string hash function (djb2) - taken from http://www.cse.yorku.ca/~oz/hash.html - //unsigned hashStr(unsigned const char* str) { - // unsigned long hash = 5381; - // char c; - // while((c = *str++)) - // hash = ((hash << 5) + hash) + c; // hash * 33 + c - // return hash; - //} - // checks if the name matches any of the filters (and can be configured what to do when empty) bool matchesAny(const char* name, const std::vector& filters, bool matchEmpty, - bool caseSensitive) { - if(filters.empty() && matchEmpty) + bool caseSensitive) { + if (filters.empty() && matchEmpty) return true; - for(auto& curr : filters) - if(wildcmp(name, curr.c_str(), caseSensitive)) + for (auto& curr : filters) + if (wildcmp(name, curr.c_str(), caseSensitive)) return true; return false; } -} // namespace -namespace detail { - Subcase::Subcase(const String& name, const char* file, int line) - : m_signature({name, file, line}) { - auto* s = g_cs; + unsigned long long hash(unsigned long long a, unsigned long long b) { + return (a << 5) + b; + } - // check subcase filters - if(s->subcasesStack.size() < size_t(s->subcase_filter_levels)) { - if(!matchesAny(m_signature.m_name.c_str(), s->filters[6], true, s->case_sensitive)) - return; - if(matchesAny(m_signature.m_name.c_str(), s->filters[7], false, s->case_sensitive)) - return; - } - - // if a Subcase on the same level has already been entered - if(s->subcasesStack.size() < size_t(s->subcasesCurrentMaxLevel)) { - s->should_reenter = true; - return; - } + // C string hash function (djb2) - taken from http://www.cse.yorku.ca/~oz/hash.html + unsigned long long hash(const char* str) { + unsigned long long hash = 5381; + char c; + while ((c = *str++)) + hash = ((hash << 5) + hash) + c; // hash * 33 + c + return hash; + } - // push the current signature to the stack so we can check if the - // current stack + the current new subcase have been traversed - s->subcasesStack.push_back(m_signature); - if(s->subcasesPassed.count(s->subcasesStack) != 0) { - // pop - revert to previous stack since we've already passed this - s->subcasesStack.pop_back(); - return; + unsigned long long hash(const SubcaseSignature& sig) { + return hash(hash(hash(sig.m_file), hash(sig.m_name.c_str())), sig.m_line); + } + + unsigned long long hash(const std::vector& sigs, size_t count) { + unsigned long long running = 0; + auto end = sigs.begin() + count; + for (auto it = sigs.begin(); it != end; it++) { + running = hash(running, hash(*it)); } + return running; + } - s->subcasesCurrentMaxLevel = s->subcasesStack.size(); - m_entered = true; + unsigned long long hash(const std::vector& sigs) { + unsigned long long running = 0; + for (const SubcaseSignature& sig : sigs) { + running = hash(running, hash(sig)); + } + return running; + } +} // namespace +namespace detail { + bool Subcase::checkFilters() { + if (g_cs->subcaseStack.size() < size_t(g_cs->subcase_filter_levels)) { + if (!matchesAny(m_signature.m_name.c_str(), g_cs->filters[6], true, g_cs->case_sensitive)) + return true; + if (matchesAny(m_signature.m_name.c_str(), g_cs->filters[7], false, g_cs->case_sensitive)) + return true; + } + return false; + } - DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_start, m_signature); + Subcase::Subcase(const String& name, const char* file, int line) + : m_signature({name, file, line}) { + if (!g_cs->reachedLeaf) { + if (g_cs->nextSubcaseStack.size() <= g_cs->subcaseStack.size() + || g_cs->nextSubcaseStack[g_cs->subcaseStack.size()] == m_signature) { + // Going down. + if (checkFilters()) { return; } + + g_cs->subcaseStack.push_back(m_signature); + g_cs->currentSubcaseDepth++; + m_entered = true; + DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_start, m_signature); + } + } else { + if (g_cs->subcaseStack[g_cs->currentSubcaseDepth] == m_signature) { + // This subcase is reentered via control flow. + g_cs->currentSubcaseDepth++; + m_entered = true; + DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_start, m_signature); + } else if (g_cs->nextSubcaseStack.size() <= g_cs->currentSubcaseDepth + && g_cs->fullyTraversedSubcases.find(hash(hash(g_cs->subcaseStack, g_cs->currentSubcaseDepth), hash(m_signature))) + == g_cs->fullyTraversedSubcases.end()) { + if (checkFilters()) { return; } + // This subcase is part of the one to be executed next. + g_cs->nextSubcaseStack.clear(); + g_cs->nextSubcaseStack.insert(g_cs->nextSubcaseStack.end(), + g_cs->subcaseStack.begin(), g_cs->subcaseStack.begin() + g_cs->currentSubcaseDepth); + g_cs->nextSubcaseStack.push_back(m_signature); + } + } } - DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4996) // std::uncaught_exception is deprecated in C++17 - DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations") + DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4996) // std::uncaught_exception is deprecated in C++17 + DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations") DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations") Subcase::~Subcase() { - if(m_entered) { - // only mark the subcase stack as passed if no subcases have been skipped - if(g_cs->should_reenter == false) - g_cs->subcasesPassed.insert(g_cs->subcasesStack); - g_cs->subcasesStack.pop_back(); + if (m_entered) { + g_cs->currentSubcaseDepth--; + + if (!g_cs->reachedLeaf) { + // Leaf. + g_cs->fullyTraversedSubcases.insert(hash(g_cs->subcaseStack)); + g_cs->nextSubcaseStack.clear(); + g_cs->reachedLeaf = true; + } else if (g_cs->nextSubcaseStack.empty()) { + // All children are finished. + g_cs->fullyTraversedSubcases.insert(hash(g_cs->subcaseStack)); + } #if defined(__cpp_lib_uncaught_exceptions) && __cpp_lib_uncaught_exceptions >= 201411L && (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200) if(std::uncaught_exceptions() > 0 #else if(std::uncaught_exception() #endif - && g_cs->shouldLogCurrentException) { + && g_cs->shouldLogCurrentException) { DOCTEST_ITERATE_THROUGH_REPORTERS( test_case_exception, {"exception thrown in subcase - will translate later " - "when the whole test case has been exited (cannot " - "translate while there is an active exception)", - false}); + "when the whole test case has been exited (cannot " + "translate while there is an active exception)", + false}); g_cs->shouldLogCurrentException = false; } + DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_end, DOCTEST_EMPTY); } } - DOCTEST_CLANG_SUPPRESS_WARNING_POP - DOCTEST_GCC_SUPPRESS_WARNING_POP + DOCTEST_CLANG_SUPPRESS_WARNING_POP + DOCTEST_GCC_SUPPRESS_WARNING_POP DOCTEST_MSVC_SUPPRESS_WARNING_POP Subcase::operator bool() const { return m_entered; } @@ -3812,20 +4233,11 @@ namespace detail { TestSuite& TestSuite::operator*(const char* in) { m_test_suite = in; - // clear state - m_description = nullptr; - m_skip = false; - m_no_breaks = false; - m_no_output = false; - m_may_fail = false; - m_should_fail = false; - m_expected_failures = 0; - m_timeout = 0; return *this; } TestCase::TestCase(funcType test, const char* file, unsigned line, const TestSuite& test_suite, - const char* type, int template_id) { + const String& type, int template_id) { m_file = file; m_line = line; m_name = nullptr; // will be later overridden in operator* @@ -3850,10 +4262,8 @@ namespace detail { } DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(26434) // hides a non-virtual function - DOCTEST_MSVC_SUPPRESS_WARNING(26437) // Do not slice TestCase& TestCase::operator=(const TestCase& other) { - static_cast(*this) = static_cast(other); - + TestCaseData::operator=(other); m_test = other.m_test; m_type = other.m_type; m_template_id = other.m_template_id; @@ -3869,7 +4279,7 @@ namespace detail { m_name = in; // make a new name with an appended type for templated test case if(m_template_id != -1) { - m_full_name = String(m_name) + m_type; + m_full_name = String(m_name) + "<" + m_type + ">"; // redirect the name to point to the newly constructed full name m_name = m_full_name.c_str(); } @@ -3925,29 +4335,6 @@ namespace { return suiteOrderComparator(lhs, rhs); } -#ifdef DOCTEST_CONFIG_COLORS_WINDOWS - HANDLE g_stdoutHandle; - WORD g_origFgAttrs; - WORD g_origBgAttrs; - bool g_attrsInitted = false; - - int colors_init() { - if(!g_attrsInitted) { - g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE); - g_attrsInitted = true; - CONSOLE_SCREEN_BUFFER_INFO csbiInfo; - GetConsoleScreenBufferInfo(g_stdoutHandle, &csbiInfo); - g_origFgAttrs = csbiInfo.wAttributes & ~(BACKGROUND_GREEN | BACKGROUND_RED | - BACKGROUND_BLUE | BACKGROUND_INTENSITY); - g_origBgAttrs = csbiInfo.wAttributes & ~(FOREGROUND_GREEN | FOREGROUND_RED | - FOREGROUND_BLUE | FOREGROUND_INTENSITY); - } - return 0; - } - - int dumy_init_console_colors = colors_init(); -#endif // DOCTEST_CONFIG_COLORS_WINDOWS - DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations") void color_to_stream(std::ostream& s, Color::Enum code) { static_cast(s); // for DOCTEST_CONFIG_COLORS_NONE or DOCTEST_CONFIG_COLORS_WINDOWS @@ -3981,10 +4368,26 @@ namespace { #ifdef DOCTEST_CONFIG_COLORS_WINDOWS if(g_no_colors || - (isatty(fileno(stdout)) == false && getContextOptions()->force_colors == false)) + (_isatty(_fileno(stdout)) == false && getContextOptions()->force_colors == false)) return; -#define DOCTEST_SET_ATTR(x) SetConsoleTextAttribute(g_stdoutHandle, x | g_origBgAttrs) + static struct ConsoleHelper { + HANDLE stdoutHandle; + WORD origFgAttrs; + WORD origBgAttrs; + + ConsoleHelper() { + stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE); + CONSOLE_SCREEN_BUFFER_INFO csbiInfo; + GetConsoleScreenBufferInfo(stdoutHandle, &csbiInfo); + origFgAttrs = csbiInfo.wAttributes & ~(BACKGROUND_GREEN | BACKGROUND_RED | + BACKGROUND_BLUE | BACKGROUND_INTENSITY); + origBgAttrs = csbiInfo.wAttributes & ~(FOREGROUND_GREEN | FOREGROUND_RED | + FOREGROUND_BLUE | FOREGROUND_INTENSITY); + } + } ch; + +#define DOCTEST_SET_ATTR(x) SetConsoleTextAttribute(ch.stdoutHandle, x | ch.origBgAttrs) // clang-format off switch (code) { @@ -4001,7 +4404,7 @@ namespace { case Color::BrightWhite: DOCTEST_SET_ATTR(FOREGROUND_INTENSITY | FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE); break; case Color::None: case Color::Bright: // invalid - default: DOCTEST_SET_ATTR(g_origFgAttrs); + default: DOCTEST_SET_ATTR(ch.origFgAttrs); } // clang-format on #endif // DOCTEST_CONFIG_COLORS_WINDOWS @@ -4118,35 +4521,22 @@ namespace detail { getExceptionTranslators().push_back(et); } -#ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING - void toStream(std::ostream* s, char* in) { *s << in; } - void toStream(std::ostream* s, const char* in) { *s << in; } -#endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING - void toStream(std::ostream* s, bool in) { *s << std::boolalpha << in << std::noboolalpha; } - void toStream(std::ostream* s, float in) { *s << in; } - void toStream(std::ostream* s, double in) { *s << in; } - void toStream(std::ostream* s, double long in) { *s << in; } - - void toStream(std::ostream* s, char in) { *s << in; } - void toStream(std::ostream* s, char signed in) { *s << in; } - void toStream(std::ostream* s, char unsigned in) { *s << in; } - void toStream(std::ostream* s, int short in) { *s << in; } - void toStream(std::ostream* s, int short unsigned in) { *s << in; } - void toStream(std::ostream* s, int in) { *s << in; } - void toStream(std::ostream* s, int unsigned in) { *s << in; } - void toStream(std::ostream* s, int long in) { *s << in; } - void toStream(std::ostream* s, int long unsigned in) { *s << in; } - void toStream(std::ostream* s, int long long in) { *s << in; } - void toStream(std::ostream* s, int long long unsigned in) { *s << in; } - DOCTEST_THREAD_LOCAL std::vector g_infoContexts; // for logging with INFO() ContextScopeBase::ContextScopeBase() { g_infoContexts.push_back(this); } - DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4996) // std::uncaught_exception is deprecated in C++17 - DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations") + ContextScopeBase::ContextScopeBase(ContextScopeBase&& other) noexcept { + if (other.need_to_destroy) { + other.destroy(); + } + other.need_to_destroy = false; + g_infoContexts.push_back(this); + } + + DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4996) // std::uncaught_exception is deprecated in C++17 + DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations") DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations") // destroy cannot be inlined into the destructor because that would mean calling stringify after @@ -4165,8 +4555,8 @@ namespace detail { g_infoContexts.pop_back(); } - DOCTEST_CLANG_SUPPRESS_WARNING_POP - DOCTEST_GCC_SUPPRESS_WARNING_POP + DOCTEST_CLANG_SUPPRESS_WARNING_POP + DOCTEST_GCC_SUPPRESS_WARNING_POP DOCTEST_MSVC_SUPPRESS_WARNING_POP } // namespace detail namespace { @@ -4207,10 +4597,10 @@ namespace { static LONG CALLBACK handleException(PEXCEPTION_POINTERS ExceptionInfo) { // Multiple threads may enter this filter/handler at once. We want the error message to be printed on the // console just once no matter how many threads have crashed. - static std::mutex mutex; + DOCTEST_DECLARE_STATIC_MUTEX(mutex) static bool execute = true; { - std::lock_guard lock(mutex); + DOCTEST_LOCK_MUTEX(mutex) if(execute) { bool reported = false; for(size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) { @@ -4313,7 +4703,7 @@ namespace { static unsigned int prev_abort_behavior; static int prev_report_mode; static _HFILE prev_report_file; - static void (*prev_sigabrt_handler)(int); + static void (DOCTEST_CDECL *prev_sigabrt_handler)(int); static std::terminate_handler original_terminate_handler; static bool isSet; static ULONG guaranteeSize; @@ -4325,7 +4715,7 @@ namespace { unsigned int FatalConditionHandler::prev_abort_behavior; int FatalConditionHandler::prev_report_mode; _HFILE FatalConditionHandler::prev_report_file; - void (*FatalConditionHandler::prev_sigabrt_handler)(int); + void (DOCTEST_CDECL *FatalConditionHandler::prev_sigabrt_handler)(int); std::terminate_handler FatalConditionHandler::original_terminate_handler; bool FatalConditionHandler::isSet = false; ULONG FatalConditionHandler::guaranteeSize = 0; @@ -4383,7 +4773,7 @@ namespace { sigStack.ss_flags = 0; sigaltstack(&sigStack, &oldSigStack); struct sigaction sa = {}; - sa.sa_handler = handleSignal; // NOLINT + sa.sa_handler = handleSignal; sa.sa_flags = SA_ONSTACK; for(std::size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) { sigaction(signalDefs[i].id, &sa, &oldSigActions[i]); @@ -4422,7 +4812,7 @@ namespace { #define DOCTEST_OUTPUT_DEBUG_STRING(text) ::OutputDebugStringA(text) #else // TODO: integration with XCode and other IDEs -#define DOCTEST_OUTPUT_DEBUG_STRING(text) // NOLINT(clang-diagnostic-unused-macros) +#define DOCTEST_OUTPUT_DEBUG_STRING(text) #endif // Platform void addAssert(assertType::Enum at) { @@ -4441,8 +4831,8 @@ namespace { DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_exception, {message.c_str(), true}); - while(g_cs->subcasesStack.size()) { - g_cs->subcasesStack.pop_back(); + while (g_cs->subcaseStack.size()) { + g_cs->subcaseStack.pop_back(); DOCTEST_ITERATE_THROUGH_REPORTERS(subcase_end, DOCTEST_EMPTY); } @@ -4454,25 +4844,26 @@ namespace { } #endif // DOCTEST_CONFIG_POSIX_SIGNALS || DOCTEST_CONFIG_WINDOWS_SEH } // namespace -namespace detail { - ResultBuilder::ResultBuilder(assertType::Enum at, const char* file, int line, const char* expr, - const char* exception_type, const char* exception_string) { - m_test_case = g_cs->currentTest; - m_at = at; - m_file = file; - m_line = line; - m_expr = expr; - m_failed = true; - m_threw = false; - m_threw_as = false; - m_exception_type = exception_type; - m_exception_string = exception_string; +AssertData::AssertData(assertType::Enum at, const char* file, int line, const char* expr, + const char* exception_type, const StringContains& exception_string) + : m_test_case(g_cs->currentTest), m_at(at), m_file(file), m_line(line), m_expr(expr), + m_failed(true), m_threw(false), m_threw_as(false), m_exception_type(exception_type), + m_exception_string(exception_string) { #if DOCTEST_MSVC - if(m_expr[0] == ' ') // this happens when variadic macros are disabled under MSVC - ++m_expr; + if (m_expr[0] == ' ') // this happens when variadic macros are disabled under MSVC + ++m_expr; #endif // MSVC - } +} + +namespace detail { + ResultBuilder::ResultBuilder(assertType::Enum at, const char* file, int line, const char* expr, + const char* exception_type, const String& exception_string) + : AssertData(at, file, line, expr, exception_type, exception_string) { } + + ResultBuilder::ResultBuilder(assertType::Enum at, const char* file, int line, const char* expr, + const char* exception_type, const Contains& exception_string) + : AssertData(at, file, line, expr, exception_type, exception_string) { } void ResultBuilder::setResult(const Result& res) { m_decomp = res.m_decomp; @@ -4488,17 +4879,17 @@ namespace detail { if(m_at & assertType::is_throws) { //!OCLINT bitwise operator in conditional m_failed = !m_threw; } else if((m_at & assertType::is_throws_as) && (m_at & assertType::is_throws_with)) { //!OCLINT - m_failed = !m_threw_as || (m_exception != m_exception_string); + m_failed = !m_threw_as || !m_exception_string.check(m_exception); } else if(m_at & assertType::is_throws_as) { //!OCLINT bitwise operator in conditional m_failed = !m_threw_as; } else if(m_at & assertType::is_throws_with) { //!OCLINT bitwise operator in conditional - m_failed = m_exception != m_exception_string; + m_failed = !m_exception_string.check(m_exception); } else if(m_at & assertType::is_nothrow) { //!OCLINT bitwise operator in conditional m_failed = m_threw; } if(m_exception.size()) - m_exception = String("\"") + m_exception + "\""; + m_exception = "\"" + m_exception + "\""; if(is_running_in_test) { addAssert(m_at); @@ -4526,8 +4917,8 @@ namespace detail { std::abort(); } - void decomp_assert(assertType::Enum at, const char* file, int line, const char* expr, - Result result) { + bool decomp_assert(assertType::Enum at, const char* file, int line, const char* expr, + const Result& result) { bool failed = !result.m_passed; // ################################################################################### @@ -4536,21 +4927,29 @@ namespace detail { // ################################################################################### DOCTEST_ASSERT_OUT_OF_TESTS(result.m_decomp); DOCTEST_ASSERT_IN_TESTS(result.m_decomp); - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) + return !failed; } MessageBuilder::MessageBuilder(const char* file, int line, assertType::Enum severity) { - m_stream = getTlsOss(); + m_stream = tlssPush(); m_file = file; m_line = line; m_severity = severity; } - IExceptionTranslator::IExceptionTranslator() = default; - IExceptionTranslator::~IExceptionTranslator() = default; + MessageBuilder::~MessageBuilder() { + if (!logged) + tlssPop(); + } + + DOCTEST_DEFINE_INTERFACE(IExceptionTranslator) bool MessageBuilder::log() { - m_string = getTlsOssResult(); + if (!logged) { + m_string = tlssPop(); + logged = true; + } + DOCTEST_ITERATE_THROUGH_REPORTERS(log_message, *this); const bool isWarn = m_severity & assertType::is_warn; @@ -4569,29 +4968,10 @@ namespace detail { if(m_severity & assertType::is_require) //!OCLINT bitwise operator in conditional throwException(); } - - MessageBuilder::~MessageBuilder() = default; } // namespace detail namespace { using namespace detail; - template - DOCTEST_NORETURN void throw_exception(Ex const& e) { -#ifndef DOCTEST_CONFIG_NO_EXCEPTIONS - throw e; -#else // DOCTEST_CONFIG_NO_EXCEPTIONS - std::cerr << "doctest will terminate because it needed to throw an exception.\n" - << "The message was: " << e.what() << '\n'; - std::terminate(); -#endif // DOCTEST_CONFIG_NO_EXCEPTIONS - } - -#ifndef DOCTEST_INTERNAL_ERROR -#define DOCTEST_INTERNAL_ERROR(msg) \ - throw_exception(std::logic_error( \ - __FILE__ ":" DOCTEST_TOSTR(__LINE__) ": Internal doctest error: " msg)) -#endif // DOCTEST_INTERNAL_ERROR - // clang-format off // ================================================================================================= @@ -4673,10 +5053,10 @@ namespace { void ensureTagClosed(); - private: - void writeDeclaration(); + private: + void newlineIfNecessary(); bool m_tagIsOpen = false; @@ -4865,7 +5245,7 @@ namespace { XmlWriter::XmlWriter( std::ostream& os ) : m_os( os ) { - writeDeclaration(); + // writeDeclaration(); // called explicitly by the reporters that use the writer class - see issue #627 } XmlWriter::~XmlWriter() { @@ -4976,8 +5356,8 @@ namespace { struct XmlReporter : public IReporter { - XmlWriter xml; - std::mutex mutex; + XmlWriter xml; + DOCTEST_DECLARE_MUTEX(mutex) // caching pointers/references to objects of these types - safe to do const ContextOptions& opt; @@ -5054,7 +5434,8 @@ namespace { xml.scopedElement("TestCase").writeAttribute("name", in.data[i]->m_name) .writeAttribute("testsuite", in.data[i]->m_test_suite) .writeAttribute("filename", skipPathFromFilename(in.data[i]->m_file.c_str())) - .writeAttribute("line", line(in.data[i]->m_line)); + .writeAttribute("line", line(in.data[i]->m_line)) + .writeAttribute("skipped", in.data[i]->m_skip); } xml.scopedElement("OverallResultsTestCases") .writeAttribute("unskipped", in.run_stats->numTestCasesPassingFilters); @@ -5070,6 +5451,8 @@ namespace { } void test_run_start() override { + xml.writeDeclaration(); + // remove .exe extension - mainly to have the same output on UNIX and Windows std::string binary_name = skipPathFromFilename(opt.binary_name.c_str()); #ifdef DOCTEST_PLATFORM_WINDOWS @@ -5124,7 +5507,8 @@ namespace { xml.startElement("OverallResultsAsserts") .writeAttribute("successes", st.numAssertsCurrentTest - st.numAssertsFailedCurrentTest) - .writeAttribute("failures", st.numAssertsFailedCurrentTest); + .writeAttribute("failures", st.numAssertsFailedCurrentTest) + .writeAttribute("test_case_success", st.testCaseSuccess); if(opt.duration) xml.writeAttribute("duration", st.seconds); if(tc->m_expected_failures) @@ -5135,7 +5519,7 @@ namespace { } void test_case_exception(const TestCaseException& e) override { - std::lock_guard lock(mutex); + DOCTEST_LOCK_MUTEX(mutex) xml.scopedElement("Exception") .writeAttribute("crash", e.is_crash) @@ -5143,8 +5527,6 @@ namespace { } void subcase_start(const SubcaseSignature& in) override { - std::lock_guard lock(mutex); - xml.startElement("SubCase") .writeAttribute("name", in.m_name) .writeAttribute("filename", skipPathFromFilename(in.m_file)) @@ -5158,7 +5540,7 @@ namespace { if(!rb.m_failed && !opt.success) return; - std::lock_guard lock(mutex); + DOCTEST_LOCK_MUTEX(mutex) xml.startElement("Expression") .writeAttribute("success", !rb.m_failed) @@ -5174,7 +5556,7 @@ namespace { if(rb.m_at & assertType::is_throws_as) xml.scopedElement("ExpectedException").writeText(rb.m_exception_type); if(rb.m_at & assertType::is_throws_with) - xml.scopedElement("ExpectedExceptionString").writeText(rb.m_exception_string); + xml.scopedElement("ExpectedExceptionString").writeText(rb.m_exception_string.c_str()); if((rb.m_at & assertType::is_normal) && !rb.m_threw) xml.scopedElement("Expanded").writeText(rb.m_decomp.c_str()); @@ -5184,7 +5566,7 @@ namespace { } void log_message(const MessageData& mb) override { - std::lock_guard lock(mutex); + DOCTEST_LOCK_MUTEX(mutex) xml.startElement("Message") .writeAttribute("type", failureString(mb.m_severity)) @@ -5220,7 +5602,8 @@ namespace { } else if((rb.m_at & assertType::is_throws_as) && (rb.m_at & assertType::is_throws_with)) { //!OCLINT s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", \"" - << rb.m_exception_string << "\", " << rb.m_exception_type << " ) " << Color::None; + << rb.m_exception_string.c_str() + << "\", " << rb.m_exception_type << " ) " << Color::None; if(rb.m_threw) { if(!rb.m_failed) { s << "threw as expected!\n"; @@ -5241,7 +5624,8 @@ namespace { } else if(rb.m_at & assertType::is_throws_with) { //!OCLINT bitwise operator in conditional s << Color::Cyan << assertString(rb.m_at) << "( " << rb.m_expr << ", \"" - << rb.m_exception_string << "\" ) " << Color::None + << rb.m_exception_string.c_str() + << "\" ) " << Color::None << (rb.m_threw ? (!rb.m_failed ? "threw as expected!" : "threw a DIFFERENT exception: ") : "did NOT throw at all!") @@ -5266,8 +5650,8 @@ namespace { // - more attributes in tags struct JUnitReporter : public IReporter { - XmlWriter xml; - std::mutex mutex; + XmlWriter xml; + DOCTEST_DECLARE_MUTEX(mutex) Timer timer; std::vector deepestSubcaseStackNames; @@ -5363,9 +5747,13 @@ namespace { // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE // ========================================================================================= - void report_query(const QueryData&) override {} + void report_query(const QueryData&) override { + xml.writeDeclaration(); + } - void test_run_start() override {} + void test_run_start() override { + xml.writeDeclaration(); + } void test_run_end(const TestRunStats& p) override { // remove .exe extension - mainly to have the same output on UNIX and Windows @@ -5435,12 +5823,11 @@ namespace { } void test_case_exception(const TestCaseException& e) override { - std::lock_guard lock(mutex); + DOCTEST_LOCK_MUTEX(mutex) testCaseData.addError("exception", e.error_string.c_str()); } void subcase_start(const SubcaseSignature& in) override { - std::lock_guard lock(mutex); deepestSubcaseStackNames.push_back(in.m_name); } @@ -5450,7 +5837,7 @@ namespace { if(!rb.m_failed) // report only failures & ignore the `success` option return; - std::lock_guard lock(mutex); + DOCTEST_LOCK_MUTEX(mutex) std::ostringstream os; os << skipPathFromFilename(rb.m_file) << (opt.gnu_file_line ? ":" : "(") @@ -5501,7 +5888,7 @@ namespace { bool hasLoggedCurrentTestStart; std::vector subcasesStack; size_t currentSubcaseLevel; - std::mutex mutex; + DOCTEST_DECLARE_MUTEX(mutex) // caching pointers/references to objects of these types - safe to do const ContextOptions& opt; @@ -5606,9 +5993,11 @@ namespace { } void printIntro() { - printVersion(); - s << Color::Cyan << "[doctest] " << Color::None - << "run with \"--" DOCTEST_OPTIONS_PREFIX_DISPLAY "help\" for options\n"; + if(opt.no_intro == false) { + printVersion(); + s << Color::Cyan << "[doctest] " << Color::None + << "run with \"--" DOCTEST_OPTIONS_PREFIX_DISPLAY "help\" for options\n"; + } } void printHelp() { @@ -5693,12 +6082,18 @@ namespace { << Whitespace(sizePrefixDisplay*1) << "exits after the tests finish\n"; s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "d, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "duration= " << Whitespace(sizePrefixDisplay*1) << "prints the time duration of each test\n"; + s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "m, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "minimal= " + << Whitespace(sizePrefixDisplay*1) << "minimal console output (only failures)\n"; + s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "q, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "quiet= " + << Whitespace(sizePrefixDisplay*1) << "no console output\n"; s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nt, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-throw= " << Whitespace(sizePrefixDisplay*1) << "skips exceptions-related assert checks\n"; s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ne, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-exitcode= " << Whitespace(sizePrefixDisplay*1) << "returns (or exits) always with success\n"; s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nr, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-run= " << Whitespace(sizePrefixDisplay*1) << "skips all runtime doctest operations\n"; + s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "ni, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-intro= " + << Whitespace(sizePrefixDisplay*1) << "omit the framework intro in the output\n"; s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nv, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-version= " << Whitespace(sizePrefixDisplay*1) << "omit the framework version in the output\n"; s << " -" DOCTEST_OPTIONS_PREFIX_DISPLAY "nc, --" DOCTEST_OPTIONS_PREFIX_DISPLAY "no-colors= " @@ -5736,22 +6131,6 @@ namespace { printReporters(getReporters(), "reporters"); } - void list_query_results() { - separator_to_stream(); - if(opt.count || opt.list_test_cases) { - s << Color::Cyan << "[doctest] " << Color::None - << "unskipped test cases passing the current filters: " - << g_cs->numTestCasesPassingFilters << "\n"; - } else if(opt.list_test_suites) { - s << Color::Cyan << "[doctest] " << Color::None - << "unskipped test cases passing the current filters: " - << g_cs->numTestCasesPassingFilters << "\n"; - s << Color::Cyan << "[doctest] " << Color::None - << "test suites with unskipped test cases passing the current filters: " - << g_cs->numTestSuitesPassingFilters << "\n"; - } - } - // ========================================================================================= // WHAT FOLLOWS ARE OVERRIDES OF THE VIRTUAL METHODS OF THE REPORTER INTERFACE // ========================================================================================= @@ -5797,9 +6176,15 @@ namespace { } } - void test_run_start() override { printIntro(); } + void test_run_start() override { + if(!opt.minimal) + printIntro(); + } void test_run_end(const TestRunStats& p) override { + if(opt.minimal && p.numTestCasesFailed == 0) + return; + separator_to_stream(); s << std::dec; @@ -5849,7 +6234,7 @@ namespace { // log the preamble of the test case only if there is something // else to print - something other than that an assert has failed if(opt.duration || - (st.failure_flags && st.failure_flags != TestCaseFailureReason::AssertFailure)) + (st.failure_flags && st.failure_flags != static_cast(TestCaseFailureReason::AssertFailure))) logTestStart(); if(opt.duration) @@ -5880,6 +6265,7 @@ namespace { } void test_case_exception(const TestCaseException& e) override { + DOCTEST_LOCK_MUTEX(mutex) if(tc->m_no_output) return; @@ -5904,14 +6290,12 @@ namespace { } void subcase_start(const SubcaseSignature& subc) override { - std::lock_guard lock(mutex); subcasesStack.push_back(subc); ++currentSubcaseLevel; hasLoggedCurrentTestStart = false; } void subcase_end() override { - std::lock_guard lock(mutex); --currentSubcaseLevel; hasLoggedCurrentTestStart = false; } @@ -5920,7 +6304,7 @@ namespace { if((!rb.m_failed && !opt.success) || tc->m_no_output) return; - std::lock_guard lock(mutex); + DOCTEST_LOCK_MUTEX(mutex) logTestStart(); @@ -5936,7 +6320,7 @@ namespace { if(tc->m_no_output) return; - std::lock_guard lock(mutex); + DOCTEST_LOCK_MUTEX(mutex) logTestStart(); @@ -6047,18 +6431,42 @@ namespace { std::vector& res) { String filtersString; if(parseOption(argc, argv, pattern, &filtersString)) { - // tokenize with "," as a separator - // cppcheck-suppress strtokCalled - DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations") - auto pch = std::strtok(filtersString.c_str(), ","); // modifies the string - while(pch != nullptr) { - if(strlen(pch)) - res.push_back(pch); - // uses the strtok() internal state to go to the next token - // cppcheck-suppress strtokCalled - pch = std::strtok(nullptr, ","); + // tokenize with "," as a separator, unless escaped with backslash + std::ostringstream s; + auto flush = [&s, &res]() { + auto string = s.str(); + if(string.size() > 0) { + res.push_back(string.c_str()); + } + s.str(""); + }; + + bool seenBackslash = false; + const char* current = filtersString.c_str(); + const char* end = current + strlen(current); + while(current != end) { + char character = *current++; + if(seenBackslash) { + seenBackslash = false; + if(character == ',' || character == '\\') { + s.put(character); + continue; + } + s.put('\\'); + } + if(character == '\\') { + seenBackslash = true; + } else if(character == ',') { + flush(); + } else { + s.put(character); + } + } + + if(seenBackslash) { + s.put('\\'); } - DOCTEST_CLANG_SUPPRESS_WARNING_POP + flush(); return true; } return false; @@ -6077,30 +6485,30 @@ namespace { if(!parseOption(argc, argv, pattern, &parsedValue)) return false; - if(type == 0) { + if(type) { + // integer + // TODO: change this to use std::stoi or something else! currently it uses undefined behavior - assumes '0' on failed parse... + int theInt = std::atoi(parsedValue.c_str()); + if (theInt != 0) { + res = theInt; //!OCLINT parameter reassignment + return true; + } + } else { // boolean - const char positive[][5] = {"1", "true", "on", "yes"}; // 5 - strlen("true") + 1 - const char negative[][6] = {"0", "false", "off", "no"}; // 6 - strlen("false") + 1 + const char positive[][5] = { "1", "true", "on", "yes" }; // 5 - strlen("true") + 1 + const char negative[][6] = { "0", "false", "off", "no" }; // 6 - strlen("false") + 1 // if the value matches any of the positive/negative possibilities - for(unsigned i = 0; i < 4; i++) { - if(parsedValue.compare(positive[i], true) == 0) { + for (unsigned i = 0; i < 4; i++) { + if (parsedValue.compare(positive[i], true) == 0) { res = 1; //!OCLINT parameter reassignment return true; } - if(parsedValue.compare(negative[i], true) == 0) { + if (parsedValue.compare(negative[i], true) == 0) { res = 0; //!OCLINT parameter reassignment return true; } } - } else { - // integer - // TODO: change this to use std::stoi or something else! currently it uses undefined behavior - assumes '0' on failed parse... - int theInt = std::atoi(parsedValue.c_str()); // NOLINT - if(theInt != 0) { - res = theInt; //!OCLINT parameter reassignment - return true; - } } return false; } @@ -6191,9 +6599,12 @@ void Context::parseArgs(int argc, const char* const* argv, bool withDefaults) { DOCTEST_PARSE_AS_BOOL_OR_FLAG("case-sensitive", "cs", case_sensitive, false); DOCTEST_PARSE_AS_BOOL_OR_FLAG("exit", "e", exit, false); DOCTEST_PARSE_AS_BOOL_OR_FLAG("duration", "d", duration, false); + DOCTEST_PARSE_AS_BOOL_OR_FLAG("minimal", "m", minimal, false); + DOCTEST_PARSE_AS_BOOL_OR_FLAG("quiet", "q", quiet, false); DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-throw", "nt", no_throw, false); DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-exitcode", "ne", no_exitcode, false); DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-run", "nr", no_run, false); + DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-intro", "ni", no_intro, false); DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-version", "nv", no_version, false); DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-colors", "nc", no_colors, false); DOCTEST_PARSE_AS_BOOL_OR_FLAG("force-colors", "fc", force_colors, false); @@ -6257,10 +6668,14 @@ void Context::clearFilters() { curr.clear(); } -// allows the user to override procedurally the int/bool options from the command line +// allows the user to override procedurally the bool options from the command line +void Context::setOption(const char* option, bool value) { + setOption(option, value ? "true" : "false"); +} + +// allows the user to override procedurally the int options from the command line void Context::setOption(const char* option, int value) { setOption(option, toString(value).c_str()); - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) } // allows the user to override procedurally the string options from the command line @@ -6277,6 +6692,31 @@ void Context::setAsDefaultForAssertsOutOfTestCases() { g_cs = p; } void Context::setAssertHandler(detail::assert_handler ah) { p->ah = ah; } +void Context::setCout(std::ostream* out) { p->cout = out; } + +static class DiscardOStream : public std::ostream +{ +private: + class : public std::streambuf + { + private: + // allowing some buffering decreases the amount of calls to overflow + char buf[1024]; + + protected: + std::streamsize xsputn(const char_type*, std::streamsize count) override { return count; } + + int_type overflow(int_type ch) override { + setp(std::begin(buf), std::end(buf)); + return traits_type::not_eof(ch); + } + } discardBuf; + +public: + DiscardOStream() + : std::ostream(&discardBuf) {} +} discardOut; + // the main function that does all the filtering and test running int Context::run() { using namespace detail; @@ -6290,15 +6730,18 @@ int Context::run() { g_no_colors = p->no_colors; p->resetRunData(); - // stdout by default - p->cout = &std::cout; - p->cerr = &std::cerr; - - // or to a file if specified std::fstream fstr; - if(p->out.size()) { - fstr.open(p->out.c_str(), std::fstream::out); - p->cout = &fstr; + if(p->cout == nullptr) { + if(p->quiet) { + p->cout = &discardOut; + } else if(p->out.size()) { + // to a file if specified + fstr.open(p->out.c_str(), std::fstream::out); + p->cout = &fstr; + } else { + // stdout by default + p->cout = &std::cout; + } } FatalConditionHandler::allocateAltStackMem(); @@ -6370,7 +6813,7 @@ int Context::run() { // random_shuffle implementation const auto first = &testArray[0]; for(size_t i = testArray.size() - 1; i > 0; --i) { - int idxToSwap = std::rand() % (i + 1); // NOLINT + int idxToSwap = std::rand() % (i + 1); const auto temp = first[i]; @@ -6457,7 +6900,7 @@ int Context::run() { p->numAssertsFailedCurrentTest_atomic = 0; p->numAssertsCurrentTest_atomic = 0; - p->subcasesPassed.clear(); + p->fullyTraversedSubcases.clear(); DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_start, tc); @@ -6467,9 +6910,10 @@ int Context::run() { do { // reset some of the fields for subcases (except for the set of fully passed ones) - p->should_reenter = false; - p->subcasesCurrentMaxLevel = 0; - p->subcasesStack.clear(); + p->reachedLeaf = false; + // May not be empty if previous subcase exited via exception. + p->subcaseStack.clear(); + p->currentSubcaseDepth = 0; p->shouldLogCurrentException = true; @@ -6503,9 +6947,9 @@ DOCTEST_MSVC_SUPPRESS_WARNING_POP p->failure_flags |= TestCaseFailureReason::TooManyFailedAsserts; } - if(p->should_reenter && run_test) + if(!p->nextSubcaseStack.empty() && run_test) DOCTEST_ITERATE_THROUGH_REPORTERS(test_case_reenter, tc); - if(!p->should_reenter) + if(p->nextSubcaseStack.empty()) run_test = false; } while(run_test); @@ -6531,17 +6975,10 @@ DOCTEST_MSVC_SUPPRESS_WARNING_POP DOCTEST_ITERATE_THROUGH_REPORTERS(report_query, qdata); } - // see these issues on the reasoning for this: - // - https://github.com/onqtam/doctest/issues/143#issuecomment-414418903 - // - https://github.com/onqtam/doctest/issues/126 - auto DOCTEST_FIX_FOR_MACOS_LIBCPP_IOSFWD_STRING_LINK_ERRORS = []() DOCTEST_NOINLINE - { std::cout << std::string(); }; - DOCTEST_FIX_FOR_MACOS_LIBCPP_IOSFWD_STRING_LINK_ERRORS(); - return cleanup_and_return(); } -IReporter::~IReporter() = default; +DOCTEST_DEFINE_INTERFACE(IReporter) int IReporter::get_num_active_contexts() { return detail::g_infoContexts.size(); } const IContextScope* const* IReporter::get_active_contexts() { @@ -6576,5 +7013,7 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP DOCTEST_MSVC_SUPPRESS_WARNING_POP DOCTEST_GCC_SUPPRESS_WARNING_POP +DOCTEST_SUPPRESS_COMMON_WARNINGS_POP + #endif // DOCTEST_LIBRARY_IMPLEMENTATION #endif // DOCTEST_CONFIG_IMPLEMENT diff --git a/src/third_party/fmt/core.h b/src/third_party/fmt/core.h index 0a81e0c..92a7aa1 100644 --- a/src/third_party/fmt/core.h +++ b/src/third_party/fmt/core.h @@ -1,4 +1,4 @@ -// Formatting library for C++ - the core API +// Formatting library for C++ - the core API for char/UTF-8 // // Copyright (c) 2012 - present, Victor Zverovich // All rights reserved. @@ -8,40 +8,45 @@ #ifndef FMT_CORE_H_ #define FMT_CORE_H_ -#include // std::FILE +#include // std::byte +#include // std::FILE #include -#include #include -#include +#include #include #include -#include // The fmt library version in the form major * 10000 + minor * 100 + patch. -#define FMT_VERSION 70103 +#define FMT_VERSION 80101 -#ifdef __clang__ +#if defined(__clang__) && !defined(__ibmxl__) # define FMT_CLANG_VERSION (__clang_major__ * 100 + __clang_minor__) #else # define FMT_CLANG_VERSION 0 #endif -#if defined(__GNUC__) && !defined(__clang__) +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) && \ + !defined(__NVCOMPILER) # define FMT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) #else # define FMT_GCC_VERSION 0 #endif -#if defined(__INTEL_COMPILER) -# define FMT_ICC_VERSION __INTEL_COMPILER -#else -# define FMT_ICC_VERSION 0 +#ifndef FMT_GCC_PRAGMA +// Workaround _Pragma bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59884. +# if FMT_GCC_VERSION >= 504 +# define FMT_GCC_PRAGMA(arg) _Pragma(arg) +# else +# define FMT_GCC_PRAGMA(arg) +# endif #endif -#if __cplusplus >= 201103L || defined(__GXX_EXPERIMENTAL_CXX0X__) -# define FMT_HAS_GXX_CXX11 FMT_GCC_VERSION +#ifdef __ICL +# define FMT_ICC_VERSION __ICL +#elif defined(__INTEL_COMPILER) +# define FMT_ICC_VERSION __INTEL_COMPILER #else -# define FMT_HAS_GXX_CXX11 0 +# define FMT_ICC_VERSION 0 #endif #ifdef __NVCC__ @@ -52,10 +57,10 @@ #ifdef _MSC_VER # define FMT_MSC_VER _MSC_VER -# define FMT_SUPPRESS_MSC_WARNING(n) __pragma(warning(suppress : n)) +# define FMT_MSC_WARNING(...) __pragma(warning(__VA_ARGS__)) #else # define FMT_MSC_VER 0 -# define FMT_SUPPRESS_MSC_WARNING(n) +# define FMT_MSC_WARNING(...) #endif #ifdef __has_feature @@ -64,7 +69,8 @@ # define FMT_HAS_FEATURE(x) 0 #endif -#if defined(__has_include) && !defined(__INTELLISENSE__) && \ +#if defined(__has_include) && \ + (!defined(__INTELLISENSE__) || FMT_MSC_VER > 1900) && \ (!FMT_ICC_VERSION || FMT_ICC_VERSION >= 1600) # define FMT_HAS_INCLUDE(x) __has_include(x) #else @@ -77,17 +83,23 @@ # define FMT_HAS_CPP_ATTRIBUTE(x) 0 #endif +#ifdef _MSVC_LANG +# define FMT_CPLUSPLUS _MSVC_LANG +#else +# define FMT_CPLUSPLUS __cplusplus +#endif + #define FMT_HAS_CPP14_ATTRIBUTE(attribute) \ - (__cplusplus >= 201402L && FMT_HAS_CPP_ATTRIBUTE(attribute)) + (FMT_CPLUSPLUS >= 201402L && FMT_HAS_CPP_ATTRIBUTE(attribute)) #define FMT_HAS_CPP17_ATTRIBUTE(attribute) \ - (__cplusplus >= 201703L && FMT_HAS_CPP_ATTRIBUTE(attribute)) + (FMT_CPLUSPLUS >= 201703L && FMT_HAS_CPP_ATTRIBUTE(attribute)) // Check if relaxed C++14 constexpr is supported. // GCC doesn't allow throw in constexpr until version 6 (bug 67371). #ifndef FMT_USE_CONSTEXPR # define FMT_USE_CONSTEXPR \ - (FMT_HAS_FEATURE(cxx_relaxed_constexpr) || FMT_MSC_VER >= 1910 || \ + (FMT_HAS_FEATURE(cxx_relaxed_constexpr) || FMT_MSC_VER >= 1912 || \ (FMT_GCC_VERSION >= 600 && __cplusplus >= 201402L)) && \ !FMT_NVCC && !FMT_ICC_VERSION #endif @@ -95,17 +107,32 @@ # define FMT_CONSTEXPR constexpr # define FMT_CONSTEXPR_DECL constexpr #else -# define FMT_CONSTEXPR inline +# define FMT_CONSTEXPR # define FMT_CONSTEXPR_DECL #endif -#ifndef FMT_OVERRIDE -# if FMT_HAS_FEATURE(cxx_override_control) || \ - (FMT_GCC_VERSION >= 408 && FMT_HAS_GXX_CXX11) || FMT_MSC_VER >= 1900 -# define FMT_OVERRIDE override -# else -# define FMT_OVERRIDE +#if ((__cplusplus >= 202002L) && \ + (!defined(_GLIBCXX_RELEASE) || _GLIBCXX_RELEASE > 9)) || \ + (__cplusplus >= 201709L && FMT_GCC_VERSION >= 1002) +# define FMT_CONSTEXPR20 constexpr +#else +# define FMT_CONSTEXPR20 +#endif + +// Check if constexpr std::char_traits<>::compare,length is supported. +#if defined(__GLIBCXX__) +# if __cplusplus >= 201703L && defined(_GLIBCXX_RELEASE) && \ + _GLIBCXX_RELEASE >= 7 // GCC 7+ libstdc++ has _GLIBCXX_RELEASE. +# define FMT_CONSTEXPR_CHAR_TRAITS constexpr # endif +#elif defined(_LIBCPP_VERSION) && __cplusplus >= 201703L && \ + _LIBCPP_VERSION >= 4000 +# define FMT_CONSTEXPR_CHAR_TRAITS constexpr +#elif FMT_MSC_VER >= 1914 && _MSVC_LANG >= 201703L +# define FMT_CONSTEXPR_CHAR_TRAITS constexpr +#endif +#ifndef FMT_CONSTEXPR_CHAR_TRAITS +# define FMT_CONSTEXPR_CHAR_TRAITS #endif // Check if exceptions are disabled. @@ -124,7 +151,7 @@ #endif #if FMT_USE_NOEXCEPT || FMT_HAS_FEATURE(cxx_noexcept) || \ - (FMT_GCC_VERSION >= 408 && FMT_HAS_GXX_CXX11) || FMT_MSC_VER >= 1900 + FMT_GCC_VERSION >= 408 || FMT_MSC_VER >= 1900 # define FMT_DETECTED_NOEXCEPT noexcept # define FMT_HAS_CXX11_NOEXCEPT 1 #else @@ -149,25 +176,39 @@ # define FMT_NORETURN #endif -#ifndef FMT_DEPRECATED -# if FMT_HAS_CPP14_ATTRIBUTE(deprecated) || FMT_MSC_VER >= 1900 -# define FMT_DEPRECATED [[deprecated]] +#if __cplusplus == 201103L || __cplusplus == 201402L +# if defined(__INTEL_COMPILER) || defined(__PGI) +# define FMT_FALLTHROUGH +# elif defined(__clang__) +# define FMT_FALLTHROUGH [[clang::fallthrough]] +# elif FMT_GCC_VERSION >= 700 && \ + (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 520) +# define FMT_FALLTHROUGH [[gnu::fallthrough]] # else -# if (defined(__GNUC__) && !defined(__LCC__)) || defined(__clang__) -# define FMT_DEPRECATED __attribute__((deprecated)) -# elif FMT_MSC_VER -# define FMT_DEPRECATED __declspec(deprecated) -# else -# define FMT_DEPRECATED /* deprecated */ -# endif +# define FMT_FALLTHROUGH # endif +#elif FMT_HAS_CPP17_ATTRIBUTE(fallthrough) +# define FMT_FALLTHROUGH [[fallthrough]] +#else +# define FMT_FALLTHROUGH #endif -// Workaround broken [[deprecated]] in the Intel, PGI and NVCC compilers. -#if FMT_ICC_VERSION || defined(__PGI) || FMT_NVCC -# define FMT_DEPRECATED_ALIAS -#else -# define FMT_DEPRECATED_ALIAS FMT_DEPRECATED +#ifndef FMT_NODISCARD +# if FMT_HAS_CPP17_ATTRIBUTE(nodiscard) +# define FMT_NODISCARD [[nodiscard]] +# else +# define FMT_NODISCARD +# endif +#endif + +#ifndef FMT_USE_FLOAT +# define FMT_USE_FLOAT 1 +#endif +#ifndef FMT_USE_DOUBLE +# define FMT_USE_DOUBLE 1 +#endif +#ifndef FMT_USE_LONG_DOUBLE +# define FMT_USE_LONG_DOUBLE 1 #endif #ifndef FMT_INLINE @@ -178,61 +219,55 @@ # endif #endif -#ifndef FMT_USE_INLINE_NAMESPACES -# if FMT_HAS_FEATURE(cxx_inline_namespaces) || FMT_GCC_VERSION >= 404 || \ - (FMT_MSC_VER >= 1900 && !_MANAGED) -# define FMT_USE_INLINE_NAMESPACES 1 +#ifndef FMT_DEPRECATED +# if FMT_HAS_CPP14_ATTRIBUTE(deprecated) || FMT_MSC_VER >= 1900 +# define FMT_DEPRECATED [[deprecated]] # else -# define FMT_USE_INLINE_NAMESPACES 0 +# if (defined(__GNUC__) && !defined(__LCC__)) || defined(__clang__) +# define FMT_DEPRECATED __attribute__((deprecated)) +# elif FMT_MSC_VER +# define FMT_DEPRECATED __declspec(deprecated) +# else +# define FMT_DEPRECATED /* deprecated */ +# endif # endif #endif #ifndef FMT_BEGIN_NAMESPACE -# if FMT_USE_INLINE_NAMESPACES -# define FMT_INLINE_NAMESPACE inline namespace -# define FMT_END_NAMESPACE \ - } \ - } -# else -# define FMT_INLINE_NAMESPACE namespace -# define FMT_END_NAMESPACE \ - } \ - using namespace v7; \ - } -# endif # define FMT_BEGIN_NAMESPACE \ namespace fmt { \ - FMT_INLINE_NAMESPACE v7 { + inline namespace v8 { +# define FMT_END_NAMESPACE \ + } \ + } +#endif + +#ifndef FMT_MODULE_EXPORT +# define FMT_MODULE_EXPORT +# define FMT_MODULE_EXPORT_BEGIN +# define FMT_MODULE_EXPORT_END +# define FMT_BEGIN_DETAIL_NAMESPACE namespace detail { +# define FMT_END_DETAIL_NAMESPACE } #endif #if !defined(FMT_HEADER_ONLY) && defined(_WIN32) -# define FMT_CLASS_API FMT_SUPPRESS_MSC_WARNING(4275) +# define FMT_CLASS_API FMT_MSC_WARNING(suppress : 4275) # ifdef FMT_EXPORT # define FMT_API __declspec(dllexport) -# define FMT_EXTERN_TEMPLATE_API FMT_API -# define FMT_EXPORTED # elif defined(FMT_SHARED) # define FMT_API __declspec(dllimport) -# define FMT_EXTERN_TEMPLATE_API FMT_API # endif #else # define FMT_CLASS_API +# if defined(FMT_EXPORT) || defined(FMT_SHARED) +# if defined(__GNUC__) || defined(__clang__) +# define FMT_API __attribute__((visibility("default"))) +# endif +# endif #endif #ifndef FMT_API # define FMT_API #endif -#ifndef FMT_EXTERN_TEMPLATE_API -# define FMT_EXTERN_TEMPLATE_API -#endif -#ifndef FMT_INSTANTIATION_DEF_API -# define FMT_INSTANTIATION_DEF_API FMT_API -#endif - -#ifndef FMT_HEADER_ONLY -# define FMT_EXTERN extern -#else -# define FMT_EXTERN -#endif // libc++ supports string_view in pre-c++17. #if (FMT_HAS_INCLUDE() && \ @@ -248,16 +283,43 @@ #ifndef FMT_UNICODE # define FMT_UNICODE !FMT_MSC_VER #endif -#if FMT_UNICODE && FMT_MSC_VER -# pragma execution_character_set("utf-8") + +#ifndef FMT_CONSTEVAL +# if ((FMT_GCC_VERSION >= 1000 || FMT_CLANG_VERSION >= 1101) && \ + __cplusplus > 201703L && !defined(__apple_build_version__)) || \ + (defined(__cpp_consteval) && \ + (!FMT_MSC_VER || _MSC_FULL_VER >= 193030704)) +// consteval is broken in MSVC before VS2022 and Apple clang 13. +# define FMT_CONSTEVAL consteval +# define FMT_HAS_CONSTEVAL +# else +# define FMT_CONSTEVAL +# endif +#endif + +#ifndef FMT_USE_NONTYPE_TEMPLATE_PARAMETERS +# if defined(__cpp_nontype_template_args) && \ + ((FMT_GCC_VERSION >= 903 && __cplusplus >= 201709L) || \ + __cpp_nontype_template_args >= 201911L) +# define FMT_USE_NONTYPE_TEMPLATE_PARAMETERS 1 +# else +# define FMT_USE_NONTYPE_TEMPLATE_PARAMETERS 0 +# endif +#endif + +// Enable minimal optimizations for more compact code in debug mode. +FMT_GCC_PRAGMA("GCC push_options") +#ifndef __OPTIMIZE__ +FMT_GCC_PRAGMA("GCC optimize(\"Og\")") #endif FMT_BEGIN_NAMESPACE +FMT_MODULE_EXPORT_BEGIN // Implementations of enable_if_t and other metafunctions for older systems. -template +template using enable_if_t = typename std::enable_if::type; -template +template using conditional_t = typename std::conditional::type; template using bool_constant = std::integral_constant; template @@ -269,17 +331,40 @@ using remove_cvref_t = typename std::remove_cv>::type; template struct type_identity { using type = T; }; template using type_identity_t = typename type_identity::type; -struct monostate {}; +struct monostate { + constexpr monostate() {} +}; // An enable_if helper to be used in template parameters which results in much // shorter symbols: https://godbolt.org/z/sWw4vP. Extra parentheses are needed // to workaround a bug in MSVC 2019 (see #1140 and #1186). -#define FMT_ENABLE_IF(...) enable_if_t<(__VA_ARGS__), int> = 0 +#ifdef FMT_DOC +# define FMT_ENABLE_IF(...) +#else +# define FMT_ENABLE_IF(...) enable_if_t<(__VA_ARGS__), int> = 0 +#endif + +FMT_BEGIN_DETAIL_NAMESPACE -namespace detail { +// Suppress "unused variable" warnings with the method described in +// https://herbsutter.com/2009/10/18/mailbag-shutting-up-compiler-warnings/. +// (void)var does not work on many Intel compilers. +template FMT_CONSTEXPR void ignore_unused(const T&...) {} + +constexpr FMT_INLINE auto is_constant_evaluated(bool default_value = false) + FMT_NOEXCEPT -> bool { +#ifdef __cpp_lib_is_constant_evaluated + ignore_unused(default_value); + return std::is_constant_evaluated(); +#else + return default_value; +#endif +} -// A helper function to suppress "conditional expression is constant" warnings. -template constexpr T const_check(T value) { return value; } +// A function to suppress "conditional expression is constant" warnings. +template constexpr FMT_INLINE auto const_check(T value) -> T { + return value; +} FMT_NORETURN FMT_API void assert_fail(const char* file, int line, const char* message); @@ -287,7 +372,8 @@ FMT_NORETURN FMT_API void assert_fail(const char* file, int line, #ifndef FMT_ASSERT # ifdef NDEBUG // FMT_ASSERT is not empty to avoid -Werror=empty-body. -# define FMT_ASSERT(condition, message) ((void)0) +# define FMT_ASSERT(condition, message) \ + ::fmt::detail::ignore_unused((condition), (message)) # else # define FMT_ASSERT(condition, message) \ ((condition) /* void() fails with -Winvalid-constexpr on clang 4.0.1 */ \ @@ -296,6 +382,12 @@ FMT_NORETURN FMT_API void assert_fail(const char* file, int line, # endif #endif +#ifdef __cpp_lib_byte +using byte = std::byte; +#else +enum class byte : unsigned char {}; +#endif + #if defined(FMT_USE_STRING_VIEW) template using std_string_view = std::basic_string_view; #elif defined(FMT_USE_EXPERIMENTAL_STRING_VIEW) @@ -312,38 +404,39 @@ template struct std_string_view {}; # define FMT_USE_INT128 1 using int128_t = __int128_t; using uint128_t = __uint128_t; +template inline auto convert_for_visit(T value) -> T { + return value; +} #else # define FMT_USE_INT128 0 #endif #if !FMT_USE_INT128 -struct int128_t {}; -struct uint128_t {}; +enum class int128_t {}; +enum class uint128_t {}; +// Reduce template instantiations. +template inline auto convert_for_visit(T) -> monostate { + return {}; +} #endif // Casts a nonnegative integer to unsigned. template -FMT_CONSTEXPR typename std::make_unsigned::type to_unsigned(Int value) { +FMT_CONSTEXPR auto to_unsigned(Int value) -> + typename std::make_unsigned::type { FMT_ASSERT(value >= 0, "negative value"); return static_cast::type>(value); } -FMT_SUPPRESS_MSC_WARNING(4566) constexpr unsigned char micro[] = "\u00B5"; +FMT_MSC_WARNING(suppress : 4566) constexpr unsigned char micro[] = "\u00B5"; -template constexpr bool is_unicode() { - return FMT_UNICODE || sizeof(Char) != 1 || - (sizeof(micro) == 3 && micro[0] == 0xC2 && micro[1] == 0xB5); +constexpr auto is_utf8() -> bool { + // Avoid buggy sign extensions in MSVC's constant evaluation mode. + // https://developercommunity.visualstudio.com/t/C-difference-in-behavior-for-unsigned/1233612 + using uchar = unsigned char; + return FMT_UNICODE || (sizeof(micro) == 3 && uchar(micro[0]) == 0xC2 && + uchar(micro[1]) == 0xB5); } - -#ifdef __cpp_char8_t -using char8_type = char8_t; -#else -enum char8_type : unsigned char {}; -#endif -} // namespace detail - -#ifdef FMT_USE_INTERNAL -namespace internal = detail; // DEPRECATED -#endif +FMT_END_DETAIL_NAMESPACE /** An implementation of ``std::basic_string_view`` for pre-C++17. It provides a @@ -374,11 +467,14 @@ template class basic_string_view { the size with ``std::char_traits::length``. \endrst */ -#if __cplusplus >= 201703L // C++17's char_traits::length() is constexpr. - FMT_CONSTEXPR -#endif + FMT_CONSTEXPR_CHAR_TRAITS + FMT_INLINE basic_string_view(const Char* s) - : data_(s), size_(std::char_traits::length(s)) {} + : data_(s), + size_(detail::const_check(std::is_same::value && + !detail::is_constant_evaluated(true)) + ? std::strlen(reinterpret_cast(s)) + : std::char_traits::length(s)) {} /** Constructs a string reference from a ``std::basic_string`` object. */ template @@ -393,23 +489,25 @@ template class basic_string_view { size_(s.size()) {} /** Returns a pointer to the string data. */ - constexpr const Char* data() const { return data_; } + constexpr auto data() const FMT_NOEXCEPT -> const Char* { return data_; } /** Returns the string size. */ - constexpr size_t size() const { return size_; } + constexpr auto size() const FMT_NOEXCEPT -> size_t { return size_; } - constexpr iterator begin() const { return data_; } - constexpr iterator end() const { return data_ + size_; } + constexpr auto begin() const FMT_NOEXCEPT -> iterator { return data_; } + constexpr auto end() const FMT_NOEXCEPT -> iterator { return data_ + size_; } - constexpr const Char& operator[](size_t pos) const { return data_[pos]; } + constexpr auto operator[](size_t pos) const FMT_NOEXCEPT -> const Char& { + return data_[pos]; + } - FMT_CONSTEXPR void remove_prefix(size_t n) { + FMT_CONSTEXPR void remove_prefix(size_t n) FMT_NOEXCEPT { data_ += n; size_ -= n; } // Lexicographically compare this string reference to other. - int compare(basic_string_view other) const { + FMT_CONSTEXPR_CHAR_TRAITS auto compare(basic_string_view other) const -> int { size_t str_size = size_ < other.size_ ? size_ : other.size_; int result = std::char_traits::compare(data_, other.data_, str_size); if (result == 0) @@ -417,72 +515,53 @@ template class basic_string_view { return result; } - friend bool operator==(basic_string_view lhs, basic_string_view rhs) { + FMT_CONSTEXPR_CHAR_TRAITS friend auto operator==(basic_string_view lhs, + basic_string_view rhs) + -> bool { return lhs.compare(rhs) == 0; } - friend bool operator!=(basic_string_view lhs, basic_string_view rhs) { + friend auto operator!=(basic_string_view lhs, basic_string_view rhs) -> bool { return lhs.compare(rhs) != 0; } - friend bool operator<(basic_string_view lhs, basic_string_view rhs) { + friend auto operator<(basic_string_view lhs, basic_string_view rhs) -> bool { return lhs.compare(rhs) < 0; } - friend bool operator<=(basic_string_view lhs, basic_string_view rhs) { + friend auto operator<=(basic_string_view lhs, basic_string_view rhs) -> bool { return lhs.compare(rhs) <= 0; } - friend bool operator>(basic_string_view lhs, basic_string_view rhs) { + friend auto operator>(basic_string_view lhs, basic_string_view rhs) -> bool { return lhs.compare(rhs) > 0; } - friend bool operator>=(basic_string_view lhs, basic_string_view rhs) { + friend auto operator>=(basic_string_view lhs, basic_string_view rhs) -> bool { return lhs.compare(rhs) >= 0; } }; using string_view = basic_string_view; -using wstring_view = basic_string_view; /** Specifies if ``T`` is a character type. Can be specialized by users. */ template struct is_char : std::false_type {}; template <> struct is_char : std::true_type {}; -template <> struct is_char : std::true_type {}; -template <> struct is_char : std::true_type {}; -template <> struct is_char : std::true_type {}; -template <> struct is_char : std::true_type {}; - -/** - \rst - Returns a string view of `s`. In order to add custom string type support to - {fmt} provide an overload of `to_string_view` for it in the same namespace as - the type for the argument-dependent lookup to work. - **Example**:: - - namespace my_ns { - inline string_view to_string_view(const my_string& s) { - return {s.data(), s.length()}; - } - } - std::string message = fmt::format(my_string("The answer is {}"), 42); - \endrst - */ +// Returns a string view of `s`. template ::value)> -inline basic_string_view to_string_view(const Char* s) { +FMT_INLINE auto to_string_view(const Char* s) -> basic_string_view { return s; } - template -inline basic_string_view to_string_view( - const std::basic_string& s) { +inline auto to_string_view(const std::basic_string& s) + -> basic_string_view { return s; } - template -inline basic_string_view to_string_view(basic_string_view s) { +constexpr auto to_string_view(basic_string_view s) + -> basic_string_view { return s; } - template >::value)> -inline basic_string_view to_string_view(detail::std_string_view s) { +inline auto to_string_view(detail::std_string_view s) + -> basic_string_view { return s; } @@ -494,13 +573,15 @@ template struct is_compile_string : std::is_base_of {}; template ::value)> -constexpr basic_string_view to_string_view(const S& s) { - return s; +constexpr auto to_string_view(const S& s) + -> basic_string_view { + return basic_string_view(s); } -namespace detail { +FMT_BEGIN_DETAIL_NAMESPACE + void to_string_view(...); -using fmt::v7::to_string_view; +using fmt::to_string_view; // Specifies whether S is a string type convertible to fmt::basic_string_view. // It should be a constexpr function but MSVC 2017 fails to compile it in @@ -527,6 +608,8 @@ FMT_INLINE void check_format_string(const S&) { template ::value)> void check_format_string(S); +FMT_NORETURN FMT_API void throw_format_error(const char* message); + struct error_handler { constexpr error_handler() = default; constexpr error_handler(const error_handler&) = default; @@ -534,7 +617,7 @@ struct error_handler { // This function is intentionally not constexpr to give a compile-time error. FMT_NORETURN FMT_API void on_error(const char* message); }; -} // namespace detail +FMT_END_DETAIL_NAMESPACE /** String's character type. */ template using char_t = typename detail::char_t_impl::type; @@ -543,16 +626,7 @@ template using char_t = typename detail::char_t_impl::type; \rst Parsing context consisting of a format string range being parsed and an argument counter for automatic indexing. - - You can use one of the following type aliases for common character types: - - +-----------------------+-------------------------------------+ - | Type | Definition | - +=======================+=====================================+ - | format_parse_context | basic_format_parse_context | - +-----------------------+-------------------------------------+ - | wformat_parse_context | basic_format_parse_context | - +-----------------------+-------------------------------------+ + You can use the ``format_parse_context`` type alias for ``char`` instead. \endrst */ template @@ -574,12 +648,16 @@ class basic_format_parse_context : private ErrorHandler { Returns an iterator to the beginning of the format string range being parsed. */ - constexpr iterator begin() const FMT_NOEXCEPT { return format_str_.begin(); } + constexpr auto begin() const FMT_NOEXCEPT -> iterator { + return format_str_.begin(); + } /** Returns an iterator past the end of the format string range being parsed. */ - constexpr iterator end() const FMT_NOEXCEPT { return format_str_.end(); } + constexpr auto end() const FMT_NOEXCEPT -> iterator { + return format_str_.end(); + } /** Advances the begin iterator to ``it``. */ FMT_CONSTEXPR void advance_to(iterator it) { @@ -590,7 +668,7 @@ class basic_format_parse_context : private ErrorHandler { Reports an error if using the manual argument indexing; otherwise returns the next argument index and switches to the automatic indexing. */ - FMT_CONSTEXPR int next_arg_id() { + FMT_CONSTEXPR auto next_arg_id() -> int { // Don't check if the argument id is valid to avoid overhead and because it // will be checked during formatting anyway. if (next_arg_id_ >= 0) return next_arg_id_++; @@ -615,11 +693,10 @@ class basic_format_parse_context : private ErrorHandler { ErrorHandler::on_error(message); } - constexpr ErrorHandler error_handler() const { return *this; } + constexpr auto error_handler() const -> ErrorHandler { return *this; } }; using format_parse_context = basic_format_parse_context; -using wformat_parse_context = basic_format_parse_context; template class basic_format_arg; template class basic_format_args; @@ -643,11 +720,30 @@ template struct is_contiguous : std::false_type {}; template struct is_contiguous> : std::true_type {}; -namespace detail { +class appender; + +FMT_BEGIN_DETAIL_NAMESPACE + +template +constexpr auto has_const_formatter_impl(T*) + -> decltype(typename Context::template formatter_type().format( + std::declval(), std::declval()), + true) { + return true; +} +template +constexpr auto has_const_formatter_impl(...) -> bool { + return false; +} +template +constexpr auto has_const_formatter() -> bool { + return has_const_formatter_impl(static_cast(nullptr)); +} // Extracts a reference to the container from back_insert_iterator. template -inline Container& get_container(std::back_insert_iterator it) { +inline auto get_container(std::back_insert_iterator it) + -> Container& { using bi_iterator = std::back_insert_iterator; struct accessor : bi_iterator { accessor(bi_iterator iter) : bi_iterator(iter) {} @@ -656,6 +752,23 @@ inline Container& get_container(std::back_insert_iterator it) { return *accessor(it).container; } +template +FMT_CONSTEXPR auto copy_str(InputIt begin, InputIt end, OutputIt out) + -> OutputIt { + while (begin != end) *out++ = static_cast(*begin++); + return out; +} + +template , U>::value&& is_char::value)> +FMT_CONSTEXPR auto copy_str(T* begin, T* end, U* out) -> U* { + if (is_constant_evaluated()) return copy_str(begin, end, out); + auto size = to_unsigned(end - begin); + memcpy(out, begin, size * sizeof(U)); + return out + size; +} + /** \rst A contiguous memory buffer with an optional growing ability. It is an internal @@ -670,24 +783,25 @@ template class buffer { protected: // Don't initialize ptr_ since it is not accessed to save a few cycles. - FMT_SUPPRESS_MSC_WARNING(26495) + FMT_MSC_WARNING(suppress : 26495) buffer(size_t sz) FMT_NOEXCEPT : size_(sz), capacity_(sz) {} - buffer(T* p = nullptr, size_t sz = 0, size_t cap = 0) FMT_NOEXCEPT - : ptr_(p), - size_(sz), - capacity_(cap) {} + FMT_CONSTEXPR20 buffer(T* p = nullptr, size_t sz = 0, + size_t cap = 0) FMT_NOEXCEPT : ptr_(p), + size_(sz), + capacity_(cap) {} - ~buffer() = default; + FMT_CONSTEXPR20 ~buffer() = default; + buffer(buffer&&) = default; /** Sets the buffer data and capacity. */ - void set(T* buf_data, size_t buf_capacity) FMT_NOEXCEPT { + FMT_CONSTEXPR void set(T* buf_data, size_t buf_capacity) FMT_NOEXCEPT { ptr_ = buf_data; capacity_ = buf_capacity; } /** Increases the buffer capacity to hold at least *capacity* elements. */ - virtual void grow(size_t capacity) = 0; + virtual FMT_CONSTEXPR20 void grow(size_t capacity) = 0; public: using value_type = T; @@ -696,30 +810,30 @@ template class buffer { buffer(const buffer&) = delete; void operator=(const buffer&) = delete; - T* begin() FMT_NOEXCEPT { return ptr_; } - T* end() FMT_NOEXCEPT { return ptr_ + size_; } + auto begin() FMT_NOEXCEPT -> T* { return ptr_; } + auto end() FMT_NOEXCEPT -> T* { return ptr_ + size_; } - const T* begin() const FMT_NOEXCEPT { return ptr_; } - const T* end() const FMT_NOEXCEPT { return ptr_ + size_; } + auto begin() const FMT_NOEXCEPT -> const T* { return ptr_; } + auto end() const FMT_NOEXCEPT -> const T* { return ptr_ + size_; } /** Returns the size of this buffer. */ - size_t size() const FMT_NOEXCEPT { return size_; } + constexpr auto size() const FMT_NOEXCEPT -> size_t { return size_; } /** Returns the capacity of this buffer. */ - size_t capacity() const FMT_NOEXCEPT { return capacity_; } + constexpr auto capacity() const FMT_NOEXCEPT -> size_t { return capacity_; } /** Returns a pointer to the buffer data. */ - T* data() FMT_NOEXCEPT { return ptr_; } + FMT_CONSTEXPR auto data() FMT_NOEXCEPT -> T* { return ptr_; } /** Returns a pointer to the buffer data. */ - const T* data() const FMT_NOEXCEPT { return ptr_; } + FMT_CONSTEXPR auto data() const FMT_NOEXCEPT -> const T* { return ptr_; } /** Clears this buffer. */ void clear() { size_ = 0; } // Tries resizing the buffer to contain *count* elements. If T is a POD type // the new elements may not be initialized. - void try_resize(size_t count) { + FMT_CONSTEXPR20 void try_resize(size_t count) { try_reserve(count); size_ = count <= capacity_ ? count : capacity_; } @@ -728,11 +842,11 @@ template class buffer { // capacity by a smaller amount than requested but guarantees there is space // for at least one additional element either by increasing the capacity or by // flushing the buffer if it is full. - void try_reserve(size_t new_capacity) { + FMT_CONSTEXPR20 void try_reserve(size_t new_capacity) { if (new_capacity > capacity_) grow(new_capacity); } - void push_back(const T& value) { + FMT_CONSTEXPR20 void push_back(const T& value) { try_reserve(size_ + 1); ptr_[size_++] = value; } @@ -740,16 +854,19 @@ template class buffer { /** Appends data to the end of the buffer. */ template void append(const U* begin, const U* end); - template T& operator[](I index) { return ptr_[index]; } - template const T& operator[](I index) const { + template FMT_CONSTEXPR auto operator[](I index) -> T& { + return ptr_[index]; + } + template + FMT_CONSTEXPR auto operator[](I index) const -> const T& { return ptr_[index]; } }; struct buffer_traits { explicit buffer_traits(size_t) {} - size_t count() const { return 0; } - size_t limit(size_t size) { return size; } + auto count() const -> size_t { return 0; } + auto limit(size_t size) -> size_t { return size; } }; class fixed_buffer_traits { @@ -759,8 +876,8 @@ class fixed_buffer_traits { public: explicit fixed_buffer_traits(size_t limit) : limit_(limit) {} - size_t count() const { return count_; } - size_t limit(size_t size) { + auto count() const -> size_t { return count_; } + auto limit(size_t size) -> size_t { size_t n = limit_ > count_ ? limit_ - count_ : 0; count_ += size; return size < n ? size : n; @@ -776,33 +893,84 @@ class iterator_buffer final : public Traits, public buffer { T data_[buffer_size]; protected: - void grow(size_t) final FMT_OVERRIDE { + FMT_CONSTEXPR20 void grow(size_t) override { if (this->size() == buffer_size) flush(); } - void flush(); + + void flush() { + auto size = this->size(); + this->clear(); + out_ = copy_str(data_, data_ + this->limit(size), out_); + } public: explicit iterator_buffer(OutputIt out, size_t n = buffer_size) - : Traits(n), - buffer(data_, 0, buffer_size), - out_(out) {} + : Traits(n), buffer(data_, 0, buffer_size), out_(out) {} + iterator_buffer(iterator_buffer&& other) + : Traits(other), buffer(data_, 0, buffer_size), out_(other.out_) {} + ~iterator_buffer() { flush(); } + + auto out() -> OutputIt { + flush(); + return out_; + } + auto count() const -> size_t { return Traits::count() + this->size(); } +}; + +template +class iterator_buffer final + : public fixed_buffer_traits, + public buffer { + private: + T* out_; + enum { buffer_size = 256 }; + T data_[buffer_size]; + + protected: + FMT_CONSTEXPR20 void grow(size_t) override { + if (this->size() == this->capacity()) flush(); + } + + void flush() { + size_t n = this->limit(this->size()); + if (this->data() == out_) { + out_ += n; + this->set(data_, buffer_size); + } + this->clear(); + } + + public: + explicit iterator_buffer(T* out, size_t n = buffer_size) + : fixed_buffer_traits(n), buffer(out, 0, n), out_(out) {} + iterator_buffer(iterator_buffer&& other) + : fixed_buffer_traits(other), + buffer(std::move(other)), + out_(other.out_) { + if (this->data() != out_) { + this->set(data_, buffer_size); + this->clear(); + } + } ~iterator_buffer() { flush(); } - OutputIt out() { + auto out() -> T* { flush(); return out_; } - size_t count() const { return Traits::count() + this->size(); } + auto count() const -> size_t { + return fixed_buffer_traits::count() + this->size(); + } }; template class iterator_buffer final : public buffer { protected: - void grow(size_t) final FMT_OVERRIDE {} + FMT_CONSTEXPR20 void grow(size_t) override {} public: explicit iterator_buffer(T* out, size_t = 0) : buffer(out, 0, ~size_t()) {} - T* out() { return &*this->end(); } + auto out() -> T* { return &*this->end(); } }; // A buffer that writes to a container with the contiguous storage. @@ -815,7 +983,7 @@ class iterator_buffer, Container& container_; protected: - void grow(size_t capacity) final FMT_OVERRIDE { + FMT_CONSTEXPR20 void grow(size_t capacity) override { container_.resize(capacity); this->set(&container_[0], capacity); } @@ -825,7 +993,7 @@ class iterator_buffer, : buffer(c.size()), container_(c) {} explicit iterator_buffer(std::back_insert_iterator out, size_t = 0) : iterator_buffer(get_container(out)) {} - std::back_insert_iterator out() { + auto out() -> std::back_insert_iterator { return std::back_inserter(container_); } }; @@ -838,7 +1006,7 @@ template class counting_buffer final : public buffer { size_t count_ = 0; protected: - void grow(size_t) final FMT_OVERRIDE { + FMT_CONSTEXPR20 void grow(size_t) override { if (this->size() != buffer_size) return; count_ += this->size(); this->clear(); @@ -847,48 +1015,24 @@ template class counting_buffer final : public buffer { public: counting_buffer() : buffer(data_, 0, buffer_size) {} - size_t count() { return count_ + this->size(); } + auto count() -> size_t { return count_ + this->size(); } }; -// An output iterator that appends to the buffer. -// It is used to reduce symbol sizes for the common case. template -class buffer_appender : public std::back_insert_iterator> { - using base = std::back_insert_iterator>; - - public: - explicit buffer_appender(buffer& buf) : base(buf) {} - buffer_appender(base it) : base(it) {} +using buffer_appender = conditional_t::value, appender, + std::back_insert_iterator>>; - buffer_appender& operator++() { - base::operator++(); - return *this; - } - - buffer_appender operator++(int) { - buffer_appender tmp = *this; - ++*this; - return tmp; - } -}; - -// Maps an output iterator into a buffer. +// Maps an output iterator to a buffer. template -iterator_buffer get_buffer(OutputIt); -template buffer& get_buffer(buffer_appender); - -template OutputIt get_buffer_init(OutputIt out) { - return out; -} -template buffer& get_buffer_init(buffer_appender out) { - return get_container(out); +auto get_buffer(OutputIt out) -> iterator_buffer { + return iterator_buffer(out); } template auto get_iterator(Buffer& buf) -> decltype(buf.out()) { return buf.out(); } -template buffer_appender get_iterator(buffer& buf) { +template auto get_iterator(buffer& buf) -> buffer_appender { return buffer_appender(buf); } @@ -898,9 +1042,9 @@ struct fallback_formatter { }; // Specifies if T has an enabled fallback_formatter specialization. -template +template using has_fallback_formatter = - std::is_constructible>; + std::is_constructible>; struct view {}; @@ -925,8 +1069,8 @@ struct arg_data { template arg_data(const U&... init) : args_{T(named_args_, NUM_NAMED_ARGS), init...} {} arg_data(const arg_data& other) = delete; - const T* args() const { return args_ + 1; } - named_arg_info* named_args() { return named_args_; } + auto args() const -> const T* { return args_ + 1; } + auto named_args() -> named_arg_info* { return named_args_; } }; template @@ -935,45 +1079,55 @@ struct arg_data { T args_[NUM_ARGS != 0 ? NUM_ARGS : +1]; template - FMT_INLINE arg_data(const U&... init) : args_{init...} {} - FMT_INLINE const T* args() const { return args_; } - FMT_INLINE std::nullptr_t named_args() { return nullptr; } + FMT_CONSTEXPR FMT_INLINE arg_data(const U&... init) : args_{init...} {} + FMT_CONSTEXPR FMT_INLINE auto args() const -> const T* { return args_; } + FMT_CONSTEXPR FMT_INLINE auto named_args() -> std::nullptr_t { + return nullptr; + } }; template inline void init_named_args(named_arg_info*, int, int) {} -template +template struct is_named_arg : std::false_type {}; +template struct is_statically_named_arg : std::false_type {}; + +template +struct is_named_arg> : std::true_type {}; + +template ::value)> void init_named_args(named_arg_info* named_args, int arg_count, int named_arg_count, const T&, const Tail&... args) { init_named_args(named_args, arg_count + 1, named_arg_count, args...); } -template +template ::value)> void init_named_args(named_arg_info* named_args, int arg_count, - int named_arg_count, const named_arg& arg, - const Tail&... args) { + int named_arg_count, const T& arg, const Tail&... args) { named_args[named_arg_count++] = {arg.name, arg_count}; init_named_args(named_args, arg_count + 1, named_arg_count, args...); } template -FMT_INLINE void init_named_args(std::nullptr_t, int, int, const Args&...) {} - -template struct is_named_arg : std::false_type {}; - -template -struct is_named_arg> : std::true_type {}; +FMT_CONSTEXPR FMT_INLINE void init_named_args(std::nullptr_t, int, int, + const Args&...) {} -template constexpr size_t count() { return B ? 1 : 0; } -template constexpr size_t count() { +template constexpr auto count() -> size_t { return B ? 1 : 0; } +template constexpr auto count() -> size_t { return (B1 ? 1 : 0) + count(); } -template constexpr size_t count_named_args() { +template constexpr auto count_named_args() -> size_t { return count::value...>(); } +template +constexpr auto count_statically_named_args() -> size_t { + return count::value...>(); +} + enum class type { none_type, // Integer types should go first, @@ -1029,6 +1183,11 @@ constexpr bool is_arithmetic_type(type t) { return t > type::none_type && t <= type::last_numeric_type; } +struct unformattable {}; +struct unformattable_char : unformattable {}; +struct unformattable_const : unformattable {}; +struct unformattable_pointer : unformattable {}; + template struct string_value { const Char* data; size_t size; @@ -1041,8 +1200,8 @@ template struct named_arg_value { template struct custom_value { using parse_context = typename Context::parse_context_type; - const void* value; - void (*format)(const void* arg, parse_context& parse_ctx, Context& ctx); + void* value; + void (*format)(void* arg, parse_context& parse_ctx, Context& ctx); }; // A formatting argument value. @@ -1051,6 +1210,7 @@ template class value { using char_type = typename Context::char_type; union { + monostate no_value; int int_value; unsigned uint_value; long long long_long_value; @@ -1068,19 +1228,23 @@ template class value { named_arg_value named_args; }; - constexpr FMT_INLINE value(int val = 0) : int_value(val) {} + constexpr FMT_INLINE value() : no_value() {} + constexpr FMT_INLINE value(int val) : int_value(val) {} constexpr FMT_INLINE value(unsigned val) : uint_value(val) {} - FMT_INLINE value(long long val) : long_long_value(val) {} - FMT_INLINE value(unsigned long long val) : ulong_long_value(val) {} + constexpr FMT_INLINE value(long long val) : long_long_value(val) {} + constexpr FMT_INLINE value(unsigned long long val) : ulong_long_value(val) {} FMT_INLINE value(int128_t val) : int128_value(val) {} FMT_INLINE value(uint128_t val) : uint128_value(val) {} - FMT_INLINE value(float val) : float_value(val) {} - FMT_INLINE value(double val) : double_value(val) {} + constexpr FMT_INLINE value(float val) : float_value(val) {} + constexpr FMT_INLINE value(double val) : double_value(val) {} FMT_INLINE value(long double val) : long_double_value(val) {} - FMT_INLINE value(bool val) : bool_value(val) {} - FMT_INLINE value(char_type val) : char_value(val) {} - FMT_INLINE value(const char_type* val) { string.data = val; } - FMT_INLINE value(basic_string_view val) { + constexpr FMT_INLINE value(bool val) : bool_value(val) {} + constexpr FMT_INLINE value(char_type val) : char_value(val) {} + FMT_CONSTEXPR FMT_INLINE value(const char_type* val) { + string.data = val; + if (is_constant_evaluated()) string.size = {}; + } + FMT_CONSTEXPR FMT_INLINE value(basic_string_view val) { string.data = val.data(); string.size = val.size(); } @@ -1088,31 +1252,39 @@ template class value { FMT_INLINE value(const named_arg_info* args, size_t size) : named_args{args, size} {} - template FMT_INLINE value(const T& val) { - custom.value = &val; + template FMT_CONSTEXPR FMT_INLINE value(T& val) { + using value_type = remove_cvref_t; + custom.value = const_cast(&val); // Get the formatter type through the context to allow different contexts // have different extension points, e.g. `formatter` for `format` and // `printf_formatter` for `printf`. custom.format = format_custom_arg< - T, conditional_t::value, - typename Context::template formatter_type, - fallback_formatter>>; + value_type, + conditional_t::value, + typename Context::template formatter_type, + fallback_formatter>>; } + value(unformattable); + value(unformattable_char); + value(unformattable_const); + value(unformattable_pointer); private: // Formats an argument of a custom type, such as a user-defined class. template - static void format_custom_arg(const void* arg, + static void format_custom_arg(void* arg, typename Context::parse_context_type& parse_ctx, Context& ctx) { - Formatter f; + auto f = Formatter(); parse_ctx.advance_to(f.parse(parse_ctx)); - ctx.advance_to(f.format(*static_cast(arg), ctx)); + using qualified_type = + conditional_t(), const T, T>; + ctx.advance_to(f.format(*static_cast(arg), ctx)); } }; template -FMT_CONSTEXPR basic_format_arg make_arg(const T& value); +FMT_CONSTEXPR auto make_arg(const T& value) -> basic_format_arg; // To minimize the number of types we need to deal with, long is translated // either to int or to long long depending on its size. @@ -1120,52 +1292,84 @@ enum { long_short = sizeof(long) == sizeof(int) }; using long_type = conditional_t; using ulong_type = conditional_t; -struct unformattable {}; - // Maps formatting arguments to core types. +// arg_mapper reports errors by returning unformattable instead of using +// static_assert because it's used in the is_formattable trait. template struct arg_mapper { using char_type = typename Context::char_type; - FMT_CONSTEXPR int map(signed char val) { return val; } - FMT_CONSTEXPR unsigned map(unsigned char val) { return val; } - FMT_CONSTEXPR int map(short val) { return val; } - FMT_CONSTEXPR unsigned map(unsigned short val) { return val; } - FMT_CONSTEXPR int map(int val) { return val; } - FMT_CONSTEXPR unsigned map(unsigned val) { return val; } - FMT_CONSTEXPR long_type map(long val) { return val; } - FMT_CONSTEXPR ulong_type map(unsigned long val) { return val; } - FMT_CONSTEXPR long long map(long long val) { return val; } - FMT_CONSTEXPR unsigned long long map(unsigned long long val) { return val; } - FMT_CONSTEXPR int128_t map(int128_t val) { return val; } - FMT_CONSTEXPR uint128_t map(uint128_t val) { return val; } - FMT_CONSTEXPR bool map(bool val) { return val; } - - template ::value)> - FMT_CONSTEXPR char_type map(T val) { - static_assert( - std::is_same::value || std::is_same::value, - "mixing character types is disallowed"); + FMT_CONSTEXPR FMT_INLINE auto map(signed char val) -> int { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(unsigned char val) -> unsigned { + return val; + } + FMT_CONSTEXPR FMT_INLINE auto map(short val) -> int { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(unsigned short val) -> unsigned { + return val; + } + FMT_CONSTEXPR FMT_INLINE auto map(int val) -> int { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(unsigned val) -> unsigned { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(long val) -> long_type { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(unsigned long val) -> ulong_type { + return val; + } + FMT_CONSTEXPR FMT_INLINE auto map(long long val) -> long long { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(unsigned long long val) + -> unsigned long long { + return val; + } + FMT_CONSTEXPR FMT_INLINE auto map(int128_t val) -> int128_t { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(uint128_t val) -> uint128_t { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(bool val) -> bool { return val; } + + template ::value || + std::is_same::value)> + FMT_CONSTEXPR FMT_INLINE auto map(T val) -> char_type { return val; } + template ::value || +#ifdef __cpp_char8_t + std::is_same::value || +#endif + std::is_same::value || + std::is_same::value) && + !std::is_same::value, + int> = 0> + FMT_CONSTEXPR FMT_INLINE auto map(T) -> unformattable_char { + return {}; + } - FMT_CONSTEXPR float map(float val) { return val; } - FMT_CONSTEXPR double map(double val) { return val; } - FMT_CONSTEXPR long double map(long double val) { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(float val) -> float { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(double val) -> double { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(long double val) -> long double { + return val; + } - FMT_CONSTEXPR const char_type* map(char_type* val) { return val; } - FMT_CONSTEXPR const char_type* map(const char_type* val) { return val; } - template ::value)> - FMT_CONSTEXPR basic_string_view map(const T& val) { - static_assert(std::is_same>::value, - "mixing character types is disallowed"); + FMT_CONSTEXPR FMT_INLINE auto map(char_type* val) -> const char_type* { + return val; + } + FMT_CONSTEXPR FMT_INLINE auto map(const char_type* val) -> const char_type* { + return val; + } + template ::value && !std::is_pointer::value && + std::is_same>::value)> + FMT_CONSTEXPR FMT_INLINE auto map(const T& val) + -> basic_string_view { return to_string_view(val); } + template ::value && !std::is_pointer::value && + !std::is_same>::value)> + FMT_CONSTEXPR FMT_INLINE auto map(const T&) -> unformattable_char { + return {}; + } template , T>::value && !is_string::value && !has_formatter::value && - !has_fallback_formatter::value)> - FMT_CONSTEXPR basic_string_view map(const T& val) { + !has_fallback_formatter::value)> + FMT_CONSTEXPR FMT_INLINE auto map(const T& val) + -> basic_string_view { return basic_string_view(val); } template < @@ -1174,63 +1378,113 @@ template struct arg_mapper { std::is_constructible, T>::value && !std::is_constructible, T>::value && !is_string::value && !has_formatter::value && - !has_fallback_formatter::value)> - FMT_CONSTEXPR basic_string_view map(const T& val) { + !has_fallback_formatter::value)> + FMT_CONSTEXPR FMT_INLINE auto map(const T& val) + -> basic_string_view { return std_string_view(val); } - FMT_CONSTEXPR const char* map(const signed char* val) { - static_assert(std::is_same::value, "invalid string type"); - return reinterpret_cast(val); + + using cstring_result = conditional_t::value, + const char*, unformattable_pointer>; + + FMT_DEPRECATED FMT_CONSTEXPR FMT_INLINE auto map(const signed char* val) + -> cstring_result { + return map(reinterpret_cast(val)); } - FMT_CONSTEXPR const char* map(const unsigned char* val) { - static_assert(std::is_same::value, "invalid string type"); - return reinterpret_cast(val); + FMT_DEPRECATED FMT_CONSTEXPR FMT_INLINE auto map(const unsigned char* val) + -> cstring_result { + return map(reinterpret_cast(val)); } - FMT_CONSTEXPR const char* map(signed char* val) { - const auto* const_val = val; - return map(const_val); + FMT_DEPRECATED FMT_CONSTEXPR FMT_INLINE auto map(signed char* val) + -> cstring_result { + return map(reinterpret_cast(val)); } - FMT_CONSTEXPR const char* map(unsigned char* val) { - const auto* const_val = val; - return map(const_val); + FMT_DEPRECATED FMT_CONSTEXPR FMT_INLINE auto map(unsigned char* val) + -> cstring_result { + return map(reinterpret_cast(val)); } - FMT_CONSTEXPR const void* map(void* val) { return val; } - FMT_CONSTEXPR const void* map(const void* val) { return val; } - FMT_CONSTEXPR const void* map(std::nullptr_t val) { return val; } - template FMT_CONSTEXPR int map(const T*) { - // Formatting of arbitrary pointers is disallowed. If you want to output - // a pointer cast it to "void *" or "const void *". In particular, this - // forbids formatting of "[const] volatile char *" which is printed as bool - // by iostreams. - static_assert(!sizeof(T), "formatting of non-void pointers is disallowed"); - return 0; + FMT_CONSTEXPR FMT_INLINE auto map(void* val) -> const void* { return val; } + FMT_CONSTEXPR FMT_INLINE auto map(const void* val) -> const void* { + return val; + } + FMT_CONSTEXPR FMT_INLINE auto map(std::nullptr_t val) -> const void* { + return val; + } + + // We use SFINAE instead of a const T* parameter to avoid conflicting with + // the C array overload. + template < + typename T, + FMT_ENABLE_IF( + std::is_member_pointer::value || + std::is_function::type>::value || + (std::is_convertible::value && + !std::is_convertible::value))> + FMT_CONSTEXPR auto map(const T&) -> unformattable_pointer { + return {}; + } + + template ::value)> + FMT_CONSTEXPR FMT_INLINE auto map(const T (&values)[N]) -> const T (&)[N] { + return values; } template ::value && - !has_formatter::value && - !has_fallback_formatter::value)> - FMT_CONSTEXPR auto map(const T& val) + FMT_ENABLE_IF( + std::is_enum::value&& std::is_convertible::value && + !has_formatter::value && + !has_fallback_formatter::value)> + FMT_CONSTEXPR FMT_INLINE auto map(const T& val) -> decltype(std::declval().map( static_cast::type>(val))) { return map(static_cast::type>(val)); } - template ::value && !is_char::value && - (has_formatter::value || - has_fallback_formatter::value))> - FMT_CONSTEXPR const T& map(const T& val) { + + FMT_CONSTEXPR FMT_INLINE auto map(detail::byte val) -> unsigned { + return map(static_cast(val)); + } + + template > + struct formattable + : bool_constant() || + !std::is_const>::value || + has_fallback_formatter::value> {}; + +#if FMT_MSC_VER != 0 && FMT_MSC_VER < 1910 + // Workaround a bug in MSVC. + template FMT_CONSTEXPR FMT_INLINE auto do_map(T&& val) -> T& { + return val; + } +#else + template ::value)> + FMT_CONSTEXPR FMT_INLINE auto do_map(T&& val) -> T& { return val; } + template ::value)> + FMT_CONSTEXPR FMT_INLINE auto do_map(T&&) -> unformattable_const { + return {}; + } +#endif + + template , + FMT_ENABLE_IF(!is_string::value && !is_char::value && + !std::is_array::value && + (has_formatter::value || + has_fallback_formatter::value))> + FMT_CONSTEXPR FMT_INLINE auto map(T&& val) + -> decltype(this->do_map(std::forward(val))) { + return do_map(std::forward(val)); + } - template - FMT_CONSTEXPR auto map(const named_arg& val) - -> decltype(std::declval().map(val.value)) { - return map(val.value); + template ::value)> + FMT_CONSTEXPR FMT_INLINE auto map(const T& named_arg) + -> decltype(std::declval().map(named_arg.value)) { + return map(named_arg.value); } - unformattable map(...) { return {}; } + auto map(...) -> unformattable { return {}; } }; // A type constant after applying arg_mapper. @@ -1244,7 +1498,28 @@ enum { packed_arg_bits = 4 }; enum { max_packed_args = 62 / packed_arg_bits }; enum : unsigned long long { is_unpacked_bit = 1ULL << 63 }; enum : unsigned long long { has_named_args_bit = 1ULL << 62 }; -} // namespace detail + +FMT_END_DETAIL_NAMESPACE + +// An output iterator that appends to a buffer. +// It is used to reduce symbol sizes for the common case. +class appender : public std::back_insert_iterator> { + using base = std::back_insert_iterator>; + + template + friend auto get_buffer(appender out) -> detail::buffer& { + return detail::get_container(out); + } + + public: + using std::back_insert_iterator>::back_insert_iterator; + appender(base it) FMT_NOEXCEPT : base(it) {} + using _Unchecked_type = appender; // Mark iterator as checked. + + auto operator++() FMT_NOEXCEPT -> appender& { return *this; } + + auto operator++(int) FMT_NOEXCEPT -> appender { return *this; } +}; // A formatting argument. It is a trivially copyable/constructible type to // allow storage in basic_memory_buffer. @@ -1254,8 +1529,8 @@ template class basic_format_arg { detail::type type_; template - friend FMT_CONSTEXPR basic_format_arg detail::make_arg( - const T& value); + friend FMT_CONSTEXPR auto detail::make_arg(const T& value) + -> basic_format_arg; template friend FMT_CONSTEXPR auto visit_format_arg(Visitor&& vis, @@ -1293,10 +1568,12 @@ template class basic_format_arg { return type_ != detail::type::none_type; } - detail::type type() const { return type_; } + auto type() const -> detail::type { return type_; } - bool is_integral() const { return detail::is_integral_type(type_); } - bool is_arithmetic() const { return detail::is_arithmetic_type(type_); } + auto is_integral() const -> bool { return detail::is_integral_type(type_); } + auto is_arithmetic() const -> bool { + return detail::is_arithmetic_type(type_); + } }; /** @@ -1307,9 +1584,8 @@ template class basic_format_arg { \endrst */ template -FMT_CONSTEXPR_DECL FMT_INLINE auto visit_format_arg( +FMT_CONSTEXPR FMT_INLINE auto visit_format_arg( Visitor&& vis, const basic_format_arg& arg) -> decltype(vis(0)) { - using char_type = typename Context::char_type; switch (arg.type_) { case detail::type::none_type: break; @@ -1321,16 +1597,10 @@ FMT_CONSTEXPR_DECL FMT_INLINE auto visit_format_arg( return vis(arg.value_.long_long_value); case detail::type::ulong_long_type: return vis(arg.value_.ulong_long_value); -#if FMT_USE_INT128 case detail::type::int128_type: - return vis(arg.value_.int128_value); + return vis(detail::convert_for_visit(arg.value_.int128_value)); case detail::type::uint128_type: - return vis(arg.value_.uint128_value); -#else - case detail::type::int128_type: - case detail::type::uint128_type: - break; -#endif + return vis(detail::convert_for_visit(arg.value_.uint128_value)); case detail::type::bool_type: return vis(arg.value_.bool_value); case detail::type::char_type: @@ -1344,8 +1614,8 @@ FMT_CONSTEXPR_DECL FMT_INLINE auto visit_format_arg( case detail::type::cstring_type: return vis(arg.value_.string.data); case detail::type::string_type: - return vis(basic_string_view(arg.value_.string.data, - arg.value_.string.size)); + using sv = basic_string_view; + return vis(sv(arg.value_.string.data, arg.value_.string.size)); case detail::type::pointer_type: return vis(arg.value_.pointer); case detail::type::custom_type: @@ -1354,14 +1624,22 @@ FMT_CONSTEXPR_DECL FMT_INLINE auto visit_format_arg( return vis(monostate()); } -template struct formattable : std::false_type {}; +FMT_BEGIN_DETAIL_NAMESPACE -namespace detail { +template +auto copy_str(InputIt begin, InputIt end, appender out) -> appender { + get_container(out).append(begin, end); + return out; +} +#if FMT_GCC_VERSION && FMT_GCC_VERSION < 500 // A workaround for gcc 4.8 to make void_t work in a SFINAE context. template struct void_t_impl { using type = void; }; template using void_t = typename detail::void_t_impl::type; +#else +template using void_t = void; +#endif template struct is_output_iterator : std::false_type {}; @@ -1384,9 +1662,8 @@ struct is_contiguous_back_insert_iterator : std::false_type {}; template struct is_contiguous_back_insert_iterator> : is_contiguous {}; -template -struct is_contiguous_back_insert_iterator> - : std::true_type {}; +template <> +struct is_contiguous_back_insert_iterator : std::true_type {}; // A type-erased reference to an std::locale to avoid heavy include. class locale_ref { @@ -1394,97 +1671,72 @@ class locale_ref { const void* locale_; // A type-erased pointer to std::locale. public: - locale_ref() : locale_(nullptr) {} + constexpr locale_ref() : locale_(nullptr) {} template explicit locale_ref(const Locale& loc); explicit operator bool() const FMT_NOEXCEPT { return locale_ != nullptr; } - template Locale get() const; + template auto get() const -> Locale; }; -template constexpr unsigned long long encode_types() { return 0; } - +template constexpr auto encode_types() -> unsigned long long { + return 0; +} + template -constexpr unsigned long long encode_types() { +constexpr auto encode_types() -> unsigned long long { return static_cast(mapped_type_constant::value) | (encode_types() << packed_arg_bits); } template -FMT_CONSTEXPR basic_format_arg make_arg(const T& value) { +FMT_CONSTEXPR auto make_arg(const T& value) -> basic_format_arg { basic_format_arg arg; arg.type_ = mapped_type_constant::value; arg.value_ = arg_mapper().map(value); return arg; } -template int check(unformattable) { - static_assert( - formattable(), - "Cannot format an argument. To make type T formattable provide a " - "formatter specialization: https://fmt.dev/latest/api.html#udt"); - return 0; -} -template inline const U& check(const U& val) { - return val; -} - // The type template parameter is there to avoid an ODR violation when using // a fallback formatter in one translation unit and an implicit conversion in // another (not recommended). template -inline value make_arg(const T& val) { - return check(arg_mapper().map(val)); +FMT_CONSTEXPR FMT_INLINE auto make_arg(T&& val) -> value { + const auto& arg = arg_mapper().map(std::forward(val)); + + constexpr bool formattable_char = + !std::is_same::value; + static_assert(formattable_char, "Mixing character types is disallowed."); + + constexpr bool formattable_const = + !std::is_same::value; + static_assert(formattable_const, "Cannot format a const argument."); + + // Formatting of arbitrary pointers is disallowed. If you want to output + // a pointer cast it to "void *" or "const void *". In particular, this + // forbids formatting of "[const] volatile char *" which is printed as bool + // by iostreams. + constexpr bool formattable_pointer = + !std::is_same::value; + static_assert(formattable_pointer, + "Formatting of non-void pointers is disallowed."); + + constexpr bool formattable = + !std::is_same::value; + static_assert( + formattable, + "Cannot format an argument. To make type T formattable provide a " + "formatter specialization: https://fmt.dev/latest/api.html#udt"); + return {arg}; } template -inline basic_format_arg make_arg(const T& value) { +inline auto make_arg(const T& value) -> basic_format_arg { return make_arg(value); } - -template struct is_reference_wrapper : std::false_type {}; -template -struct is_reference_wrapper> : std::true_type {}; - -template const T& unwrap(const T& v) { return v; } -template const T& unwrap(const std::reference_wrapper& v) { - return static_cast(v); -} - -class dynamic_arg_list { - // Workaround for clang's -Wweak-vtables. Unlike for regular classes, for - // templates it doesn't complain about inability to deduce single translation - // unit for placing vtable. So storage_node_base is made a fake template. - template struct node { - virtual ~node() = default; - std::unique_ptr> next; - }; - - template struct typed_node : node<> { - T value; - - template - FMT_CONSTEXPR typed_node(const Arg& arg) : value(arg) {} - - template - FMT_CONSTEXPR typed_node(const basic_string_view& arg) - : value(arg.data(), arg.size()) {} - }; - - std::unique_ptr> head_; - - public: - template const T& push(const Arg& arg) { - auto new_node = std::unique_ptr>(new typed_node(arg)); - auto& value = new_node->value; - new_node->next = std::move(head_); - head_ = std::move(new_node); - return value; - } -}; -} // namespace detail +FMT_END_DETAIL_NAMESPACE // Formatting context. template class basic_format_context { @@ -1503,46 +1755,59 @@ template class basic_format_context { using parse_context_type = basic_format_parse_context; template using formatter_type = formatter; + basic_format_context(basic_format_context&&) = default; basic_format_context(const basic_format_context&) = delete; void operator=(const basic_format_context&) = delete; /** Constructs a ``basic_format_context`` object. References to the arguments are stored in the object so make sure they have appropriate lifetimes. */ - basic_format_context(OutputIt out, - basic_format_args ctx_args, - detail::locale_ref loc = detail::locale_ref()) + constexpr basic_format_context( + OutputIt out, basic_format_args ctx_args, + detail::locale_ref loc = detail::locale_ref()) : out_(out), args_(ctx_args), loc_(loc) {} - format_arg arg(int id) const { return args_.get(id); } - format_arg arg(basic_string_view name) { return args_.get(name); } - int arg_id(basic_string_view name) { return args_.get_id(name); } - const basic_format_args& args() const { return args_; } + constexpr auto arg(int id) const -> format_arg { return args_.get(id); } + FMT_CONSTEXPR auto arg(basic_string_view name) -> format_arg { + return args_.get(name); + } + FMT_CONSTEXPR auto arg_id(basic_string_view name) -> int { + return args_.get_id(name); + } + auto args() const -> const basic_format_args& { + return args_; + } - detail::error_handler error_handler() { return {}; } + FMT_CONSTEXPR auto error_handler() -> detail::error_handler { return {}; } void on_error(const char* message) { error_handler().on_error(message); } // Returns an iterator to the beginning of the output range. - iterator out() { return out_; } + FMT_CONSTEXPR auto out() -> iterator { return out_; } // Advances the begin iterator to ``it``. void advance_to(iterator it) { if (!detail::is_back_insert_iterator()) out_ = it; } - detail::locale_ref locale() { return loc_; } + FMT_CONSTEXPR auto locale() -> detail::locale_ref { return loc_; } }; template using buffer_context = basic_format_context, Char>; using format_context = buffer_context; -using wformat_context = buffer_context; // Workaround an alias issue: https://stackoverflow.com/q/62767544/471164. #define FMT_BUFFER_CONTEXT(Char) \ basic_format_context, Char> +template +using is_formattable = bool_constant< + !std::is_base_of>().map( + std::declval()))>::value && + !detail::has_fallback_formatter::value>; + /** \rst An array of references to arguments. It can be implicitly converted into @@ -1579,14 +1844,16 @@ class format_arg_store : 0); public: - format_arg_store(const Args&... args) + template + FMT_CONSTEXPR FMT_INLINE format_arg_store(T&&... args) : #if FMT_GCC_VERSION && FMT_GCC_VERSION < 409 basic_format_args(*this), #endif data_{detail::make_arg< is_packed, Context, - detail::mapped_type_constant::value>(args)...} { + detail::mapped_type_constant, Context>::value>( + std::forward(args))...} { detail::init_named_args(data_.named_args(), 0, 0, args...); } }; @@ -1600,36 +1867,16 @@ class format_arg_store \endrst */ template -inline format_arg_store make_format_args( - const Args&... args) { - return {args...}; -} - -/** - \rst - Constructs a `~fmt::format_arg_store` object that contains references - to arguments and can be implicitly converted to `~fmt::format_args`. - If ``format_str`` is a compile-time string then `make_args_checked` checks - its validity at compile time. - \endrst - */ -template > -inline auto make_args_checked(const S& format_str, - const remove_reference_t&... args) - -> format_arg_store, remove_reference_t...> { - static_assert( - detail::count<( - std::is_base_of>::value && - std::is_reference::value)...>() == 0, - "passing views as lvalues is disallowed"); - detail::check_format_string(format_str); - return {args...}; +constexpr auto make_format_args(Args&&... args) + -> format_arg_store...> { + return {std::forward(args)...}; } /** \rst - Returns a named argument to be used in a formatting function. It should only - be used in a call to a formatting function. + Returns a named argument to be used in a formatting function. + It should only be used in a call to a formatting function or + `dynamic_format_arg_store::push_back`. **Example**:: @@ -1637,184 +1884,11 @@ inline auto make_args_checked(const S& format_str, \endrst */ template -inline detail::named_arg arg(const Char* name, const T& arg) { +inline auto arg(const Char* name, const T& arg) -> detail::named_arg { static_assert(!detail::is_named_arg(), "nested named arguments"); return {name, arg}; } -/** - \rst - A dynamic version of `fmt::format_arg_store`. - It's equipped with a storage to potentially temporary objects which lifetimes - could be shorter than the format arguments object. - - It can be implicitly converted into `~fmt::basic_format_args` for passing - into type-erased formatting functions such as `~fmt::vformat`. - \endrst - */ -template -class dynamic_format_arg_store -#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409 - // Workaround a GCC template argument substitution bug. - : public basic_format_args -#endif -{ - private: - using char_type = typename Context::char_type; - - template struct need_copy { - static constexpr detail::type mapped_type = - detail::mapped_type_constant::value; - - enum { - value = !(detail::is_reference_wrapper::value || - std::is_same>::value || - std::is_same>::value || - (mapped_type != detail::type::cstring_type && - mapped_type != detail::type::string_type && - mapped_type != detail::type::custom_type)) - }; - }; - - template - using stored_type = conditional_t::value, - std::basic_string, T>; - - // Storage of basic_format_arg must be contiguous. - std::vector> data_; - std::vector> named_info_; - - // Storage of arguments not fitting into basic_format_arg must grow - // without relocation because items in data_ refer to it. - detail::dynamic_arg_list dynamic_args_; - - friend class basic_format_args; - - unsigned long long get_types() const { - return detail::is_unpacked_bit | data_.size() | - (named_info_.empty() - ? 0ULL - : static_cast(detail::has_named_args_bit)); - } - - const basic_format_arg* data() const { - return named_info_.empty() ? data_.data() : data_.data() + 1; - } - - template void emplace_arg(const T& arg) { - data_.emplace_back(detail::make_arg(arg)); - } - - template - void emplace_arg(const detail::named_arg& arg) { - if (named_info_.empty()) { - constexpr const detail::named_arg_info* zero_ptr{nullptr}; - data_.insert(data_.begin(), {zero_ptr, 0}); - } - data_.emplace_back(detail::make_arg(detail::unwrap(arg.value))); - auto pop_one = [](std::vector>* data) { - data->pop_back(); - }; - std::unique_ptr>, decltype(pop_one)> - guard{&data_, pop_one}; - named_info_.push_back({arg.name, static_cast(data_.size() - 2u)}); - data_[0].value_.named_args = {named_info_.data(), named_info_.size()}; - guard.release(); - } - - public: - /** - \rst - Adds an argument into the dynamic store for later passing to a formatting - function. - - Note that custom types and string types (but not string views) are copied - into the store dynamically allocating memory if necessary. - - **Example**:: - - fmt::dynamic_format_arg_store store; - store.push_back(42); - store.push_back("abc"); - store.push_back(1.5f); - std::string result = fmt::vformat("{} and {} and {}", store); - \endrst - */ - template void push_back(const T& arg) { - if (detail::const_check(need_copy::value)) - emplace_arg(dynamic_args_.push>(arg)); - else - emplace_arg(detail::unwrap(arg)); - } - - /** - \rst - Adds a reference to the argument into the dynamic store for later passing to - a formatting function. Supports named arguments wrapped in - ``std::reference_wrapper`` via ``std::ref()``/``std::cref()``. - - **Example**:: - - fmt::dynamic_format_arg_store store; - char str[] = "1234567890"; - store.push_back(std::cref(str)); - int a1_val{42}; - auto a1 = fmt::arg("a1_", a1_val); - store.push_back(std::cref(a1)); - - // Changing str affects the output but only for string and custom types. - str[0] = 'X'; - - std::string result = fmt::vformat("{} and {a1_}"); - assert(result == "X234567890 and 42"); - \endrst - */ - template void push_back(std::reference_wrapper arg) { - static_assert( - detail::is_named_arg::type>::value || - need_copy::value, - "objects of built-in types and string views are always copied"); - emplace_arg(arg.get()); - } - - /** - Adds named argument into the dynamic store for later passing to a formatting - function. ``std::reference_wrapper`` is supported to avoid copying of the - argument. - */ - template - void push_back(const detail::named_arg& arg) { - const char_type* arg_name = - dynamic_args_.push>(arg.name).c_str(); - if (detail::const_check(need_copy::value)) { - emplace_arg( - fmt::arg(arg_name, dynamic_args_.push>(arg.value))); - } else { - emplace_arg(fmt::arg(arg_name, arg.value)); - } - } - - /** Erase all elements from the store */ - void clear() { - data_.clear(); - named_info_.clear(); - dynamic_args_ = detail::dynamic_arg_list(); - } - - /** - \rst - Reserves space to store at least *new_cap* arguments including - *new_cap_named* named arguments. - \endrst - */ - void reserve(size_t new_cap, size_t new_cap_named) { - FMT_ASSERT(new_cap >= new_cap_named, - "Set of arguments includes set of named arguments"); - data_.reserve(new_cap); - named_info_.reserve(new_cap_named); - } -}; - /** \rst A view of a collection of formatting arguments. To avoid lifetime issues it @@ -1846,25 +1920,27 @@ template class basic_format_args { const format_arg* args_; }; - bool is_packed() const { return (desc_ & detail::is_unpacked_bit) == 0; } - bool has_named_args() const { + constexpr auto is_packed() const -> bool { + return (desc_ & detail::is_unpacked_bit) == 0; + } + auto has_named_args() const -> bool { return (desc_ & detail::has_named_args_bit) != 0; } - detail::type type(int index) const { + FMT_CONSTEXPR auto type(int index) const -> detail::type { int shift = index * detail::packed_arg_bits; unsigned int mask = (1 << detail::packed_arg_bits) - 1; return static_cast((desc_ >> shift) & mask); } - basic_format_args(unsigned long long desc, - const detail::value* values) + constexpr FMT_INLINE basic_format_args(unsigned long long desc, + const detail::value* values) : desc_(desc), values_(values) {} - basic_format_args(unsigned long long desc, const format_arg* args) + constexpr basic_format_args(unsigned long long desc, const format_arg* args) : desc_(desc), args_(args) {} public: - basic_format_args() : desc_(0) {} + constexpr basic_format_args() : desc_(0), args_(nullptr) {} /** \rst @@ -1872,8 +1948,10 @@ template class basic_format_args { \endrst */ template - FMT_INLINE basic_format_args(const format_arg_store& store) - : basic_format_args(store.desc, store.data_.args()) {} + constexpr FMT_INLINE basic_format_args( + const format_arg_store& store) + : basic_format_args(format_arg_store::desc, + store.data_.args()) {} /** \rst @@ -1881,7 +1959,8 @@ template class basic_format_args { `~fmt::dynamic_format_arg_store`. \endrst */ - FMT_INLINE basic_format_args(const dynamic_format_arg_store& store) + constexpr FMT_INLINE basic_format_args( + const dynamic_format_arg_store& store) : basic_format_args(store.get_types(), store.data()) {} /** @@ -1889,12 +1968,12 @@ template class basic_format_args { Constructs a `basic_format_args` object from a dynamic set of arguments. \endrst */ - basic_format_args(const format_arg* args, int count) + constexpr basic_format_args(const format_arg* args, int count) : basic_format_args(detail::is_unpacked_bit | detail::to_unsigned(count), args) {} /** Returns the argument with the specified id. */ - format_arg get(int id) const { + FMT_CONSTEXPR auto get(int id) const -> format_arg { format_arg arg; if (!is_packed()) { if (id < max_size()) arg = args_[id]; @@ -1907,12 +1986,14 @@ template class basic_format_args { return arg; } - template format_arg get(basic_string_view name) const { + template + auto get(basic_string_view name) const -> format_arg { int id = get_id(name); return id >= 0 ? get(id) : format_arg(); } - template int get_id(basic_string_view name) const { + template + auto get_id(basic_string_view name) const -> int { if (!has_named_args()) return -1; const auto& named_args = (is_packed() ? values_[-1] : args_[-1].value_).named_args; @@ -1922,87 +2003,1149 @@ template class basic_format_args { return -1; } - int max_size() const { + auto max_size() const -> int { unsigned long long max_packed = detail::max_packed_args; return static_cast(is_packed() ? max_packed : desc_ & ~detail::is_unpacked_bit); } }; -#ifdef FMT_ARM_ABI_COMPATIBILITY /** An alias to ``basic_format_args``. */ -// Separate types would result in shorter symbols but break ABI compatibility +// A separate type would result in shorter symbols but break ABI compatibility // between clang and gcc on ARM (#1919). using format_args = basic_format_args; -using wformat_args = basic_format_args; -#else -// DEPRECATED! These are kept for ABI compatibility. -// It is a separate type rather than an alias to make symbols readable. -struct format_args : basic_format_args { - template - FMT_INLINE format_args(const Args&... args) : basic_format_args(args...) {} + +// We cannot use enum classes as bit fields because of a gcc bug +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414. +namespace align { +enum type { none, left, right, center, numeric }; +} +using align_t = align::type; +namespace sign { +enum type { none, minus, plus, space }; +} +using sign_t = sign::type; + +FMT_BEGIN_DETAIL_NAMESPACE + +// Workaround an array initialization issue in gcc 4.8. +template struct fill_t { + private: + enum { max_size = 4 }; + Char data_[max_size] = {Char(' '), Char(0), Char(0), Char(0)}; + unsigned char size_ = 1; + + public: + FMT_CONSTEXPR void operator=(basic_string_view s) { + auto size = s.size(); + if (size > max_size) return throw_format_error("invalid fill"); + for (size_t i = 0; i < size; ++i) data_[i] = s[i]; + size_ = static_cast(size); + } + + constexpr auto size() const -> size_t { return size_; } + constexpr auto data() const -> const Char* { return data_; } + + FMT_CONSTEXPR auto operator[](size_t index) -> Char& { return data_[index]; } + FMT_CONSTEXPR auto operator[](size_t index) const -> const Char& { + return data_[index]; + } +}; +FMT_END_DETAIL_NAMESPACE + +enum class presentation_type : unsigned char { + none, + // Integer types should go first, + dec, // 'd' + oct, // 'o' + hex_lower, // 'x' + hex_upper, // 'X' + bin_lower, // 'b' + bin_upper, // 'B' + hexfloat_lower, // 'a' + hexfloat_upper, // 'A' + exp_lower, // 'e' + exp_upper, // 'E' + fixed_lower, // 'f' + fixed_upper, // 'F' + general_lower, // 'g' + general_upper, // 'G' + chr, // 'c' + string, // 's' + pointer // 'p' +}; + +// Format specifiers for built-in and string types. +template struct basic_format_specs { + int width; + int precision; + presentation_type type; + align_t align : 4; + sign_t sign : 3; + bool alt : 1; // Alternate form ('#'). + bool localized : 1; + detail::fill_t fill; + + constexpr basic_format_specs() + : width(0), + precision(-1), + type(presentation_type::none), + align(align::none), + sign(sign::none), + alt(false), + localized(false) {} +}; + +using format_specs = basic_format_specs; + +FMT_BEGIN_DETAIL_NAMESPACE + +enum class arg_id_kind { none, index, name }; + +// An argument reference. +template struct arg_ref { + FMT_CONSTEXPR arg_ref() : kind(arg_id_kind::none), val() {} + + FMT_CONSTEXPR explicit arg_ref(int index) + : kind(arg_id_kind::index), val(index) {} + FMT_CONSTEXPR explicit arg_ref(basic_string_view name) + : kind(arg_id_kind::name), val(name) {} + + FMT_CONSTEXPR auto operator=(int idx) -> arg_ref& { + kind = arg_id_kind::index; + val.index = idx; + return *this; + } + + arg_id_kind kind; + union value { + FMT_CONSTEXPR value(int id = 0) : index{id} {} + FMT_CONSTEXPR value(basic_string_view n) : name(n) {} + + int index; + basic_string_view name; + } val; +}; + +// Format specifiers with width and precision resolved at formatting rather +// than parsing time to allow re-using the same parsed specifiers with +// different sets of arguments (precompilation of format strings). +template +struct dynamic_format_specs : basic_format_specs { + arg_ref width_ref; + arg_ref precision_ref; +}; + +struct auto_id {}; + +// A format specifier handler that sets fields in basic_format_specs. +template class specs_setter { + protected: + basic_format_specs& specs_; + + public: + explicit FMT_CONSTEXPR specs_setter(basic_format_specs& specs) + : specs_(specs) {} + + FMT_CONSTEXPR specs_setter(const specs_setter& other) + : specs_(other.specs_) {} + + FMT_CONSTEXPR void on_align(align_t align) { specs_.align = align; } + FMT_CONSTEXPR void on_fill(basic_string_view fill) { + specs_.fill = fill; + } + FMT_CONSTEXPR void on_sign(sign_t s) { specs_.sign = s; } + FMT_CONSTEXPR void on_hash() { specs_.alt = true; } + FMT_CONSTEXPR void on_localized() { specs_.localized = true; } + + FMT_CONSTEXPR void on_zero() { + if (specs_.align == align::none) specs_.align = align::numeric; + specs_.fill[0] = Char('0'); + } + + FMT_CONSTEXPR void on_width(int width) { specs_.width = width; } + FMT_CONSTEXPR void on_precision(int precision) { + specs_.precision = precision; + } + FMT_CONSTEXPR void end_precision() {} + + FMT_CONSTEXPR void on_type(presentation_type type) { specs_.type = type; } +}; + +// Format spec handler that saves references to arguments representing dynamic +// width and precision to be resolved at formatting time. +template +class dynamic_specs_handler + : public specs_setter { + public: + using char_type = typename ParseContext::char_type; + + FMT_CONSTEXPR dynamic_specs_handler(dynamic_format_specs& specs, + ParseContext& ctx) + : specs_setter(specs), specs_(specs), context_(ctx) {} + + FMT_CONSTEXPR dynamic_specs_handler(const dynamic_specs_handler& other) + : specs_setter(other), + specs_(other.specs_), + context_(other.context_) {} + + template FMT_CONSTEXPR void on_dynamic_width(Id arg_id) { + specs_.width_ref = make_arg_ref(arg_id); + } + + template FMT_CONSTEXPR void on_dynamic_precision(Id arg_id) { + specs_.precision_ref = make_arg_ref(arg_id); + } + + FMT_CONSTEXPR void on_error(const char* message) { + context_.on_error(message); + } + + private: + dynamic_format_specs& specs_; + ParseContext& context_; + + using arg_ref_type = arg_ref; + + FMT_CONSTEXPR auto make_arg_ref(int arg_id) -> arg_ref_type { + context_.check_arg_id(arg_id); + return arg_ref_type(arg_id); + } + + FMT_CONSTEXPR auto make_arg_ref(auto_id) -> arg_ref_type { + return arg_ref_type(context_.next_arg_id()); + } + + FMT_CONSTEXPR auto make_arg_ref(basic_string_view arg_id) + -> arg_ref_type { + context_.check_arg_id(arg_id); + basic_string_view format_str( + context_.begin(), to_unsigned(context_.end() - context_.begin())); + return arg_ref_type(arg_id); + } +}; + +template constexpr bool is_ascii_letter(Char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +// Converts a character to ASCII. Returns a number > 127 on conversion failure. +template ::value)> +constexpr auto to_ascii(Char value) -> Char { + return value; +} +template ::value)> +constexpr auto to_ascii(Char value) -> + typename std::underlying_type::type { + return value; +} + +template +FMT_CONSTEXPR auto code_point_length(const Char* begin) -> int { + if (const_check(sizeof(Char) != 1)) return 1; + auto lengths = + "\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0\0\0\2\2\2\2\3\3\4"; + int len = lengths[static_cast(*begin) >> 3]; + + // Compute the pointer to the next character early so that the next + // iteration can start working on the next character. Neither Clang + // nor GCC figure out this reordering on their own. + return len + !len; +} + +// Return the result via the out param to workaround gcc bug 77539. +template +FMT_CONSTEXPR auto find(Ptr first, Ptr last, T value, Ptr& out) -> bool { + for (out = first; out != last; ++out) { + if (*out == value) return true; + } + return false; +} + +template <> +inline auto find(const char* first, const char* last, char value, + const char*& out) -> bool { + out = static_cast( + std::memchr(first, value, to_unsigned(last - first))); + return out != nullptr; +} + +// Parses the range [begin, end) as an unsigned integer. This function assumes +// that the range is non-empty and the first character is a digit. +template +FMT_CONSTEXPR auto parse_nonnegative_int(const Char*& begin, const Char* end, + int error_value) noexcept -> int { + FMT_ASSERT(begin != end && '0' <= *begin && *begin <= '9', ""); + unsigned value = 0, prev = 0; + auto p = begin; + do { + prev = value; + value = value * 10 + unsigned(*p - '0'); + ++p; + } while (p != end && '0' <= *p && *p <= '9'); + auto num_digits = p - begin; + begin = p; + if (num_digits <= std::numeric_limits::digits10) + return static_cast(value); + // Check for overflow. + const unsigned max = to_unsigned((std::numeric_limits::max)()); + return num_digits == std::numeric_limits::digits10 + 1 && + prev * 10ull + unsigned(p[-1] - '0') <= max + ? static_cast(value) + : error_value; +} + +// Parses fill and alignment. +template +FMT_CONSTEXPR auto parse_align(const Char* begin, const Char* end, + Handler&& handler) -> const Char* { + FMT_ASSERT(begin != end, ""); + auto align = align::none; + auto p = begin + code_point_length(begin); + if (p >= end) p = begin; + for (;;) { + switch (to_ascii(*p)) { + case '<': + align = align::left; + break; + case '>': + align = align::right; + break; + case '^': + align = align::center; + break; + default: + break; + } + if (align != align::none) { + if (p != begin) { + auto c = *begin; + if (c == '{') + return handler.on_error("invalid fill character '{'"), begin; + handler.on_fill(basic_string_view(begin, to_unsigned(p - begin))); + begin = p + 1; + } else + ++begin; + handler.on_align(align); + break; + } else if (p == begin) { + break; + } + p = begin; + } + return begin; +} + +template FMT_CONSTEXPR bool is_name_start(Char c) { + return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || '_' == c; +} + +template +FMT_CONSTEXPR auto do_parse_arg_id(const Char* begin, const Char* end, + IDHandler&& handler) -> const Char* { + FMT_ASSERT(begin != end, ""); + Char c = *begin; + if (c >= '0' && c <= '9') { + int index = 0; + if (c != '0') + index = + parse_nonnegative_int(begin, end, (std::numeric_limits::max)()); + else + ++begin; + if (begin == end || (*begin != '}' && *begin != ':')) + handler.on_error("invalid format string"); + else + handler(index); + return begin; + } + if (!is_name_start(c)) { + handler.on_error("invalid format string"); + return begin; + } + auto it = begin; + do { + ++it; + } while (it != end && (is_name_start(c = *it) || ('0' <= c && c <= '9'))); + handler(basic_string_view(begin, to_unsigned(it - begin))); + return it; +} + +template +FMT_CONSTEXPR FMT_INLINE auto parse_arg_id(const Char* begin, const Char* end, + IDHandler&& handler) -> const Char* { + Char c = *begin; + if (c != '}' && c != ':') return do_parse_arg_id(begin, end, handler); + handler(); + return begin; +} + +template +FMT_CONSTEXPR auto parse_width(const Char* begin, const Char* end, + Handler&& handler) -> const Char* { + using detail::auto_id; + struct width_adapter { + Handler& handler; + + FMT_CONSTEXPR void operator()() { handler.on_dynamic_width(auto_id()); } + FMT_CONSTEXPR void operator()(int id) { handler.on_dynamic_width(id); } + FMT_CONSTEXPR void operator()(basic_string_view id) { + handler.on_dynamic_width(id); + } + FMT_CONSTEXPR void on_error(const char* message) { + if (message) handler.on_error(message); + } + }; + + FMT_ASSERT(begin != end, ""); + if ('0' <= *begin && *begin <= '9') { + int width = parse_nonnegative_int(begin, end, -1); + if (width != -1) + handler.on_width(width); + else + handler.on_error("number is too big"); + } else if (*begin == '{') { + ++begin; + if (begin != end) begin = parse_arg_id(begin, end, width_adapter{handler}); + if (begin == end || *begin != '}') + return handler.on_error("invalid format string"), begin; + ++begin; + } + return begin; +} + +template +FMT_CONSTEXPR auto parse_precision(const Char* begin, const Char* end, + Handler&& handler) -> const Char* { + using detail::auto_id; + struct precision_adapter { + Handler& handler; + + FMT_CONSTEXPR void operator()() { handler.on_dynamic_precision(auto_id()); } + FMT_CONSTEXPR void operator()(int id) { handler.on_dynamic_precision(id); } + FMT_CONSTEXPR void operator()(basic_string_view id) { + handler.on_dynamic_precision(id); + } + FMT_CONSTEXPR void on_error(const char* message) { + if (message) handler.on_error(message); + } + }; + + ++begin; + auto c = begin != end ? *begin : Char(); + if ('0' <= c && c <= '9') { + auto precision = parse_nonnegative_int(begin, end, -1); + if (precision != -1) + handler.on_precision(precision); + else + handler.on_error("number is too big"); + } else if (c == '{') { + ++begin; + if (begin != end) + begin = parse_arg_id(begin, end, precision_adapter{handler}); + if (begin == end || *begin++ != '}') + return handler.on_error("invalid format string"), begin; + } else { + return handler.on_error("missing precision specifier"), begin; + } + handler.end_precision(); + return begin; +} + +template +FMT_CONSTEXPR auto parse_presentation_type(Char type) -> presentation_type { + switch (to_ascii(type)) { + case 'd': + return presentation_type::dec; + case 'o': + return presentation_type::oct; + case 'x': + return presentation_type::hex_lower; + case 'X': + return presentation_type::hex_upper; + case 'b': + return presentation_type::bin_lower; + case 'B': + return presentation_type::bin_upper; + case 'a': + return presentation_type::hexfloat_lower; + case 'A': + return presentation_type::hexfloat_upper; + case 'e': + return presentation_type::exp_lower; + case 'E': + return presentation_type::exp_upper; + case 'f': + return presentation_type::fixed_lower; + case 'F': + return presentation_type::fixed_upper; + case 'g': + return presentation_type::general_lower; + case 'G': + return presentation_type::general_upper; + case 'c': + return presentation_type::chr; + case 's': + return presentation_type::string; + case 'p': + return presentation_type::pointer; + default: + return presentation_type::none; + } +} + +// Parses standard format specifiers and sends notifications about parsed +// components to handler. +template +FMT_CONSTEXPR FMT_INLINE auto parse_format_specs(const Char* begin, + const Char* end, + SpecHandler&& handler) + -> const Char* { + if (1 < end - begin && begin[1] == '}' && is_ascii_letter(*begin) && + *begin != 'L') { + presentation_type type = parse_presentation_type(*begin++); + if (type == presentation_type::none) + handler.on_error("invalid type specifier"); + handler.on_type(type); + return begin; + } + + if (begin == end) return begin; + + begin = parse_align(begin, end, handler); + if (begin == end) return begin; + + // Parse sign. + switch (to_ascii(*begin)) { + case '+': + handler.on_sign(sign::plus); + ++begin; + break; + case '-': + handler.on_sign(sign::minus); + ++begin; + break; + case ' ': + handler.on_sign(sign::space); + ++begin; + break; + default: + break; + } + if (begin == end) return begin; + + if (*begin == '#') { + handler.on_hash(); + if (++begin == end) return begin; + } + + // Parse zero flag. + if (*begin == '0') { + handler.on_zero(); + if (++begin == end) return begin; + } + + begin = parse_width(begin, end, handler); + if (begin == end) return begin; + + // Parse precision. + if (*begin == '.') { + begin = parse_precision(begin, end, handler); + if (begin == end) return begin; + } + + if (*begin == 'L') { + handler.on_localized(); + ++begin; + } + + // Parse type. + if (begin != end && *begin != '}') { + presentation_type type = parse_presentation_type(*begin++); + if (type == presentation_type::none) + handler.on_error("invalid type specifier"); + handler.on_type(type); + } + return begin; +} + +template +FMT_CONSTEXPR auto parse_replacement_field(const Char* begin, const Char* end, + Handler&& handler) -> const Char* { + struct id_adapter { + Handler& handler; + int arg_id; + + FMT_CONSTEXPR void operator()() { arg_id = handler.on_arg_id(); } + FMT_CONSTEXPR void operator()(int id) { arg_id = handler.on_arg_id(id); } + FMT_CONSTEXPR void operator()(basic_string_view id) { + arg_id = handler.on_arg_id(id); + } + FMT_CONSTEXPR void on_error(const char* message) { + if (message) handler.on_error(message); + } + }; + + ++begin; + if (begin == end) return handler.on_error("invalid format string"), end; + if (*begin == '}') { + handler.on_replacement_field(handler.on_arg_id(), begin); + } else if (*begin == '{') { + handler.on_text(begin, begin + 1); + } else { + auto adapter = id_adapter{handler, 0}; + begin = parse_arg_id(begin, end, adapter); + Char c = begin != end ? *begin : Char(); + if (c == '}') { + handler.on_replacement_field(adapter.arg_id, begin); + } else if (c == ':') { + begin = handler.on_format_specs(adapter.arg_id, begin + 1, end); + if (begin == end || *begin != '}') + return handler.on_error("unknown format specifier"), end; + } else { + return handler.on_error("missing '}' in format string"), end; + } + } + return begin + 1; +} + +template +FMT_CONSTEXPR FMT_INLINE void parse_format_string( + basic_string_view format_str, Handler&& handler) { + // Workaround a name-lookup bug in MSVC's modules implementation. + using detail::find; + + auto begin = format_str.data(); + auto end = begin + format_str.size(); + if (end - begin < 32) { + // Use a simple loop instead of memchr for small strings. + const Char* p = begin; + while (p != end) { + auto c = *p++; + if (c == '{') { + handler.on_text(begin, p - 1); + begin = p = parse_replacement_field(p - 1, end, handler); + } else if (c == '}') { + if (p == end || *p != '}') + return handler.on_error("unmatched '}' in format string"); + handler.on_text(begin, p); + begin = ++p; + } + } + handler.on_text(begin, end); + return; + } + struct writer { + FMT_CONSTEXPR void operator()(const Char* pbegin, const Char* pend) { + if (pbegin == pend) return; + for (;;) { + const Char* p = nullptr; + if (!find(pbegin, pend, Char('}'), p)) + return handler_.on_text(pbegin, pend); + ++p; + if (p == pend || *p != '}') + return handler_.on_error("unmatched '}' in format string"); + handler_.on_text(pbegin, p); + pbegin = p + 1; + } + } + Handler& handler_; + } write{handler}; + while (begin != end) { + // Doing two passes with memchr (one for '{' and another for '}') is up to + // 2.5x faster than the naive one-pass implementation on big format strings. + const Char* p = begin; + if (*begin != '{' && !find(begin + 1, end, Char('{'), p)) + return write(begin, end); + write(begin, p); + begin = parse_replacement_field(p, end, handler); + } +} + +template +FMT_CONSTEXPR auto parse_format_specs(ParseContext& ctx) + -> decltype(ctx.begin()) { + using char_type = typename ParseContext::char_type; + using context = buffer_context; + using mapped_type = conditional_t< + mapped_type_constant::value != type::custom_type, + decltype(arg_mapper().map(std::declval())), T>; + auto f = conditional_t::value, + formatter, + fallback_formatter>(); + return f.parse(ctx); +} + +// A parse context with extra argument id checks. It is only used at compile +// time because adding checks at runtime would introduce substantial overhead +// and would be redundant since argument ids are checked when arguments are +// retrieved anyway. +template +class compile_parse_context + : public basic_format_parse_context { + private: + int num_args_; + using base = basic_format_parse_context; + + public: + explicit FMT_CONSTEXPR compile_parse_context( + basic_string_view format_str, + int num_args = (std::numeric_limits::max)(), ErrorHandler eh = {}) + : base(format_str, eh), num_args_(num_args) {} + + FMT_CONSTEXPR auto next_arg_id() -> int { + int id = base::next_arg_id(); + if (id >= num_args_) this->on_error("argument not found"); + return id; + } + + FMT_CONSTEXPR void check_arg_id(int id) { + base::check_arg_id(id); + if (id >= num_args_) this->on_error("argument not found"); + } + using base::check_arg_id; }; -struct wformat_args : basic_format_args { - using basic_format_args::basic_format_args; + +template +FMT_CONSTEXPR void check_int_type_spec(presentation_type type, + ErrorHandler&& eh) { + if (type > presentation_type::bin_upper && type != presentation_type::chr) + eh.on_error("invalid type specifier"); +} + +// Checks char specs and returns true if the type spec is char (and not int). +template +FMT_CONSTEXPR auto check_char_specs(const basic_format_specs& specs, + ErrorHandler&& eh = {}) -> bool { + if (specs.type != presentation_type::none && + specs.type != presentation_type::chr) { + check_int_type_spec(specs.type, eh); + return false; + } + if (specs.align == align::numeric || specs.sign != sign::none || specs.alt) + eh.on_error("invalid format specifier for char"); + return true; +} + +// A floating-point presentation format. +enum class float_format : unsigned char { + general, // General: exponent notation or fixed point based on magnitude. + exp, // Exponent notation with the default precision of 6, e.g. 1.2e-3. + fixed, // Fixed point with the default precision of 6, e.g. 0.0012. + hex +}; + +struct float_specs { + int precision; + float_format format : 8; + sign_t sign : 8; + bool upper : 1; + bool locale : 1; + bool binary32 : 1; + bool fallback : 1; + bool showpoint : 1; +}; + +template +FMT_CONSTEXPR auto parse_float_type_spec(const basic_format_specs& specs, + ErrorHandler&& eh = {}) + -> float_specs { + auto result = float_specs(); + result.showpoint = specs.alt; + result.locale = specs.localized; + switch (specs.type) { + case presentation_type::none: + result.format = float_format::general; + break; + case presentation_type::general_upper: + result.upper = true; + FMT_FALLTHROUGH; + case presentation_type::general_lower: + result.format = float_format::general; + break; + case presentation_type::exp_upper: + result.upper = true; + FMT_FALLTHROUGH; + case presentation_type::exp_lower: + result.format = float_format::exp; + result.showpoint |= specs.precision != 0; + break; + case presentation_type::fixed_upper: + result.upper = true; + FMT_FALLTHROUGH; + case presentation_type::fixed_lower: + result.format = float_format::fixed; + result.showpoint |= specs.precision != 0; + break; + case presentation_type::hexfloat_upper: + result.upper = true; + FMT_FALLTHROUGH; + case presentation_type::hexfloat_lower: + result.format = float_format::hex; + break; + default: + eh.on_error("invalid type specifier"); + break; + } + return result; +} + +template +FMT_CONSTEXPR auto check_cstring_type_spec(presentation_type type, + ErrorHandler&& eh = {}) -> bool { + if (type == presentation_type::none || type == presentation_type::string) + return true; + if (type != presentation_type::pointer) eh.on_error("invalid type specifier"); + return false; +} + +template +FMT_CONSTEXPR void check_string_type_spec(presentation_type type, + ErrorHandler&& eh = {}) { + if (type != presentation_type::none && type != presentation_type::string) + eh.on_error("invalid type specifier"); +} + +template +FMT_CONSTEXPR void check_pointer_type_spec(presentation_type type, + ErrorHandler&& eh) { + if (type != presentation_type::none && type != presentation_type::pointer) + eh.on_error("invalid type specifier"); +} + +// A parse_format_specs handler that checks if specifiers are consistent with +// the argument type. +template class specs_checker : public Handler { + private: + detail::type arg_type_; + + FMT_CONSTEXPR void require_numeric_argument() { + if (!is_arithmetic_type(arg_type_)) + this->on_error("format specifier requires numeric argument"); + } + + public: + FMT_CONSTEXPR specs_checker(const Handler& handler, detail::type arg_type) + : Handler(handler), arg_type_(arg_type) {} + + FMT_CONSTEXPR void on_align(align_t align) { + if (align == align::numeric) require_numeric_argument(); + Handler::on_align(align); + } + + FMT_CONSTEXPR void on_sign(sign_t s) { + require_numeric_argument(); + if (is_integral_type(arg_type_) && arg_type_ != type::int_type && + arg_type_ != type::long_long_type && arg_type_ != type::char_type) { + this->on_error("format specifier requires signed argument"); + } + Handler::on_sign(s); + } + + FMT_CONSTEXPR void on_hash() { + require_numeric_argument(); + Handler::on_hash(); + } + + FMT_CONSTEXPR void on_localized() { + require_numeric_argument(); + Handler::on_localized(); + } + + FMT_CONSTEXPR void on_zero() { + require_numeric_argument(); + Handler::on_zero(); + } + + FMT_CONSTEXPR void end_precision() { + if (is_integral_type(arg_type_) || arg_type_ == type::pointer_type) + this->on_error("precision not allowed for this argument type"); + } }; + +constexpr int invalid_arg_index = -1; + +#if FMT_USE_NONTYPE_TEMPLATE_PARAMETERS +template +constexpr auto get_arg_index_by_name(basic_string_view name) -> int { + if constexpr (detail::is_statically_named_arg()) { + if (name == T::name) return N; + } + if constexpr (sizeof...(Args) > 0) + return get_arg_index_by_name(name); + (void)name; // Workaround an MSVC bug about "unused" parameter. + return invalid_arg_index; +} +#endif + +template +FMT_CONSTEXPR auto get_arg_index_by_name(basic_string_view name) -> int { +#if FMT_USE_NONTYPE_TEMPLATE_PARAMETERS + if constexpr (sizeof...(Args) > 0) + return get_arg_index_by_name<0, Args...>(name); +#endif + (void)name; + return invalid_arg_index; +} + +template +class format_string_checker { + private: + using parse_context_type = compile_parse_context; + enum { num_args = sizeof...(Args) }; + + // Format specifier parsing function. + using parse_func = const Char* (*)(parse_context_type&); + + parse_context_type context_; + parse_func parse_funcs_[num_args > 0 ? num_args : 1]; + + public: + explicit FMT_CONSTEXPR format_string_checker( + basic_string_view format_str, ErrorHandler eh) + : context_(format_str, num_args, eh), + parse_funcs_{&parse_format_specs...} {} + + FMT_CONSTEXPR void on_text(const Char*, const Char*) {} + + FMT_CONSTEXPR auto on_arg_id() -> int { return context_.next_arg_id(); } + FMT_CONSTEXPR auto on_arg_id(int id) -> int { + return context_.check_arg_id(id), id; + } + FMT_CONSTEXPR auto on_arg_id(basic_string_view id) -> int { +#if FMT_USE_NONTYPE_TEMPLATE_PARAMETERS + auto index = get_arg_index_by_name(id); + if (index == invalid_arg_index) on_error("named argument is not found"); + return context_.check_arg_id(index), index; +#else + (void)id; + on_error("compile-time checks for named arguments require C++20 support"); + return 0; #endif + } + + FMT_CONSTEXPR void on_replacement_field(int, const Char*) {} -namespace detail { + FMT_CONSTEXPR auto on_format_specs(int id, const Char* begin, const Char*) + -> const Char* { + context_.advance_to(context_.begin() + (begin - &*context_.begin())); + // id >= 0 check is a workaround for gcc 10 bug (#2065). + return id >= 0 && id < num_args ? parse_funcs_[id](context_) : begin; + } -template ::value)> -std::basic_string vformat( - basic_string_view format_str, - basic_format_args>> args); + FMT_CONSTEXPR void on_error(const char* message) { + context_.on_error(message); + } +}; -FMT_API std::string vformat(string_view format_str, format_args args); +template ::value), int>> +void check_format_string(S format_str) { + FMT_CONSTEXPR auto s = to_string_view(format_str); + using checker = format_string_checker...>; + FMT_CONSTEXPR bool invalid_format = + (parse_format_string(s, checker(s, {})), true); + ignore_unused(invalid_format); +} template void vformat_to( - buffer& buf, basic_string_view format_str, + buffer& buf, basic_string_view fmt, basic_format_args)> args, - detail::locale_ref loc = {}); - -template ::value)> -inline void vprint_mojibake(std::FILE*, basic_string_view, const Args&) {} + locale_ref loc = {}); FMT_API void vprint_mojibake(std::FILE*, string_view, format_args); #ifndef _WIN32 inline void vprint_mojibake(std::FILE*, string_view, format_args) {} #endif -} // namespace detail +FMT_END_DETAIL_NAMESPACE + +// A formatter specialization for the core types corresponding to detail::type +// constants. +template +struct formatter::value != + detail::type::custom_type>> { + private: + detail::dynamic_format_specs specs_; + + public: + // Parses format specifiers stopping either at the end of the range or at the + // terminating '}'. + template + FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) { + auto begin = ctx.begin(), end = ctx.end(); + if (begin == end) return begin; + using handler_type = detail::dynamic_specs_handler; + auto type = detail::type_constant::value; + auto checker = + detail::specs_checker(handler_type(specs_, ctx), type); + auto it = detail::parse_format_specs(begin, end, checker); + auto eh = ctx.error_handler(); + switch (type) { + case detail::type::none_type: + FMT_ASSERT(false, "invalid argument type"); + break; + case detail::type::bool_type: + if (specs_.type == presentation_type::none || + specs_.type == presentation_type::string) { + break; + } + FMT_FALLTHROUGH; + case detail::type::int_type: + case detail::type::uint_type: + case detail::type::long_long_type: + case detail::type::ulong_long_type: + case detail::type::int128_type: + case detail::type::uint128_type: + detail::check_int_type_spec(specs_.type, eh); + break; + case detail::type::char_type: + detail::check_char_specs(specs_, eh); + break; + case detail::type::float_type: + if (detail::const_check(FMT_USE_FLOAT)) + detail::parse_float_type_spec(specs_, eh); + else + FMT_ASSERT(false, "float support disabled"); + break; + case detail::type::double_type: + if (detail::const_check(FMT_USE_DOUBLE)) + detail::parse_float_type_spec(specs_, eh); + else + FMT_ASSERT(false, "double support disabled"); + break; + case detail::type::long_double_type: + if (detail::const_check(FMT_USE_LONG_DOUBLE)) + detail::parse_float_type_spec(specs_, eh); + else + FMT_ASSERT(false, "long double support disabled"); + break; + case detail::type::cstring_type: + detail::check_cstring_type_spec(specs_.type, eh); + break; + case detail::type::string_type: + detail::check_string_type_spec(specs_.type, eh); + break; + case detail::type::pointer_type: + detail::check_pointer_type_spec(specs_.type, eh); + break; + case detail::type::custom_type: + // Custom format specifiers are checked in parse functions of + // formatter specializations. + break; + } + return it; + } + + template + FMT_CONSTEXPR auto format(const T& val, FormatContext& ctx) const + -> decltype(ctx.out()); +}; + +template struct basic_runtime { basic_string_view str; }; + +/** A compile-time format string. */ +template class basic_format_string { + private: + basic_string_view str_; + + public: + template >::value)> + FMT_CONSTEVAL FMT_INLINE basic_format_string(const S& s) : str_(s) { + static_assert( + detail::count< + (std::is_base_of>::value && + std::is_reference::value)...>() == 0, + "passing views as lvalues is disallowed"); +#ifdef FMT_HAS_CONSTEVAL + if constexpr (detail::count_named_args() == + detail::count_statically_named_args()) { + using checker = detail::format_string_checker...>; + detail::parse_format_string(str_, checker(s, {})); + } +#else + detail::check_format_string(s); +#endif + } + basic_format_string(basic_runtime r) : str_(r.str) {} + + FMT_INLINE operator basic_string_view() const { return str_; } +}; + +#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409 +// Workaround broken conversion on older gcc. +template using format_string = string_view; +template auto runtime(const S& s) -> basic_string_view> { + return s; +} +#else +template +using format_string = basic_format_string...>; +/** + \rst + Creates a runtime format string. + + **Example**:: + + // Check format string at runtime instead of compile-time. + fmt::print(fmt::runtime("{:d}"), "I am not a number"); + \endrst + */ +template auto runtime(const S& s) -> basic_runtime> { + return {{s}}; +} +#endif + +FMT_API auto vformat(string_view fmt, format_args args) -> std::string; + +/** + \rst + Formats ``args`` according to specifications in ``fmt`` and returns the result + as a string. + + **Example**:: + + #include + std::string message = fmt::format("The answer is {}.", 42); + \endrst +*/ +template +FMT_NODISCARD FMT_INLINE auto format(format_string fmt, T&&... args) + -> std::string { + return vformat(fmt, fmt::make_format_args(args...)); +} /** Formats a string and writes the output to ``out``. */ -// GCC 8 and earlier cannot handle std::back_insert_iterator with -// vformat_to(...) overload, so SFINAE on iterator type instead. -template , - bool enable = detail::is_output_iterator::value> -auto vformat_to(OutputIt out, const S& format_str, - basic_format_args>> args) - -> typename std::enable_if::type { - decltype(detail::get_buffer(out)) buf(detail::get_buffer_init(out)); - detail::vformat_to(buf, to_string_view(format_str), args); +template ::value)> +auto vformat_to(OutputIt out, string_view fmt, format_args args) -> OutputIt { + using detail::get_buffer; + auto&& buf = get_buffer(out); + detail::vformat_to(buf, fmt, args, {}); return detail::get_iterator(buf); } /** \rst - Formats arguments, writes the result to the output iterator ``out`` and returns - the iterator past the end of the output range. + Formats ``args`` according to specifications in ``fmt``, writes the result to + the output iterator ``out`` and returns the iterator past the end of the output + range. `format_to` does not append a terminating null character. **Example**:: - std::vector out; + auto out = std::vector(); fmt::format_to(std::back_inserter(out), "{}", 42); \endrst */ -// We cannot use FMT_ENABLE_IF because of a bug in gcc 8.3. -template >::value> -inline auto format_to(OutputIt out, const S& format_str, Args&&... args) -> - typename std::enable_if::type { - const auto& vargs = fmt::make_args_checked(format_str, args...); - return vformat_to(out, to_string_view(format_str), vargs); +template ::value)> +FMT_INLINE auto format_to(OutputIt out, format_string fmt, T&&... args) + -> OutputIt { + return vformat_to(out, fmt, fmt::make_format_args(args...)); } template struct format_to_n_result { @@ -2012,111 +3155,82 @@ template struct format_to_n_result { size_t size; }; -template ::value)> -inline format_to_n_result vformat_to_n( - OutputIt out, size_t n, basic_string_view format_str, - basic_format_args>> args) { - detail::iterator_buffer buf(out, - n); - detail::vformat_to(buf, format_str, args); +template ::value)> +auto vformat_to_n(OutputIt out, size_t n, string_view fmt, format_args args) + -> format_to_n_result { + using traits = detail::fixed_buffer_traits; + auto buf = detail::iterator_buffer(out, n); + detail::vformat_to(buf, fmt, args, {}); return {buf.out(), buf.count()}; } /** - \rst - Formats arguments, writes up to ``n`` characters of the result to the output - iterator ``out`` and returns the total output size and the iterator past the - end of the output range. - \endrst + \rst + Formats ``args`` according to specifications in ``fmt``, writes up to ``n`` + characters of the result to the output iterator ``out`` and returns the total + (not truncated) output size and the iterator past the end of the output range. + `format_to_n` does not append a terminating null character. + \endrst */ -template >::value> -inline auto format_to_n(OutputIt out, size_t n, const S& format_str, - const Args&... args) -> - typename std::enable_if>::type { - const auto& vargs = fmt::make_args_checked(format_str, args...); - return vformat_to_n(out, n, to_string_view(format_str), vargs); +template ::value)> +FMT_INLINE auto format_to_n(OutputIt out, size_t n, format_string fmt, + T&&... args) -> format_to_n_result { + return vformat_to_n(out, n, fmt, fmt::make_format_args(args...)); } -/** - Returns the number of characters in the output of - ``format(format_str, args...)``. - */ -template -inline size_t formatted_size(string_view format_str, Args&&... args) { - const auto& vargs = fmt::make_args_checked(format_str, args...); - detail::counting_buffer<> buf; - detail::vformat_to(buf, format_str, vargs); +/** Returns the number of chars in the output of ``format(fmt, args...)``. */ +template +FMT_NODISCARD FMT_INLINE auto formatted_size(format_string fmt, + T&&... args) -> size_t { + auto buf = detail::counting_buffer<>(); + detail::vformat_to(buf, string_view(fmt), fmt::make_format_args(args...), {}); return buf.count(); } -template > -FMT_INLINE std::basic_string vformat( - const S& format_str, - basic_format_args>> args) { - return detail::vformat(to_string_view(format_str), args); -} +FMT_API void vprint(string_view fmt, format_args args); +FMT_API void vprint(std::FILE* f, string_view fmt, format_args args); /** \rst - Formats arguments and returns the result as a string. + Formats ``args`` according to specifications in ``fmt`` and writes the output + to ``stdout``. **Example**:: - #include - std::string message = fmt::format("The answer is {}", 42); + fmt::print("Elapsed time: {0:.2f} seconds", 1.23); \endrst -*/ -// Pass char_t as a default template parameter instead of using -// std::basic_string> to reduce the symbol size. -template > -FMT_INLINE std::basic_string format(const S& format_str, Args&&... args) { - const auto& vargs = fmt::make_args_checked(format_str, args...); - return detail::vformat(to_string_view(format_str), vargs); + */ +template +FMT_INLINE void print(format_string fmt, T&&... args) { + const auto& vargs = fmt::make_format_args(args...); + return detail::is_utf8() ? vprint(fmt, vargs) + : detail::vprint_mojibake(stdout, fmt, vargs); } -FMT_API void vprint(string_view, format_args); -FMT_API void vprint(std::FILE*, string_view, format_args); - /** \rst - Formats ``args`` according to specifications in ``format_str`` and writes the - output to the file ``f``. Strings are assumed to be Unicode-encoded unless the - ``FMT_UNICODE`` macro is set to 0. + Formats ``args`` according to specifications in ``fmt`` and writes the + output to the file ``f``. **Example**:: fmt::print(stderr, "Don't {}!", "panic"); \endrst */ -template > -inline void print(std::FILE* f, const S& format_str, Args&&... args) { - const auto& vargs = fmt::make_args_checked(format_str, args...); - return detail::is_unicode() - ? vprint(f, to_string_view(format_str), vargs) - : detail::vprint_mojibake(f, to_string_view(format_str), vargs); +template +FMT_INLINE void print(std::FILE* f, format_string fmt, T&&... args) { + const auto& vargs = fmt::make_format_args(args...); + return detail::is_utf8() ? vprint(f, fmt, vargs) + : detail::vprint_mojibake(f, fmt, vargs); } -/** - \rst - Formats ``args`` according to specifications in ``format_str`` and writes - the output to ``stdout``. Strings are assumed to be Unicode-encoded unless - the ``FMT_UNICODE`` macro is set to 0. - - **Example**:: - - fmt::print("Elapsed time: {0:.2f} seconds", 1.23); - \endrst - */ -template > -inline void print(const S& format_str, Args&&... args) { - const auto& vargs = fmt::make_args_checked(format_str, args...); - return detail::is_unicode() - ? vprint(to_string_view(format_str), vargs) - : detail::vprint_mojibake(stdout, to_string_view(format_str), - vargs); -} +FMT_MODULE_EXPORT_END +FMT_GCC_PRAGMA("GCC pop_options") FMT_END_NAMESPACE +#ifdef FMT_HEADER_ONLY +# include "format.h" +#endif #endif // FMT_CORE_H_ diff --git a/src/third_party/fmt/format-inl.h b/src/third_party/fmt/format-inl.h index 8f2fe73..2c51c50 100644 --- a/src/third_party/fmt/format-inl.h +++ b/src/third_party/fmt/format-inl.h @@ -8,8 +8,9 @@ #ifndef FMT_FORMAT_INL_H_ #define FMT_FORMAT_INL_H_ -#include +#include #include +#include // errno #include #include #include @@ -27,11 +28,6 @@ #include "format.h" -// Dummy implementations of strerror_r and strerror_s called if corresponding -// system functions are not available. -inline fmt::detail::null<> strerror_r(int, char*, ...) { return {}; } -inline fmt::detail::null<> strerror_s(char*, size_t, ...) { return {}; } - FMT_BEGIN_NAMESPACE namespace detail { @@ -44,6 +40,10 @@ FMT_FUNC void assert_fail(const char* file, int line, const char* message) { std::terminate(); } +FMT_FUNC void throw_format_error(const char* message) { + FMT_THROW(format_error(message)); +} + #ifndef _MSC_VER # define FMT_SNPRINTF snprintf #else // _MSC_VER @@ -57,76 +57,6 @@ inline int fmt_snprintf(char* buffer, size_t size, const char* format, ...) { # define FMT_SNPRINTF fmt_snprintf #endif // _MSC_VER -// A portable thread-safe version of strerror. -// Sets buffer to point to a string describing the error code. -// This can be either a pointer to a string stored in buffer, -// or a pointer to some static immutable string. -// Returns one of the following values: -// 0 - success -// ERANGE - buffer is not large enough to store the error message -// other - failure -// Buffer should be at least of size 1. -inline int safe_strerror(int error_code, char*& buffer, - size_t buffer_size) FMT_NOEXCEPT { - FMT_ASSERT(buffer != nullptr && buffer_size != 0, "invalid buffer"); - - class dispatcher { - private: - int error_code_; - char*& buffer_; - size_t buffer_size_; - - // A noop assignment operator to avoid bogus warnings. - void operator=(const dispatcher&) {} - - // Handle the result of XSI-compliant version of strerror_r. - int handle(int result) { - // glibc versions before 2.13 return result in errno. - return result == -1 ? errno : result; - } - - // Handle the result of GNU-specific version of strerror_r. - FMT_MAYBE_UNUSED - int handle(char* message) { - // If the buffer is full then the message is probably truncated. - if (message == buffer_ && strlen(buffer_) == buffer_size_ - 1) - return ERANGE; - buffer_ = message; - return 0; - } - - // Handle the case when strerror_r is not available. - FMT_MAYBE_UNUSED - int handle(detail::null<>) { - return fallback(strerror_s(buffer_, buffer_size_, error_code_)); - } - - // Fallback to strerror_s when strerror_r is not available. - FMT_MAYBE_UNUSED - int fallback(int result) { - // If the buffer is full then the message is probably truncated. - return result == 0 && strlen(buffer_) == buffer_size_ - 1 ? ERANGE - : result; - } - -#if !FMT_MSC_VER - // Fallback to strerror if strerror_r and strerror_s are not available. - int fallback(detail::null<>) { - errno = 0; - buffer_ = strerror(error_code_); - return errno; - } -#endif - - public: - dispatcher(int err_code, char*& buf, size_t buf_size) - : error_code_(err_code), buffer_(buf), buffer_size_(buf_size) {} - - int run() { return handle(strerror_r(error_code_, buffer_, buffer_size_)); } - }; - return dispatcher(error_code, buffer, buffer_size).run(); -} - FMT_FUNC void format_error_code(detail::buffer& out, int error_code, string_view message) FMT_NOEXCEPT { // Report error code making sure that the output fits into @@ -145,18 +75,18 @@ FMT_FUNC void format_error_code(detail::buffer& out, int error_code, error_code_size += detail::to_unsigned(detail::count_digits(abs_value)); auto it = buffer_appender(out); if (message.size() <= inline_buffer_size - error_code_size) - format_to(it, "{}{}", message, SEP); - format_to(it, "{}{}", ERROR_STR, error_code); - assert(out.size() <= inline_buffer_size); + format_to(it, FMT_STRING("{}{}"), message, SEP); + format_to(it, FMT_STRING("{}{}"), ERROR_STR, error_code); + FMT_ASSERT(out.size() <= inline_buffer_size, ""); } FMT_FUNC void report_error(format_func func, int error_code, - string_view message) FMT_NOEXCEPT { + const char* message) FMT_NOEXCEPT { memory_buffer full_message; func(full_message, error_code, message); // Don't use fwrite_fully because the latter may throw. - (void)std::fwrite(full_message.data(), full_message.size(), 1, stderr); - std::fputc('\n', stderr); + if (std::fwrite(full_message.data(), full_message.size(), 1, stderr) > 0) + std::fputc('\n', stderr); } // A wrapper around fwrite that throws on error. @@ -165,11 +95,8 @@ inline void fwrite_fully(const void* ptr, size_t size, size_t count, size_t written = std::fwrite(ptr, size, count, stream); if (written < count) FMT_THROW(system_error(errno, "cannot write to file")); } -} // namespace detail - -#if !defined(FMT_STATIC_THOUSANDS_SEPARATOR) -namespace detail { +#ifndef FMT_STATIC_THOUSANDS_SEPARATOR template locale_ref::locale_ref(const Locale& loc) : locale_(&loc) { static_assert(std::is_same::value, ""); @@ -180,41 +107,36 @@ template Locale locale_ref::get() const { return locale_ ? *static_cast(locale_) : std::locale(); } -template FMT_FUNC std::string grouping_impl(locale_ref loc) { - return std::use_facet>(loc.get()).grouping(); -} -template FMT_FUNC Char thousands_sep_impl(locale_ref loc) { - return std::use_facet>(loc.get()) - .thousands_sep(); +template +FMT_FUNC auto thousands_sep_impl(locale_ref loc) -> thousands_sep_result { + auto& facet = std::use_facet>(loc.get()); + auto grouping = facet.grouping(); + auto thousands_sep = grouping.empty() ? Char() : facet.thousands_sep(); + return {std::move(grouping), thousands_sep}; } template FMT_FUNC Char decimal_point_impl(locale_ref loc) { return std::use_facet>(loc.get()) .decimal_point(); } -} // namespace detail #else template -FMT_FUNC std::string detail::grouping_impl(locale_ref) { - return "\03"; -} -template FMT_FUNC Char detail::thousands_sep_impl(locale_ref) { - return FMT_STATIC_THOUSANDS_SEPARATOR; +FMT_FUNC auto thousands_sep_impl(locale_ref) -> thousands_sep_result { + return {"\03", FMT_STATIC_THOUSANDS_SEPARATOR}; } -template FMT_FUNC Char detail::decimal_point_impl(locale_ref) { +template FMT_FUNC Char decimal_point_impl(locale_ref) { return '.'; } #endif +} // namespace detail +#if !FMT_MSC_VER FMT_API FMT_FUNC format_error::~format_error() FMT_NOEXCEPT = default; -FMT_API FMT_FUNC system_error::~system_error() FMT_NOEXCEPT = default; +#endif -FMT_FUNC void system_error::init(int err_code, string_view format_str, - format_args args) { - error_code_ = err_code; - memory_buffer buffer; - format_system_error(buffer, err_code, vformat(format_str, args)); - std::runtime_error& base = *this; - base = std::runtime_error(to_string(buffer)); +FMT_FUNC std::system_error vsystem_error(int error_code, string_view format_str, + format_args args) { + auto ec = std::error_code(error_code, std::generic_category()); + return std::system_error(ec, vformat(format_str, args)); } namespace detail { @@ -227,946 +149,153 @@ template <> FMT_FUNC int count_digits<4>(detail::fallback_uintptr n) { return i >= 0 ? i * char_digits + count_digits<4, unsigned>(n.value[i]) : 1; } -template -const typename basic_data::digit_pair basic_data::digits[] = { - {'0', '0'}, {'0', '1'}, {'0', '2'}, {'0', '3'}, {'0', '4'}, {'0', '5'}, - {'0', '6'}, {'0', '7'}, {'0', '8'}, {'0', '9'}, {'1', '0'}, {'1', '1'}, - {'1', '2'}, {'1', '3'}, {'1', '4'}, {'1', '5'}, {'1', '6'}, {'1', '7'}, - {'1', '8'}, {'1', '9'}, {'2', '0'}, {'2', '1'}, {'2', '2'}, {'2', '3'}, - {'2', '4'}, {'2', '5'}, {'2', '6'}, {'2', '7'}, {'2', '8'}, {'2', '9'}, - {'3', '0'}, {'3', '1'}, {'3', '2'}, {'3', '3'}, {'3', '4'}, {'3', '5'}, - {'3', '6'}, {'3', '7'}, {'3', '8'}, {'3', '9'}, {'4', '0'}, {'4', '1'}, - {'4', '2'}, {'4', '3'}, {'4', '4'}, {'4', '5'}, {'4', '6'}, {'4', '7'}, - {'4', '8'}, {'4', '9'}, {'5', '0'}, {'5', '1'}, {'5', '2'}, {'5', '3'}, - {'5', '4'}, {'5', '5'}, {'5', '6'}, {'5', '7'}, {'5', '8'}, {'5', '9'}, - {'6', '0'}, {'6', '1'}, {'6', '2'}, {'6', '3'}, {'6', '4'}, {'6', '5'}, - {'6', '6'}, {'6', '7'}, {'6', '8'}, {'6', '9'}, {'7', '0'}, {'7', '1'}, - {'7', '2'}, {'7', '3'}, {'7', '4'}, {'7', '5'}, {'7', '6'}, {'7', '7'}, - {'7', '8'}, {'7', '9'}, {'8', '0'}, {'8', '1'}, {'8', '2'}, {'8', '3'}, - {'8', '4'}, {'8', '5'}, {'8', '6'}, {'8', '7'}, {'8', '8'}, {'8', '9'}, - {'9', '0'}, {'9', '1'}, {'9', '2'}, {'9', '3'}, {'9', '4'}, {'9', '5'}, - {'9', '6'}, {'9', '7'}, {'9', '8'}, {'9', '9'}}; - -template -const char basic_data::hex_digits[] = "0123456789abcdef"; - -#define FMT_POWERS_OF_10(factor) \ - factor * 10, (factor)*100, (factor)*1000, (factor)*10000, (factor)*100000, \ - (factor)*1000000, (factor)*10000000, (factor)*100000000, \ - (factor)*1000000000 - -template -const uint64_t basic_data::powers_of_10_64[] = { - 1, FMT_POWERS_OF_10(1), FMT_POWERS_OF_10(1000000000ULL), - 10000000000000000000ULL}; - -template -const uint32_t basic_data::zero_or_powers_of_10_32[] = {0, - FMT_POWERS_OF_10(1)}; -template -const uint64_t basic_data::zero_or_powers_of_10_64[] = { - 0, FMT_POWERS_OF_10(1), FMT_POWERS_OF_10(1000000000ULL), - 10000000000000000000ULL}; - -template -const uint32_t basic_data::zero_or_powers_of_10_32_new[] = { - 0, 0, FMT_POWERS_OF_10(1)}; - -template -const uint64_t basic_data::zero_or_powers_of_10_64_new[] = { - 0, 0, FMT_POWERS_OF_10(1), FMT_POWERS_OF_10(1000000000ULL), - 10000000000000000000ULL}; - -// Normalized 64-bit significands of pow(10, k), for k = -348, -340, ..., 340. -// These are generated by support/compute-powers.py. -template -const uint64_t basic_data::grisu_pow10_significands[] = { - 0xfa8fd5a0081c0288, 0xbaaee17fa23ebf76, 0x8b16fb203055ac76, - 0xcf42894a5dce35ea, 0x9a6bb0aa55653b2d, 0xe61acf033d1a45df, - 0xab70fe17c79ac6ca, 0xff77b1fcbebcdc4f, 0xbe5691ef416bd60c, - 0x8dd01fad907ffc3c, 0xd3515c2831559a83, 0x9d71ac8fada6c9b5, - 0xea9c227723ee8bcb, 0xaecc49914078536d, 0x823c12795db6ce57, - 0xc21094364dfb5637, 0x9096ea6f3848984f, 0xd77485cb25823ac7, - 0xa086cfcd97bf97f4, 0xef340a98172aace5, 0xb23867fb2a35b28e, - 0x84c8d4dfd2c63f3b, 0xc5dd44271ad3cdba, 0x936b9fcebb25c996, - 0xdbac6c247d62a584, 0xa3ab66580d5fdaf6, 0xf3e2f893dec3f126, - 0xb5b5ada8aaff80b8, 0x87625f056c7c4a8b, 0xc9bcff6034c13053, - 0x964e858c91ba2655, 0xdff9772470297ebd, 0xa6dfbd9fb8e5b88f, - 0xf8a95fcf88747d94, 0xb94470938fa89bcf, 0x8a08f0f8bf0f156b, - 0xcdb02555653131b6, 0x993fe2c6d07b7fac, 0xe45c10c42a2b3b06, - 0xaa242499697392d3, 0xfd87b5f28300ca0e, 0xbce5086492111aeb, - 0x8cbccc096f5088cc, 0xd1b71758e219652c, 0x9c40000000000000, - 0xe8d4a51000000000, 0xad78ebc5ac620000, 0x813f3978f8940984, - 0xc097ce7bc90715b3, 0x8f7e32ce7bea5c70, 0xd5d238a4abe98068, - 0x9f4f2726179a2245, 0xed63a231d4c4fb27, 0xb0de65388cc8ada8, - 0x83c7088e1aab65db, 0xc45d1df942711d9a, 0x924d692ca61be758, - 0xda01ee641a708dea, 0xa26da3999aef774a, 0xf209787bb47d6b85, - 0xb454e4a179dd1877, 0x865b86925b9bc5c2, 0xc83553c5c8965d3d, - 0x952ab45cfa97a0b3, 0xde469fbd99a05fe3, 0xa59bc234db398c25, - 0xf6c69a72a3989f5c, 0xb7dcbf5354e9bece, 0x88fcf317f22241e2, - 0xcc20ce9bd35c78a5, 0x98165af37b2153df, 0xe2a0b5dc971f303a, - 0xa8d9d1535ce3b396, 0xfb9b7cd9a4a7443c, 0xbb764c4ca7a44410, - 0x8bab8eefb6409c1a, 0xd01fef10a657842c, 0x9b10a4e5e9913129, - 0xe7109bfba19c0c9d, 0xac2820d9623bf429, 0x80444b5e7aa7cf85, - 0xbf21e44003acdd2d, 0x8e679c2f5e44ff8f, 0xd433179d9c8cb841, - 0x9e19db92b4e31ba9, 0xeb96bf6ebadf77d9, 0xaf87023b9bf0ee6b, -}; - -// Binary exponents of pow(10, k), for k = -348, -340, ..., 340, corresponding -// to significands above. -template -const int16_t basic_data::grisu_pow10_exponents[] = { - -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, -954, - -927, -901, -874, -847, -821, -794, -768, -741, -715, -688, -661, - -635, -608, -582, -555, -529, -502, -475, -449, -422, -396, -369, - -343, -316, -289, -263, -236, -210, -183, -157, -130, -103, -77, - -50, -24, 3, 30, 56, 83, 109, 136, 162, 189, 216, - 242, 269, 295, 322, 348, 375, 402, 428, 455, 481, 508, - 534, 561, 588, 614, 641, 667, 694, 720, 747, 774, 800, - 827, 853, 880, 907, 933, 960, 986, 1013, 1039, 1066}; - -template -const divtest_table_entry basic_data::divtest_table_for_pow5_32[] = - {{0x00000001, 0xffffffff}, {0xcccccccd, 0x33333333}, - {0xc28f5c29, 0x0a3d70a3}, {0x26e978d5, 0x020c49ba}, - {0x3afb7e91, 0x0068db8b}, {0x0bcbe61d, 0x0014f8b5}, - {0x68c26139, 0x000431bd}, {0xae8d46a5, 0x0000d6bf}, - {0x22e90e21, 0x00002af3}, {0x3a2e9c6d, 0x00000897}, - {0x3ed61f49, 0x000001b7}}; - -template -const divtest_table_entry basic_data::divtest_table_for_pow5_64[] = - {{0x0000000000000001, 0xffffffffffffffff}, - {0xcccccccccccccccd, 0x3333333333333333}, - {0x8f5c28f5c28f5c29, 0x0a3d70a3d70a3d70}, - {0x1cac083126e978d5, 0x020c49ba5e353f7c}, - {0xd288ce703afb7e91, 0x0068db8bac710cb2}, - {0x5d4e8fb00bcbe61d, 0x0014f8b588e368f0}, - {0x790fb65668c26139, 0x000431bde82d7b63}, - {0xe5032477ae8d46a5, 0x0000d6bf94d5e57a}, - {0xc767074b22e90e21, 0x00002af31dc46118}, - {0x8e47ce423a2e9c6d, 0x0000089705f4136b}, - {0x4fa7f60d3ed61f49, 0x000001b7cdfd9d7b}, - {0x0fee64690c913975, 0x00000057f5ff85e5}, - {0x3662e0e1cf503eb1, 0x000000119799812d}, - {0xa47a2cf9f6433fbd, 0x0000000384b84d09}, - {0x54186f653140a659, 0x00000000b424dc35}, - {0x7738164770402145, 0x0000000024075f3d}, - {0xe4a4d1417cd9a041, 0x000000000734aca5}, - {0xc75429d9e5c5200d, 0x000000000170ef54}, - {0xc1773b91fac10669, 0x000000000049c977}, - {0x26b172506559ce15, 0x00000000000ec1e4}, - {0xd489e3a9addec2d1, 0x000000000002f394}, - {0x90e860bb892c8d5d, 0x000000000000971d}, - {0x502e79bf1b6f4f79, 0x0000000000001e39}, - {0xdcd618596be30fe5, 0x000000000000060b}}; - -template -const uint64_t basic_data::dragonbox_pow10_significands_64[] = { - 0x81ceb32c4b43fcf5, 0xa2425ff75e14fc32, 0xcad2f7f5359a3b3f, - 0xfd87b5f28300ca0e, 0x9e74d1b791e07e49, 0xc612062576589ddb, - 0xf79687aed3eec552, 0x9abe14cd44753b53, 0xc16d9a0095928a28, - 0xf1c90080baf72cb2, 0x971da05074da7bef, 0xbce5086492111aeb, - 0xec1e4a7db69561a6, 0x9392ee8e921d5d08, 0xb877aa3236a4b44a, - 0xe69594bec44de15c, 0x901d7cf73ab0acda, 0xb424dc35095cd810, - 0xe12e13424bb40e14, 0x8cbccc096f5088cc, 0xafebff0bcb24aaff, - 0xdbe6fecebdedd5bf, 0x89705f4136b4a598, 0xabcc77118461cefd, - 0xd6bf94d5e57a42bd, 0x8637bd05af6c69b6, 0xa7c5ac471b478424, - 0xd1b71758e219652c, 0x83126e978d4fdf3c, 0xa3d70a3d70a3d70b, - 0xcccccccccccccccd, 0x8000000000000000, 0xa000000000000000, - 0xc800000000000000, 0xfa00000000000000, 0x9c40000000000000, - 0xc350000000000000, 0xf424000000000000, 0x9896800000000000, - 0xbebc200000000000, 0xee6b280000000000, 0x9502f90000000000, - 0xba43b74000000000, 0xe8d4a51000000000, 0x9184e72a00000000, - 0xb5e620f480000000, 0xe35fa931a0000000, 0x8e1bc9bf04000000, - 0xb1a2bc2ec5000000, 0xde0b6b3a76400000, 0x8ac7230489e80000, - 0xad78ebc5ac620000, 0xd8d726b7177a8000, 0x878678326eac9000, - 0xa968163f0a57b400, 0xd3c21bcecceda100, 0x84595161401484a0, - 0xa56fa5b99019a5c8, 0xcecb8f27f4200f3a, 0x813f3978f8940984, - 0xa18f07d736b90be5, 0xc9f2c9cd04674ede, 0xfc6f7c4045812296, - 0x9dc5ada82b70b59d, 0xc5371912364ce305, 0xf684df56c3e01bc6, - 0x9a130b963a6c115c, 0xc097ce7bc90715b3, 0xf0bdc21abb48db20, - 0x96769950b50d88f4, 0xbc143fa4e250eb31, 0xeb194f8e1ae525fd, - 0x92efd1b8d0cf37be, 0xb7abc627050305ad, 0xe596b7b0c643c719, - 0x8f7e32ce7bea5c6f, 0xb35dbf821ae4f38b, 0xe0352f62a19e306e}; +// log10(2) = 0x0.4d104d427de7fbcc... +static constexpr uint64_t log10_2_significand = 0x4d104d427de7fbcc; + +template struct basic_impl_data { + // Normalized 64-bit significands of pow(10, k), for k = -348, -340, ..., 340. + // These are generated by support/compute-powers.py. + static constexpr uint64_t pow10_significands[87] = { + 0xfa8fd5a0081c0288, 0xbaaee17fa23ebf76, 0x8b16fb203055ac76, + 0xcf42894a5dce35ea, 0x9a6bb0aa55653b2d, 0xe61acf033d1a45df, + 0xab70fe17c79ac6ca, 0xff77b1fcbebcdc4f, 0xbe5691ef416bd60c, + 0x8dd01fad907ffc3c, 0xd3515c2831559a83, 0x9d71ac8fada6c9b5, + 0xea9c227723ee8bcb, 0xaecc49914078536d, 0x823c12795db6ce57, + 0xc21094364dfb5637, 0x9096ea6f3848984f, 0xd77485cb25823ac7, + 0xa086cfcd97bf97f4, 0xef340a98172aace5, 0xb23867fb2a35b28e, + 0x84c8d4dfd2c63f3b, 0xc5dd44271ad3cdba, 0x936b9fcebb25c996, + 0xdbac6c247d62a584, 0xa3ab66580d5fdaf6, 0xf3e2f893dec3f126, + 0xb5b5ada8aaff80b8, 0x87625f056c7c4a8b, 0xc9bcff6034c13053, + 0x964e858c91ba2655, 0xdff9772470297ebd, 0xa6dfbd9fb8e5b88f, + 0xf8a95fcf88747d94, 0xb94470938fa89bcf, 0x8a08f0f8bf0f156b, + 0xcdb02555653131b6, 0x993fe2c6d07b7fac, 0xe45c10c42a2b3b06, + 0xaa242499697392d3, 0xfd87b5f28300ca0e, 0xbce5086492111aeb, + 0x8cbccc096f5088cc, 0xd1b71758e219652c, 0x9c40000000000000, + 0xe8d4a51000000000, 0xad78ebc5ac620000, 0x813f3978f8940984, + 0xc097ce7bc90715b3, 0x8f7e32ce7bea5c70, 0xd5d238a4abe98068, + 0x9f4f2726179a2245, 0xed63a231d4c4fb27, 0xb0de65388cc8ada8, + 0x83c7088e1aab65db, 0xc45d1df942711d9a, 0x924d692ca61be758, + 0xda01ee641a708dea, 0xa26da3999aef774a, 0xf209787bb47d6b85, + 0xb454e4a179dd1877, 0x865b86925b9bc5c2, 0xc83553c5c8965d3d, + 0x952ab45cfa97a0b3, 0xde469fbd99a05fe3, 0xa59bc234db398c25, + 0xf6c69a72a3989f5c, 0xb7dcbf5354e9bece, 0x88fcf317f22241e2, + 0xcc20ce9bd35c78a5, 0x98165af37b2153df, 0xe2a0b5dc971f303a, + 0xa8d9d1535ce3b396, 0xfb9b7cd9a4a7443c, 0xbb764c4ca7a44410, + 0x8bab8eefb6409c1a, 0xd01fef10a657842c, 0x9b10a4e5e9913129, + 0xe7109bfba19c0c9d, 0xac2820d9623bf429, 0x80444b5e7aa7cf85, + 0xbf21e44003acdd2d, 0x8e679c2f5e44ff8f, 0xd433179d9c8cb841, + 0x9e19db92b4e31ba9, 0xeb96bf6ebadf77d9, 0xaf87023b9bf0ee6b, + }; -template -const uint128_wrapper basic_data::dragonbox_pow10_significands_128[] = { -#if FMT_USE_FULL_CACHE_DRAGONBOX - {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b}, - {0x9faacf3df73609b1, 0x77b191618c54e9ad}, - {0xc795830d75038c1d, 0xd59df5b9ef6a2418}, - {0xf97ae3d0d2446f25, 0x4b0573286b44ad1e}, - {0x9becce62836ac577, 0x4ee367f9430aec33}, - {0xc2e801fb244576d5, 0x229c41f793cda740}, - {0xf3a20279ed56d48a, 0x6b43527578c11110}, - {0x9845418c345644d6, 0x830a13896b78aaaa}, - {0xbe5691ef416bd60c, 0x23cc986bc656d554}, - {0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa9}, - {0x94b3a202eb1c3f39, 0x7bf7d71432f3d6aa}, - {0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc54}, - {0xe858ad248f5c22c9, 0xd1b3400f8f9cff69}, - {0x91376c36d99995be, 0x23100809b9c21fa2}, - {0xb58547448ffffb2d, 0xabd40a0c2832a78b}, - {0xe2e69915b3fff9f9, 0x16c90c8f323f516d}, - {0x8dd01fad907ffc3b, 0xae3da7d97f6792e4}, - {0xb1442798f49ffb4a, 0x99cd11cfdf41779d}, - {0xdd95317f31c7fa1d, 0x40405643d711d584}, - {0x8a7d3eef7f1cfc52, 0x482835ea666b2573}, - {0xad1c8eab5ee43b66, 0xda3243650005eed0}, - {0xd863b256369d4a40, 0x90bed43e40076a83}, - {0x873e4f75e2224e68, 0x5a7744a6e804a292}, - {0xa90de3535aaae202, 0x711515d0a205cb37}, - {0xd3515c2831559a83, 0x0d5a5b44ca873e04}, - {0x8412d9991ed58091, 0xe858790afe9486c3}, - {0xa5178fff668ae0b6, 0x626e974dbe39a873}, - {0xce5d73ff402d98e3, 0xfb0a3d212dc81290}, - {0x80fa687f881c7f8e, 0x7ce66634bc9d0b9a}, - {0xa139029f6a239f72, 0x1c1fffc1ebc44e81}, - {0xc987434744ac874e, 0xa327ffb266b56221}, - {0xfbe9141915d7a922, 0x4bf1ff9f0062baa9}, - {0x9d71ac8fada6c9b5, 0x6f773fc3603db4aa}, - {0xc4ce17b399107c22, 0xcb550fb4384d21d4}, - {0xf6019da07f549b2b, 0x7e2a53a146606a49}, - {0x99c102844f94e0fb, 0x2eda7444cbfc426e}, - {0xc0314325637a1939, 0xfa911155fefb5309}, - {0xf03d93eebc589f88, 0x793555ab7eba27cb}, - {0x96267c7535b763b5, 0x4bc1558b2f3458df}, - {0xbbb01b9283253ca2, 0x9eb1aaedfb016f17}, - {0xea9c227723ee8bcb, 0x465e15a979c1cadd}, - {0x92a1958a7675175f, 0x0bfacd89ec191eca}, - {0xb749faed14125d36, 0xcef980ec671f667c}, - {0xe51c79a85916f484, 0x82b7e12780e7401b}, - {0x8f31cc0937ae58d2, 0xd1b2ecb8b0908811}, - {0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa16}, - {0xdfbdcece67006ac9, 0x67a791e093e1d49b}, - {0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e1}, - {0xaecc49914078536d, 0x58fae9f773886e19}, - {0xda7f5bf590966848, 0xaf39a475506a899f}, - {0x888f99797a5e012d, 0x6d8406c952429604}, - {0xaab37fd7d8f58178, 0xc8e5087ba6d33b84}, - {0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a65}, - {0x855c3be0a17fcd26, 0x5cf2eea09a550680}, - {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f}, - {0xd0601d8efc57b08b, 0xf13b94daf124da27}, - {0x823c12795db6ce57, 0x76c53d08d6b70859}, - {0xa2cb1717b52481ed, 0x54768c4b0c64ca6f}, - {0xcb7ddcdda26da268, 0xa9942f5dcf7dfd0a}, - {0xfe5d54150b090b02, 0xd3f93b35435d7c4d}, - {0x9efa548d26e5a6e1, 0xc47bc5014a1a6db0}, - {0xc6b8e9b0709f109a, 0x359ab6419ca1091c}, - {0xf867241c8cc6d4c0, 0xc30163d203c94b63}, - {0x9b407691d7fc44f8, 0x79e0de63425dcf1e}, - {0xc21094364dfb5636, 0x985915fc12f542e5}, - {0xf294b943e17a2bc4, 0x3e6f5b7b17b2939e}, - {0x979cf3ca6cec5b5a, 0xa705992ceecf9c43}, - {0xbd8430bd08277231, 0x50c6ff782a838354}, - {0xece53cec4a314ebd, 0xa4f8bf5635246429}, - {0x940f4613ae5ed136, 0x871b7795e136be9a}, - {0xb913179899f68584, 0x28e2557b59846e40}, - {0xe757dd7ec07426e5, 0x331aeada2fe589d0}, - {0x9096ea6f3848984f, 0x3ff0d2c85def7622}, - {0xb4bca50b065abe63, 0x0fed077a756b53aa}, - {0xe1ebce4dc7f16dfb, 0xd3e8495912c62895}, - {0x8d3360f09cf6e4bd, 0x64712dd7abbbd95d}, - {0xb080392cc4349dec, 0xbd8d794d96aacfb4}, - {0xdca04777f541c567, 0xecf0d7a0fc5583a1}, - {0x89e42caaf9491b60, 0xf41686c49db57245}, - {0xac5d37d5b79b6239, 0x311c2875c522ced6}, - {0xd77485cb25823ac7, 0x7d633293366b828c}, - {0x86a8d39ef77164bc, 0xae5dff9c02033198}, - {0xa8530886b54dbdeb, 0xd9f57f830283fdfd}, - {0xd267caa862a12d66, 0xd072df63c324fd7c}, - {0x8380dea93da4bc60, 0x4247cb9e59f71e6e}, - {0xa46116538d0deb78, 0x52d9be85f074e609}, - {0xcd795be870516656, 0x67902e276c921f8c}, - {0x806bd9714632dff6, 0x00ba1cd8a3db53b7}, - {0xa086cfcd97bf97f3, 0x80e8a40eccd228a5}, - {0xc8a883c0fdaf7df0, 0x6122cd128006b2ce}, - {0xfad2a4b13d1b5d6c, 0x796b805720085f82}, - {0x9cc3a6eec6311a63, 0xcbe3303674053bb1}, - {0xc3f490aa77bd60fc, 0xbedbfc4411068a9d}, - {0xf4f1b4d515acb93b, 0xee92fb5515482d45}, - {0x991711052d8bf3c5, 0x751bdd152d4d1c4b}, - {0xbf5cd54678eef0b6, 0xd262d45a78a0635e}, - {0xef340a98172aace4, 0x86fb897116c87c35}, - {0x9580869f0e7aac0e, 0xd45d35e6ae3d4da1}, - {0xbae0a846d2195712, 0x8974836059cca10a}, - {0xe998d258869facd7, 0x2bd1a438703fc94c}, - {0x91ff83775423cc06, 0x7b6306a34627ddd0}, - {0xb67f6455292cbf08, 0x1a3bc84c17b1d543}, - {0xe41f3d6a7377eeca, 0x20caba5f1d9e4a94}, - {0x8e938662882af53e, 0x547eb47b7282ee9d}, - {0xb23867fb2a35b28d, 0xe99e619a4f23aa44}, - {0xdec681f9f4c31f31, 0x6405fa00e2ec94d5}, - {0x8b3c113c38f9f37e, 0xde83bc408dd3dd05}, - {0xae0b158b4738705e, 0x9624ab50b148d446}, - {0xd98ddaee19068c76, 0x3badd624dd9b0958}, - {0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d7}, - {0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4d}, - {0xd47487cc8470652b, 0x7647c32000696720}, - {0x84c8d4dfd2c63f3b, 0x29ecd9f40041e074}, - {0xa5fb0a17c777cf09, 0xf468107100525891}, - {0xcf79cc9db955c2cc, 0x7182148d4066eeb5}, - {0x81ac1fe293d599bf, 0xc6f14cd848405531}, - {0xa21727db38cb002f, 0xb8ada00e5a506a7d}, - {0xca9cf1d206fdc03b, 0xa6d90811f0e4851d}, - {0xfd442e4688bd304a, 0x908f4a166d1da664}, - {0x9e4a9cec15763e2e, 0x9a598e4e043287ff}, - {0xc5dd44271ad3cdba, 0x40eff1e1853f29fe}, - {0xf7549530e188c128, 0xd12bee59e68ef47d}, - {0x9a94dd3e8cf578b9, 0x82bb74f8301958cf}, - {0xc13a148e3032d6e7, 0xe36a52363c1faf02}, - {0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac2}, - {0x96f5600f15a7b7e5, 0x29ab103a5ef8c0ba}, - {0xbcb2b812db11a5de, 0x7415d448f6b6f0e8}, - {0xebdf661791d60f56, 0x111b495b3464ad22}, - {0x936b9fcebb25c995, 0xcab10dd900beec35}, - {0xb84687c269ef3bfb, 0x3d5d514f40eea743}, - {0xe65829b3046b0afa, 0x0cb4a5a3112a5113}, - {0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ac}, - {0xb3f4e093db73a093, 0x59ed216765690f57}, - {0xe0f218b8d25088b8, 0x306869c13ec3532d}, - {0x8c974f7383725573, 0x1e414218c73a13fc}, - {0xafbd2350644eeacf, 0xe5d1929ef90898fb}, - {0xdbac6c247d62a583, 0xdf45f746b74abf3a}, - {0x894bc396ce5da772, 0x6b8bba8c328eb784}, - {0xab9eb47c81f5114f, 0x066ea92f3f326565}, - {0xd686619ba27255a2, 0xc80a537b0efefebe}, - {0x8613fd0145877585, 0xbd06742ce95f5f37}, - {0xa798fc4196e952e7, 0x2c48113823b73705}, - {0xd17f3b51fca3a7a0, 0xf75a15862ca504c6}, - {0x82ef85133de648c4, 0x9a984d73dbe722fc}, - {0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebbb}, - {0xcc963fee10b7d1b3, 0x318df905079926a9}, - {0xffbbcfe994e5c61f, 0xfdf17746497f7053}, - {0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa634}, - {0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc1}, - {0xf9bd690a1b68637b, 0x3dfdce7aa3c673b1}, - {0x9c1661a651213e2d, 0x06bea10ca65c084f}, - {0xc31bfa0fe5698db8, 0x486e494fcff30a63}, - {0xf3e2f893dec3f126, 0x5a89dba3c3efccfb}, - {0x986ddb5c6b3a76b7, 0xf89629465a75e01d}, - {0xbe89523386091465, 0xf6bbb397f1135824}, - {0xee2ba6c0678b597f, 0x746aa07ded582e2d}, - {0x94db483840b717ef, 0xa8c2a44eb4571cdd}, - {0xba121a4650e4ddeb, 0x92f34d62616ce414}, - {0xe896a0d7e51e1566, 0x77b020baf9c81d18}, - {0x915e2486ef32cd60, 0x0ace1474dc1d122f}, - {0xb5b5ada8aaff80b8, 0x0d819992132456bb}, - {0xe3231912d5bf60e6, 0x10e1fff697ed6c6a}, - {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2}, - {0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb3}, - {0xddd0467c64bce4a0, 0xac7cb3f6d05ddbdf}, - {0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96c}, - {0xad4ab7112eb3929d, 0x86c16c98d2c953c7}, - {0xd89d64d57a607744, 0xe871c7bf077ba8b8}, - {0x87625f056c7c4a8b, 0x11471cd764ad4973}, - {0xa93af6c6c79b5d2d, 0xd598e40d3dd89bd0}, - {0xd389b47879823479, 0x4aff1d108d4ec2c4}, - {0x843610cb4bf160cb, 0xcedf722a585139bb}, - {0xa54394fe1eedb8fe, 0xc2974eb4ee658829}, - {0xce947a3da6a9273e, 0x733d226229feea33}, - {0x811ccc668829b887, 0x0806357d5a3f5260}, - {0xa163ff802a3426a8, 0xca07c2dcb0cf26f8}, - {0xc9bcff6034c13052, 0xfc89b393dd02f0b6}, - {0xfc2c3f3841f17c67, 0xbbac2078d443ace3}, - {0x9d9ba7832936edc0, 0xd54b944b84aa4c0e}, - {0xc5029163f384a931, 0x0a9e795e65d4df12}, - {0xf64335bcf065d37d, 0x4d4617b5ff4a16d6}, - {0x99ea0196163fa42e, 0x504bced1bf8e4e46}, - {0xc06481fb9bcf8d39, 0xe45ec2862f71e1d7}, - {0xf07da27a82c37088, 0x5d767327bb4e5a4d}, - {0x964e858c91ba2655, 0x3a6a07f8d510f870}, - {0xbbe226efb628afea, 0x890489f70a55368c}, - {0xeadab0aba3b2dbe5, 0x2b45ac74ccea842f}, - {0x92c8ae6b464fc96f, 0x3b0b8bc90012929e}, - {0xb77ada0617e3bbcb, 0x09ce6ebb40173745}, - {0xe55990879ddcaabd, 0xcc420a6a101d0516}, - {0x8f57fa54c2a9eab6, 0x9fa946824a12232e}, - {0xb32df8e9f3546564, 0x47939822dc96abfa}, - {0xdff9772470297ebd, 0x59787e2b93bc56f8}, - {0x8bfbea76c619ef36, 0x57eb4edb3c55b65b}, - {0xaefae51477a06b03, 0xede622920b6b23f2}, - {0xdab99e59958885c4, 0xe95fab368e45ecee}, - {0x88b402f7fd75539b, 0x11dbcb0218ebb415}, - {0xaae103b5fcd2a881, 0xd652bdc29f26a11a}, - {0xd59944a37c0752a2, 0x4be76d3346f04960}, - {0x857fcae62d8493a5, 0x6f70a4400c562ddc}, - {0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb953}, - {0xd097ad07a71f26b2, 0x7e2000a41346a7a8}, - {0x825ecc24c873782f, 0x8ed400668c0c28c9}, - {0xa2f67f2dfa90563b, 0x728900802f0f32fb}, - {0xcbb41ef979346bca, 0x4f2b40a03ad2ffba}, - {0xfea126b7d78186bc, 0xe2f610c84987bfa9}, - {0x9f24b832e6b0f436, 0x0dd9ca7d2df4d7ca}, - {0xc6ede63fa05d3143, 0x91503d1c79720dbc}, - {0xf8a95fcf88747d94, 0x75a44c6397ce912b}, - {0x9b69dbe1b548ce7c, 0xc986afbe3ee11abb}, - {0xc24452da229b021b, 0xfbe85badce996169}, - {0xf2d56790ab41c2a2, 0xfae27299423fb9c4}, - {0x97c560ba6b0919a5, 0xdccd879fc967d41b}, - {0xbdb6b8e905cb600f, 0x5400e987bbc1c921}, - {0xed246723473e3813, 0x290123e9aab23b69}, - {0x9436c0760c86e30b, 0xf9a0b6720aaf6522}, - {0xb94470938fa89bce, 0xf808e40e8d5b3e6a}, - {0xe7958cb87392c2c2, 0xb60b1d1230b20e05}, - {0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c3}, - {0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af4}, - {0xe2280b6c20dd5232, 0x25c6da63c38de1b1}, - {0x8d590723948a535f, 0x579c487e5a38ad0f}, - {0xb0af48ec79ace837, 0x2d835a9df0c6d852}, - {0xdcdb1b2798182244, 0xf8e431456cf88e66}, - {0x8a08f0f8bf0f156b, 0x1b8e9ecb641b5900}, - {0xac8b2d36eed2dac5, 0xe272467e3d222f40}, - {0xd7adf884aa879177, 0x5b0ed81dcc6abb10}, - {0x86ccbb52ea94baea, 0x98e947129fc2b4ea}, - {0xa87fea27a539e9a5, 0x3f2398d747b36225}, - {0xd29fe4b18e88640e, 0x8eec7f0d19a03aae}, - {0x83a3eeeef9153e89, 0x1953cf68300424ad}, - {0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd8}, - {0xcdb02555653131b6, 0x3792f412cb06794e}, - {0x808e17555f3ebf11, 0xe2bbd88bbee40bd1}, - {0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec5}, - {0xc8de047564d20a8b, 0xf245825a5a445276}, - {0xfb158592be068d2e, 0xeed6e2f0f0d56713}, - {0x9ced737bb6c4183d, 0x55464dd69685606c}, - {0xc428d05aa4751e4c, 0xaa97e14c3c26b887}, - {0xf53304714d9265df, 0xd53dd99f4b3066a9}, - {0x993fe2c6d07b7fab, 0xe546a8038efe402a}, - {0xbf8fdb78849a5f96, 0xde98520472bdd034}, - {0xef73d256a5c0f77c, 0x963e66858f6d4441}, - {0x95a8637627989aad, 0xdde7001379a44aa9}, - {0xbb127c53b17ec159, 0x5560c018580d5d53}, - {0xe9d71b689dde71af, 0xaab8f01e6e10b4a7}, - {0x9226712162ab070d, 0xcab3961304ca70e9}, - {0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d23}, - {0xe45c10c42a2b3b05, 0x8cb89a7db77c506b}, - {0x8eb98a7a9a5b04e3, 0x77f3608e92adb243}, - {0xb267ed1940f1c61c, 0x55f038b237591ed4}, - {0xdf01e85f912e37a3, 0x6b6c46dec52f6689}, - {0x8b61313bbabce2c6, 0x2323ac4b3b3da016}, - {0xae397d8aa96c1b77, 0xabec975e0a0d081b}, - {0xd9c7dced53c72255, 0x96e7bd358c904a22}, - {0x881cea14545c7575, 0x7e50d64177da2e55}, - {0xaa242499697392d2, 0xdde50bd1d5d0b9ea}, - {0xd4ad2dbfc3d07787, 0x955e4ec64b44e865}, - {0x84ec3c97da624ab4, 0xbd5af13bef0b113f}, - {0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58f}, - {0xcfb11ead453994ba, 0x67de18eda5814af3}, - {0x81ceb32c4b43fcf4, 0x80eacf948770ced8}, - {0xa2425ff75e14fc31, 0xa1258379a94d028e}, - {0xcad2f7f5359a3b3e, 0x096ee45813a04331}, - {0xfd87b5f28300ca0d, 0x8bca9d6e188853fd}, - {0x9e74d1b791e07e48, 0x775ea264cf55347e}, - {0xc612062576589dda, 0x95364afe032a819e}, - {0xf79687aed3eec551, 0x3a83ddbd83f52205}, - {0x9abe14cd44753b52, 0xc4926a9672793543}, - {0xc16d9a0095928a27, 0x75b7053c0f178294}, - {0xf1c90080baf72cb1, 0x5324c68b12dd6339}, - {0x971da05074da7bee, 0xd3f6fc16ebca5e04}, - {0xbce5086492111aea, 0x88f4bb1ca6bcf585}, - {0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6}, - {0x9392ee8e921d5d07, 0x3aff322e62439fd0}, - {0xb877aa3236a4b449, 0x09befeb9fad487c3}, - {0xe69594bec44de15b, 0x4c2ebe687989a9b4}, - {0x901d7cf73ab0acd9, 0x0f9d37014bf60a11}, - {0xb424dc35095cd80f, 0x538484c19ef38c95}, - {0xe12e13424bb40e13, 0x2865a5f206b06fba}, - {0x8cbccc096f5088cb, 0xf93f87b7442e45d4}, - {0xafebff0bcb24aafe, 0xf78f69a51539d749}, - {0xdbe6fecebdedd5be, 0xb573440e5a884d1c}, - {0x89705f4136b4a597, 0x31680a88f8953031}, - {0xabcc77118461cefc, 0xfdc20d2b36ba7c3e}, - {0xd6bf94d5e57a42bc, 0x3d32907604691b4d}, - {0x8637bd05af6c69b5, 0xa63f9a49c2c1b110}, - {0xa7c5ac471b478423, 0x0fcf80dc33721d54}, - {0xd1b71758e219652b, 0xd3c36113404ea4a9}, - {0x83126e978d4fdf3b, 0x645a1cac083126ea}, - {0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4}, - {0xcccccccccccccccc, 0xcccccccccccccccd}, - {0x8000000000000000, 0x0000000000000000}, - {0xa000000000000000, 0x0000000000000000}, - {0xc800000000000000, 0x0000000000000000}, - {0xfa00000000000000, 0x0000000000000000}, - {0x9c40000000000000, 0x0000000000000000}, - {0xc350000000000000, 0x0000000000000000}, - {0xf424000000000000, 0x0000000000000000}, - {0x9896800000000000, 0x0000000000000000}, - {0xbebc200000000000, 0x0000000000000000}, - {0xee6b280000000000, 0x0000000000000000}, - {0x9502f90000000000, 0x0000000000000000}, - {0xba43b74000000000, 0x0000000000000000}, - {0xe8d4a51000000000, 0x0000000000000000}, - {0x9184e72a00000000, 0x0000000000000000}, - {0xb5e620f480000000, 0x0000000000000000}, - {0xe35fa931a0000000, 0x0000000000000000}, - {0x8e1bc9bf04000000, 0x0000000000000000}, - {0xb1a2bc2ec5000000, 0x0000000000000000}, - {0xde0b6b3a76400000, 0x0000000000000000}, - {0x8ac7230489e80000, 0x0000000000000000}, - {0xad78ebc5ac620000, 0x0000000000000000}, - {0xd8d726b7177a8000, 0x0000000000000000}, - {0x878678326eac9000, 0x0000000000000000}, - {0xa968163f0a57b400, 0x0000000000000000}, - {0xd3c21bcecceda100, 0x0000000000000000}, - {0x84595161401484a0, 0x0000000000000000}, - {0xa56fa5b99019a5c8, 0x0000000000000000}, - {0xcecb8f27f4200f3a, 0x0000000000000000}, - {0x813f3978f8940984, 0x4000000000000000}, - {0xa18f07d736b90be5, 0x5000000000000000}, - {0xc9f2c9cd04674ede, 0xa400000000000000}, - {0xfc6f7c4045812296, 0x4d00000000000000}, - {0x9dc5ada82b70b59d, 0xf020000000000000}, - {0xc5371912364ce305, 0x6c28000000000000}, - {0xf684df56c3e01bc6, 0xc732000000000000}, - {0x9a130b963a6c115c, 0x3c7f400000000000}, - {0xc097ce7bc90715b3, 0x4b9f100000000000}, - {0xf0bdc21abb48db20, 0x1e86d40000000000}, - {0x96769950b50d88f4, 0x1314448000000000}, - {0xbc143fa4e250eb31, 0x17d955a000000000}, - {0xeb194f8e1ae525fd, 0x5dcfab0800000000}, - {0x92efd1b8d0cf37be, 0x5aa1cae500000000}, - {0xb7abc627050305ad, 0xf14a3d9e40000000}, - {0xe596b7b0c643c719, 0x6d9ccd05d0000000}, - {0x8f7e32ce7bea5c6f, 0xe4820023a2000000}, - {0xb35dbf821ae4f38b, 0xdda2802c8a800000}, - {0xe0352f62a19e306e, 0xd50b2037ad200000}, - {0x8c213d9da502de45, 0x4526f422cc340000}, - {0xaf298d050e4395d6, 0x9670b12b7f410000}, - {0xdaf3f04651d47b4c, 0x3c0cdd765f114000}, - {0x88d8762bf324cd0f, 0xa5880a69fb6ac800}, - {0xab0e93b6efee0053, 0x8eea0d047a457a00}, - {0xd5d238a4abe98068, 0x72a4904598d6d880}, - {0x85a36366eb71f041, 0x47a6da2b7f864750}, - {0xa70c3c40a64e6c51, 0x999090b65f67d924}, - {0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d}, - {0x82818f1281ed449f, 0xbff8f10e7a8921a4}, - {0xa321f2d7226895c7, 0xaff72d52192b6a0d}, - {0xcbea6f8ceb02bb39, 0x9bf4f8a69f764490}, - {0xfee50b7025c36a08, 0x02f236d04753d5b4}, - {0x9f4f2726179a2245, 0x01d762422c946590}, - {0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef5}, - {0xf8ebad2b84e0d58b, 0xd2e0898765a7deb2}, - {0x9b934c3b330c8577, 0x63cc55f49f88eb2f}, - {0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fb}, - {0xf316271c7fc3908a, 0x8bef464e3945ef7a}, - {0x97edd871cfda3a56, 0x97758bf0e3cbb5ac}, - {0xbde94e8e43d0c8ec, 0x3d52eeed1cbea317}, - {0xed63a231d4c4fb27, 0x4ca7aaa863ee4bdd}, - {0x945e455f24fb1cf8, 0x8fe8caa93e74ef6a}, - {0xb975d6b6ee39e436, 0xb3e2fd538e122b44}, - {0xe7d34c64a9c85d44, 0x60dbbca87196b616}, - {0x90e40fbeea1d3a4a, 0xbc8955e946fe31cd}, - {0xb51d13aea4a488dd, 0x6babab6398bdbe41}, - {0xe264589a4dcdab14, 0xc696963c7eed2dd1}, - {0x8d7eb76070a08aec, 0xfc1e1de5cf543ca2}, - {0xb0de65388cc8ada8, 0x3b25a55f43294bcb}, - {0xdd15fe86affad912, 0x49ef0eb713f39ebe}, - {0x8a2dbf142dfcc7ab, 0x6e3569326c784337}, - {0xacb92ed9397bf996, 0x49c2c37f07965404}, - {0xd7e77a8f87daf7fb, 0xdc33745ec97be906}, - {0x86f0ac99b4e8dafd, 0x69a028bb3ded71a3}, - {0xa8acd7c0222311bc, 0xc40832ea0d68ce0c}, - {0xd2d80db02aabd62b, 0xf50a3fa490c30190}, - {0x83c7088e1aab65db, 0x792667c6da79e0fa}, - {0xa4b8cab1a1563f52, 0x577001b891185938}, - {0xcde6fd5e09abcf26, 0xed4c0226b55e6f86}, - {0x80b05e5ac60b6178, 0x544f8158315b05b4}, - {0xa0dc75f1778e39d6, 0x696361ae3db1c721}, - {0xc913936dd571c84c, 0x03bc3a19cd1e38e9}, - {0xfb5878494ace3a5f, 0x04ab48a04065c723}, - {0x9d174b2dcec0e47b, 0x62eb0d64283f9c76}, - {0xc45d1df942711d9a, 0x3ba5d0bd324f8394}, - {0xf5746577930d6500, 0xca8f44ec7ee36479}, - {0x9968bf6abbe85f20, 0x7e998b13cf4e1ecb}, - {0xbfc2ef456ae276e8, 0x9e3fedd8c321a67e}, - {0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101e}, - {0x95d04aee3b80ece5, 0xbba1f1d158724a12}, - {0xbb445da9ca61281f, 0x2a8a6e45ae8edc97}, - {0xea1575143cf97226, 0xf52d09d71a3293bd}, - {0x924d692ca61be758, 0x593c2626705f9c56}, - {0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836c}, - {0xe498f455c38b997a, 0x0b6dfb9c0f956447}, - {0x8edf98b59a373fec, 0x4724bd4189bd5eac}, - {0xb2977ee300c50fe7, 0x58edec91ec2cb657}, - {0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ed}, - {0x8b865b215899f46c, 0xbd79e0d20082ee74}, - {0xae67f1e9aec07187, 0xecd8590680a3aa11}, - {0xda01ee641a708de9, 0xe80e6f4820cc9495}, - {0x884134fe908658b2, 0x3109058d147fdcdd}, - {0xaa51823e34a7eede, 0xbd4b46f0599fd415}, - {0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91a}, - {0x850fadc09923329e, 0x03e2cf6bc604ddb0}, - {0xa6539930bf6bff45, 0x84db8346b786151c}, - {0xcfe87f7cef46ff16, 0xe612641865679a63}, - {0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07e}, - {0xa26da3999aef7749, 0xe3be5e330f38f09d}, - {0xcb090c8001ab551c, 0x5cadf5bfd3072cc5}, - {0xfdcb4fa002162a63, 0x73d9732fc7c8f7f6}, - {0x9e9f11c4014dda7e, 0x2867e7fddcdd9afa}, - {0xc646d63501a1511d, 0xb281e1fd541501b8}, - {0xf7d88bc24209a565, 0x1f225a7ca91a4226}, - {0x9ae757596946075f, 0x3375788de9b06958}, - {0xc1a12d2fc3978937, 0x0052d6b1641c83ae}, - {0xf209787bb47d6b84, 0xc0678c5dbd23a49a}, - {0x9745eb4d50ce6332, 0xf840b7ba963646e0}, - {0xbd176620a501fbff, 0xb650e5a93bc3d898}, - {0xec5d3fa8ce427aff, 0xa3e51f138ab4cebe}, - {0x93ba47c980e98cdf, 0xc66f336c36b10137}, - {0xb8a8d9bbe123f017, 0xb80b0047445d4184}, - {0xe6d3102ad96cec1d, 0xa60dc059157491e5}, - {0x9043ea1ac7e41392, 0x87c89837ad68db2f}, - {0xb454e4a179dd1877, 0x29babe4598c311fb}, - {0xe16a1dc9d8545e94, 0xf4296dd6fef3d67a}, - {0x8ce2529e2734bb1d, 0x1899e4a65f58660c}, - {0xb01ae745b101e9e4, 0x5ec05dcff72e7f8f}, - {0xdc21a1171d42645d, 0x76707543f4fa1f73}, - {0x899504ae72497eba, 0x6a06494a791c53a8}, - {0xabfa45da0edbde69, 0x0487db9d17636892}, - {0xd6f8d7509292d603, 0x45a9d2845d3c42b6}, - {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b2}, - {0xa7f26836f282b732, 0x8e6cac7768d7141e}, - {0xd1ef0244af2364ff, 0x3207d795430cd926}, - {0x8335616aed761f1f, 0x7f44e6bd49e807b8}, - {0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a6}, - {0xcd036837130890a1, 0x36dba887c37a8c0f}, - {0x802221226be55a64, 0xc2494954da2c9789}, - {0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6c}, - {0xc83553c5c8965d3d, 0x6f92829494e5acc7}, - {0xfa42a8b73abbf48c, 0xcb772339ba1f17f9}, - {0x9c69a97284b578d7, 0xff2a760414536efb}, - {0xc38413cf25e2d70d, 0xfef5138519684aba}, - {0xf46518c2ef5b8cd1, 0x7eb258665fc25d69}, - {0x98bf2f79d5993802, 0xef2f773ffbd97a61}, - {0xbeeefb584aff8603, 0xaafb550ffacfd8fa}, - {0xeeaaba2e5dbf6784, 0x95ba2a53f983cf38}, - {0x952ab45cfa97a0b2, 0xdd945a747bf26183}, - {0xba756174393d88df, 0x94f971119aeef9e4}, - {0xe912b9d1478ceb17, 0x7a37cd5601aab85d}, - {0x91abb422ccb812ee, 0xac62e055c10ab33a}, - {0xb616a12b7fe617aa, 0x577b986b314d6009}, - {0xe39c49765fdf9d94, 0xed5a7e85fda0b80b}, - {0x8e41ade9fbebc27d, 0x14588f13be847307}, - {0xb1d219647ae6b31c, 0x596eb2d8ae258fc8}, - {0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bb}, - {0x8aec23d680043bee, 0x25de7bb9480d5854}, - {0xada72ccc20054ae9, 0xaf561aa79a10ae6a}, - {0xd910f7ff28069da4, 0x1b2ba1518094da04}, - {0x87aa9aff79042286, 0x90fb44d2f05d0842}, - {0xa99541bf57452b28, 0x353a1607ac744a53}, - {0xd3fa922f2d1675f2, 0x42889b8997915ce8}, - {0x847c9b5d7c2e09b7, 0x69956135febada11}, - {0xa59bc234db398c25, 0x43fab9837e699095}, - {0xcf02b2c21207ef2e, 0x94f967e45e03f4bb}, - {0x8161afb94b44f57d, 0x1d1be0eebac278f5}, - {0xa1ba1ba79e1632dc, 0x6462d92a69731732}, - {0xca28a291859bbf93, 0x7d7b8f7503cfdcfe}, - {0xfcb2cb35e702af78, 0x5cda735244c3d43e}, - {0x9defbf01b061adab, 0x3a0888136afa64a7}, - {0xc56baec21c7a1916, 0x088aaa1845b8fdd0}, - {0xf6c69a72a3989f5b, 0x8aad549e57273d45}, - {0x9a3c2087a63f6399, 0x36ac54e2f678864b}, - {0xc0cb28a98fcf3c7f, 0x84576a1bb416a7dd}, - {0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d5}, - {0x969eb7c47859e743, 0x9f644ae5a4b1b325}, - {0xbc4665b596706114, 0x873d5d9f0dde1fee}, - {0xeb57ff22fc0c7959, 0xa90cb506d155a7ea}, - {0x9316ff75dd87cbd8, 0x09a7f12442d588f2}, - {0xb7dcbf5354e9bece, 0x0c11ed6d538aeb2f}, - {0xe5d3ef282a242e81, 0x8f1668c8a86da5fa}, - {0x8fa475791a569d10, 0xf96e017d694487bc}, - {0xb38d92d760ec4455, 0x37c981dcc395a9ac}, - {0xe070f78d3927556a, 0x85bbe253f47b1417}, - {0x8c469ab843b89562, 0x93956d7478ccec8e}, - {0xaf58416654a6babb, 0x387ac8d1970027b2}, - {0xdb2e51bfe9d0696a, 0x06997b05fcc0319e}, - {0x88fcf317f22241e2, 0x441fece3bdf81f03}, - {0xab3c2fddeeaad25a, 0xd527e81cad7626c3}, - {0xd60b3bd56a5586f1, 0x8a71e223d8d3b074}, - {0x85c7056562757456, 0xf6872d5667844e49}, - {0xa738c6bebb12d16c, 0xb428f8ac016561db}, - {0xd106f86e69d785c7, 0xe13336d701beba52}, - {0x82a45b450226b39c, 0xecc0024661173473}, - {0xa34d721642b06084, 0x27f002d7f95d0190}, - {0xcc20ce9bd35c78a5, 0x31ec038df7b441f4}, - {0xff290242c83396ce, 0x7e67047175a15271}, - {0x9f79a169bd203e41, 0x0f0062c6e984d386}, - {0xc75809c42c684dd1, 0x52c07b78a3e60868}, - {0xf92e0c3537826145, 0xa7709a56ccdf8a82}, - {0x9bbcc7a142b17ccb, 0x88a66076400bb691}, - {0xc2abf989935ddbfe, 0x6acff893d00ea435}, - {0xf356f7ebf83552fe, 0x0583f6b8c4124d43}, - {0x98165af37b2153de, 0xc3727a337a8b704a}, - {0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5c}, - {0xeda2ee1c7064130c, 0x1162def06f79df73}, - {0x9485d4d1c63e8be7, 0x8addcb5645ac2ba8}, - {0xb9a74a0637ce2ee1, 0x6d953e2bd7173692}, - {0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0437}, - {0x910ab1d4db9914a0, 0x1d9c9892400a22a2}, - {0xb54d5e4a127f59c8, 0x2503beb6d00cab4b}, - {0xe2a0b5dc971f303a, 0x2e44ae64840fd61d}, - {0x8da471a9de737e24, 0x5ceaecfed289e5d2}, - {0xb10d8e1456105dad, 0x7425a83e872c5f47}, - {0xdd50f1996b947518, 0xd12f124e28f77719}, - {0x8a5296ffe33cc92f, 0x82bd6b70d99aaa6f}, - {0xace73cbfdc0bfb7b, 0x636cc64d1001550b}, - {0xd8210befd30efa5a, 0x3c47f7e05401aa4e}, - {0x8714a775e3e95c78, 0x65acfaec34810a71}, - {0xa8d9d1535ce3b396, 0x7f1839a741a14d0d}, - {0xd31045a8341ca07c, 0x1ede48111209a050}, - {0x83ea2b892091e44d, 0x934aed0aab460432}, - {0xa4e4b66b68b65d60, 0xf81da84d5617853f}, - {0xce1de40642e3f4b9, 0x36251260ab9d668e}, - {0x80d2ae83e9ce78f3, 0xc1d72b7c6b426019}, - {0xa1075a24e4421730, 0xb24cf65b8612f81f}, - {0xc94930ae1d529cfc, 0xdee033f26797b627}, - {0xfb9b7cd9a4a7443c, 0x169840ef017da3b1}, - {0x9d412e0806e88aa5, 0x8e1f289560ee864e}, - {0xc491798a08a2ad4e, 0xf1a6f2bab92a27e2}, - {0xf5b5d7ec8acb58a2, 0xae10af696774b1db}, - {0x9991a6f3d6bf1765, 0xacca6da1e0a8ef29}, - {0xbff610b0cc6edd3f, 0x17fd090a58d32af3}, - {0xeff394dcff8a948e, 0xddfc4b4cef07f5b0}, - {0x95f83d0a1fb69cd9, 0x4abdaf101564f98e}, - {0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f1}, - {0xea53df5fd18d5513, 0x84c86189216dc5ed}, - {0x92746b9be2f8552c, 0x32fd3cf5b4e49bb4}, - {0xb7118682dbb66a77, 0x3fbc8c33221dc2a1}, - {0xe4d5e82392a40515, 0x0fabaf3feaa5334a}, - {0x8f05b1163ba6832d, 0x29cb4d87f2a7400e}, - {0xb2c71d5bca9023f8, 0x743e20e9ef511012}, - {0xdf78e4b2bd342cf6, 0x914da9246b255416}, - {0x8bab8eefb6409c1a, 0x1ad089b6c2f7548e}, - {0xae9672aba3d0c320, 0xa184ac2473b529b1}, - {0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741e}, - {0x8865899617fb1871, 0x7e2fa67c7a658892}, - {0xaa7eebfb9df9de8d, 0xddbb901b98feeab7}, - {0xd51ea6fa85785631, 0x552a74227f3ea565}, - {0x8533285c936b35de, 0xd53a88958f87275f}, - {0xa67ff273b8460356, 0x8a892abaf368f137}, - {0xd01fef10a657842c, 0x2d2b7569b0432d85}, - {0x8213f56a67f6b29b, 0x9c3b29620e29fc73}, - {0xa298f2c501f45f42, 0x8349f3ba91b47b8f}, - {0xcb3f2f7642717713, 0x241c70a936219a73}, - {0xfe0efb53d30dd4d7, 0xed238cd383aa0110}, - {0x9ec95d1463e8a506, 0xf4363804324a40aa}, - {0xc67bb4597ce2ce48, 0xb143c6053edcd0d5}, - {0xf81aa16fdc1b81da, 0xdd94b7868e94050a}, - {0x9b10a4e5e9913128, 0xca7cf2b4191c8326}, - {0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f0}, - {0xf24a01a73cf2dccf, 0xbc633b39673c8cec}, - {0x976e41088617ca01, 0xd5be0503e085d813}, - {0xbd49d14aa79dbc82, 0x4b2d8644d8a74e18}, - {0xec9c459d51852ba2, 0xddf8e7d60ed1219e}, - {0x93e1ab8252f33b45, 0xcabb90e5c942b503}, - {0xb8da1662e7b00a17, 0x3d6a751f3b936243}, - {0xe7109bfba19c0c9d, 0x0cc512670a783ad4}, - {0x906a617d450187e2, 0x27fb2b80668b24c5}, - {0xb484f9dc9641e9da, 0xb1f9f660802dedf6}, - {0xe1a63853bbd26451, 0x5e7873f8a0396973}, - {0x8d07e33455637eb2, 0xdb0b487b6423e1e8}, - {0xb049dc016abc5e5f, 0x91ce1a9a3d2cda62}, - {0xdc5c5301c56b75f7, 0x7641a140cc7810fb}, - {0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9d}, - {0xac2820d9623bf429, 0x546345fa9fbdcd44}, - {0xd732290fbacaf133, 0xa97c177947ad4095}, - {0x867f59a9d4bed6c0, 0x49ed8eabcccc485d}, - {0xa81f301449ee8c70, 0x5c68f256bfff5a74}, - {0xd226fc195c6a2f8c, 0x73832eec6fff3111}, - {0x83585d8fd9c25db7, 0xc831fd53c5ff7eab}, - {0xa42e74f3d032f525, 0xba3e7ca8b77f5e55}, - {0xcd3a1230c43fb26f, 0x28ce1bd2e55f35eb}, - {0x80444b5e7aa7cf85, 0x7980d163cf5b81b3}, - {0xa0555e361951c366, 0xd7e105bcc332621f}, - {0xc86ab5c39fa63440, 0x8dd9472bf3fefaa7}, - {0xfa856334878fc150, 0xb14f98f6f0feb951}, - {0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d3}, - {0xc3b8358109e84f07, 0x0a862f80ec4700c8}, - {0xf4a642e14c6262c8, 0xcd27bb612758c0fa}, - {0x98e7e9cccfbd7dbd, 0x8038d51cb897789c}, - {0xbf21e44003acdd2c, 0xe0470a63e6bd56c3}, - {0xeeea5d5004981478, 0x1858ccfce06cac74}, - {0x95527a5202df0ccb, 0x0f37801e0c43ebc8}, - {0xbaa718e68396cffd, 0xd30560258f54e6ba}, - {0xe950df20247c83fd, 0x47c6b82ef32a2069}, - {0x91d28b7416cdd27e, 0x4cdc331d57fa5441}, - {0xb6472e511c81471d, 0xe0133fe4adf8e952}, - {0xe3d8f9e563a198e5, 0x58180fddd97723a6}, - {0x8e679c2f5e44ff8f, 0x570f09eaa7ea7648}, - {0xb201833b35d63f73, 0x2cd2cc6551e513da}, - {0xde81e40a034bcf4f, 0xf8077f7ea65e58d1}, - {0x8b112e86420f6191, 0xfb04afaf27faf782}, - {0xadd57a27d29339f6, 0x79c5db9af1f9b563}, - {0xd94ad8b1c7380874, 0x18375281ae7822bc}, - {0x87cec76f1c830548, 0x8f2293910d0b15b5}, - {0xa9c2794ae3a3c69a, 0xb2eb3875504ddb22}, - {0xd433179d9c8cb841, 0x5fa60692a46151eb}, - {0x849feec281d7f328, 0xdbc7c41ba6bcd333}, - {0xa5c7ea73224deff3, 0x12b9b522906c0800}, - {0xcf39e50feae16bef, 0xd768226b34870a00}, - {0x81842f29f2cce375, 0xe6a1158300d46640}, - {0xa1e53af46f801c53, 0x60495ae3c1097fd0}, - {0xca5e89b18b602368, 0x385bb19cb14bdfc4}, - {0xfcf62c1dee382c42, 0x46729e03dd9ed7b5}, - {0x9e19db92b4e31ba9, 0x6c07a2c26a8346d1}, - {0xc5a05277621be293, 0xc7098b7305241885}, - {0xf70867153aa2db38, 0xb8cbee4fc66d1ea7} -#else - {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b}, - {0xce5d73ff402d98e3, 0xfb0a3d212dc81290}, - {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f}, - {0x86a8d39ef77164bc, 0xae5dff9c02033198}, - {0xd98ddaee19068c76, 0x3badd624dd9b0958}, - {0xafbd2350644eeacf, 0xe5d1929ef90898fb}, - {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2}, - {0xe55990879ddcaabd, 0xcc420a6a101d0516}, - {0xb94470938fa89bce, 0xf808e40e8d5b3e6a}, - {0x95a8637627989aad, 0xdde7001379a44aa9}, - {0xf1c90080baf72cb1, 0x5324c68b12dd6339}, - {0xc350000000000000, 0x0000000000000000}, - {0x9dc5ada82b70b59d, 0xf020000000000000}, - {0xfee50b7025c36a08, 0x02f236d04753d5b4}, - {0xcde6fd5e09abcf26, 0xed4c0226b55e6f86}, - {0xa6539930bf6bff45, 0x84db8346b786151c}, - {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b2}, - {0xd910f7ff28069da4, 0x1b2ba1518094da04}, - {0xaf58416654a6babb, 0x387ac8d1970027b2}, - {0x8da471a9de737e24, 0x5ceaecfed289e5d2}, - {0xe4d5e82392a40515, 0x0fabaf3feaa5334a}, - {0xb8da1662e7b00a17, 0x3d6a751f3b936243}, - {0x95527a5202df0ccb, 0x0f37801e0c43ebc8} +#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wnarrowing" #endif + // Binary exponents of pow(10, k), for k = -348, -340, ..., 340, corresponding + // to significands above. + static constexpr int16_t pow10_exponents[87] = { + -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, -954, + -927, -901, -874, -847, -821, -794, -768, -741, -715, -688, -661, + -635, -608, -582, -555, -529, -502, -475, -449, -422, -396, -369, + -343, -316, -289, -263, -236, -210, -183, -157, -130, -103, -77, + -50, -24, 3, 30, 56, 83, 109, 136, 162, 189, 216, + 242, 269, 295, 322, 348, 375, 402, 428, 455, 481, 508, + 534, 561, 588, 614, 641, 667, 694, 720, 747, 774, 800, + 827, 853, 880, 907, 933, 960, 986, 1013, 1039, 1066}; +#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409 +# pragma GCC diagnostic pop +#endif + + static constexpr uint64_t power_of_10_64[20] = { + 1, FMT_POWERS_OF_10(1ULL), FMT_POWERS_OF_10(1000000000ULL), + 10000000000000000000ULL}; }; -#if !FMT_USE_FULL_CACHE_DRAGONBOX -template -const uint64_t basic_data::powers_of_5_64[] = { - 0x0000000000000001, 0x0000000000000005, 0x0000000000000019, - 0x000000000000007d, 0x0000000000000271, 0x0000000000000c35, - 0x0000000000003d09, 0x000000000001312d, 0x000000000005f5e1, - 0x00000000001dcd65, 0x00000000009502f9, 0x0000000002e90edd, - 0x000000000e8d4a51, 0x0000000048c27395, 0x000000016bcc41e9, - 0x000000071afd498d, 0x0000002386f26fc1, 0x000000b1a2bc2ec5, - 0x000003782dace9d9, 0x00001158e460913d, 0x000056bc75e2d631, - 0x0001b1ae4d6e2ef5, 0x000878678326eac9, 0x002a5a058fc295ed, - 0x00d3c21bcecceda1, 0x0422ca8b0a00a425, 0x14adf4b7320334b9}; +// This is a struct rather than an alias to avoid shadowing warnings in gcc. +struct impl_data : basic_impl_data<> {}; +#if __cplusplus < 201703L template -const uint32_t basic_data::dragonbox_pow10_recovery_errors[] = { - 0x50001400, 0x54044100, 0x54014555, 0x55954415, 0x54115555, 0x00000001, - 0x50000000, 0x00104000, 0x54010004, 0x05004001, 0x55555544, 0x41545555, - 0x54040551, 0x15445545, 0x51555514, 0x10000015, 0x00101100, 0x01100015, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x04450514, 0x45414110, - 0x55555145, 0x50544050, 0x15040155, 0x11054140, 0x50111514, 0x11451454, - 0x00400541, 0x00000000, 0x55555450, 0x10056551, 0x10054011, 0x55551014, - 0x69514555, 0x05151109, 0x00155555}; +constexpr uint64_t basic_impl_data::pow10_significands[]; +template constexpr int16_t basic_impl_data::pow10_exponents[]; +template constexpr uint64_t basic_impl_data::power_of_10_64[]; #endif -template -const char basic_data::foreground_color[] = "\x1b[38;2;"; -template -const char basic_data::background_color[] = "\x1b[48;2;"; -template const char basic_data::reset_color[] = "\x1b[0m"; -template const wchar_t basic_data::wreset_color[] = L"\x1b[0m"; -template const char basic_data::signs[] = {0, '-', '+', ' '}; -template -const char basic_data::left_padding_shifts[] = {31, 31, 0, 1, 0}; -template -const char basic_data::right_padding_shifts[] = {0, 31, 0, 1, 0}; - template struct bits { static FMT_CONSTEXPR_DECL const int value = static_cast(sizeof(T) * std::numeric_limits::digits); }; -class fp; -template fp normalize(fp value); - -// Lower (upper) boundary is a value half way between a floating-point value -// and its predecessor (successor). Boundaries have the same exponent as the -// value so only significands are stored. -struct boundaries { - uint64_t lower; - uint64_t upper; -}; - -// A handmade floating-point number f * pow(2, e). -class fp { - private: - using significand_type = uint64_t; - - template - using is_supported_float = bool_constant; +// Returns the number of significand bits in Float excluding the implicit bit. +template constexpr int num_significand_bits() { + // Subtract 1 to account for an implicit most significant bit in the + // normalized form. + return std::numeric_limits::digits - 1; +} - public: - significand_type f; +// A floating-point number f * pow(2, e). +struct fp { + uint64_t f; int e; - // All sizes are in bits. - // Subtract 1 to account for an implicit most significant bit in the - // normalized form. - static FMT_CONSTEXPR_DECL const int double_significand_size = - std::numeric_limits::digits - 1; - static FMT_CONSTEXPR_DECL const uint64_t implicit_bit = - 1ULL << double_significand_size; - static FMT_CONSTEXPR_DECL const int significand_size = - bits::value; + static constexpr const int num_significand_bits = bits::value; - fp() : f(0), e(0) {} - fp(uint64_t f_val, int e_val) : f(f_val), e(e_val) {} + constexpr fp() : f(0), e(0) {} + constexpr fp(uint64_t f_val, int e_val) : f(f_val), e(e_val) {} - // Constructs fp from an IEEE754 double. It is a template to prevent compile - // errors on platforms where double is not IEEE754. - template explicit fp(Double d) { assign(d); } + // Constructs fp from an IEEE754 floating-point number. It is a template to + // prevent compile errors on systems where n is not IEEE754. + template explicit FMT_CONSTEXPR fp(Float n) { assign(n); } + + template + using is_supported = bool_constant; // Assigns d to this and return true iff predecessor is closer than successor. - template ::value)> - bool assign(Float d) { + template ::value)> + FMT_CONSTEXPR bool assign(Float n) { // Assume float is in the format [sign][exponent][significand]. - using limits = std::numeric_limits; - const int float_significand_size = limits::digits - 1; - const int exponent_size = - bits::value - float_significand_size - 1; // -1 for sign - const uint64_t float_implicit_bit = 1ULL << float_significand_size; - const uint64_t significand_mask = float_implicit_bit - 1; - const uint64_t exponent_mask = (~0ULL >> 1) & ~significand_mask; - const int exponent_bias = (1 << exponent_size) - limits::max_exponent - 1; + const int num_float_significand_bits = + detail::num_significand_bits(); + const uint64_t implicit_bit = 1ULL << num_float_significand_bits; + const uint64_t significand_mask = implicit_bit - 1; constexpr bool is_double = sizeof(Float) == sizeof(uint64_t); - auto u = bit_cast>(d); + auto u = bit_cast>(n); f = u & significand_mask; + const uint64_t exponent_mask = (~0ULL >> 1) & ~significand_mask; int biased_e = - static_cast((u & exponent_mask) >> float_significand_size); - // Predecessor is closer if d is a normalized power of 2 (f == 0) other than - // the smallest normalized number (biased_e > 1). + static_cast((u & exponent_mask) >> num_float_significand_bits); + // The predecessor is closer if n is a normalized power of 2 (f == 0) other + // than the smallest normalized number (biased_e > 1). bool is_predecessor_closer = f == 0 && biased_e > 1; if (biased_e != 0) - f += float_implicit_bit; + f += implicit_bit; else biased_e = 1; // Subnormals use biased exponent 1 (min exponent). - e = biased_e - exponent_bias - float_significand_size; + const int exponent_bias = std::numeric_limits::max_exponent - 1; + e = biased_e - exponent_bias - num_float_significand_bits; return is_predecessor_closer; } - template ::value)> + template ::value)> bool assign(Float) { - *this = fp(); + FMT_ASSERT(false, ""); return false; } }; // Normalizes the value converted from double and multiplied by (1 << SHIFT). -template fp normalize(fp value) { +template FMT_CONSTEXPR fp normalize(fp value) { // Handle subnormals. - const auto shifted_implicit_bit = fp::implicit_bit << SHIFT; + const uint64_t implicit_bit = 1ULL << num_significand_bits(); + const auto shifted_implicit_bit = implicit_bit << SHIFT; while ((value.f & shifted_implicit_bit) == 0) { value.f <<= 1; --value.e; } // Subtract 1 to account for hidden bit. const auto offset = - fp::significand_size - fp::double_significand_size - SHIFT - 1; + fp::num_significand_bits - num_significand_bits() - SHIFT - 1; value.f <<= offset; value.e -= offset; return value; @@ -1175,7 +304,7 @@ template fp normalize(fp value) { inline bool operator==(fp x, fp y) { return x.f == y.f && x.e == y.e; } // Computes lhs * rhs / pow(2, 64) rounded to nearest with half-up tie breaking. -inline uint64_t multiply(uint64_t lhs, uint64_t rhs) { +FMT_CONSTEXPR inline uint64_t multiply(uint64_t lhs, uint64_t rhs) { #if FMT_USE_INT128 auto product = static_cast<__uint128_t>(lhs) * rhs; auto f = static_cast(product >> 64); @@ -1192,15 +321,18 @@ inline uint64_t multiply(uint64_t lhs, uint64_t rhs) { #endif } -inline fp operator*(fp x, fp y) { return {multiply(x.f, y.f), x.e + y.e + 64}; } +FMT_CONSTEXPR inline fp operator*(fp x, fp y) { + return {multiply(x.f, y.f), x.e + y.e + 64}; +} // Returns a cached power of 10 `c_k = c_k.f * pow(2, c_k.e)` such that its // (binary) exponent satisfies `min_exponent <= c_k.e <= min_exponent + 28`. -inline fp get_cached_power(int min_exponent, int& pow10_exponent) { +FMT_CONSTEXPR inline fp get_cached_power(int min_exponent, + int& pow10_exponent) { const int shift = 32; - const auto significand = static_cast(data::log10_2_significand); + const auto significand = static_cast(log10_2_significand); int index = static_cast( - ((min_exponent + fp::significand_size - 1) * (significand >> shift) + + ((min_exponent + fp::num_significand_bits - 1) * (significand >> shift) + ((int64_t(1) << shift) - 1)) // ceil >> 32 // arithmetic shift ); @@ -1210,8 +342,8 @@ inline fp get_cached_power(int min_exponent, int& pow10_exponent) { const int dec_exp_step = 8; index = (index - first_dec_exp - 1) / dec_exp_step + 1; pow10_exponent = first_dec_exp + index * dec_exp_step; - return {data::grisu_pow10_significands[index], - data::grisu_pow10_exponents[index]}; + return {impl_data::pow10_significands[index], + impl_data::pow10_exponents[index]}; } // A simple accumulator to hold the sums of terms in bigint::square if uint128_t @@ -1220,15 +352,17 @@ struct accumulator { uint64_t lower; uint64_t upper; - accumulator() : lower(0), upper(0) {} - explicit operator uint32_t() const { return static_cast(lower); } + constexpr accumulator() : lower(0), upper(0) {} + constexpr explicit operator uint32_t() const { + return static_cast(lower); + } - void operator+=(uint64_t n) { + FMT_CONSTEXPR void operator+=(uint64_t n) { lower += n; if (lower < n) ++upper; } - void operator>>=(int shift) { - assert(shift == 32); + FMT_CONSTEXPR void operator>>=(int shift) { + FMT_ASSERT(shift == 32, ""); (void)shift; lower = (upper << 32) | (lower >> 32); upper >>= 32; @@ -1245,27 +379,31 @@ class bigint { basic_memory_buffer bigits_; int exp_; - bigit operator[](int index) const { return bigits_[to_unsigned(index)]; } - bigit& operator[](int index) { return bigits_[to_unsigned(index)]; } + FMT_CONSTEXPR20 bigit operator[](int index) const { + return bigits_[to_unsigned(index)]; + } + FMT_CONSTEXPR20 bigit& operator[](int index) { + return bigits_[to_unsigned(index)]; + } static FMT_CONSTEXPR_DECL const int bigit_bits = bits::value; friend struct formatter; - void subtract_bigits(int index, bigit other, bigit& borrow) { + FMT_CONSTEXPR20 void subtract_bigits(int index, bigit other, bigit& borrow) { auto result = static_cast((*this)[index]) - other - borrow; (*this)[index] = static_cast(result); borrow = static_cast(result >> (bigit_bits * 2 - 1)); } - void remove_leading_zeros() { + FMT_CONSTEXPR20 void remove_leading_zeros() { int num_bigits = static_cast(bigits_.size()) - 1; while (num_bigits > 0 && (*this)[num_bigits] == 0) --num_bigits; bigits_.resize(to_unsigned(num_bigits + 1)); } // Computes *this -= other assuming aligned bigints and *this >= other. - void subtract_aligned(const bigint& other) { + FMT_CONSTEXPR20 void subtract_aligned(const bigint& other) { FMT_ASSERT(other.exp_ >= exp_, "unaligned bigints"); FMT_ASSERT(compare(*this, other) >= 0, ""); bigit borrow = 0; @@ -1276,7 +414,7 @@ class bigint { remove_leading_zeros(); } - void multiply(uint32_t value) { + FMT_CONSTEXPR20 void multiply(uint32_t value) { const double_bigit wide_value = value; bigit carry = 0; for (size_t i = 0, n = bigits_.size(); i < n; ++i) { @@ -1287,7 +425,7 @@ class bigint { if (carry != 0) bigits_.push_back(carry); } - void multiply(uint64_t value) { + FMT_CONSTEXPR20 void multiply(uint64_t value) { const bigit mask = ~bigit(0); const double_bigit lower = value & mask; const double_bigit upper = value >> bigit_bits; @@ -1305,14 +443,16 @@ class bigint { } public: - bigint() : exp_(0) {} + FMT_CONSTEXPR20 bigint() : exp_(0) {} explicit bigint(uint64_t n) { assign(n); } - ~bigint() { assert(bigits_.capacity() <= bigits_capacity); } + FMT_CONSTEXPR20 ~bigint() { + FMT_ASSERT(bigits_.capacity() <= bigits_capacity, ""); + } bigint(const bigint&) = delete; void operator=(const bigint&) = delete; - void assign(const bigint& other) { + FMT_CONSTEXPR20 void assign(const bigint& other) { auto size = other.bigits_.size(); bigits_.resize(size); auto data = other.bigits_.data(); @@ -1320,7 +460,7 @@ class bigint { exp_ = other.exp_; } - void assign(uint64_t n) { + FMT_CONSTEXPR20 void assign(uint64_t n) { size_t num_bigits = 0; do { bigits_[num_bigits++] = n & ~bigit(0); @@ -1330,10 +470,12 @@ class bigint { exp_ = 0; } - int num_bigits() const { return static_cast(bigits_.size()) + exp_; } + FMT_CONSTEXPR20 int num_bigits() const { + return static_cast(bigits_.size()) + exp_; + } - FMT_NOINLINE bigint& operator<<=(int shift) { - assert(shift >= 0); + FMT_NOINLINE FMT_CONSTEXPR20 bigint& operator<<=(int shift) { + FMT_ASSERT(shift >= 0, ""); exp_ += shift / bigit_bits; shift %= bigit_bits; if (shift == 0) return *this; @@ -1347,13 +489,13 @@ class bigint { return *this; } - template bigint& operator*=(Int value) { + template FMT_CONSTEXPR20 bigint& operator*=(Int value) { FMT_ASSERT(value > 0, ""); multiply(uint32_or_64_or_128_t(value)); return *this; } - friend int compare(const bigint& lhs, const bigint& rhs) { + friend FMT_CONSTEXPR20 int compare(const bigint& lhs, const bigint& rhs) { int num_lhs_bigits = lhs.num_bigits(), num_rhs_bigits = rhs.num_bigits(); if (num_lhs_bigits != num_rhs_bigits) return num_lhs_bigits > num_rhs_bigits ? 1 : -1; @@ -1370,8 +512,8 @@ class bigint { } // Returns compare(lhs1 + lhs2, rhs). - friend int add_compare(const bigint& lhs1, const bigint& lhs2, - const bigint& rhs) { + friend FMT_CONSTEXPR20 int add_compare(const bigint& lhs1, const bigint& lhs2, + const bigint& rhs) { int max_lhs_bigits = (std::max)(lhs1.num_bigits(), lhs2.num_bigits()); int num_rhs_bigits = rhs.num_bigits(); if (max_lhs_bigits + 1 < num_rhs_bigits) return -1; @@ -1394,8 +536,8 @@ class bigint { } // Assigns pow(10, exp) to this bigint. - void assign_pow10(int exp) { - assert(exp >= 0); + FMT_CONSTEXPR20 void assign_pow10(int exp) { + FMT_ASSERT(exp >= 0, ""); if (exp == 0) return assign(1); // Find the top bit. int bitmask = 1; @@ -1413,10 +555,10 @@ class bigint { *this <<= exp; // Multiply by pow(2, exp) by shifting. } - void square() { - basic_memory_buffer n(std::move(bigits_)); + FMT_CONSTEXPR20 void square() { int num_bigits = static_cast(bigits_.size()); int num_result_bigits = 2 * num_bigits; + basic_memory_buffer n(std::move(bigits_)); bigits_.resize(to_unsigned(num_result_bigits)); using accumulator_t = conditional_t; auto sum = accumulator_t(); @@ -1438,14 +580,13 @@ class bigint { (*this)[bigit_index] = static_cast(sum); sum >>= bits::value; } - --num_result_bigits; remove_leading_zeros(); exp_ *= 2; } // If this bigint has a bigger exponent than other, adds trailing zero to make // exponents equal. This simplifies some operations such as subtraction. - void align(const bigint& other) { + FMT_CONSTEXPR20 void align(const bigint& other) { int exp_difference = exp_ - other.exp_; if (exp_difference <= 0) return; int num_bigits = static_cast(bigits_.size()); @@ -1458,7 +599,7 @@ class bigint { // Divides this bignum by divisor, assigning the remainder to this and // returning the quotient. - int divmod_assign(const bigint& divisor) { + FMT_CONSTEXPR20 int divmod_assign(const bigint& divisor) { FMT_ASSERT(this != &divisor, ""); if (compare(*this, divisor) < 0) return 0; FMT_ASSERT(divisor.bigits_[divisor.bigits_.size() - 1u] != 0, ""); @@ -1478,8 +619,9 @@ enum class round_direction { unknown, up, down }; // some number v and the error, returns whether v should be rounded up, down, or // whether the rounding direction can't be determined due to error. // error should be less than divisor / 2. -inline round_direction get_round_direction(uint64_t divisor, uint64_t remainder, - uint64_t error) { +FMT_CONSTEXPR inline round_direction get_round_direction(uint64_t divisor, + uint64_t remainder, + uint64_t error) { FMT_ASSERT(remainder < divisor, ""); // divisor - remainder won't overflow. FMT_ASSERT(error < divisor, ""); // divisor - error won't overflow. FMT_ASSERT(error < divisor - error, ""); // error * 2 won't overflow. @@ -1502,12 +644,52 @@ enum result { }; } +struct gen_digits_handler { + char* buf; + int size; + int precision; + int exp10; + bool fixed; + + FMT_CONSTEXPR digits::result on_digit(char digit, uint64_t divisor, + uint64_t remainder, uint64_t error, + bool integral) { + FMT_ASSERT(remainder < divisor, ""); + buf[size++] = digit; + if (!integral && error >= remainder) return digits::error; + if (size < precision) return digits::more; + if (!integral) { + // Check if error * 2 < divisor with overflow prevention. + // The check is not needed for the integral part because error = 1 + // and divisor > (1 << 32) there. + if (error >= divisor || error >= divisor - error) return digits::error; + } else { + FMT_ASSERT(error == 1 && divisor > 2, ""); + } + auto dir = get_round_direction(divisor, remainder, error); + if (dir != round_direction::up) + return dir == round_direction::down ? digits::done : digits::error; + ++buf[size - 1]; + for (int i = size - 1; i > 0 && buf[i] > '9'; --i) { + buf[i] = '0'; + ++buf[i - 1]; + } + if (buf[0] > '9') { + buf[0] = '1'; + if (fixed) + buf[size++] = '0'; + else + ++exp10; + } + return digits::done; + } +}; + // Generates output using the Grisu digit-gen algorithm. // error: the size of the region (lower, upper) outside of which numbers // definitely do not round to value (Delta in Grisu3). -template -FMT_ALWAYS_INLINE digits::result grisu_gen_digits(fp value, uint64_t error, - int& exp, Handler& handler) { +FMT_INLINE FMT_CONSTEXPR20 digits::result grisu_gen_digits( + fp value, uint64_t error, int& exp, gen_digits_handler& handler) { const fp one(1ULL << -value.e, value.e); // The integral part of scaled value (p1 in Grisu) = value / one. It cannot be // zero because it contains a product of two 64-bit numbers with MSB set (due @@ -1518,10 +700,28 @@ FMT_ALWAYS_INLINE digits::result grisu_gen_digits(fp value, uint64_t error, // The fractional part of scaled value (p2 in Grisu) c = value % one. uint64_t fractional = value.f & (one.f - 1); exp = count_digits(integral); // kappa in Grisu. - // Divide by 10 to prevent overflow. - auto result = handler.on_start(data::powers_of_10_64[exp - 1] << -one.e, - value.f / 10, error * 10, exp); - if (result != digits::more) return result; + // Non-fixed formats require at least one digit and no precision adjustment. + if (handler.fixed) { + // Adjust fixed precision by exponent because it is relative to decimal + // point. + int precision_offset = exp + handler.exp10; + if (precision_offset > 0 && + handler.precision > max_value() - precision_offset) { + FMT_THROW(format_error("number is too big")); + } + handler.precision += precision_offset; + // Check if precision is satisfied just by leading zeros, e.g. + // format("{:.2f}", 0.001) gives "0.00" without generating any digits. + if (handler.precision <= 0) { + if (handler.precision < 0) return digits::done; + // Divide by 10 to prevent overflow. + uint64_t divisor = impl_data::power_of_10_64[exp - 1] << -one.e; + auto dir = get_round_direction(divisor, value.f / 10, error * 10); + if (dir == round_direction::unknown) return digits::error; + handler.buf[handler.size++] = dir == round_direction::up ? '1' : '0'; + return digits::done; + } + } // Generate digits for the integral part. This can produce up to 10 digits. do { uint32_t digit = 0; @@ -1568,9 +768,9 @@ FMT_ALWAYS_INLINE digits::result grisu_gen_digits(fp value, uint64_t error, } --exp; auto remainder = (static_cast(integral) << -one.e) + fractional; - result = handler.on_digit(static_cast('0' + digit), - data::powers_of_10_64[exp] << -one.e, remainder, - error, exp, true); + auto result = handler.on_digit(static_cast('0' + digit), + impl_data::power_of_10_64[exp] << -one.e, + remainder, error, true); if (result != digits::more) return result; } while (exp > 0); // Generate digits for the fractional part. @@ -1580,74 +780,63 @@ FMT_ALWAYS_INLINE digits::result grisu_gen_digits(fp value, uint64_t error, char digit = static_cast('0' + (fractional >> -one.e)); fractional &= one.f - 1; --exp; - result = handler.on_digit(digit, one.f, fractional, error, exp, false); + auto result = handler.on_digit(digit, one.f, fractional, error, false); if (result != digits::more) return result; } } -// The fixed precision digit handler. -struct fixed_handler { - char* buf; - int size; - int precision; - int exp10; - bool fixed; +// A 128-bit integer type used internally, +struct uint128_wrapper { + uint128_wrapper() = default; - digits::result on_start(uint64_t divisor, uint64_t remainder, uint64_t error, - int& exp) { - // Non-fixed formats require at least one digit and no precision adjustment. - if (!fixed) return digits::more; - // Adjust fixed precision by exponent because it is relative to decimal - // point. - precision += exp + exp10; - // Check if precision is satisfied just by leading zeros, e.g. - // format("{:.2f}", 0.001) gives "0.00" without generating any digits. - if (precision > 0) return digits::more; - if (precision < 0) return digits::done; - auto dir = get_round_direction(divisor, remainder, error); - if (dir == round_direction::unknown) return digits::error; - buf[size++] = dir == round_direction::up ? '1' : '0'; - return digits::done; +#if FMT_USE_INT128 + uint128_t internal_; + + constexpr uint128_wrapper(uint64_t high, uint64_t low) FMT_NOEXCEPT + : internal_{static_cast(low) | + (static_cast(high) << 64)} {} + + constexpr uint128_wrapper(uint128_t u) : internal_{u} {} + + constexpr uint64_t high() const FMT_NOEXCEPT { + return uint64_t(internal_ >> 64); } + constexpr uint64_t low() const FMT_NOEXCEPT { return uint64_t(internal_); } - digits::result on_digit(char digit, uint64_t divisor, uint64_t remainder, - uint64_t error, int, bool integral) { - FMT_ASSERT(remainder < divisor, ""); - buf[size++] = digit; - if (!integral && error >= remainder) return digits::error; - if (size < precision) return digits::more; - if (!integral) { - // Check if error * 2 < divisor with overflow prevention. - // The check is not needed for the integral part because error = 1 - // and divisor > (1 << 32) there. - if (error >= divisor || error >= divisor - error) return digits::error; - } else { - FMT_ASSERT(error == 1 && divisor > 2, ""); - } - auto dir = get_round_direction(divisor, remainder, error); - if (dir != round_direction::up) - return dir == round_direction::down ? digits::done : digits::error; - ++buf[size - 1]; - for (int i = size - 1; i > 0 && buf[i] > '9'; --i) { - buf[i] = '0'; - ++buf[i - 1]; - } - if (buf[0] > '9') { - buf[0] = '1'; - if (fixed) - buf[size++] = '0'; - else - ++exp10; - } - return digits::done; + uint128_wrapper& operator+=(uint64_t n) FMT_NOEXCEPT { + internal_ += n; + return *this; } +#else + uint64_t high_; + uint64_t low_; + + constexpr uint128_wrapper(uint64_t high, uint64_t low) FMT_NOEXCEPT + : high_{high}, + low_{low} {} + + constexpr uint64_t high() const FMT_NOEXCEPT { return high_; } + constexpr uint64_t low() const FMT_NOEXCEPT { return low_; } + + uint128_wrapper& operator+=(uint64_t n) FMT_NOEXCEPT { +# if defined(_MSC_VER) && defined(_M_X64) + unsigned char carry = _addcarry_u64(0, low_, n, &low_); + _addcarry_u64(carry, high_, 0, &high_); + return *this; +# else + uint64_t sum = low_ + n; + high_ += (sum < low_ ? 1 : 0); + low_ = sum; + return *this; +# endif + } +#endif }; // Implementation of Dragonbox algorithm: https://github.com/jk-jeon/dragonbox. namespace dragonbox { // Computes 128-bit result of multiplication of two 64-bit unsigned integers. -FMT_SAFEBUFFERS inline uint128_wrapper umul128(uint64_t x, - uint64_t y) FMT_NOEXCEPT { +inline uint128_wrapper umul128(uint64_t x, uint64_t y) FMT_NOEXCEPT { #if FMT_USE_INT128 return static_cast(x) * static_cast(y); #elif defined(_MSC_VER) && defined(_M_X64) @@ -1675,8 +864,7 @@ FMT_SAFEBUFFERS inline uint128_wrapper umul128(uint64_t x, } // Computes upper 64 bits of multiplication of two 64-bit unsigned integers. -FMT_SAFEBUFFERS inline uint64_t umul128_upper64(uint64_t x, - uint64_t y) FMT_NOEXCEPT { +inline uint64_t umul128_upper64(uint64_t x, uint64_t y) FMT_NOEXCEPT { #if FMT_USE_INT128 auto p = static_cast(x) * static_cast(y); return static_cast(p >> 64); @@ -1689,8 +877,7 @@ FMT_SAFEBUFFERS inline uint64_t umul128_upper64(uint64_t x, // Computes upper 64 bits of multiplication of a 64-bit unsigned integer and a // 128-bit unsigned integer. -FMT_SAFEBUFFERS inline uint64_t umul192_upper64(uint64_t x, uint128_wrapper y) - FMT_NOEXCEPT { +inline uint64_t umul192_upper64(uint64_t x, uint128_wrapper y) FMT_NOEXCEPT { uint128_wrapper g0 = umul128(x, y.high()); g0 += umul128_upper64(x, y.low()); return g0.high(); @@ -1704,8 +891,7 @@ inline uint32_t umul96_upper32(uint32_t x, uint64_t y) FMT_NOEXCEPT { // Computes middle 64 bits of multiplication of a 64-bit unsigned integer and a // 128-bit unsigned integer. -FMT_SAFEBUFFERS inline uint64_t umul192_middle64(uint64_t x, uint128_wrapper y) - FMT_NOEXCEPT { +inline uint64_t umul192_middle64(uint64_t x, uint128_wrapper y) FMT_NOEXCEPT { uint64_t g01 = x * y.high(); uint64_t g10 = umul128_upper64(x, y.low()); return g01 + g10; @@ -1722,8 +908,7 @@ inline uint64_t umul96_lower64(uint32_t x, uint64_t y) FMT_NOEXCEPT { inline int floor_log10_pow2(int e) FMT_NOEXCEPT { FMT_ASSERT(e <= 1700 && e >= -1700, "too large exponent"); const int shift = 22; - return (e * static_cast(data::log10_2_significand >> (64 - shift))) >> - shift; + return (e * static_cast(log10_2_significand >> (64 - shift))) >> shift; } // Various fast log computations. @@ -1741,8 +926,7 @@ inline int floor_log10_pow2_minus_log10_4_over_3(int e) FMT_NOEXCEPT { FMT_ASSERT(e <= 1700 && e >= -1700, "too large exponent"); const uint64_t log10_4_over_3_fractional_digits = 0x1ffbfc2bbc780375; const int shift_amount = 22; - return (e * static_cast(data::log10_2_significand >> - (64 - shift_amount)) - + return (e * static_cast(log10_2_significand >> (64 - shift_amount)) - static_cast(log10_4_over_3_fractional_digits >> (64 - shift_amount))) >> shift_amount; @@ -1768,16 +952,52 @@ inline bool divisible_by_power_of_2(uint64_t x, int exp) FMT_NOEXCEPT { #endif } +// Table entry type for divisibility test. +template struct divtest_table_entry { + T mod_inv; + T max_quotient; +}; + // Returns true iff x is divisible by pow(5, exp). inline bool divisible_by_power_of_5(uint32_t x, int exp) FMT_NOEXCEPT { FMT_ASSERT(exp <= 10, "too large exponent"); - return x * data::divtest_table_for_pow5_32[exp].mod_inv <= - data::divtest_table_for_pow5_32[exp].max_quotient; + static constexpr const divtest_table_entry divtest_table[] = { + {0x00000001, 0xffffffff}, {0xcccccccd, 0x33333333}, + {0xc28f5c29, 0x0a3d70a3}, {0x26e978d5, 0x020c49ba}, + {0x3afb7e91, 0x0068db8b}, {0x0bcbe61d, 0x0014f8b5}, + {0x68c26139, 0x000431bd}, {0xae8d46a5, 0x0000d6bf}, + {0x22e90e21, 0x00002af3}, {0x3a2e9c6d, 0x00000897}, + {0x3ed61f49, 0x000001b7}}; + return x * divtest_table[exp].mod_inv <= divtest_table[exp].max_quotient; } inline bool divisible_by_power_of_5(uint64_t x, int exp) FMT_NOEXCEPT { FMT_ASSERT(exp <= 23, "too large exponent"); - return x * data::divtest_table_for_pow5_64[exp].mod_inv <= - data::divtest_table_for_pow5_64[exp].max_quotient; + static constexpr const divtest_table_entry divtest_table[] = { + {0x0000000000000001, 0xffffffffffffffff}, + {0xcccccccccccccccd, 0x3333333333333333}, + {0x8f5c28f5c28f5c29, 0x0a3d70a3d70a3d70}, + {0x1cac083126e978d5, 0x020c49ba5e353f7c}, + {0xd288ce703afb7e91, 0x0068db8bac710cb2}, + {0x5d4e8fb00bcbe61d, 0x0014f8b588e368f0}, + {0x790fb65668c26139, 0x000431bde82d7b63}, + {0xe5032477ae8d46a5, 0x0000d6bf94d5e57a}, + {0xc767074b22e90e21, 0x00002af31dc46118}, + {0x8e47ce423a2e9c6d, 0x0000089705f4136b}, + {0x4fa7f60d3ed61f49, 0x000001b7cdfd9d7b}, + {0x0fee64690c913975, 0x00000057f5ff85e5}, + {0x3662e0e1cf503eb1, 0x000000119799812d}, + {0xa47a2cf9f6433fbd, 0x0000000384b84d09}, + {0x54186f653140a659, 0x00000000b424dc35}, + {0x7738164770402145, 0x0000000024075f3d}, + {0xe4a4d1417cd9a041, 0x000000000734aca5}, + {0xc75429d9e5c5200d, 0x000000000170ef54}, + {0xc1773b91fac10669, 0x000000000049c977}, + {0x26b172506559ce15, 0x00000000000ec1e4}, + {0xd489e3a9addec2d1, 0x000000000002f394}, + {0x90e860bb892c8d5d, 0x000000000000971d}, + {0x502e79bf1b6f4f79, 0x0000000000001e39}, + {0xdcd618596be30fe5, 0x000000000000060b}}; + return x * divtest_table[exp].mod_inv <= divtest_table[exp].max_quotient; } // Replaces n by floor(n / pow(5, N)) returning true if and only if n is @@ -1831,7 +1051,34 @@ template <> struct cache_accessor { static uint64_t get_cached_power(int k) FMT_NOEXCEPT { FMT_ASSERT(k >= float_info::min_k && k <= float_info::max_k, "k is out of range"); - return data::dragonbox_pow10_significands_64[k - float_info::min_k]; + static constexpr const uint64_t pow10_significands[] = { + 0x81ceb32c4b43fcf5, 0xa2425ff75e14fc32, 0xcad2f7f5359a3b3f, + 0xfd87b5f28300ca0e, 0x9e74d1b791e07e49, 0xc612062576589ddb, + 0xf79687aed3eec552, 0x9abe14cd44753b53, 0xc16d9a0095928a28, + 0xf1c90080baf72cb2, 0x971da05074da7bef, 0xbce5086492111aeb, + 0xec1e4a7db69561a6, 0x9392ee8e921d5d08, 0xb877aa3236a4b44a, + 0xe69594bec44de15c, 0x901d7cf73ab0acda, 0xb424dc35095cd810, + 0xe12e13424bb40e14, 0x8cbccc096f5088cc, 0xafebff0bcb24aaff, + 0xdbe6fecebdedd5bf, 0x89705f4136b4a598, 0xabcc77118461cefd, + 0xd6bf94d5e57a42bd, 0x8637bd05af6c69b6, 0xa7c5ac471b478424, + 0xd1b71758e219652c, 0x83126e978d4fdf3c, 0xa3d70a3d70a3d70b, + 0xcccccccccccccccd, 0x8000000000000000, 0xa000000000000000, + 0xc800000000000000, 0xfa00000000000000, 0x9c40000000000000, + 0xc350000000000000, 0xf424000000000000, 0x9896800000000000, + 0xbebc200000000000, 0xee6b280000000000, 0x9502f90000000000, + 0xba43b74000000000, 0xe8d4a51000000000, 0x9184e72a00000000, + 0xb5e620f480000000, 0xe35fa931a0000000, 0x8e1bc9bf04000000, + 0xb1a2bc2ec5000000, 0xde0b6b3a76400000, 0x8ac7230489e80000, + 0xad78ebc5ac620000, 0xd8d726b7177a8000, 0x878678326eac9000, + 0xa968163f0a57b400, 0xd3c21bcecceda100, 0x84595161401484a0, + 0xa56fa5b99019a5c8, 0xcecb8f27f4200f3a, 0x813f3978f8940984, + 0xa18f07d736b90be5, 0xc9f2c9cd04674ede, 0xfc6f7c4045812296, + 0x9dc5ada82b70b59d, 0xc5371912364ce305, 0xf684df56c3e01bc6, + 0x9a130b963a6c115c, 0xc097ce7bc90715b3, 0xf0bdc21abb48db20, + 0x96769950b50d88f4, 0xbc143fa4e250eb31, 0xeb194f8e1ae525fd, + 0x92efd1b8d0cf37be, 0xb7abc627050305ad, 0xe596b7b0c643c719, + 0x8f7e32ce7bea5c6f, 0xb35dbf821ae4f38b, 0xe0352f62a19e306e}; + return pow10_significands[k - float_info::min_k]; } static carrier_uint compute_mul(carrier_uint u, @@ -1885,10 +1132,679 @@ template <> struct cache_accessor { FMT_ASSERT(k >= float_info::min_k && k <= float_info::max_k, "k is out of range"); + static constexpr const uint128_wrapper pow10_significands[] = { +#if FMT_USE_FULL_CACHE_DRAGONBOX + {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b}, + {0x9faacf3df73609b1, 0x77b191618c54e9ad}, + {0xc795830d75038c1d, 0xd59df5b9ef6a2418}, + {0xf97ae3d0d2446f25, 0x4b0573286b44ad1e}, + {0x9becce62836ac577, 0x4ee367f9430aec33}, + {0xc2e801fb244576d5, 0x229c41f793cda740}, + {0xf3a20279ed56d48a, 0x6b43527578c11110}, + {0x9845418c345644d6, 0x830a13896b78aaaa}, + {0xbe5691ef416bd60c, 0x23cc986bc656d554}, + {0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa9}, + {0x94b3a202eb1c3f39, 0x7bf7d71432f3d6aa}, + {0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc54}, + {0xe858ad248f5c22c9, 0xd1b3400f8f9cff69}, + {0x91376c36d99995be, 0x23100809b9c21fa2}, + {0xb58547448ffffb2d, 0xabd40a0c2832a78b}, + {0xe2e69915b3fff9f9, 0x16c90c8f323f516d}, + {0x8dd01fad907ffc3b, 0xae3da7d97f6792e4}, + {0xb1442798f49ffb4a, 0x99cd11cfdf41779d}, + {0xdd95317f31c7fa1d, 0x40405643d711d584}, + {0x8a7d3eef7f1cfc52, 0x482835ea666b2573}, + {0xad1c8eab5ee43b66, 0xda3243650005eed0}, + {0xd863b256369d4a40, 0x90bed43e40076a83}, + {0x873e4f75e2224e68, 0x5a7744a6e804a292}, + {0xa90de3535aaae202, 0x711515d0a205cb37}, + {0xd3515c2831559a83, 0x0d5a5b44ca873e04}, + {0x8412d9991ed58091, 0xe858790afe9486c3}, + {0xa5178fff668ae0b6, 0x626e974dbe39a873}, + {0xce5d73ff402d98e3, 0xfb0a3d212dc81290}, + {0x80fa687f881c7f8e, 0x7ce66634bc9d0b9a}, + {0xa139029f6a239f72, 0x1c1fffc1ebc44e81}, + {0xc987434744ac874e, 0xa327ffb266b56221}, + {0xfbe9141915d7a922, 0x4bf1ff9f0062baa9}, + {0x9d71ac8fada6c9b5, 0x6f773fc3603db4aa}, + {0xc4ce17b399107c22, 0xcb550fb4384d21d4}, + {0xf6019da07f549b2b, 0x7e2a53a146606a49}, + {0x99c102844f94e0fb, 0x2eda7444cbfc426e}, + {0xc0314325637a1939, 0xfa911155fefb5309}, + {0xf03d93eebc589f88, 0x793555ab7eba27cb}, + {0x96267c7535b763b5, 0x4bc1558b2f3458df}, + {0xbbb01b9283253ca2, 0x9eb1aaedfb016f17}, + {0xea9c227723ee8bcb, 0x465e15a979c1cadd}, + {0x92a1958a7675175f, 0x0bfacd89ec191eca}, + {0xb749faed14125d36, 0xcef980ec671f667c}, + {0xe51c79a85916f484, 0x82b7e12780e7401b}, + {0x8f31cc0937ae58d2, 0xd1b2ecb8b0908811}, + {0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa16}, + {0xdfbdcece67006ac9, 0x67a791e093e1d49b}, + {0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e1}, + {0xaecc49914078536d, 0x58fae9f773886e19}, + {0xda7f5bf590966848, 0xaf39a475506a899f}, + {0x888f99797a5e012d, 0x6d8406c952429604}, + {0xaab37fd7d8f58178, 0xc8e5087ba6d33b84}, + {0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a65}, + {0x855c3be0a17fcd26, 0x5cf2eea09a550680}, + {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f}, + {0xd0601d8efc57b08b, 0xf13b94daf124da27}, + {0x823c12795db6ce57, 0x76c53d08d6b70859}, + {0xa2cb1717b52481ed, 0x54768c4b0c64ca6f}, + {0xcb7ddcdda26da268, 0xa9942f5dcf7dfd0a}, + {0xfe5d54150b090b02, 0xd3f93b35435d7c4d}, + {0x9efa548d26e5a6e1, 0xc47bc5014a1a6db0}, + {0xc6b8e9b0709f109a, 0x359ab6419ca1091c}, + {0xf867241c8cc6d4c0, 0xc30163d203c94b63}, + {0x9b407691d7fc44f8, 0x79e0de63425dcf1e}, + {0xc21094364dfb5636, 0x985915fc12f542e5}, + {0xf294b943e17a2bc4, 0x3e6f5b7b17b2939e}, + {0x979cf3ca6cec5b5a, 0xa705992ceecf9c43}, + {0xbd8430bd08277231, 0x50c6ff782a838354}, + {0xece53cec4a314ebd, 0xa4f8bf5635246429}, + {0x940f4613ae5ed136, 0x871b7795e136be9a}, + {0xb913179899f68584, 0x28e2557b59846e40}, + {0xe757dd7ec07426e5, 0x331aeada2fe589d0}, + {0x9096ea6f3848984f, 0x3ff0d2c85def7622}, + {0xb4bca50b065abe63, 0x0fed077a756b53aa}, + {0xe1ebce4dc7f16dfb, 0xd3e8495912c62895}, + {0x8d3360f09cf6e4bd, 0x64712dd7abbbd95d}, + {0xb080392cc4349dec, 0xbd8d794d96aacfb4}, + {0xdca04777f541c567, 0xecf0d7a0fc5583a1}, + {0x89e42caaf9491b60, 0xf41686c49db57245}, + {0xac5d37d5b79b6239, 0x311c2875c522ced6}, + {0xd77485cb25823ac7, 0x7d633293366b828c}, + {0x86a8d39ef77164bc, 0xae5dff9c02033198}, + {0xa8530886b54dbdeb, 0xd9f57f830283fdfd}, + {0xd267caa862a12d66, 0xd072df63c324fd7c}, + {0x8380dea93da4bc60, 0x4247cb9e59f71e6e}, + {0xa46116538d0deb78, 0x52d9be85f074e609}, + {0xcd795be870516656, 0x67902e276c921f8c}, + {0x806bd9714632dff6, 0x00ba1cd8a3db53b7}, + {0xa086cfcd97bf97f3, 0x80e8a40eccd228a5}, + {0xc8a883c0fdaf7df0, 0x6122cd128006b2ce}, + {0xfad2a4b13d1b5d6c, 0x796b805720085f82}, + {0x9cc3a6eec6311a63, 0xcbe3303674053bb1}, + {0xc3f490aa77bd60fc, 0xbedbfc4411068a9d}, + {0xf4f1b4d515acb93b, 0xee92fb5515482d45}, + {0x991711052d8bf3c5, 0x751bdd152d4d1c4b}, + {0xbf5cd54678eef0b6, 0xd262d45a78a0635e}, + {0xef340a98172aace4, 0x86fb897116c87c35}, + {0x9580869f0e7aac0e, 0xd45d35e6ae3d4da1}, + {0xbae0a846d2195712, 0x8974836059cca10a}, + {0xe998d258869facd7, 0x2bd1a438703fc94c}, + {0x91ff83775423cc06, 0x7b6306a34627ddd0}, + {0xb67f6455292cbf08, 0x1a3bc84c17b1d543}, + {0xe41f3d6a7377eeca, 0x20caba5f1d9e4a94}, + {0x8e938662882af53e, 0x547eb47b7282ee9d}, + {0xb23867fb2a35b28d, 0xe99e619a4f23aa44}, + {0xdec681f9f4c31f31, 0x6405fa00e2ec94d5}, + {0x8b3c113c38f9f37e, 0xde83bc408dd3dd05}, + {0xae0b158b4738705e, 0x9624ab50b148d446}, + {0xd98ddaee19068c76, 0x3badd624dd9b0958}, + {0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d7}, + {0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4d}, + {0xd47487cc8470652b, 0x7647c32000696720}, + {0x84c8d4dfd2c63f3b, 0x29ecd9f40041e074}, + {0xa5fb0a17c777cf09, 0xf468107100525891}, + {0xcf79cc9db955c2cc, 0x7182148d4066eeb5}, + {0x81ac1fe293d599bf, 0xc6f14cd848405531}, + {0xa21727db38cb002f, 0xb8ada00e5a506a7d}, + {0xca9cf1d206fdc03b, 0xa6d90811f0e4851d}, + {0xfd442e4688bd304a, 0x908f4a166d1da664}, + {0x9e4a9cec15763e2e, 0x9a598e4e043287ff}, + {0xc5dd44271ad3cdba, 0x40eff1e1853f29fe}, + {0xf7549530e188c128, 0xd12bee59e68ef47d}, + {0x9a94dd3e8cf578b9, 0x82bb74f8301958cf}, + {0xc13a148e3032d6e7, 0xe36a52363c1faf02}, + {0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac2}, + {0x96f5600f15a7b7e5, 0x29ab103a5ef8c0ba}, + {0xbcb2b812db11a5de, 0x7415d448f6b6f0e8}, + {0xebdf661791d60f56, 0x111b495b3464ad22}, + {0x936b9fcebb25c995, 0xcab10dd900beec35}, + {0xb84687c269ef3bfb, 0x3d5d514f40eea743}, + {0xe65829b3046b0afa, 0x0cb4a5a3112a5113}, + {0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ac}, + {0xb3f4e093db73a093, 0x59ed216765690f57}, + {0xe0f218b8d25088b8, 0x306869c13ec3532d}, + {0x8c974f7383725573, 0x1e414218c73a13fc}, + {0xafbd2350644eeacf, 0xe5d1929ef90898fb}, + {0xdbac6c247d62a583, 0xdf45f746b74abf3a}, + {0x894bc396ce5da772, 0x6b8bba8c328eb784}, + {0xab9eb47c81f5114f, 0x066ea92f3f326565}, + {0xd686619ba27255a2, 0xc80a537b0efefebe}, + {0x8613fd0145877585, 0xbd06742ce95f5f37}, + {0xa798fc4196e952e7, 0x2c48113823b73705}, + {0xd17f3b51fca3a7a0, 0xf75a15862ca504c6}, + {0x82ef85133de648c4, 0x9a984d73dbe722fc}, + {0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebbb}, + {0xcc963fee10b7d1b3, 0x318df905079926a9}, + {0xffbbcfe994e5c61f, 0xfdf17746497f7053}, + {0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa634}, + {0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc1}, + {0xf9bd690a1b68637b, 0x3dfdce7aa3c673b1}, + {0x9c1661a651213e2d, 0x06bea10ca65c084f}, + {0xc31bfa0fe5698db8, 0x486e494fcff30a63}, + {0xf3e2f893dec3f126, 0x5a89dba3c3efccfb}, + {0x986ddb5c6b3a76b7, 0xf89629465a75e01d}, + {0xbe89523386091465, 0xf6bbb397f1135824}, + {0xee2ba6c0678b597f, 0x746aa07ded582e2d}, + {0x94db483840b717ef, 0xa8c2a44eb4571cdd}, + {0xba121a4650e4ddeb, 0x92f34d62616ce414}, + {0xe896a0d7e51e1566, 0x77b020baf9c81d18}, + {0x915e2486ef32cd60, 0x0ace1474dc1d122f}, + {0xb5b5ada8aaff80b8, 0x0d819992132456bb}, + {0xe3231912d5bf60e6, 0x10e1fff697ed6c6a}, + {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2}, + {0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb3}, + {0xddd0467c64bce4a0, 0xac7cb3f6d05ddbdf}, + {0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96c}, + {0xad4ab7112eb3929d, 0x86c16c98d2c953c7}, + {0xd89d64d57a607744, 0xe871c7bf077ba8b8}, + {0x87625f056c7c4a8b, 0x11471cd764ad4973}, + {0xa93af6c6c79b5d2d, 0xd598e40d3dd89bd0}, + {0xd389b47879823479, 0x4aff1d108d4ec2c4}, + {0x843610cb4bf160cb, 0xcedf722a585139bb}, + {0xa54394fe1eedb8fe, 0xc2974eb4ee658829}, + {0xce947a3da6a9273e, 0x733d226229feea33}, + {0x811ccc668829b887, 0x0806357d5a3f5260}, + {0xa163ff802a3426a8, 0xca07c2dcb0cf26f8}, + {0xc9bcff6034c13052, 0xfc89b393dd02f0b6}, + {0xfc2c3f3841f17c67, 0xbbac2078d443ace3}, + {0x9d9ba7832936edc0, 0xd54b944b84aa4c0e}, + {0xc5029163f384a931, 0x0a9e795e65d4df12}, + {0xf64335bcf065d37d, 0x4d4617b5ff4a16d6}, + {0x99ea0196163fa42e, 0x504bced1bf8e4e46}, + {0xc06481fb9bcf8d39, 0xe45ec2862f71e1d7}, + {0xf07da27a82c37088, 0x5d767327bb4e5a4d}, + {0x964e858c91ba2655, 0x3a6a07f8d510f870}, + {0xbbe226efb628afea, 0x890489f70a55368c}, + {0xeadab0aba3b2dbe5, 0x2b45ac74ccea842f}, + {0x92c8ae6b464fc96f, 0x3b0b8bc90012929e}, + {0xb77ada0617e3bbcb, 0x09ce6ebb40173745}, + {0xe55990879ddcaabd, 0xcc420a6a101d0516}, + {0x8f57fa54c2a9eab6, 0x9fa946824a12232e}, + {0xb32df8e9f3546564, 0x47939822dc96abfa}, + {0xdff9772470297ebd, 0x59787e2b93bc56f8}, + {0x8bfbea76c619ef36, 0x57eb4edb3c55b65b}, + {0xaefae51477a06b03, 0xede622920b6b23f2}, + {0xdab99e59958885c4, 0xe95fab368e45ecee}, + {0x88b402f7fd75539b, 0x11dbcb0218ebb415}, + {0xaae103b5fcd2a881, 0xd652bdc29f26a11a}, + {0xd59944a37c0752a2, 0x4be76d3346f04960}, + {0x857fcae62d8493a5, 0x6f70a4400c562ddc}, + {0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb953}, + {0xd097ad07a71f26b2, 0x7e2000a41346a7a8}, + {0x825ecc24c873782f, 0x8ed400668c0c28c9}, + {0xa2f67f2dfa90563b, 0x728900802f0f32fb}, + {0xcbb41ef979346bca, 0x4f2b40a03ad2ffba}, + {0xfea126b7d78186bc, 0xe2f610c84987bfa9}, + {0x9f24b832e6b0f436, 0x0dd9ca7d2df4d7ca}, + {0xc6ede63fa05d3143, 0x91503d1c79720dbc}, + {0xf8a95fcf88747d94, 0x75a44c6397ce912b}, + {0x9b69dbe1b548ce7c, 0xc986afbe3ee11abb}, + {0xc24452da229b021b, 0xfbe85badce996169}, + {0xf2d56790ab41c2a2, 0xfae27299423fb9c4}, + {0x97c560ba6b0919a5, 0xdccd879fc967d41b}, + {0xbdb6b8e905cb600f, 0x5400e987bbc1c921}, + {0xed246723473e3813, 0x290123e9aab23b69}, + {0x9436c0760c86e30b, 0xf9a0b6720aaf6522}, + {0xb94470938fa89bce, 0xf808e40e8d5b3e6a}, + {0xe7958cb87392c2c2, 0xb60b1d1230b20e05}, + {0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c3}, + {0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af4}, + {0xe2280b6c20dd5232, 0x25c6da63c38de1b1}, + {0x8d590723948a535f, 0x579c487e5a38ad0f}, + {0xb0af48ec79ace837, 0x2d835a9df0c6d852}, + {0xdcdb1b2798182244, 0xf8e431456cf88e66}, + {0x8a08f0f8bf0f156b, 0x1b8e9ecb641b5900}, + {0xac8b2d36eed2dac5, 0xe272467e3d222f40}, + {0xd7adf884aa879177, 0x5b0ed81dcc6abb10}, + {0x86ccbb52ea94baea, 0x98e947129fc2b4ea}, + {0xa87fea27a539e9a5, 0x3f2398d747b36225}, + {0xd29fe4b18e88640e, 0x8eec7f0d19a03aae}, + {0x83a3eeeef9153e89, 0x1953cf68300424ad}, + {0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd8}, + {0xcdb02555653131b6, 0x3792f412cb06794e}, + {0x808e17555f3ebf11, 0xe2bbd88bbee40bd1}, + {0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec5}, + {0xc8de047564d20a8b, 0xf245825a5a445276}, + {0xfb158592be068d2e, 0xeed6e2f0f0d56713}, + {0x9ced737bb6c4183d, 0x55464dd69685606c}, + {0xc428d05aa4751e4c, 0xaa97e14c3c26b887}, + {0xf53304714d9265df, 0xd53dd99f4b3066a9}, + {0x993fe2c6d07b7fab, 0xe546a8038efe402a}, + {0xbf8fdb78849a5f96, 0xde98520472bdd034}, + {0xef73d256a5c0f77c, 0x963e66858f6d4441}, + {0x95a8637627989aad, 0xdde7001379a44aa9}, + {0xbb127c53b17ec159, 0x5560c018580d5d53}, + {0xe9d71b689dde71af, 0xaab8f01e6e10b4a7}, + {0x9226712162ab070d, 0xcab3961304ca70e9}, + {0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d23}, + {0xe45c10c42a2b3b05, 0x8cb89a7db77c506b}, + {0x8eb98a7a9a5b04e3, 0x77f3608e92adb243}, + {0xb267ed1940f1c61c, 0x55f038b237591ed4}, + {0xdf01e85f912e37a3, 0x6b6c46dec52f6689}, + {0x8b61313bbabce2c6, 0x2323ac4b3b3da016}, + {0xae397d8aa96c1b77, 0xabec975e0a0d081b}, + {0xd9c7dced53c72255, 0x96e7bd358c904a22}, + {0x881cea14545c7575, 0x7e50d64177da2e55}, + {0xaa242499697392d2, 0xdde50bd1d5d0b9ea}, + {0xd4ad2dbfc3d07787, 0x955e4ec64b44e865}, + {0x84ec3c97da624ab4, 0xbd5af13bef0b113f}, + {0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58f}, + {0xcfb11ead453994ba, 0x67de18eda5814af3}, + {0x81ceb32c4b43fcf4, 0x80eacf948770ced8}, + {0xa2425ff75e14fc31, 0xa1258379a94d028e}, + {0xcad2f7f5359a3b3e, 0x096ee45813a04331}, + {0xfd87b5f28300ca0d, 0x8bca9d6e188853fd}, + {0x9e74d1b791e07e48, 0x775ea264cf55347e}, + {0xc612062576589dda, 0x95364afe032a819e}, + {0xf79687aed3eec551, 0x3a83ddbd83f52205}, + {0x9abe14cd44753b52, 0xc4926a9672793543}, + {0xc16d9a0095928a27, 0x75b7053c0f178294}, + {0xf1c90080baf72cb1, 0x5324c68b12dd6339}, + {0x971da05074da7bee, 0xd3f6fc16ebca5e04}, + {0xbce5086492111aea, 0x88f4bb1ca6bcf585}, + {0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6}, + {0x9392ee8e921d5d07, 0x3aff322e62439fd0}, + {0xb877aa3236a4b449, 0x09befeb9fad487c3}, + {0xe69594bec44de15b, 0x4c2ebe687989a9b4}, + {0x901d7cf73ab0acd9, 0x0f9d37014bf60a11}, + {0xb424dc35095cd80f, 0x538484c19ef38c95}, + {0xe12e13424bb40e13, 0x2865a5f206b06fba}, + {0x8cbccc096f5088cb, 0xf93f87b7442e45d4}, + {0xafebff0bcb24aafe, 0xf78f69a51539d749}, + {0xdbe6fecebdedd5be, 0xb573440e5a884d1c}, + {0x89705f4136b4a597, 0x31680a88f8953031}, + {0xabcc77118461cefc, 0xfdc20d2b36ba7c3e}, + {0xd6bf94d5e57a42bc, 0x3d32907604691b4d}, + {0x8637bd05af6c69b5, 0xa63f9a49c2c1b110}, + {0xa7c5ac471b478423, 0x0fcf80dc33721d54}, + {0xd1b71758e219652b, 0xd3c36113404ea4a9}, + {0x83126e978d4fdf3b, 0x645a1cac083126ea}, + {0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4}, + {0xcccccccccccccccc, 0xcccccccccccccccd}, + {0x8000000000000000, 0x0000000000000000}, + {0xa000000000000000, 0x0000000000000000}, + {0xc800000000000000, 0x0000000000000000}, + {0xfa00000000000000, 0x0000000000000000}, + {0x9c40000000000000, 0x0000000000000000}, + {0xc350000000000000, 0x0000000000000000}, + {0xf424000000000000, 0x0000000000000000}, + {0x9896800000000000, 0x0000000000000000}, + {0xbebc200000000000, 0x0000000000000000}, + {0xee6b280000000000, 0x0000000000000000}, + {0x9502f90000000000, 0x0000000000000000}, + {0xba43b74000000000, 0x0000000000000000}, + {0xe8d4a51000000000, 0x0000000000000000}, + {0x9184e72a00000000, 0x0000000000000000}, + {0xb5e620f480000000, 0x0000000000000000}, + {0xe35fa931a0000000, 0x0000000000000000}, + {0x8e1bc9bf04000000, 0x0000000000000000}, + {0xb1a2bc2ec5000000, 0x0000000000000000}, + {0xde0b6b3a76400000, 0x0000000000000000}, + {0x8ac7230489e80000, 0x0000000000000000}, + {0xad78ebc5ac620000, 0x0000000000000000}, + {0xd8d726b7177a8000, 0x0000000000000000}, + {0x878678326eac9000, 0x0000000000000000}, + {0xa968163f0a57b400, 0x0000000000000000}, + {0xd3c21bcecceda100, 0x0000000000000000}, + {0x84595161401484a0, 0x0000000000000000}, + {0xa56fa5b99019a5c8, 0x0000000000000000}, + {0xcecb8f27f4200f3a, 0x0000000000000000}, + {0x813f3978f8940984, 0x4000000000000000}, + {0xa18f07d736b90be5, 0x5000000000000000}, + {0xc9f2c9cd04674ede, 0xa400000000000000}, + {0xfc6f7c4045812296, 0x4d00000000000000}, + {0x9dc5ada82b70b59d, 0xf020000000000000}, + {0xc5371912364ce305, 0x6c28000000000000}, + {0xf684df56c3e01bc6, 0xc732000000000000}, + {0x9a130b963a6c115c, 0x3c7f400000000000}, + {0xc097ce7bc90715b3, 0x4b9f100000000000}, + {0xf0bdc21abb48db20, 0x1e86d40000000000}, + {0x96769950b50d88f4, 0x1314448000000000}, + {0xbc143fa4e250eb31, 0x17d955a000000000}, + {0xeb194f8e1ae525fd, 0x5dcfab0800000000}, + {0x92efd1b8d0cf37be, 0x5aa1cae500000000}, + {0xb7abc627050305ad, 0xf14a3d9e40000000}, + {0xe596b7b0c643c719, 0x6d9ccd05d0000000}, + {0x8f7e32ce7bea5c6f, 0xe4820023a2000000}, + {0xb35dbf821ae4f38b, 0xdda2802c8a800000}, + {0xe0352f62a19e306e, 0xd50b2037ad200000}, + {0x8c213d9da502de45, 0x4526f422cc340000}, + {0xaf298d050e4395d6, 0x9670b12b7f410000}, + {0xdaf3f04651d47b4c, 0x3c0cdd765f114000}, + {0x88d8762bf324cd0f, 0xa5880a69fb6ac800}, + {0xab0e93b6efee0053, 0x8eea0d047a457a00}, + {0xd5d238a4abe98068, 0x72a4904598d6d880}, + {0x85a36366eb71f041, 0x47a6da2b7f864750}, + {0xa70c3c40a64e6c51, 0x999090b65f67d924}, + {0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d}, + {0x82818f1281ed449f, 0xbff8f10e7a8921a4}, + {0xa321f2d7226895c7, 0xaff72d52192b6a0d}, + {0xcbea6f8ceb02bb39, 0x9bf4f8a69f764490}, + {0xfee50b7025c36a08, 0x02f236d04753d5b4}, + {0x9f4f2726179a2245, 0x01d762422c946590}, + {0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef5}, + {0xf8ebad2b84e0d58b, 0xd2e0898765a7deb2}, + {0x9b934c3b330c8577, 0x63cc55f49f88eb2f}, + {0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fb}, + {0xf316271c7fc3908a, 0x8bef464e3945ef7a}, + {0x97edd871cfda3a56, 0x97758bf0e3cbb5ac}, + {0xbde94e8e43d0c8ec, 0x3d52eeed1cbea317}, + {0xed63a231d4c4fb27, 0x4ca7aaa863ee4bdd}, + {0x945e455f24fb1cf8, 0x8fe8caa93e74ef6a}, + {0xb975d6b6ee39e436, 0xb3e2fd538e122b44}, + {0xe7d34c64a9c85d44, 0x60dbbca87196b616}, + {0x90e40fbeea1d3a4a, 0xbc8955e946fe31cd}, + {0xb51d13aea4a488dd, 0x6babab6398bdbe41}, + {0xe264589a4dcdab14, 0xc696963c7eed2dd1}, + {0x8d7eb76070a08aec, 0xfc1e1de5cf543ca2}, + {0xb0de65388cc8ada8, 0x3b25a55f43294bcb}, + {0xdd15fe86affad912, 0x49ef0eb713f39ebe}, + {0x8a2dbf142dfcc7ab, 0x6e3569326c784337}, + {0xacb92ed9397bf996, 0x49c2c37f07965404}, + {0xd7e77a8f87daf7fb, 0xdc33745ec97be906}, + {0x86f0ac99b4e8dafd, 0x69a028bb3ded71a3}, + {0xa8acd7c0222311bc, 0xc40832ea0d68ce0c}, + {0xd2d80db02aabd62b, 0xf50a3fa490c30190}, + {0x83c7088e1aab65db, 0x792667c6da79e0fa}, + {0xa4b8cab1a1563f52, 0x577001b891185938}, + {0xcde6fd5e09abcf26, 0xed4c0226b55e6f86}, + {0x80b05e5ac60b6178, 0x544f8158315b05b4}, + {0xa0dc75f1778e39d6, 0x696361ae3db1c721}, + {0xc913936dd571c84c, 0x03bc3a19cd1e38e9}, + {0xfb5878494ace3a5f, 0x04ab48a04065c723}, + {0x9d174b2dcec0e47b, 0x62eb0d64283f9c76}, + {0xc45d1df942711d9a, 0x3ba5d0bd324f8394}, + {0xf5746577930d6500, 0xca8f44ec7ee36479}, + {0x9968bf6abbe85f20, 0x7e998b13cf4e1ecb}, + {0xbfc2ef456ae276e8, 0x9e3fedd8c321a67e}, + {0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101e}, + {0x95d04aee3b80ece5, 0xbba1f1d158724a12}, + {0xbb445da9ca61281f, 0x2a8a6e45ae8edc97}, + {0xea1575143cf97226, 0xf52d09d71a3293bd}, + {0x924d692ca61be758, 0x593c2626705f9c56}, + {0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836c}, + {0xe498f455c38b997a, 0x0b6dfb9c0f956447}, + {0x8edf98b59a373fec, 0x4724bd4189bd5eac}, + {0xb2977ee300c50fe7, 0x58edec91ec2cb657}, + {0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ed}, + {0x8b865b215899f46c, 0xbd79e0d20082ee74}, + {0xae67f1e9aec07187, 0xecd8590680a3aa11}, + {0xda01ee641a708de9, 0xe80e6f4820cc9495}, + {0x884134fe908658b2, 0x3109058d147fdcdd}, + {0xaa51823e34a7eede, 0xbd4b46f0599fd415}, + {0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91a}, + {0x850fadc09923329e, 0x03e2cf6bc604ddb0}, + {0xa6539930bf6bff45, 0x84db8346b786151c}, + {0xcfe87f7cef46ff16, 0xe612641865679a63}, + {0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07e}, + {0xa26da3999aef7749, 0xe3be5e330f38f09d}, + {0xcb090c8001ab551c, 0x5cadf5bfd3072cc5}, + {0xfdcb4fa002162a63, 0x73d9732fc7c8f7f6}, + {0x9e9f11c4014dda7e, 0x2867e7fddcdd9afa}, + {0xc646d63501a1511d, 0xb281e1fd541501b8}, + {0xf7d88bc24209a565, 0x1f225a7ca91a4226}, + {0x9ae757596946075f, 0x3375788de9b06958}, + {0xc1a12d2fc3978937, 0x0052d6b1641c83ae}, + {0xf209787bb47d6b84, 0xc0678c5dbd23a49a}, + {0x9745eb4d50ce6332, 0xf840b7ba963646e0}, + {0xbd176620a501fbff, 0xb650e5a93bc3d898}, + {0xec5d3fa8ce427aff, 0xa3e51f138ab4cebe}, + {0x93ba47c980e98cdf, 0xc66f336c36b10137}, + {0xb8a8d9bbe123f017, 0xb80b0047445d4184}, + {0xe6d3102ad96cec1d, 0xa60dc059157491e5}, + {0x9043ea1ac7e41392, 0x87c89837ad68db2f}, + {0xb454e4a179dd1877, 0x29babe4598c311fb}, + {0xe16a1dc9d8545e94, 0xf4296dd6fef3d67a}, + {0x8ce2529e2734bb1d, 0x1899e4a65f58660c}, + {0xb01ae745b101e9e4, 0x5ec05dcff72e7f8f}, + {0xdc21a1171d42645d, 0x76707543f4fa1f73}, + {0x899504ae72497eba, 0x6a06494a791c53a8}, + {0xabfa45da0edbde69, 0x0487db9d17636892}, + {0xd6f8d7509292d603, 0x45a9d2845d3c42b6}, + {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b2}, + {0xa7f26836f282b732, 0x8e6cac7768d7141e}, + {0xd1ef0244af2364ff, 0x3207d795430cd926}, + {0x8335616aed761f1f, 0x7f44e6bd49e807b8}, + {0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a6}, + {0xcd036837130890a1, 0x36dba887c37a8c0f}, + {0x802221226be55a64, 0xc2494954da2c9789}, + {0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6c}, + {0xc83553c5c8965d3d, 0x6f92829494e5acc7}, + {0xfa42a8b73abbf48c, 0xcb772339ba1f17f9}, + {0x9c69a97284b578d7, 0xff2a760414536efb}, + {0xc38413cf25e2d70d, 0xfef5138519684aba}, + {0xf46518c2ef5b8cd1, 0x7eb258665fc25d69}, + {0x98bf2f79d5993802, 0xef2f773ffbd97a61}, + {0xbeeefb584aff8603, 0xaafb550ffacfd8fa}, + {0xeeaaba2e5dbf6784, 0x95ba2a53f983cf38}, + {0x952ab45cfa97a0b2, 0xdd945a747bf26183}, + {0xba756174393d88df, 0x94f971119aeef9e4}, + {0xe912b9d1478ceb17, 0x7a37cd5601aab85d}, + {0x91abb422ccb812ee, 0xac62e055c10ab33a}, + {0xb616a12b7fe617aa, 0x577b986b314d6009}, + {0xe39c49765fdf9d94, 0xed5a7e85fda0b80b}, + {0x8e41ade9fbebc27d, 0x14588f13be847307}, + {0xb1d219647ae6b31c, 0x596eb2d8ae258fc8}, + {0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bb}, + {0x8aec23d680043bee, 0x25de7bb9480d5854}, + {0xada72ccc20054ae9, 0xaf561aa79a10ae6a}, + {0xd910f7ff28069da4, 0x1b2ba1518094da04}, + {0x87aa9aff79042286, 0x90fb44d2f05d0842}, + {0xa99541bf57452b28, 0x353a1607ac744a53}, + {0xd3fa922f2d1675f2, 0x42889b8997915ce8}, + {0x847c9b5d7c2e09b7, 0x69956135febada11}, + {0xa59bc234db398c25, 0x43fab9837e699095}, + {0xcf02b2c21207ef2e, 0x94f967e45e03f4bb}, + {0x8161afb94b44f57d, 0x1d1be0eebac278f5}, + {0xa1ba1ba79e1632dc, 0x6462d92a69731732}, + {0xca28a291859bbf93, 0x7d7b8f7503cfdcfe}, + {0xfcb2cb35e702af78, 0x5cda735244c3d43e}, + {0x9defbf01b061adab, 0x3a0888136afa64a7}, + {0xc56baec21c7a1916, 0x088aaa1845b8fdd0}, + {0xf6c69a72a3989f5b, 0x8aad549e57273d45}, + {0x9a3c2087a63f6399, 0x36ac54e2f678864b}, + {0xc0cb28a98fcf3c7f, 0x84576a1bb416a7dd}, + {0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d5}, + {0x969eb7c47859e743, 0x9f644ae5a4b1b325}, + {0xbc4665b596706114, 0x873d5d9f0dde1fee}, + {0xeb57ff22fc0c7959, 0xa90cb506d155a7ea}, + {0x9316ff75dd87cbd8, 0x09a7f12442d588f2}, + {0xb7dcbf5354e9bece, 0x0c11ed6d538aeb2f}, + {0xe5d3ef282a242e81, 0x8f1668c8a86da5fa}, + {0x8fa475791a569d10, 0xf96e017d694487bc}, + {0xb38d92d760ec4455, 0x37c981dcc395a9ac}, + {0xe070f78d3927556a, 0x85bbe253f47b1417}, + {0x8c469ab843b89562, 0x93956d7478ccec8e}, + {0xaf58416654a6babb, 0x387ac8d1970027b2}, + {0xdb2e51bfe9d0696a, 0x06997b05fcc0319e}, + {0x88fcf317f22241e2, 0x441fece3bdf81f03}, + {0xab3c2fddeeaad25a, 0xd527e81cad7626c3}, + {0xd60b3bd56a5586f1, 0x8a71e223d8d3b074}, + {0x85c7056562757456, 0xf6872d5667844e49}, + {0xa738c6bebb12d16c, 0xb428f8ac016561db}, + {0xd106f86e69d785c7, 0xe13336d701beba52}, + {0x82a45b450226b39c, 0xecc0024661173473}, + {0xa34d721642b06084, 0x27f002d7f95d0190}, + {0xcc20ce9bd35c78a5, 0x31ec038df7b441f4}, + {0xff290242c83396ce, 0x7e67047175a15271}, + {0x9f79a169bd203e41, 0x0f0062c6e984d386}, + {0xc75809c42c684dd1, 0x52c07b78a3e60868}, + {0xf92e0c3537826145, 0xa7709a56ccdf8a82}, + {0x9bbcc7a142b17ccb, 0x88a66076400bb691}, + {0xc2abf989935ddbfe, 0x6acff893d00ea435}, + {0xf356f7ebf83552fe, 0x0583f6b8c4124d43}, + {0x98165af37b2153de, 0xc3727a337a8b704a}, + {0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5c}, + {0xeda2ee1c7064130c, 0x1162def06f79df73}, + {0x9485d4d1c63e8be7, 0x8addcb5645ac2ba8}, + {0xb9a74a0637ce2ee1, 0x6d953e2bd7173692}, + {0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0437}, + {0x910ab1d4db9914a0, 0x1d9c9892400a22a2}, + {0xb54d5e4a127f59c8, 0x2503beb6d00cab4b}, + {0xe2a0b5dc971f303a, 0x2e44ae64840fd61d}, + {0x8da471a9de737e24, 0x5ceaecfed289e5d2}, + {0xb10d8e1456105dad, 0x7425a83e872c5f47}, + {0xdd50f1996b947518, 0xd12f124e28f77719}, + {0x8a5296ffe33cc92f, 0x82bd6b70d99aaa6f}, + {0xace73cbfdc0bfb7b, 0x636cc64d1001550b}, + {0xd8210befd30efa5a, 0x3c47f7e05401aa4e}, + {0x8714a775e3e95c78, 0x65acfaec34810a71}, + {0xa8d9d1535ce3b396, 0x7f1839a741a14d0d}, + {0xd31045a8341ca07c, 0x1ede48111209a050}, + {0x83ea2b892091e44d, 0x934aed0aab460432}, + {0xa4e4b66b68b65d60, 0xf81da84d5617853f}, + {0xce1de40642e3f4b9, 0x36251260ab9d668e}, + {0x80d2ae83e9ce78f3, 0xc1d72b7c6b426019}, + {0xa1075a24e4421730, 0xb24cf65b8612f81f}, + {0xc94930ae1d529cfc, 0xdee033f26797b627}, + {0xfb9b7cd9a4a7443c, 0x169840ef017da3b1}, + {0x9d412e0806e88aa5, 0x8e1f289560ee864e}, + {0xc491798a08a2ad4e, 0xf1a6f2bab92a27e2}, + {0xf5b5d7ec8acb58a2, 0xae10af696774b1db}, + {0x9991a6f3d6bf1765, 0xacca6da1e0a8ef29}, + {0xbff610b0cc6edd3f, 0x17fd090a58d32af3}, + {0xeff394dcff8a948e, 0xddfc4b4cef07f5b0}, + {0x95f83d0a1fb69cd9, 0x4abdaf101564f98e}, + {0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f1}, + {0xea53df5fd18d5513, 0x84c86189216dc5ed}, + {0x92746b9be2f8552c, 0x32fd3cf5b4e49bb4}, + {0xb7118682dbb66a77, 0x3fbc8c33221dc2a1}, + {0xe4d5e82392a40515, 0x0fabaf3feaa5334a}, + {0x8f05b1163ba6832d, 0x29cb4d87f2a7400e}, + {0xb2c71d5bca9023f8, 0x743e20e9ef511012}, + {0xdf78e4b2bd342cf6, 0x914da9246b255416}, + {0x8bab8eefb6409c1a, 0x1ad089b6c2f7548e}, + {0xae9672aba3d0c320, 0xa184ac2473b529b1}, + {0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741e}, + {0x8865899617fb1871, 0x7e2fa67c7a658892}, + {0xaa7eebfb9df9de8d, 0xddbb901b98feeab7}, + {0xd51ea6fa85785631, 0x552a74227f3ea565}, + {0x8533285c936b35de, 0xd53a88958f87275f}, + {0xa67ff273b8460356, 0x8a892abaf368f137}, + {0xd01fef10a657842c, 0x2d2b7569b0432d85}, + {0x8213f56a67f6b29b, 0x9c3b29620e29fc73}, + {0xa298f2c501f45f42, 0x8349f3ba91b47b8f}, + {0xcb3f2f7642717713, 0x241c70a936219a73}, + {0xfe0efb53d30dd4d7, 0xed238cd383aa0110}, + {0x9ec95d1463e8a506, 0xf4363804324a40aa}, + {0xc67bb4597ce2ce48, 0xb143c6053edcd0d5}, + {0xf81aa16fdc1b81da, 0xdd94b7868e94050a}, + {0x9b10a4e5e9913128, 0xca7cf2b4191c8326}, + {0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f0}, + {0xf24a01a73cf2dccf, 0xbc633b39673c8cec}, + {0x976e41088617ca01, 0xd5be0503e085d813}, + {0xbd49d14aa79dbc82, 0x4b2d8644d8a74e18}, + {0xec9c459d51852ba2, 0xddf8e7d60ed1219e}, + {0x93e1ab8252f33b45, 0xcabb90e5c942b503}, + {0xb8da1662e7b00a17, 0x3d6a751f3b936243}, + {0xe7109bfba19c0c9d, 0x0cc512670a783ad4}, + {0x906a617d450187e2, 0x27fb2b80668b24c5}, + {0xb484f9dc9641e9da, 0xb1f9f660802dedf6}, + {0xe1a63853bbd26451, 0x5e7873f8a0396973}, + {0x8d07e33455637eb2, 0xdb0b487b6423e1e8}, + {0xb049dc016abc5e5f, 0x91ce1a9a3d2cda62}, + {0xdc5c5301c56b75f7, 0x7641a140cc7810fb}, + {0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9d}, + {0xac2820d9623bf429, 0x546345fa9fbdcd44}, + {0xd732290fbacaf133, 0xa97c177947ad4095}, + {0x867f59a9d4bed6c0, 0x49ed8eabcccc485d}, + {0xa81f301449ee8c70, 0x5c68f256bfff5a74}, + {0xd226fc195c6a2f8c, 0x73832eec6fff3111}, + {0x83585d8fd9c25db7, 0xc831fd53c5ff7eab}, + {0xa42e74f3d032f525, 0xba3e7ca8b77f5e55}, + {0xcd3a1230c43fb26f, 0x28ce1bd2e55f35eb}, + {0x80444b5e7aa7cf85, 0x7980d163cf5b81b3}, + {0xa0555e361951c366, 0xd7e105bcc332621f}, + {0xc86ab5c39fa63440, 0x8dd9472bf3fefaa7}, + {0xfa856334878fc150, 0xb14f98f6f0feb951}, + {0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d3}, + {0xc3b8358109e84f07, 0x0a862f80ec4700c8}, + {0xf4a642e14c6262c8, 0xcd27bb612758c0fa}, + {0x98e7e9cccfbd7dbd, 0x8038d51cb897789c}, + {0xbf21e44003acdd2c, 0xe0470a63e6bd56c3}, + {0xeeea5d5004981478, 0x1858ccfce06cac74}, + {0x95527a5202df0ccb, 0x0f37801e0c43ebc8}, + {0xbaa718e68396cffd, 0xd30560258f54e6ba}, + {0xe950df20247c83fd, 0x47c6b82ef32a2069}, + {0x91d28b7416cdd27e, 0x4cdc331d57fa5441}, + {0xb6472e511c81471d, 0xe0133fe4adf8e952}, + {0xe3d8f9e563a198e5, 0x58180fddd97723a6}, + {0x8e679c2f5e44ff8f, 0x570f09eaa7ea7648}, + {0xb201833b35d63f73, 0x2cd2cc6551e513da}, + {0xde81e40a034bcf4f, 0xf8077f7ea65e58d1}, + {0x8b112e86420f6191, 0xfb04afaf27faf782}, + {0xadd57a27d29339f6, 0x79c5db9af1f9b563}, + {0xd94ad8b1c7380874, 0x18375281ae7822bc}, + {0x87cec76f1c830548, 0x8f2293910d0b15b5}, + {0xa9c2794ae3a3c69a, 0xb2eb3875504ddb22}, + {0xd433179d9c8cb841, 0x5fa60692a46151eb}, + {0x849feec281d7f328, 0xdbc7c41ba6bcd333}, + {0xa5c7ea73224deff3, 0x12b9b522906c0800}, + {0xcf39e50feae16bef, 0xd768226b34870a00}, + {0x81842f29f2cce375, 0xe6a1158300d46640}, + {0xa1e53af46f801c53, 0x60495ae3c1097fd0}, + {0xca5e89b18b602368, 0x385bb19cb14bdfc4}, + {0xfcf62c1dee382c42, 0x46729e03dd9ed7b5}, + {0x9e19db92b4e31ba9, 0x6c07a2c26a8346d1}, + {0xc5a05277621be293, 0xc7098b7305241885}, + { 0xf70867153aa2db38, + 0xb8cbee4fc66d1ea7 } +#else + {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b}, + {0xce5d73ff402d98e3, 0xfb0a3d212dc81290}, + {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f}, + {0x86a8d39ef77164bc, 0xae5dff9c02033198}, + {0xd98ddaee19068c76, 0x3badd624dd9b0958}, + {0xafbd2350644eeacf, 0xe5d1929ef90898fb}, + {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2}, + {0xe55990879ddcaabd, 0xcc420a6a101d0516}, + {0xb94470938fa89bce, 0xf808e40e8d5b3e6a}, + {0x95a8637627989aad, 0xdde7001379a44aa9}, + {0xf1c90080baf72cb1, 0x5324c68b12dd6339}, + {0xc350000000000000, 0x0000000000000000}, + {0x9dc5ada82b70b59d, 0xf020000000000000}, + {0xfee50b7025c36a08, 0x02f236d04753d5b4}, + {0xcde6fd5e09abcf26, 0xed4c0226b55e6f86}, + {0xa6539930bf6bff45, 0x84db8346b786151c}, + {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b2}, + {0xd910f7ff28069da4, 0x1b2ba1518094da04}, + {0xaf58416654a6babb, 0x387ac8d1970027b2}, + {0x8da471a9de737e24, 0x5ceaecfed289e5d2}, + {0xe4d5e82392a40515, 0x0fabaf3feaa5334a}, + {0xb8da1662e7b00a17, 0x3d6a751f3b936243}, + { 0x95527a5202df0ccb, + 0x0f37801e0c43ebc8 } +#endif + }; + #if FMT_USE_FULL_CACHE_DRAGONBOX - return data::dragonbox_pow10_significands_128[k - - float_info::min_k]; + return pow10_significands[k - float_info::min_k]; #else + static constexpr const uint64_t powers_of_5_64[] = { + 0x0000000000000001, 0x0000000000000005, 0x0000000000000019, + 0x000000000000007d, 0x0000000000000271, 0x0000000000000c35, + 0x0000000000003d09, 0x000000000001312d, 0x000000000005f5e1, + 0x00000000001dcd65, 0x00000000009502f9, 0x0000000002e90edd, + 0x000000000e8d4a51, 0x0000000048c27395, 0x000000016bcc41e9, + 0x000000071afd498d, 0x0000002386f26fc1, 0x000000b1a2bc2ec5, + 0x000003782dace9d9, 0x00001158e460913d, 0x000056bc75e2d631, + 0x0001b1ae4d6e2ef5, 0x000878678326eac9, 0x002a5a058fc295ed, + 0x00d3c21bcecceda1, 0x0422ca8b0a00a425, 0x14adf4b7320334b9}; + + static constexpr const uint32_t pow10_recovery_errors[] = { + 0x50001400, 0x54044100, 0x54014555, 0x55954415, 0x54115555, 0x00000001, + 0x50000000, 0x00104000, 0x54010004, 0x05004001, 0x55555544, 0x41545555, + 0x54040551, 0x15445545, 0x51555514, 0x10000015, 0x00101100, 0x01100015, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x04450514, 0x45414110, + 0x55555145, 0x50544050, 0x15040155, 0x11054140, 0x50111514, 0x11451454, + 0x00400541, 0x00000000, 0x55555450, 0x10056551, 0x10054011, 0x55551014, + 0x69514555, 0x05151109, 0x00155555}; + static const int compression_ratio = 27; // Compute base index. @@ -1897,8 +1813,7 @@ template <> struct cache_accessor { int offset = k - kb; // Get base cache. - uint128_wrapper base_cache = - data::dragonbox_pow10_significands_128[cache_index]; + uint128_wrapper base_cache = pow10_significands[cache_index]; if (offset == 0) return base_cache; // Compute the required amount of bit-shift. @@ -1906,7 +1821,7 @@ template <> struct cache_accessor { FMT_ASSERT(alpha > 0 && alpha < 64, "shifting error detected"); // Try to recover the real cache. - uint64_t pow5 = data::powers_of_5_64[offset]; + uint64_t pow5 = powers_of_5_64[offset]; uint128_wrapper recovered_cache = umul128(base_cache.high(), pow5); uint128_wrapper middle_low = umul128(base_cache.low() - (kb < 0 ? 1u : 0u), pow5); @@ -1924,7 +1839,7 @@ template <> struct cache_accessor { // Get error. int error_idx = (k - float_info::min_k) / 16; - uint32_t error = (data::dragonbox_pow10_recovery_errors[error_idx] >> + uint32_t error = (pow10_recovery_errors[error_idx] >> ((k - float_info::min_k) % 16) * 2) & 0x3; @@ -2010,7 +1925,7 @@ bool is_center_integer(typename float_info::carrier_uint two_f, int exponent, } // Remove trailing zeros from n and return the number of zeros removed (float) -FMT_ALWAYS_INLINE int remove_trailing_zeros(uint32_t& n) FMT_NOEXCEPT { +FMT_INLINE int remove_trailing_zeros(uint32_t& n) FMT_NOEXCEPT { #ifdef FMT_BUILTIN_CTZ int t = FMT_BUILTIN_CTZ(n); #else @@ -2038,7 +1953,7 @@ FMT_ALWAYS_INLINE int remove_trailing_zeros(uint32_t& n) FMT_NOEXCEPT { } // Removes trailing zeros and returns the number of zeros removed (double) -FMT_ALWAYS_INLINE int remove_trailing_zeros(uint64_t& n) FMT_NOEXCEPT { +FMT_INLINE int remove_trailing_zeros(uint64_t& n) FMT_NOEXCEPT { #ifdef FMT_BUILTIN_CTZLL int t = FMT_BUILTIN_CTZLL(n); #else @@ -2124,8 +2039,7 @@ FMT_ALWAYS_INLINE int remove_trailing_zeros(uint64_t& n) FMT_NOEXCEPT { // The main algorithm for shorter interval case template -FMT_ALWAYS_INLINE FMT_SAFEBUFFERS decimal_fp shorter_interval_case( - int exponent) FMT_NOEXCEPT { +FMT_INLINE decimal_fp shorter_interval_case(int exponent) FMT_NOEXCEPT { decimal_fp ret_value; // Compute k and beta const int minus_k = floor_log10_pow2_minus_log10_4_over_3(exponent); @@ -2171,8 +2085,7 @@ FMT_ALWAYS_INLINE FMT_SAFEBUFFERS decimal_fp shorter_interval_case( return ret_value; } -template -FMT_SAFEBUFFERS decimal_fp to_decimal(T x) FMT_NOEXCEPT { +template decimal_fp to_decimal(T x) FMT_NOEXCEPT { // Step 1: integer promotion & Schubfach multiplier calculation. using carrier_uint = typename float_info::carrier_uint; @@ -2306,24 +2219,21 @@ small_divisor_case_label: } } // namespace dragonbox -// Formats value using a variation of the Fixed-Precision Positive -// Floating-Point Printout ((FPP)^2) algorithm by Steele & White: -// https://fmt.dev/p372-steele.pdf. -template -void fallback_format(Double d, int num_digits, bool binary32, buffer& buf, - int& exp10) { +// Formats a floating-point number using a variation of the Fixed-Precision +// Positive Floating-Point Printout ((FPP)^2) algorithm by Steele & White: +// https://fmt.dev/papers/p372-steele.pdf. +FMT_CONSTEXPR20 inline void format_dragon(fp value, bool is_predecessor_closer, + int num_digits, buffer& buf, + int& exp10) { bigint numerator; // 2 * R in (FPP)^2. bigint denominator; // 2 * S in (FPP)^2. // lower and upper are differences between value and corresponding boundaries. bigint lower; // (M^- in (FPP)^2). bigint upper_store; // upper's value if different from lower. bigint* upper = nullptr; // (M^+ in (FPP)^2). - fp value; // Shift numerator and denominator by an extra bit or two (if lower boundary // is closer) to make lower and upper integers. This eliminates multiplication // by 2 during later computations. - const bool is_predecessor_closer = - binary32 ? value.assign(static_cast(d)) : value.assign(d); int shift = is_predecessor_closer ? 2 : 1; uint64_t significand = value.f << shift; if (value.e >= 0) { @@ -2393,9 +2303,9 @@ void fallback_format(Double d, int num_digits, bool binary32, buffer& buf, // Generate the given number of digits. exp10 -= num_digits - 1; if (num_digits == 0) { - buf.try_resize(1); denominator *= 10; - buf[0] = add_compare(numerator, numerator, denominator) > 0 ? '1' : '0'; + auto digit = add_compare(numerator, numerator, denominator) > 0 ? '1' : '0'; + buf.push_back(digit); return; } buf.try_resize(to_unsigned(num_digits)); @@ -2426,9 +2336,12 @@ void fallback_format(Double d, int num_digits, bool binary32, buffer& buf, buf[num_digits - 1] = static_cast('0' + digit); } -template -int format_float(T value, int precision, float_specs specs, buffer& buf) { - static_assert(!std::is_same::value, ""); +template +FMT_HEADER_ONLY_CONSTEXPR20 int format_float(Float value, int precision, + float_specs specs, + buffer& buf) { + // float is passed as double to reduce the number of instantiations. + static_assert(!std::is_same::value, ""); FMT_ASSERT(value >= 0, "value is negative"); const bool fixed = specs.format == float_format::fixed; @@ -2438,13 +2351,13 @@ int format_float(T value, int precision, float_specs specs, buffer& buf) { return 0; } buf.try_resize(to_unsigned(precision)); - std::uninitialized_fill_n(buf.data(), precision, '0'); + fill_n(buf.data(), precision, '0'); return -precision; } - if (!specs.use_grisu) return snprintf_float(value, precision, specs, buf); + if (specs.fallback) return snprintf_float(value, precision, specs, buf); - if (precision < 0) { + if (!is_constant_evaluated() && precision < 0) { // Use Dragonbox for the shortest format. if (specs.binary32) { auto dec = dragonbox::to_decimal(static_cast(value)); @@ -2456,26 +2369,37 @@ int format_float(T value, int precision, float_specs specs, buffer& buf) { return dec.exponent; } - // Use Grisu + Dragon4 for the given precision: - // https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf. int exp = 0; - const int min_exp = -60; // alpha in Grisu. - int cached_exp10 = 0; // K in Grisu. - fp normalized = normalize(fp(value)); - const auto cached_pow = get_cached_power( - min_exp - (normalized.e + fp::significand_size), cached_exp10); - normalized = normalized * cached_pow; - // Limit precision to the maximum possible number of significant digits in an - // IEEE754 double because we don't need to generate zeros. - const int max_double_digits = 767; - if (precision > max_double_digits) precision = max_double_digits; - fixed_handler handler{buf.data(), 0, precision, -cached_exp10, fixed}; - if (grisu_gen_digits(normalized, 1, exp, handler) == digits::error) { - exp += handler.size - cached_exp10 - 1; - fallback_format(value, handler.precision, specs.binary32, buf, exp); - } else { - exp += handler.exp10; - buf.try_resize(to_unsigned(handler.size)); + bool use_dragon = true; + if (is_fast_float()) { + // Use Grisu + Dragon4 for the given precision: + // https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf. + const int min_exp = -60; // alpha in Grisu. + int cached_exp10 = 0; // K in Grisu. + fp normalized = normalize(fp(value)); + const auto cached_pow = get_cached_power( + min_exp - (normalized.e + fp::num_significand_bits), cached_exp10); + normalized = normalized * cached_pow; + gen_digits_handler handler{buf.data(), 0, precision, -cached_exp10, fixed}; + if (grisu_gen_digits(normalized, 1, exp, handler) != digits::error && + !is_constant_evaluated()) { + exp += handler.exp10; + buf.try_resize(to_unsigned(handler.size)); + use_dragon = false; + } else { + exp += handler.size - cached_exp10 - 1; + precision = handler.precision; + } + } + if (use_dragon) { + auto f = fp(); + bool is_predecessor_closer = + specs.binary32 ? f.assign(static_cast(value)) : f.assign(value); + // Limit precision to the maximum possible number of significant digits in + // an IEEE754 double because we don't need to generate zeros. + const int max_double_digits = 767; + if (precision > max_double_digits) precision = max_double_digits; + format_dragon(f, is_predecessor_closer, precision, buf, exp); } if (!fixed && !specs.showpoint) { // Remove trailing zeros. @@ -2487,7 +2411,7 @@ int format_float(T value, int precision, float_specs specs, buffer& buf) { buf.try_resize(num_digits); } return exp; -} // namespace detail +} template int snprintf_float(T value, int precision, float_specs specs, @@ -2571,11 +2495,11 @@ int snprintf_float(T value, int precision, float_specs specs, --exp_pos; } while (*exp_pos != 'e'); char sign = exp_pos[1]; - assert(sign == '+' || sign == '-'); + FMT_ASSERT(sign == '+' || sign == '-', ""); int exp = 0; auto p = exp_pos + 2; // Skip 'e' and sign. do { - assert(is_digit(*p)); + FMT_ASSERT(is_digit(*p), ""); exp = exp * 10 + (*p++ - '0'); } while (p != end); if (sign == '-') exp = -exp; @@ -2592,71 +2516,11 @@ int snprintf_float(T value, int precision, float_specs specs, return exp - fraction_size; } } - -// A public domain branchless UTF-8 decoder by Christopher Wellons: -// https://github.com/skeeto/branchless-utf8 -/* Decode the next character, c, from buf, reporting errors in e. - * - * Since this is a branchless decoder, four bytes will be read from the - * buffer regardless of the actual length of the next character. This - * means the buffer _must_ have at least three bytes of zero padding - * following the end of the data stream. - * - * Errors are reported in e, which will be non-zero if the parsed - * character was somehow invalid: invalid byte sequence, non-canonical - * encoding, or a surrogate half. - * - * The function returns a pointer to the next character. When an error - * occurs, this pointer will be a guess that depends on the particular - * error, but it will always advance at least one byte. - */ -inline const char* utf8_decode(const char* buf, uint32_t* c, int* e) { - static const int masks[] = {0x00, 0x7f, 0x1f, 0x0f, 0x07}; - static const uint32_t mins[] = {4194304, 0, 128, 2048, 65536}; - static const int shiftc[] = {0, 18, 12, 6, 0}; - static const int shifte[] = {0, 6, 4, 2, 0}; - - int len = code_point_length(buf); - const char* next = buf + len; - - // Assume a four-byte character and load four bytes. Unused bits are - // shifted out. - auto s = reinterpret_cast(buf); - *c = uint32_t(s[0] & masks[len]) << 18; - *c |= uint32_t(s[1] & 0x3f) << 12; - *c |= uint32_t(s[2] & 0x3f) << 6; - *c |= uint32_t(s[3] & 0x3f) << 0; - *c >>= shiftc[len]; - - // Accumulate the various error conditions. - *e = (*c < mins[len]) << 6; // non-canonical encoding - *e |= ((*c >> 11) == 0x1b) << 7; // surrogate half? - *e |= (*c > 0x10FFFF) << 8; // out of range? - *e |= (s[1] & 0xc0) >> 2; - *e |= (s[2] & 0xc0) >> 4; - *e |= (s[3]) >> 6; - *e ^= 0x2a; // top two bits of each tail byte correct? - *e >>= shifte[len]; - - return next; -} - -struct stringifier { - template FMT_INLINE std::string operator()(T value) const { - return to_string(value); - } - std::string operator()(basic_format_arg::handle h) const { - memory_buffer buf; - format_parse_context parse_ctx({}); - format_context format_ctx(buffer_appender(buf), {}, {}); - h.format(parse_ctx, format_ctx); - return to_string(buf); - } -}; } // namespace detail template <> struct formatter { - format_parse_context::iterator parse(format_parse_context& ctx) { + FMT_CONSTEXPR format_parse_context::iterator parse( + format_parse_context& ctx) { return ctx.begin(); } @@ -2667,24 +2531,22 @@ template <> struct formatter { for (auto i = n.bigits_.size(); i > 0; --i) { auto value = n.bigits_[i - 1u]; if (first) { - out = format_to(out, "{:x}", value); + out = format_to(out, FMT_STRING("{:x}"), value); first = false; continue; } - out = format_to(out, "{:08x}", value); + out = format_to(out, FMT_STRING("{:08x}"), value); } if (n.exp_ > 0) - out = format_to(out, "p{}", n.exp_ * detail::bigint::bigit_bits); + out = format_to(out, FMT_STRING("p{}"), + n.exp_ * detail::bigint::bigit_bits); return out; } }; FMT_FUNC detail::utf8_to_utf16::utf8_to_utf16(string_view s) { - auto transcode = [this](const char* p) { - auto cp = uint32_t(); - auto error = 0; - p = utf8_decode(p, &cp, &error); - if (error != 0) FMT_THROW(std::runtime_error("invalid utf8")); + for_each_codepoint(s, [this](uint32_t cp, string_view) { + if (cp == invalid_code_point) FMT_THROW(std::runtime_error("invalid utf8")); if (cp <= 0xFFFF) { buffer_.push_back(static_cast(cp)); } else { @@ -2692,64 +2554,38 @@ FMT_FUNC detail::utf8_to_utf16::utf8_to_utf16(string_view s) { buffer_.push_back(static_cast(0xD800 + (cp >> 10))); buffer_.push_back(static_cast(0xDC00 + (cp & 0x3FF))); } - return p; - }; - auto p = s.data(); - const size_t block_size = 4; // utf8_decode always reads blocks of 4 chars. - if (s.size() >= block_size) { - for (auto end = p + s.size() - block_size + 1; p < end;) p = transcode(p); - } - if (auto num_chars_left = s.data() + s.size() - p) { - char buf[2 * block_size - 1] = {}; - memcpy(buf, p, to_unsigned(num_chars_left)); - p = buf; - do { - p = transcode(p); - } while (p - buf < num_chars_left); - } + return true; + }); buffer_.push_back(0); } FMT_FUNC void format_system_error(detail::buffer& out, int error_code, - string_view message) FMT_NOEXCEPT { + const char* message) FMT_NOEXCEPT { FMT_TRY { - memory_buffer buf; - buf.resize(inline_buffer_size); - for (;;) { - char* system_message = &buf[0]; - int result = - detail::safe_strerror(error_code, system_message, buf.size()); - if (result == 0) { - format_to(detail::buffer_appender(out), "{}: {}", message, - system_message); - return; - } - if (result != ERANGE) - break; // Can't get error message, report error code instead. - buf.resize(buf.size() * 2); - } + auto ec = std::error_code(error_code, std::generic_category()); + write(std::back_inserter(out), std::system_error(ec, message).what()); + return; } FMT_CATCH(...) {} format_error_code(out, error_code, message); } -FMT_FUNC void detail::error_handler::on_error(const char* message) { - FMT_THROW(format_error(message)); -} - FMT_FUNC void report_system_error(int error_code, - fmt::string_view message) FMT_NOEXCEPT { + const char* message) FMT_NOEXCEPT { report_error(format_system_error, error_code, message); } -FMT_FUNC std::string detail::vformat(string_view format_str, format_args args) { - if (format_str.size() == 2 && equal2(format_str.data(), "{}")) { - auto arg = args.get(0); - if (!arg) error_handler().on_error("argument not found"); - return visit_format_arg(stringifier(), arg); - } - memory_buffer buffer; - detail::vformat_to(buffer, format_str, args); +// DEPRECATED! +// This function is defined here and not inline for ABI compatiblity. +FMT_FUNC void detail::error_handler::on_error(const char* message) { + throw_format_error(message); +} + +FMT_FUNC std::string vformat(string_view fmt, format_args args) { + // Don't optimize the "{}" case to keep the binary size small and because it + // can be better optimized in fmt::format anyway. + auto buffer = memory_buffer(); + detail::vformat_to(buffer, fmt, args); return to_string(buffer); } @@ -2761,24 +2597,30 @@ extern "C" __declspec(dllimport) int __stdcall WriteConsoleW( // } // namespace detail #endif -FMT_FUNC void vprint(std::FILE* f, string_view format_str, format_args args) { - memory_buffer buffer; - detail::vformat_to(buffer, format_str, - basic_format_args>(args)); +namespace detail { +FMT_FUNC void print(std::FILE* f, string_view text) { #ifdef _WIN32 auto fd = _fileno(f); if (_isatty(fd)) { - detail::utf8_to_utf16 u16(string_view(buffer.data(), buffer.size())); + detail::utf8_to_utf16 u16(string_view(text.data(), text.size())); auto written = detail::dword(); - if (!detail::WriteConsoleW(reinterpret_cast(_get_osfhandle(fd)), - u16.c_str(), static_cast(u16.size()), - &written, nullptr)) { - FMT_THROW(format_error("failed to write to console")); + if (detail::WriteConsoleW(reinterpret_cast(_get_osfhandle(fd)), + u16.c_str(), static_cast(u16.size()), + &written, nullptr)) { + return; } - return; + // Fallback to fwrite on failure. It can happen if the output has been + // redirected to NUL. } #endif - detail::fwrite_fully(buffer.data(), 1, buffer.size(), f); + detail::fwrite_fully(text.data(), 1, text.size(), f); +} +} // namespace detail + +FMT_FUNC void vprint(std::FILE* f, string_view format_str, format_args args) { + memory_buffer buffer; + detail::vformat_to(buffer, format_str, args); + detail::print(f, {buffer.data(), buffer.size()}); } #ifdef _WIN32 diff --git a/src/third_party/fmt/format.h b/src/third_party/fmt/format.h index 1a037b0..ee69651 100644 --- a/src/third_party/fmt/format.h +++ b/src/third_party/fmt/format.h @@ -33,22 +33,24 @@ #ifndef FMT_FORMAT_H_ #define FMT_FORMAT_H_ -#include -#include -#include -#include -#include -#include -#include +#include // std::signbit +#include // uint32_t +#include // std::numeric_limits +#include // std::uninitialized_copy +#include // std::runtime_error +#include // std::system_error +#include // std::swap + +#ifdef __cpp_lib_bit_cast +# include // std::bitcast +#endif #include "core.h" -#ifdef __INTEL_COMPILER -# define FMT_ICC_VERSION __INTEL_COMPILER -#elif defined(__ICL) -# define FMT_ICC_VERSION __ICL +#if FMT_GCC_VERSION +# define FMT_GCC_VISIBILITY_HIDDEN __attribute__((visibility("hidden"))) #else -# define FMT_ICC_VERSION 0 +# define FMT_GCC_VISIBILITY_HIDDEN #endif #ifdef __NVCC__ @@ -69,30 +71,10 @@ # define FMT_NOINLINE #endif -#if __cplusplus == 201103L || __cplusplus == 201402L -# if defined(__INTEL_COMPILER) || defined(__PGI) -# define FMT_FALLTHROUGH -# elif defined(__clang__) -# define FMT_FALLTHROUGH [[clang::fallthrough]] -# elif FMT_GCC_VERSION >= 700 && \ - (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 520) -# define FMT_FALLTHROUGH [[gnu::fallthrough]] -# else -# define FMT_FALLTHROUGH -# endif -#elif FMT_HAS_CPP17_ATTRIBUTE(fallthrough) || \ - (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) -# define FMT_FALLTHROUGH [[fallthrough]] +#if FMT_MSC_VER +# define FMT_MSC_DEFAULT = default #else -# define FMT_FALLTHROUGH -#endif - -#ifndef FMT_MAYBE_UNUSED -# if FMT_HAS_CPP17_ATTRIBUTE(maybe_unused) -# define FMT_MAYBE_UNUSED [[maybe_unused]] -# else -# define FMT_MAYBE_UNUSED -# endif +# define FMT_MSC_DEFAULT #endif #ifndef FMT_THROW @@ -113,10 +95,9 @@ FMT_END_NAMESPACE # define FMT_THROW(x) throw x # endif # else -# define FMT_THROW(x) \ - do { \ - static_cast(sizeof(x)); \ - FMT_ASSERT(false, ""); \ +# define FMT_THROW(x) \ + do { \ + FMT_ASSERT(false, (x).what()); \ } while (false) # endif #endif @@ -129,6 +110,21 @@ FMT_END_NAMESPACE # define FMT_CATCH(x) if (false) #endif +#ifndef FMT_MAYBE_UNUSED +# if FMT_HAS_CPP17_ATTRIBUTE(maybe_unused) +# define FMT_MAYBE_UNUSED [[maybe_unused]] +# else +# define FMT_MAYBE_UNUSED +# endif +#endif + +// Workaround broken [[deprecated]] in the Intel, PGI and NVCC compilers. +#if FMT_ICC_VERSION || defined(__PGI) || FMT_NVCC +# define FMT_DEPRECATED_ALIAS +#else +# define FMT_DEPRECATED_ALIAS FMT_DEPRECATED +#endif + #ifndef FMT_USE_USER_DEFINED_LITERALS // EDG based compilers (Intel, NVIDIA, Elbrus, etc), GCC and MSVC support UDLs. # if (FMT_HAS_FEATURE(cxx_user_literals) || FMT_GCC_VERSION >= 407 || \ @@ -140,53 +136,34 @@ FMT_END_NAMESPACE # endif #endif -#ifndef FMT_USE_UDL_TEMPLATE -// EDG frontend based compilers (icc, nvcc, PGI, etc) and GCC < 6.4 do not -// properly support UDL templates and GCC >= 9 warns about them. -# if FMT_USE_USER_DEFINED_LITERALS && \ - (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 501) && \ - ((FMT_GCC_VERSION >= 604 && __cplusplus >= 201402L) || \ - FMT_CLANG_VERSION >= 304) && \ - !defined(__PGI) && !defined(__NVCC__) -# define FMT_USE_UDL_TEMPLATE 1 -# else -# define FMT_USE_UDL_TEMPLATE 0 -# endif -#endif - -#ifndef FMT_USE_FLOAT -# define FMT_USE_FLOAT 1 -#endif - -#ifndef FMT_USE_DOUBLE -# define FMT_USE_DOUBLE 1 -#endif - -#ifndef FMT_USE_LONG_DOUBLE -# define FMT_USE_LONG_DOUBLE 1 -#endif - // Defining FMT_REDUCE_INT_INSTANTIATIONS to 1, will reduce the number of -// int_writer template instances to just one by only using the largest integer -// type. This results in a reduction in binary size but will cause a decrease in -// integer formatting performance. +// integer formatter template instantiations to just one by only using the +// largest integer type. This results in a reduction in binary size but will +// cause a decrease in integer formatting performance. #if !defined(FMT_REDUCE_INT_INSTANTIATIONS) # define FMT_REDUCE_INT_INSTANTIATIONS 0 #endif // __builtin_clz is broken in clang with Microsoft CodeGen: -// https://github.com/fmtlib/fmt/issues/519 -#if (FMT_GCC_VERSION || FMT_HAS_BUILTIN(__builtin_clz)) && !FMT_MSC_VER -# define FMT_BUILTIN_CLZ(n) __builtin_clz(n) -#endif -#if (FMT_GCC_VERSION || FMT_HAS_BUILTIN(__builtin_clzll)) && !FMT_MSC_VER -# define FMT_BUILTIN_CLZLL(n) __builtin_clzll(n) -#endif -#if (FMT_GCC_VERSION || FMT_HAS_BUILTIN(__builtin_ctz)) -# define FMT_BUILTIN_CTZ(n) __builtin_ctz(n) +// https://github.com/fmtlib/fmt/issues/519. +#if !FMT_MSC_VER +# if FMT_HAS_BUILTIN(__builtin_clz) || FMT_GCC_VERSION || FMT_ICC_VERSION +# define FMT_BUILTIN_CLZ(n) __builtin_clz(n) +# endif +# if FMT_HAS_BUILTIN(__builtin_clzll) || FMT_GCC_VERSION || FMT_ICC_VERSION +# define FMT_BUILTIN_CLZLL(n) __builtin_clzll(n) +# endif #endif -#if (FMT_GCC_VERSION || FMT_HAS_BUILTIN(__builtin_ctzll)) -# define FMT_BUILTIN_CTZLL(n) __builtin_ctzll(n) + +// __builtin_ctz is broken in Intel Compiler Classic on Windows: +// https://github.com/fmtlib/fmt/issues/2510. +#ifndef __ICL +# if FMT_HAS_BUILTIN(__builtin_ctz) || FMT_GCC_VERSION || FMT_ICC_VERSION +# define FMT_BUILTIN_CTZ(n) __builtin_ctz(n) +# endif +# if FMT_HAS_BUILTIN(__builtin_ctzll) || FMT_GCC_VERSION || FMT_ICC_VERSION +# define FMT_BUILTIN_CTZLL(n) __builtin_ctzll(n) +# endif #endif #if FMT_MSC_VER @@ -196,33 +173,32 @@ FMT_END_NAMESPACE // Some compilers masquerade as both MSVC and GCC-likes or otherwise support // __builtin_clz and __builtin_clzll, so only define FMT_BUILTIN_CLZ using the // MSVC intrinsics if the clz and clzll builtins are not available. -#if FMT_MSC_VER && !defined(FMT_BUILTIN_CLZLL) && \ - !defined(FMT_BUILTIN_CTZLL) && !defined(_MANAGED) +#if FMT_MSC_VER && !defined(FMT_BUILTIN_CLZLL) && !defined(FMT_BUILTIN_CTZLL) FMT_BEGIN_NAMESPACE namespace detail { // Avoid Clang with Microsoft CodeGen's -Wunknown-pragmas warning. -# ifndef __clang__ +# if !defined(__clang__) # pragma intrinsic(_BitScanForward) # pragma intrinsic(_BitScanReverse) -# endif -# if defined(_WIN64) && !defined(__clang__) -# pragma intrinsic(_BitScanForward64) -# pragma intrinsic(_BitScanReverse64) +# if defined(_WIN64) +# pragma intrinsic(_BitScanForward64) +# pragma intrinsic(_BitScanReverse64) +# endif # endif -inline int clz(uint32_t x) { +inline auto clz(uint32_t x) -> int { unsigned long r = 0; _BitScanReverse(&r, x); FMT_ASSERT(x != 0, ""); // Static analysis complains about using uninitialized data // "r", but the only way that can happen is if "x" is 0, // which the callers guarantee to not happen. - FMT_SUPPRESS_MSC_WARNING(6102) + FMT_MSC_WARNING(suppress : 6102) return 31 ^ static_cast(r); } # define FMT_BUILTIN_CLZ(n) detail::clz(n) -inline int clzll(uint64_t x) { +inline auto clzll(uint64_t x) -> int { unsigned long r = 0; # ifdef _WIN64 _BitScanReverse64(&r, x); @@ -233,24 +209,24 @@ inline int clzll(uint64_t x) { _BitScanReverse(&r, static_cast(x)); # endif FMT_ASSERT(x != 0, ""); - FMT_SUPPRESS_MSC_WARNING(6102) // Suppress a bogus static analysis warning. + FMT_MSC_WARNING(suppress : 6102) // Suppress a bogus static analysis warning. return 63 ^ static_cast(r); } # define FMT_BUILTIN_CLZLL(n) detail::clzll(n) -inline int ctz(uint32_t x) { +inline auto ctz(uint32_t x) -> int { unsigned long r = 0; _BitScanForward(&r, x); FMT_ASSERT(x != 0, ""); - FMT_SUPPRESS_MSC_WARNING(6102) // Suppress a bogus static analysis warning. + FMT_MSC_WARNING(suppress : 6102) // Suppress a bogus static analysis warning. return static_cast(r); } # define FMT_BUILTIN_CTZ(n) detail::ctz(n) -inline int ctzll(uint64_t x) { +inline auto ctzll(uint64_t x) -> int { unsigned long r = 0; FMT_ASSERT(x != 0, ""); - FMT_SUPPRESS_MSC_WARNING(6102) // Suppress a bogus static analysis warning. + FMT_MSC_WARNING(suppress : 6102) // Suppress a bogus static analysis warning. # ifdef _WIN64 _BitScanForward64(&r, x); # else @@ -267,31 +243,71 @@ inline int ctzll(uint64_t x) { FMT_END_NAMESPACE #endif -// Enable the deprecated numeric alignment. -#ifndef FMT_DEPRECATED_NUMERIC_ALIGN -# define FMT_DEPRECATED_NUMERIC_ALIGN 0 +#ifdef FMT_HEADER_ONLY +# define FMT_HEADER_ONLY_CONSTEXPR20 FMT_CONSTEXPR20 +#else +# define FMT_HEADER_ONLY_CONSTEXPR20 #endif FMT_BEGIN_NAMESPACE namespace detail { -// An equivalent of `*reinterpret_cast(&source)` that doesn't have -// undefined behavior (e.g. due to type aliasing). -// Example: uint64_t d = bit_cast(2.718); -template -inline Dest bit_cast(const Source& source) { - static_assert(sizeof(Dest) == sizeof(Source), "size mismatch"); - Dest dest; - std::memcpy(&dest, &source, sizeof(dest)); - return dest; +template class formatbuf : public Streambuf { + private: + using char_type = typename Streambuf::char_type; + using streamsize = decltype(std::declval().sputn(nullptr, 0)); + using int_type = typename Streambuf::int_type; + using traits_type = typename Streambuf::traits_type; + + buffer& buffer_; + + public: + explicit formatbuf(buffer& buf) : buffer_(buf) {} + + protected: + // The put area is always empty. This makes the implementation simpler and has + // the advantage that the streambuf and the buffer are always in sync and + // sputc never writes into uninitialized memory. A disadvantage is that each + // call to sputc always results in a (virtual) call to overflow. There is no + // disadvantage here for sputn since this always results in a call to xsputn. + + auto overflow(int_type ch) -> int_type override { + if (!traits_type::eq_int_type(ch, traits_type::eof())) + buffer_.push_back(static_cast(ch)); + return ch; + } + + auto xsputn(const char_type* s, streamsize count) -> streamsize override { + buffer_.append(s, s + count); + return count; + } +}; + +// Implementation of std::bit_cast for pre-C++20. +template +FMT_CONSTEXPR20 auto bit_cast(const From& from) -> To { + static_assert(sizeof(To) == sizeof(From), "size mismatch"); +#ifdef __cpp_lib_bit_cast + if (is_constant_evaluated()) return std::bit_cast(from); +#endif + auto to = To(); + std::memcpy(&to, &from, sizeof(to)); + return to; } -inline bool is_big_endian() { - const auto u = 1u; +inline auto is_big_endian() -> bool { +#ifdef _WIN32 + return false; +#elif defined(__BIG_ENDIAN__) + return true; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) + return __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__; +#else struct bytes { - char data[sizeof(u)]; + char data[sizeof(int)]; }; - return bit_cast(u).data[0] == 0; + return bit_cast(1).data[0] == 0; +#endif } // A fallback implementation of uintptr_t for systems that lack it. @@ -301,7 +317,7 @@ struct fallback_uintptr { fallback_uintptr() = default; explicit fallback_uintptr(const void* p) { *this = bit_cast(p); - if (is_big_endian()) { + if (const_check(is_big_endian())) { for (size_t i = 0, j = sizeof(void*) - 1; i < j; ++i, --j) std::swap(value[i], value[j]); } @@ -309,26 +325,28 @@ struct fallback_uintptr { }; #ifdef UINTPTR_MAX using uintptr_t = ::uintptr_t; -inline uintptr_t to_uintptr(const void* p) { return bit_cast(p); } +inline auto to_uintptr(const void* p) -> uintptr_t { + return bit_cast(p); +} #else using uintptr_t = fallback_uintptr; -inline fallback_uintptr to_uintptr(const void* p) { +inline auto to_uintptr(const void* p) -> fallback_uintptr { return fallback_uintptr(p); } #endif // Returns the largest possible value for type T. Same as // std::numeric_limits::max() but shorter and not affected by the max macro. -template constexpr T max_value() { +template constexpr auto max_value() -> T { return (std::numeric_limits::max)(); } -template constexpr int num_bits() { +template constexpr auto num_bits() -> int { return std::numeric_limits::digits; } // std::numeric_limits::digits may return 0 for 128-bit ints. -template <> constexpr int num_bits() { return 128; } -template <> constexpr int num_bits() { return 128; } -template <> constexpr int num_bits() { +template <> constexpr auto num_bits() -> int { return 128; } +template <> constexpr auto num_bits() -> int { return 128; } +template <> constexpr auto num_bits() -> int { return static_cast(sizeof(void*) * std::numeric_limits::digits); } @@ -346,31 +364,38 @@ using iterator_t = decltype(std::begin(std::declval())); template using sentinel_t = decltype(std::end(std::declval())); // A workaround for std::string not having mutable data() until C++17. -template inline Char* get_data(std::basic_string& s) { +template +inline auto get_data(std::basic_string& s) -> Char* { return &s[0]; } template -inline typename Container::value_type* get_data(Container& c) { +inline auto get_data(Container& c) -> typename Container::value_type* { return c.data(); } #if defined(_SECURE_SCL) && _SECURE_SCL // Make a checked iterator to avoid MSVC warnings. template using checked_ptr = stdext::checked_array_iterator; -template checked_ptr make_checked(T* p, size_t size) { +template +constexpr auto make_checked(T* p, size_t size) -> checked_ptr { return {p, size}; } #else template using checked_ptr = T*; -template inline T* make_checked(T* p, size_t) { return p; } +template constexpr auto make_checked(T* p, size_t) -> T* { + return p; +} #endif +// Attempts to reserve space for n extra characters in the output range. +// Returns a pointer to the reserved range or a reference to it. template ::value)> -#if FMT_CLANG_VERSION +#if FMT_CLANG_VERSION >= 307 && !FMT_ICC_VERSION __attribute__((no_sanitize("undefined"))) #endif -inline checked_ptr -reserve(std::back_insert_iterator it, size_t n) { +inline auto +reserve(std::back_insert_iterator it, size_t n) + -> checked_ptr { Container& c = get_container(it); size_t size = c.size(); c.resize(size + n); @@ -378,21 +403,26 @@ reserve(std::back_insert_iterator it, size_t n) { } template -inline buffer_appender reserve(buffer_appender it, size_t n) { +inline auto reserve(buffer_appender it, size_t n) -> buffer_appender { buffer& buf = get_container(it); buf.try_reserve(buf.size() + n); return it; } -template inline Iterator& reserve(Iterator& it, size_t) { +template +constexpr auto reserve(Iterator& it, size_t) -> Iterator& { return it; } +template +using reserve_iterator = + remove_reference_t(), 0))>; + template -constexpr T* to_pointer(OutputIt, size_t) { +constexpr auto to_pointer(OutputIt, size_t) -> T* { return nullptr; } -template T* to_pointer(buffer_appender it, size_t n) { +template auto to_pointer(buffer_appender it, size_t n) -> T* { buffer& buf = get_container(it); auto size = buf.size(); if (buf.capacity() < size + n) return nullptr; @@ -401,195 +431,195 @@ template T* to_pointer(buffer_appender it, size_t n) { } template ::value)> -inline std::back_insert_iterator base_iterator( - std::back_insert_iterator& it, - checked_ptr) { +inline auto base_iterator(std::back_insert_iterator& it, + checked_ptr) + -> std::back_insert_iterator { return it; } template -inline Iterator base_iterator(Iterator, Iterator it) { +constexpr auto base_iterator(Iterator, Iterator it) -> Iterator { return it; } -// An output iterator that counts the number of objects written to it and -// discards them. -class counting_iterator { - private: - size_t count_; - - public: - using iterator_category = std::output_iterator_tag; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = void; - using _Unchecked_type = counting_iterator; // Mark iterator as checked. - - struct value_type { - template void operator=(const T&) {} - }; - - counting_iterator() : count_(0) {} - - size_t count() const { return count_; } - - counting_iterator& operator++() { - ++count_; - return *this; - } - counting_iterator operator++(int) { - auto it = *this; - ++*this; - return it; - } - - friend counting_iterator operator+(counting_iterator it, difference_type n) { - it.count_ += static_cast(n); - return it; - } - - value_type operator*() const { return {}; } -}; - -template class truncating_iterator_base { - protected: - OutputIt out_; - size_t limit_; - size_t count_; - - truncating_iterator_base(OutputIt out, size_t limit) - : out_(out), limit_(limit), count_(0) {} - - public: - using iterator_category = std::output_iterator_tag; - using value_type = typename std::iterator_traits::value_type; - using difference_type = void; - using pointer = void; - using reference = void; - using _Unchecked_type = - truncating_iterator_base; // Mark iterator as checked. - - OutputIt base() const { return out_; } - size_t count() const { return count_; } -}; - -// An output iterator that truncates the output and counts the number of objects -// written to it. -template ::value_type>::type> -class truncating_iterator; - -template -class truncating_iterator - : public truncating_iterator_base { - mutable typename truncating_iterator_base::value_type blackhole_; - - public: - using value_type = typename truncating_iterator_base::value_type; - - truncating_iterator(OutputIt out, size_t limit) - : truncating_iterator_base(out, limit) {} - - truncating_iterator& operator++() { - if (this->count_++ < this->limit_) ++this->out_; - return *this; +// is spectacularly slow to compile in C++20 so use a simple fill_n +// instead (#1998). +template +FMT_CONSTEXPR auto fill_n(OutputIt out, Size count, const T& value) + -> OutputIt { + for (Size i = 0; i < count; ++i) *out++ = value; + return out; +} +template +FMT_CONSTEXPR20 auto fill_n(T* out, Size count, char value) -> T* { + if (is_constant_evaluated()) { + return fill_n(out, count, value); } + std::memset(out, value, to_unsigned(count)); + return out + count; +} - truncating_iterator operator++(int) { - auto it = *this; - ++*this; - return it; - } +#ifdef __cpp_char8_t +using char8_type = char8_t; +#else +enum char8_type : unsigned char {}; +#endif - value_type& operator*() const { - return this->count_ < this->limit_ ? *this->out_ : blackhole_; +template +FMT_CONSTEXPR FMT_NOINLINE auto copy_str_noinline(InputIt begin, InputIt end, + OutputIt out) -> OutputIt { + return copy_str(begin, end, out); +} + +// A public domain branchless UTF-8 decoder by Christopher Wellons: +// https://github.com/skeeto/branchless-utf8 +/* Decode the next character, c, from s, reporting errors in e. + * + * Since this is a branchless decoder, four bytes will be read from the + * buffer regardless of the actual length of the next character. This + * means the buffer _must_ have at least three bytes of zero padding + * following the end of the data stream. + * + * Errors are reported in e, which will be non-zero if the parsed + * character was somehow invalid: invalid byte sequence, non-canonical + * encoding, or a surrogate half. + * + * The function returns a pointer to the next character. When an error + * occurs, this pointer will be a guess that depends on the particular + * error, but it will always advance at least one byte. + */ +FMT_CONSTEXPR inline auto utf8_decode(const char* s, uint32_t* c, int* e) + -> const char* { + constexpr const int masks[] = {0x00, 0x7f, 0x1f, 0x0f, 0x07}; + constexpr const uint32_t mins[] = {4194304, 0, 128, 2048, 65536}; + constexpr const int shiftc[] = {0, 18, 12, 6, 0}; + constexpr const int shifte[] = {0, 6, 4, 2, 0}; + + int len = code_point_length(s); + const char* next = s + len; + + // Assume a four-byte character and load four bytes. Unused bits are + // shifted out. + *c = uint32_t(s[0] & masks[len]) << 18; + *c |= uint32_t(s[1] & 0x3f) << 12; + *c |= uint32_t(s[2] & 0x3f) << 6; + *c |= uint32_t(s[3] & 0x3f) << 0; + *c >>= shiftc[len]; + + // Accumulate the various error conditions. + using uchar = unsigned char; + *e = (*c < mins[len]) << 6; // non-canonical encoding + *e |= ((*c >> 11) == 0x1b) << 7; // surrogate half? + *e |= (*c > 0x10FFFF) << 8; // out of range? + *e |= (uchar(s[1]) & 0xc0) >> 2; + *e |= (uchar(s[2]) & 0xc0) >> 4; + *e |= uchar(s[3]) >> 6; + *e ^= 0x2a; // top two bits of each tail byte correct? + *e >>= shifte[len]; + + return next; +} + +constexpr uint32_t invalid_code_point = ~uint32_t(); + +// Invokes f(cp, sv) for every code point cp in s with sv being the string view +// corresponding to the code point. cp is invalid_code_point on error. +template +FMT_CONSTEXPR void for_each_codepoint(string_view s, F f) { + auto decode = [f](const char* buf_ptr, const char* ptr) { + auto cp = uint32_t(); + auto error = 0; + auto end = utf8_decode(buf_ptr, &cp, &error); + bool result = f(error ? invalid_code_point : cp, + string_view(ptr, to_unsigned(end - buf_ptr))); + return result ? end : nullptr; + }; + auto p = s.data(); + const size_t block_size = 4; // utf8_decode always reads blocks of 4 chars. + if (s.size() >= block_size) { + for (auto end = p + s.size() - block_size + 1; p < end;) { + p = decode(p, p); + if (!p) return; + } } -}; - -template -class truncating_iterator - : public truncating_iterator_base { - public: - truncating_iterator(OutputIt out, size_t limit) - : truncating_iterator_base(out, limit) {} - - template truncating_iterator& operator=(T val) { - if (this->count_++ < this->limit_) *this->out_++ = val; - return *this; + if (auto num_chars_left = s.data() + s.size() - p) { + char buf[2 * block_size - 1] = {}; + copy_str(p, p + num_chars_left, buf); + const char* buf_ptr = buf; + do { + auto end = decode(buf_ptr, p); + if (!end) return; + p += end - buf_ptr; + buf_ptr = end; + } while (buf_ptr - buf < num_chars_left); } - - truncating_iterator& operator++() { return *this; } - truncating_iterator& operator++(int) { return *this; } - truncating_iterator& operator*() { return *this; } -}; +} template -inline size_t count_code_points(basic_string_view s) { +inline auto compute_width(basic_string_view s) -> size_t { return s.size(); } -// Counts the number of code points in a UTF-8 string. -inline size_t count_code_points(basic_string_view s) { - const char* data = s.data(); +// Computes approximate display width of a UTF-8 string. +FMT_CONSTEXPR inline size_t compute_width(string_view s) { size_t num_code_points = 0; - for (size_t i = 0, size = s.size(); i != size; ++i) { - if ((data[i] & 0xc0) != 0x80) ++num_code_points; - } + // It is not a lambda for compatibility with C++14. + struct count_code_points { + size_t* count; + FMT_CONSTEXPR auto operator()(uint32_t cp, string_view) const -> bool { + *count += detail::to_unsigned( + 1 + + (cp >= 0x1100 && + (cp <= 0x115f || // Hangul Jamo init. consonants + cp == 0x2329 || // LEFT-POINTING ANGLE BRACKET + cp == 0x232a || // RIGHT-POINTING ANGLE BRACKET + // CJK ... Yi except IDEOGRAPHIC HALF FILL SPACE: + (cp >= 0x2e80 && cp <= 0xa4cf && cp != 0x303f) || + (cp >= 0xac00 && cp <= 0xd7a3) || // Hangul Syllables + (cp >= 0xf900 && cp <= 0xfaff) || // CJK Compatibility Ideographs + (cp >= 0xfe10 && cp <= 0xfe19) || // Vertical Forms + (cp >= 0xfe30 && cp <= 0xfe6f) || // CJK Compatibility Forms + (cp >= 0xff00 && cp <= 0xff60) || // Fullwidth Forms + (cp >= 0xffe0 && cp <= 0xffe6) || // Fullwidth Forms + (cp >= 0x20000 && cp <= 0x2fffd) || // CJK + (cp >= 0x30000 && cp <= 0x3fffd) || + // Miscellaneous Symbols and Pictographs + Emoticons: + (cp >= 0x1f300 && cp <= 0x1f64f) || + // Supplemental Symbols and Pictographs: + (cp >= 0x1f900 && cp <= 0x1f9ff)))); + return true; + } + }; + for_each_codepoint(s, count_code_points{&num_code_points}); return num_code_points; } -inline size_t count_code_points(basic_string_view s) { - return count_code_points(basic_string_view( +inline auto compute_width(basic_string_view s) -> size_t { + return compute_width(basic_string_view( reinterpret_cast(s.data()), s.size())); } template -inline size_t code_point_index(basic_string_view s, size_t n) { +inline auto code_point_index(basic_string_view s, size_t n) -> size_t { size_t size = s.size(); return n < size ? n : size; } // Calculates the index of the nth code point in a UTF-8 string. -inline size_t code_point_index(basic_string_view s, size_t n) { +inline auto code_point_index(basic_string_view s, size_t n) + -> size_t { const char8_type* data = s.data(); size_t num_code_points = 0; for (size_t i = 0, size = s.size(); i != size; ++i) { - if ((data[i] & 0xc0) != 0x80 && ++num_code_points > n) { - return i; - } + if ((data[i] & 0xc0) != 0x80 && ++num_code_points > n) return i; } return s.size(); } -template -using needs_conversion = bool_constant< - std::is_same::value_type, - char>::value && - std::is_same::value>; - -template ::value)> -OutputIt copy_str(InputIt begin, InputIt end, OutputIt it) { - return std::copy(begin, end, it); -} - -template ::value)> -OutputIt copy_str(InputIt begin, InputIt end, OutputIt it) { - return std::transform(begin, end, it, - [](char c) { return static_cast(c); }); -} - -template -inline counting_iterator copy_str(InputIt begin, InputIt end, - counting_iterator it) { - return it + (end - begin); -} - -template -using is_fast_float = bool_constant::is_iec559 && - sizeof(T) <= sizeof(double)>; +template ::value> +struct is_fast_float : bool_constant::is_iec559 && + sizeof(T) <= sizeof(double)> {}; +template struct is_fast_float : std::false_type {}; #ifndef FMT_USE_FULL_CACHE_DRAGONBOX # define FMT_USE_FULL_CACHE_DRAGONBOX 0 @@ -598,7 +628,7 @@ using is_fast_float = bool_constant::is_iec559 && template template void buffer::append(const U* begin, const U* end) { - do { + while (begin != end) { auto count = to_unsigned(end - begin); try_reserve(size_ + count); auto free_cap = capacity_ - size_; @@ -606,16 +636,17 @@ void buffer::append(const U* begin, const U* end) { std::uninitialized_copy_n(begin, count, make_checked(ptr_ + size_, count)); size_ += count; begin += count; - } while (begin != end); + } } -template -void iterator_buffer::flush() { - out_ = std::copy_n(data_, this->limit(this->size()), out_); - this->clear(); -} +template +struct is_locale : std::false_type {}; +template +struct is_locale> : std::true_type {}; } // namespace detail +FMT_MODULE_EXPORT_BEGIN + // The number of characters to store in the basic_memory_buffer object itself // to avoid dynamic memory allocation. enum { inline_buffer_size = 500 }; @@ -625,20 +656,12 @@ enum { inline_buffer_size = 500 }; A dynamically growing memory buffer for trivially copyable/constructible types with the first ``SIZE`` elements stored in the object itself. - You can use one of the following type aliases for common character types: - - +----------------+------------------------------+ - | Type | Definition | - +================+==============================+ - | memory_buffer | basic_memory_buffer | - +----------------+------------------------------+ - | wmemory_buffer | basic_memory_buffer | - +----------------+------------------------------+ + You can use the ``memory_buffer`` type alias for ``char`` instead. **Example**:: - fmt::memory_buffer out; - format_to(out, "The answer is {}.", 42); + auto out = fmt::memory_buffer(); + format_to(std::back_inserter(out), "The answer is {}.", 42); This will append the following output to the ``out`` object: @@ -659,34 +682,43 @@ class basic_memory_buffer final : public detail::buffer { Allocator alloc_; // Deallocate memory allocated by the buffer. - void deallocate() { + FMT_CONSTEXPR20 void deallocate() { T* data = this->data(); if (data != store_) alloc_.deallocate(data, this->capacity()); } protected: - void grow(size_t size) final FMT_OVERRIDE; + FMT_CONSTEXPR20 void grow(size_t size) override; public: using value_type = T; using const_reference = const T&; - explicit basic_memory_buffer(const Allocator& alloc = Allocator()) + FMT_CONSTEXPR20 explicit basic_memory_buffer( + const Allocator& alloc = Allocator()) : alloc_(alloc) { this->set(store_, SIZE); + if (detail::is_constant_evaluated()) { + detail::fill_n(store_, SIZE, T{}); + } } - ~basic_memory_buffer() { deallocate(); } + FMT_CONSTEXPR20 ~basic_memory_buffer() { deallocate(); } private: // Move data from other to this buffer. - void move(basic_memory_buffer& other) { + FMT_CONSTEXPR20 void move(basic_memory_buffer& other) { alloc_ = std::move(other.alloc_); T* data = other.data(); size_t size = other.size(), capacity = other.capacity(); if (data == other.store_) { this->set(store_, capacity); - std::uninitialized_copy(other.store_, other.store_ + size, - detail::make_checked(store_, capacity)); + if (detail::is_constant_evaluated()) { + detail::copy_str(other.store_, other.store_ + size, + detail::make_checked(store_, capacity)); + } else { + std::uninitialized_copy(other.store_, other.store_ + size, + detail::make_checked(store_, capacity)); + } } else { this->set(data, capacity); // Set pointer to the inline array so that delete is not called @@ -703,14 +735,18 @@ class basic_memory_buffer final : public detail::buffer { of the other object to it. \endrst */ - basic_memory_buffer(basic_memory_buffer&& other) FMT_NOEXCEPT { move(other); } + FMT_CONSTEXPR20 basic_memory_buffer(basic_memory_buffer&& other) + FMT_NOEXCEPT { + move(other); + } /** \rst Moves the content of the other ``basic_memory_buffer`` object to this one. \endrst */ - basic_memory_buffer& operator=(basic_memory_buffer&& other) FMT_NOEXCEPT { + auto operator=(basic_memory_buffer&& other) FMT_NOEXCEPT + -> basic_memory_buffer& { FMT_ASSERT(this != &other, ""); deallocate(); move(other); @@ -718,13 +754,13 @@ class basic_memory_buffer final : public detail::buffer { } // Returns a copy of the allocator associated with this buffer. - Allocator get_allocator() const { return alloc_; } + auto get_allocator() const -> Allocator { return alloc_; } /** Resizes the buffer to contain *count* elements. If T is a POD type new elements may not be initialized. */ - void resize(size_t count) { this->try_resize(count); } + FMT_CONSTEXPR20 void resize(size_t count) { this->try_resize(count); } /** Increases the buffer capacity to *new_capacity*. */ void reserve(size_t new_capacity) { this->try_reserve(new_capacity); } @@ -738,13 +774,18 @@ class basic_memory_buffer final : public detail::buffer { }; template -void basic_memory_buffer::grow(size_t size) { +FMT_CONSTEXPR20 void basic_memory_buffer::grow( + size_t size) { #ifdef FMT_FUZZ if (size > 5000) throw std::runtime_error("fuzz mode - won't grow that much"); #endif + const size_t max_size = std::allocator_traits::max_size(alloc_); size_t old_capacity = this->capacity(); size_t new_capacity = old_capacity + old_capacity / 2; - if (size > new_capacity) new_capacity = size; + if (size > new_capacity) + new_capacity = size; + else if (new_capacity > max_size) + new_capacity = size > max_size ? size : max_size; T* old_data = this->data(); T* new_data = std::allocator_traits::allocate(alloc_, new_capacity); @@ -759,12 +800,15 @@ void basic_memory_buffer::grow(size_t size) { } using memory_buffer = basic_memory_buffer; -using wmemory_buffer = basic_memory_buffer; template struct is_contiguous> : std::true_type { }; +namespace detail { +FMT_API void print(std::FILE*, string_view); +} + /** A formatting error such as invalid format string. */ FMT_CLASS_API class FMT_API format_error : public std::runtime_error { @@ -776,10 +820,62 @@ class FMT_API format_error : public std::runtime_error { format_error& operator=(const format_error&) = default; format_error(format_error&&) = default; format_error& operator=(format_error&&) = default; - ~format_error() FMT_NOEXCEPT FMT_OVERRIDE; + ~format_error() FMT_NOEXCEPT override FMT_MSC_DEFAULT; }; -namespace detail { +/** + \rst + Constructs a `~fmt::format_arg_store` object that contains references + to arguments and can be implicitly converted to `~fmt::format_args`. + If ``fmt`` is a compile-time string then `make_args_checked` checks + its validity at compile time. + \endrst + */ +template > +FMT_INLINE auto make_args_checked(const S& fmt, + const remove_reference_t&... args) + -> format_arg_store, remove_reference_t...> { + static_assert( + detail::count<( + std::is_base_of>::value && + std::is_reference::value)...>() == 0, + "passing views as lvalues is disallowed"); + detail::check_format_string(fmt); + return {args...}; +} + +// compile-time support +namespace detail_exported { +#if FMT_USE_NONTYPE_TEMPLATE_PARAMETERS +template struct fixed_string { + constexpr fixed_string(const Char (&str)[N]) { + detail::copy_str(static_cast(str), + str + N, data); + } + Char data[N]{}; +}; +#endif + +// Converts a compile-time string to basic_string_view. +template +constexpr auto compile_string_to_view(const Char (&s)[N]) + -> basic_string_view { + // Remove trailing NUL character if needed. Won't be present if this is used + // with a raw character array (i.e. not defined as a string). + return {s, N - (std::char_traits::to_int_type(s[N - 1]) == 0 ? 1 : 0)}; +} +template +constexpr auto compile_string_to_view(detail::std_string_view s) + -> basic_string_view { + return {s.data(), s.size()}; +} +} // namespace detail_exported + +FMT_BEGIN_DETAIL_NAMESPACE + +template struct is_integral : std::is_integral {}; +template <> struct is_integral : std::true_type {}; +template <> struct is_integral : std::true_type {}; template using is_signed = @@ -789,16 +885,16 @@ using is_signed = // Returns true if value is negative, false otherwise. // Same as `value < 0` but doesn't produce warnings if T is an unsigned type. template ::value)> -FMT_CONSTEXPR bool is_negative(T value) { +FMT_CONSTEXPR auto is_negative(T value) -> bool { return value < 0; } template ::value)> -FMT_CONSTEXPR bool is_negative(T) { +FMT_CONSTEXPR auto is_negative(T) -> bool { return false; } template ::value)> -FMT_CONSTEXPR bool is_supported_floating_point(T) { +FMT_CONSTEXPR auto is_supported_floating_point(T) -> uint16_t { return (std::is_same::value && FMT_USE_FLOAT) || (std::is_same::value && FMT_USE_DOUBLE) || (std::is_same::value && FMT_USE_LONG_DOUBLE); @@ -811,121 +907,33 @@ using uint32_or_64_or_128_t = conditional_t() <= 32 && !FMT_REDUCE_INT_INSTANTIATIONS, uint32_t, conditional_t() <= 64, uint64_t, uint128_t>>; - -// 128-bit integer type used internally -struct FMT_EXTERN_TEMPLATE_API uint128_wrapper { - uint128_wrapper() = default; - -#if FMT_USE_INT128 - uint128_t internal_; - - uint128_wrapper(uint64_t high, uint64_t low) FMT_NOEXCEPT - : internal_{static_cast(low) | - (static_cast(high) << 64)} {} - - uint128_wrapper(uint128_t u) : internal_{u} {} - - uint64_t high() const FMT_NOEXCEPT { return uint64_t(internal_ >> 64); } - uint64_t low() const FMT_NOEXCEPT { return uint64_t(internal_); } - - uint128_wrapper& operator+=(uint64_t n) FMT_NOEXCEPT { - internal_ += n; - return *this; - } -#else - uint64_t high_; - uint64_t low_; - - uint128_wrapper(uint64_t high, uint64_t low) FMT_NOEXCEPT : high_{high}, - low_{low} {} - - uint64_t high() const FMT_NOEXCEPT { return high_; } - uint64_t low() const FMT_NOEXCEPT { return low_; } - - uint128_wrapper& operator+=(uint64_t n) FMT_NOEXCEPT { -# if defined(_MSC_VER) && defined(_M_X64) - unsigned char carry = _addcarry_u64(0, low_, n, &low_); - _addcarry_u64(carry, high_, 0, &high_); - return *this; -# else - uint64_t sum = low_ + n; - high_ += (sum < low_ ? 1 : 0); - low_ = sum; - return *this; -# endif - } -#endif -}; - -// Table entry type for divisibility test used internally -template struct FMT_EXTERN_TEMPLATE_API divtest_table_entry { - T mod_inv; - T max_quotient; -}; - -// Static data is placed in this class template for the header-only config. -template struct FMT_EXTERN_TEMPLATE_API basic_data { - static const uint64_t powers_of_10_64[]; - static const uint32_t zero_or_powers_of_10_32_new[]; - static const uint64_t zero_or_powers_of_10_64_new[]; - static const uint64_t grisu_pow10_significands[]; - static const int16_t grisu_pow10_exponents[]; - static const divtest_table_entry divtest_table_for_pow5_32[]; - static const divtest_table_entry divtest_table_for_pow5_64[]; - static const uint64_t dragonbox_pow10_significands_64[]; - static const uint128_wrapper dragonbox_pow10_significands_128[]; - // log10(2) = 0x0.4d104d427de7fbcc... - static const uint64_t log10_2_significand = 0x4d104d427de7fbcc; -#if !FMT_USE_FULL_CACHE_DRAGONBOX - static const uint64_t powers_of_5_64[]; - static const uint32_t dragonbox_pow10_recovery_errors[]; +template +using uint64_or_128_t = conditional_t() <= 64, uint64_t, uint128_t>; + +#define FMT_POWERS_OF_10(factor) \ + factor * 10, (factor)*100, (factor)*1000, (factor)*10000, (factor)*100000, \ + (factor)*1000000, (factor)*10000000, (factor)*100000000, \ + (factor)*1000000000 + +// Converts value in the range [0, 100) to a string. +constexpr const char* digits2(size_t value) { + // GCC generates slightly better code when value is pointer-size. + return &"0001020304050607080910111213141516171819" + "2021222324252627282930313233343536373839" + "4041424344454647484950515253545556575859" + "6061626364656667686970717273747576777879" + "8081828384858687888990919293949596979899"[value * 2]; +} + +// Sign is a template parameter to workaround a bug in gcc 4.8. +template constexpr Char sign(Sign s) { +#if !FMT_GCC_VERSION || FMT_GCC_VERSION >= 604 + static_assert(std::is_same::value, ""); #endif - // GCC generates slightly better code for pairs than chars. - using digit_pair = char[2]; - static const digit_pair digits[]; - static const char hex_digits[]; - static const char foreground_color[]; - static const char background_color[]; - static const char reset_color[5]; - static const wchar_t wreset_color[5]; - static const char signs[]; - static const char left_padding_shifts[5]; - static const char right_padding_shifts[5]; - - // DEPRECATED! These are for ABI compatibility. - static const uint32_t zero_or_powers_of_10_32[]; - static const uint64_t zero_or_powers_of_10_64[]; -}; - -// Maps bsr(n) to ceil(log10(pow(2, bsr(n) + 1) - 1)). -// This is a function instead of an array to workaround a bug in GCC10 (#1810). -FMT_INLINE uint16_t bsr2log10(int bsr) { - static constexpr uint16_t data[] = { - 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, - 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10, - 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 15, 15, - 15, 16, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 19, 20}; - return data[bsr]; + return static_cast("\0-+ "[s]); } -#ifndef FMT_EXPORTED -FMT_EXTERN template struct basic_data; -#endif - -// This is a struct rather than an alias to avoid shadowing warnings in gcc. -struct data : basic_data<> {}; - -#ifdef FMT_BUILTIN_CLZLL -// Returns the number of decimal digits in n. Leading zeros are not counted -// except for n == 0 in which case count_digits returns 1. -inline int count_digits(uint64_t n) { - // https://github.com/fmtlib/format-benchmark/blob/master/digits10 - auto t = bsr2log10(FMT_BUILTIN_CLZLL(n | 1) ^ 63); - return t - (n < data::zero_or_powers_of_10_64_new[t]); -} -#else -// Fallback version of count_digits used when __builtin_clz is not available. -inline int count_digits(uint64_t n) { +template FMT_CONSTEXPR auto count_digits_fallback(T n) -> int { int count = 1; for (;;) { // Integer division is slow so do it for a group of four digits instead @@ -939,103 +947,152 @@ inline int count_digits(uint64_t n) { count += 4; } } -#endif - #if FMT_USE_INT128 -inline int count_digits(uint128_t n) { - int count = 1; - for (;;) { - // Integer division is slow so do it for a group of four digits instead - // of for every digit. The idea comes from the talk by Alexandrescu - // "Three Optimization Tips for C++". See speed-test for a comparison. - if (n < 10) return count; - if (n < 100) return count + 1; - if (n < 1000) return count + 2; - if (n < 10000) return count + 3; - n /= 10000U; - count += 4; - } +FMT_CONSTEXPR inline auto count_digits(uint128_t n) -> int { + return count_digits_fallback(n); } #endif -// Counts the number of digits in n. BITS = log2(radix). -template inline int count_digits(UInt n) { - int num_digits = 0; - do { - ++num_digits; - } while ((n >>= BITS) != 0); - return num_digits; +#ifdef FMT_BUILTIN_CLZLL +// It is a separate function rather than a part of count_digits to workaround +// the lack of static constexpr in constexpr functions. +inline auto do_count_digits(uint64_t n) -> int { + // This has comparable performance to the version by Kendall Willets + // (https://github.com/fmtlib/format-benchmark/blob/master/digits10) + // but uses smaller tables. + // Maps bsr(n) to ceil(log10(pow(2, bsr(n) + 1) - 1)). + static constexpr uint8_t bsr2log10[] = { + 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, + 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10, + 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 15, 15, + 15, 16, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 19, 20}; + auto t = bsr2log10[FMT_BUILTIN_CLZLL(n | 1) ^ 63]; + static constexpr const uint64_t zero_or_powers_of_10[] = { + 0, 0, FMT_POWERS_OF_10(1U), FMT_POWERS_OF_10(1000000000ULL), + 10000000000000000000ULL}; + return t - (n < zero_or_powers_of_10[t]); } +#endif -template <> int count_digits<4>(detail::fallback_uintptr n); - -#if FMT_GCC_VERSION || FMT_CLANG_VERSION -# define FMT_ALWAYS_INLINE inline __attribute__((always_inline)) -#elif FMT_MSC_VER -# define FMT_ALWAYS_INLINE __forceinline -#else -# define FMT_ALWAYS_INLINE inline +// Returns the number of decimal digits in n. Leading zeros are not counted +// except for n == 0 in which case count_digits returns 1. +FMT_CONSTEXPR20 inline auto count_digits(uint64_t n) -> int { +#ifdef FMT_BUILTIN_CLZLL + if (!is_constant_evaluated()) { + return do_count_digits(n); + } #endif + return count_digits_fallback(n); +} -// To suppress unnecessary security cookie checks -#if FMT_MSC_VER && !FMT_CLANG_VERSION -# define FMT_SAFEBUFFERS __declspec(safebuffers) -#else -# define FMT_SAFEBUFFERS +// Counts the number of digits in n. BITS = log2(radix). +template +FMT_CONSTEXPR auto count_digits(UInt n) -> int { +#ifdef FMT_BUILTIN_CLZ + if (num_bits() == 32) + return (FMT_BUILTIN_CLZ(static_cast(n) | 1) ^ 31) / BITS + 1; #endif + // Lambda avoids unreachable code warnings from NVHPC. + return [](UInt m) { + int num_digits = 0; + do { + ++num_digits; + } while ((m >>= BITS) != 0); + return num_digits; + }(n); +} + +template <> auto count_digits<4>(detail::fallback_uintptr n) -> int; #ifdef FMT_BUILTIN_CLZ -// Optional version of count_digits for better performance on 32-bit platforms. -inline int count_digits(uint32_t n) { - auto t = bsr2log10(FMT_BUILTIN_CLZ(n | 1) ^ 31); - return t - (n < data::zero_or_powers_of_10_32_new[t]); +// It is a separate function rather than a part of count_digits to workaround +// the lack of static constexpr in constexpr functions. +FMT_INLINE auto do_count_digits(uint32_t n) -> int { +// An optimization by Kendall Willets from https://bit.ly/3uOIQrB. +// This increments the upper 32 bits (log10(T) - 1) when >= T is added. +# define FMT_INC(T) (((sizeof(# T) - 1ull) << 32) - T) + static constexpr uint64_t table[] = { + FMT_INC(0), FMT_INC(0), FMT_INC(0), // 8 + FMT_INC(10), FMT_INC(10), FMT_INC(10), // 64 + FMT_INC(100), FMT_INC(100), FMT_INC(100), // 512 + FMT_INC(1000), FMT_INC(1000), FMT_INC(1000), // 4096 + FMT_INC(10000), FMT_INC(10000), FMT_INC(10000), // 32k + FMT_INC(100000), FMT_INC(100000), FMT_INC(100000), // 256k + FMT_INC(1000000), FMT_INC(1000000), FMT_INC(1000000), // 2048k + FMT_INC(10000000), FMT_INC(10000000), FMT_INC(10000000), // 16M + FMT_INC(100000000), FMT_INC(100000000), FMT_INC(100000000), // 128M + FMT_INC(1000000000), FMT_INC(1000000000), FMT_INC(1000000000), // 1024M + FMT_INC(1000000000), FMT_INC(1000000000) // 4B + }; + auto inc = table[FMT_BUILTIN_CLZ(n | 1) ^ 31]; + return static_cast((n + inc) >> 32); } #endif -template constexpr int digits10() FMT_NOEXCEPT { - return std::numeric_limits::digits10; +// Optional version of count_digits for better performance on 32-bit platforms. +FMT_CONSTEXPR20 inline auto count_digits(uint32_t n) -> int { +#ifdef FMT_BUILTIN_CLZ + if (!is_constant_evaluated()) { + return do_count_digits(n); + } +#endif + return count_digits_fallback(n); } -template <> constexpr int digits10() FMT_NOEXCEPT { return 38; } -template <> constexpr int digits10() FMT_NOEXCEPT { return 38; } -template FMT_API std::string grouping_impl(locale_ref loc); -template inline std::string grouping(locale_ref loc) { - return grouping_impl(loc); +template constexpr auto digits10() FMT_NOEXCEPT -> int { + return std::numeric_limits::digits10; +} +template <> constexpr auto digits10() FMT_NOEXCEPT -> int { + return 38; } -template <> inline std::string grouping(locale_ref loc) { - return grouping_impl(loc); +template <> constexpr auto digits10() FMT_NOEXCEPT -> int { + return 38; } -template FMT_API Char thousands_sep_impl(locale_ref loc); -template inline Char thousands_sep(locale_ref loc) { - return Char(thousands_sep_impl(loc)); +template struct thousands_sep_result { + std::string grouping; + Char thousands_sep; +}; + +template +FMT_API auto thousands_sep_impl(locale_ref loc) -> thousands_sep_result; +template +inline auto thousands_sep(locale_ref loc) -> thousands_sep_result { + auto result = thousands_sep_impl(loc); + return {result.grouping, Char(result.thousands_sep)}; } -template <> inline wchar_t thousands_sep(locale_ref loc) { +template <> +inline auto thousands_sep(locale_ref loc) -> thousands_sep_result { return thousands_sep_impl(loc); } -template FMT_API Char decimal_point_impl(locale_ref loc); -template inline Char decimal_point(locale_ref loc) { +template +FMT_API auto decimal_point_impl(locale_ref loc) -> Char; +template inline auto decimal_point(locale_ref loc) -> Char { return Char(decimal_point_impl(loc)); } -template <> inline wchar_t decimal_point(locale_ref loc) { +template <> inline auto decimal_point(locale_ref loc) -> wchar_t { return decimal_point_impl(loc); } // Compares two characters for equality. -template bool equal2(const Char* lhs, const char* rhs) { - return lhs[0] == rhs[0] && lhs[1] == rhs[1]; +template auto equal2(const Char* lhs, const char* rhs) -> bool { + return lhs[0] == Char(rhs[0]) && lhs[1] == Char(rhs[1]); } -inline bool equal2(const char* lhs, const char* rhs) { +inline auto equal2(const char* lhs, const char* rhs) -> bool { return memcmp(lhs, rhs, 2) == 0; } // Copies two characters from src to dst. -template void copy2(Char* dst, const char* src) { +template +FMT_CONSTEXPR20 FMT_INLINE void copy2(Char* dst, const char* src) { + if (!is_constant_evaluated() && sizeof(Char) == sizeof(char)) { + memcpy(dst, src, 2); + return; + } *dst++ = static_cast(*src++); *dst = static_cast(*src); } -FMT_INLINE void copy2(char* dst, const char* src) { memcpy(dst, src, 2); } template struct format_decimal_result { Iterator begin; @@ -1046,8 +1103,8 @@ template struct format_decimal_result { // buffer of specified size. The caller must ensure that the buffer is large // enough. template -inline format_decimal_result format_decimal(Char* out, UInt value, - int size) { +FMT_CONSTEXPR20 auto format_decimal(Char* out, UInt value, int size) + -> format_decimal_result { FMT_ASSERT(size >= count_digits(value), "invalid digit count"); out += size; Char* end = out; @@ -1056,7 +1113,7 @@ inline format_decimal_result format_decimal(Char* out, UInt value, // of for every digit. The idea comes from the talk by Alexandrescu // "Three Optimization Tips for C++". See speed-test for a comparison. out -= 2; - copy2(out, data::digits[value % 100]); + copy2(out, digits2(static_cast(value % 100))); value /= 100; } if (value < 10) { @@ -1064,27 +1121,27 @@ inline format_decimal_result format_decimal(Char* out, UInt value, return {out, end}; } out -= 2; - copy2(out, data::digits[value]); + copy2(out, digits2(static_cast(value))); return {out, end}; } template >::value)> -inline format_decimal_result format_decimal(Iterator out, UInt value, - int size) { +inline auto format_decimal(Iterator out, UInt value, int size) + -> format_decimal_result { // Buffer is large enough to hold all digits (digits10 + 1). Char buffer[digits10() + 1]; auto end = format_decimal(buffer, value, size).end; - return {out, detail::copy_str(buffer, end, out)}; + return {out, detail::copy_str_noinline(buffer, end, out)}; } template -inline Char* format_uint(Char* buffer, UInt value, int num_digits, - bool upper = false) { +FMT_CONSTEXPR auto format_uint(Char* buffer, UInt value, int num_digits, + bool upper = false) -> Char* { buffer += num_digits; Char* end = buffer; do { - const char* digits = upper ? "0123456789ABCDEF" : data::hex_digits; + const char* digits = upper ? "0123456789ABCDEF" : "0123456789abcdef"; unsigned digit = (value & ((1 << BASE_BITS) - 1)); *--buffer = static_cast(BASE_BITS < 4 ? static_cast('0' + digit) : digits[digit]); @@ -1093,8 +1150,8 @@ inline Char* format_uint(Char* buffer, UInt value, int num_digits, } template -Char* format_uint(Char* buffer, detail::fallback_uintptr n, int num_digits, - bool = false) { +auto format_uint(Char* buffer, detail::fallback_uintptr n, int num_digits, + bool = false) -> Char* { auto char_digits = std::numeric_limits::digits / 4; int start = (num_digits + char_digits - 1) / char_digits - 1; if (int start_digits = num_digits % char_digits) { @@ -1107,7 +1164,7 @@ Char* format_uint(Char* buffer, detail::fallback_uintptr n, int num_digits, auto p = buffer; for (int i = 0; i < char_digits; ++i) { unsigned digit = (value & ((1 << BASE_BITS) - 1)); - *--p = static_cast(data::hex_digits[digit]); + *--p = static_cast("0123456789abcdef"[digit]); value >>= BASE_BITS; } } @@ -1115,7 +1172,8 @@ Char* format_uint(Char* buffer, detail::fallback_uintptr n, int num_digits, } template -inline It format_uint(It out, UInt value, int num_digits, bool upper = false) { +inline auto format_uint(It out, UInt value, int num_digits, bool upper = false) + -> It { if (auto ptr = to_pointer(out, to_unsigned(num_digits))) { format_uint(ptr, value, num_digits, upper); return out; @@ -1123,90 +1181,26 @@ inline It format_uint(It out, UInt value, int num_digits, bool upper = false) { // Buffer should be large enough to hold all digits (digits / BASE_BITS + 1). char buffer[num_bits() / BASE_BITS + 1]; format_uint(buffer, value, num_digits, upper); - return detail::copy_str(buffer, buffer + num_digits, out); + return detail::copy_str_noinline(buffer, buffer + num_digits, out); } // A converter from UTF-8 to UTF-16. class utf8_to_utf16 { private: - wmemory_buffer buffer_; + basic_memory_buffer buffer_; public: FMT_API explicit utf8_to_utf16(string_view s); - operator wstring_view() const { return {&buffer_[0], size()}; } - size_t size() const { return buffer_.size() - 1; } - const wchar_t* c_str() const { return &buffer_[0]; } - std::wstring str() const { return {&buffer_[0], size()}; } + operator basic_string_view() const { return {&buffer_[0], size()}; } + auto size() const -> size_t { return buffer_.size() - 1; } + auto c_str() const -> const wchar_t* { return &buffer_[0]; } + auto str() const -> std::wstring { return {&buffer_[0], size()}; } }; -template struct null {}; +namespace dragonbox { -// Workaround an array initialization issue in gcc 4.8. -template struct fill_t { - private: - enum { max_size = 4 }; - Char data_[max_size] = {Char(' '), Char(0), Char(0), Char(0)}; - unsigned char size_ = 1; - - public: - FMT_CONSTEXPR void operator=(basic_string_view s) { - auto size = s.size(); - if (size > max_size) { - FMT_THROW(format_error("invalid fill")); - return; - } - for (size_t i = 0; i < size; ++i) data_[i] = s[i]; - size_ = static_cast(size); - } - - size_t size() const { return size_; } - const Char* data() const { return data_; } - - FMT_CONSTEXPR Char& operator[](size_t index) { return data_[index]; } - FMT_CONSTEXPR const Char& operator[](size_t index) const { - return data_[index]; - } -}; -} // namespace detail - -// We cannot use enum classes as bit fields because of a gcc bug -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414. -namespace align { -enum type { none, left, right, center, numeric }; -} -using align_t = align::type; - -namespace sign { -enum type { none, minus, plus, space }; -} -using sign_t = sign::type; - -// Format specifiers for built-in and string types. -template struct basic_format_specs { - int width; - int precision; - char type; - align_t align : 4; - sign_t sign : 3; - bool alt : 1; // Alternate form ('#'). - detail::fill_t fill; - - constexpr basic_format_specs() - : width(0), - precision(-1), - type(0), - align(align::none), - sign(sign::none), - alt(false) {} -}; - -using format_specs = basic_format_specs; - -namespace detail { -namespace dragonbox { - -// Type-specific information that Dragonbox uses. -template struct float_info; +// Type-specific information that Dragonbox uses. +template struct float_info; template <> struct float_info { using carrier_uint = uint32_t; @@ -1266,37 +1260,21 @@ template struct decimal_fp { int exponent; }; -template FMT_API decimal_fp to_decimal(T x) FMT_NOEXCEPT; +template +FMT_API auto to_decimal(T x) FMT_NOEXCEPT -> decimal_fp; } // namespace dragonbox template -constexpr typename dragonbox::float_info::carrier_uint exponent_mask() { +constexpr auto exponent_mask() -> + typename dragonbox::float_info::carrier_uint { using uint = typename dragonbox::float_info::carrier_uint; return ((uint(1) << dragonbox::float_info::exponent_bits) - 1) << dragonbox::float_info::significand_bits; } -// A floating-point presentation format. -enum class float_format : unsigned char { - general, // General: exponent notation or fixed point based on magnitude. - exp, // Exponent notation with the default precision of 6, e.g. 1.2e-3. - fixed, // Fixed point with the default precision of 6, e.g. 0.0012. - hex -}; - -struct float_specs { - int precision; - float_format format : 8; - sign_t sign : 8; - bool upper : 1; - bool locale : 1; - bool binary32 : 1; - bool use_grisu : 1; - bool showpoint : 1; -}; - // Writes the exponent exp in the form "[+-]d{2,3}" to buffer. -template It write_exponent(int exp, It it) { +template +FMT_CONSTEXPR auto write_exponent(int exp, It it) -> It { FMT_ASSERT(-10000 < exp && exp < 10000, "exponent out of range"); if (exp < 0) { *it++ = static_cast('-'); @@ -1305,185 +1283,42 @@ template It write_exponent(int exp, It it) { *it++ = static_cast('+'); } if (exp >= 100) { - const char* top = data::digits[exp / 100]; + const char* top = digits2(to_unsigned(exp / 100)); if (exp >= 1000) *it++ = static_cast(top[0]); *it++ = static_cast(top[1]); exp %= 100; } - const char* d = data::digits[exp]; + const char* d = digits2(to_unsigned(exp)); *it++ = static_cast(d[0]); *it++ = static_cast(d[1]); return it; } template -int format_float(T value, int precision, float_specs specs, buffer& buf); +FMT_HEADER_ONLY_CONSTEXPR20 auto format_float(T value, int precision, + float_specs specs, + buffer& buf) -> int; // Formats a floating-point number with snprintf. template -int snprintf_float(T value, int precision, float_specs specs, - buffer& buf); - -template T promote_float(T value) { return value; } -inline double promote_float(float value) { return static_cast(value); } - -template -FMT_CONSTEXPR void handle_int_type_spec(char spec, Handler&& handler) { - switch (spec) { - case 0: - case 'd': - handler.on_dec(); - break; - case 'x': - case 'X': - handler.on_hex(); - break; - case 'b': - case 'B': - handler.on_bin(); - break; - case 'o': - handler.on_oct(); - break; -#ifdef FMT_DEPRECATED_N_SPECIFIER - case 'n': -#endif - case 'L': - handler.on_num(); - break; - case 'c': - handler.on_chr(); - break; - default: - handler.on_error(); - } -} - -template -FMT_CONSTEXPR float_specs parse_float_type_spec( - const basic_format_specs& specs, ErrorHandler&& eh = {}) { - auto result = float_specs(); - result.showpoint = specs.alt; - switch (specs.type) { - case 0: - result.format = float_format::general; - result.showpoint |= specs.precision > 0; - break; - case 'G': - result.upper = true; - FMT_FALLTHROUGH; - case 'g': - result.format = float_format::general; - break; - case 'E': - result.upper = true; - FMT_FALLTHROUGH; - case 'e': - result.format = float_format::exp; - result.showpoint |= specs.precision != 0; - break; - case 'F': - result.upper = true; - FMT_FALLTHROUGH; - case 'f': - result.format = float_format::fixed; - result.showpoint |= specs.precision != 0; - break; - case 'A': - result.upper = true; - FMT_FALLTHROUGH; - case 'a': - result.format = float_format::hex; - break; -#ifdef FMT_DEPRECATED_N_SPECIFIER - case 'n': -#endif - case 'L': - result.locale = true; - break; - default: - eh.on_error("invalid type specifier"); - break; - } - return result; -} - -template -FMT_CONSTEXPR void handle_char_specs(const basic_format_specs* specs, - Handler&& handler) { - if (!specs) return handler.on_char(); - if (specs->type && specs->type != 'c') return handler.on_int(); - if (specs->align == align::numeric || specs->sign != sign::none || specs->alt) - handler.on_error("invalid format specifier for char"); - handler.on_char(); -} - -template -FMT_CONSTEXPR void handle_cstring_type_spec(Char spec, Handler&& handler) { - if (spec == 0 || spec == 's') - handler.on_string(); - else if (spec == 'p') - handler.on_pointer(); - else - handler.on_error("invalid type specifier"); -} +auto snprintf_float(T value, int precision, float_specs specs, + buffer& buf) -> int; -template -FMT_CONSTEXPR void check_string_type_spec(Char spec, ErrorHandler&& eh) { - if (spec != 0 && spec != 's') eh.on_error("invalid type specifier"); +template constexpr auto promote_float(T value) -> T { + return value; } - -template -FMT_CONSTEXPR void check_pointer_type_spec(Char spec, ErrorHandler&& eh) { - if (spec != 0 && spec != 'p') eh.on_error("invalid type specifier"); +constexpr auto promote_float(float value) -> double { + return static_cast(value); } -template class int_type_checker : private ErrorHandler { - public: - FMT_CONSTEXPR explicit int_type_checker(ErrorHandler eh) : ErrorHandler(eh) {} - - FMT_CONSTEXPR void on_dec() {} - FMT_CONSTEXPR void on_hex() {} - FMT_CONSTEXPR void on_bin() {} - FMT_CONSTEXPR void on_oct() {} - FMT_CONSTEXPR void on_num() {} - FMT_CONSTEXPR void on_chr() {} - - FMT_CONSTEXPR void on_error() { - ErrorHandler::on_error("invalid type specifier"); - } -}; - -template -class char_specs_checker : public ErrorHandler { - private: - char type_; - - public: - FMT_CONSTEXPR char_specs_checker(char type, ErrorHandler eh) - : ErrorHandler(eh), type_(type) {} - - FMT_CONSTEXPR void on_int() { - handle_int_type_spec(type_, int_type_checker(*this)); - } - FMT_CONSTEXPR void on_char() {} -}; - -template -class cstring_type_checker : public ErrorHandler { - public: - FMT_CONSTEXPR explicit cstring_type_checker(ErrorHandler eh) - : ErrorHandler(eh) {} - - FMT_CONSTEXPR void on_string() {} - FMT_CONSTEXPR void on_pointer() {} -}; - template -FMT_NOINLINE OutputIt fill(OutputIt it, size_t n, const fill_t& fill) { +FMT_NOINLINE FMT_CONSTEXPR auto fill(OutputIt it, size_t n, + const fill_t& fill) -> OutputIt { auto fill_size = fill.size(); - if (fill_size == 1) return std::fill_n(it, n, fill[0]); - for (size_t i = 0; i < n; ++i) it = std::copy_n(fill.data(), fill_size, it); + if (fill_size == 1) return detail::fill_n(it, n, fill[0]); + auto data = fill.data(); + for (size_t i = 0; i < n; ++i) + it = copy_str(data, data + fill_size, it); return it; } @@ -1492,39 +1327,73 @@ FMT_NOINLINE OutputIt fill(OutputIt it, size_t n, const fill_t& fill) { // width: output display width in (terminal) column positions. template -inline OutputIt write_padded(OutputIt out, - const basic_format_specs& specs, size_t size, - size_t width, F&& f) { +FMT_CONSTEXPR auto write_padded(OutputIt out, + const basic_format_specs& specs, + size_t size, size_t width, F&& f) -> OutputIt { static_assert(align == align::left || align == align::right, ""); unsigned spec_width = to_unsigned(specs.width); size_t padding = spec_width > width ? spec_width - width : 0; - auto* shifts = align == align::left ? data::left_padding_shifts - : data::right_padding_shifts; + // Shifts are encoded as string literals because static constexpr is not + // supported in constexpr functions. + auto* shifts = align == align::left ? "\x1f\x1f\x00\x01" : "\x00\x1f\x00\x01"; size_t left_padding = padding >> shifts[specs.align]; + size_t right_padding = padding - left_padding; auto it = reserve(out, size + padding * specs.fill.size()); - it = fill(it, left_padding, specs.fill); + if (left_padding != 0) it = fill(it, left_padding, specs.fill); it = f(it); - it = fill(it, padding - left_padding, specs.fill); + if (right_padding != 0) it = fill(it, right_padding, specs.fill); return base_iterator(out, it); } template -inline OutputIt write_padded(OutputIt out, - const basic_format_specs& specs, size_t size, - F&& f) { +constexpr auto write_padded(OutputIt out, const basic_format_specs& specs, + size_t size, F&& f) -> OutputIt { return write_padded(out, specs, size, size, f); } +template +FMT_CONSTEXPR auto write_bytes(OutputIt out, string_view bytes, + const basic_format_specs& specs) + -> OutputIt { + return write_padded( + out, specs, bytes.size(), [bytes](reserve_iterator it) { + const char* data = bytes.data(); + return copy_str(data, data + bytes.size(), it); + }); +} + +template +auto write_ptr(OutputIt out, UIntPtr value, + const basic_format_specs* specs) -> OutputIt { + int num_digits = count_digits<4>(value); + auto size = to_unsigned(num_digits) + size_t(2); + auto write = [=](reserve_iterator it) { + *it++ = static_cast('0'); + *it++ = static_cast('x'); + return format_uint<4, Char>(it, value, num_digits); + }; + return specs ? write_padded(out, *specs, size, write) + : base_iterator(out, write(reserve(out, size))); +} + template -OutputIt write_bytes(OutputIt out, string_view bytes, - const basic_format_specs& specs) { - using iterator = remove_reference_t; - return write_padded(out, specs, bytes.size(), [bytes](iterator it) { - const char* data = bytes.data(); - return copy_str(data, data + bytes.size(), it); +FMT_CONSTEXPR auto write_char(OutputIt out, Char value, + const basic_format_specs& specs) + -> OutputIt { + return write_padded(out, specs, 1, [=](reserve_iterator it) { + *it++ = value; + return it; }); } +template +FMT_CONSTEXPR auto write(OutputIt out, Char value, + const basic_format_specs& specs, + locale_ref loc = {}) -> OutputIt { + return check_char_specs(specs) + ? write_char(out, value, specs) + : write(out, static_cast(value), specs, loc); +} // Data for write_int that doesn't depend on output iterator type. It is used to // avoid template code bloat. @@ -1532,9 +1401,9 @@ template struct write_int_data { size_t size; size_t padding; - write_int_data(int num_digits, string_view prefix, - const basic_format_specs& specs) - : size(prefix.size() + to_unsigned(num_digits)), padding(0) { + FMT_CONSTEXPR write_int_data(int num_digits, unsigned prefix, + const basic_format_specs& specs) + : size((prefix >> 24) + to_unsigned(num_digits)), padding(0) { if (specs.align == align::numeric) { auto width = to_unsigned(specs.width); if (width > size) { @@ -1542,7 +1411,7 @@ template struct write_int_data { size = width; } } else if (specs.precision > num_digits) { - size = prefix.size() + to_unsigned(specs.precision); + size = (prefix >> 24) + to_unsigned(specs.precision); padding = to_unsigned(specs.precision - num_digits); } } @@ -1550,183 +1419,280 @@ template struct write_int_data { // Writes an integer in the format // -// where are written by f(it). -template -OutputIt write_int(OutputIt out, int num_digits, string_view prefix, - const basic_format_specs& specs, F f) { +// where are written by write_digits(it). +// prefix contains chars in three lower bytes and the size in the fourth byte. +template +FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, int num_digits, + unsigned prefix, + const basic_format_specs& specs, + W write_digits) -> OutputIt { + // Slightly faster check for specs.width == 0 && specs.precision == -1. + if ((specs.width | (specs.precision + 1)) == 0) { + auto it = reserve(out, to_unsigned(num_digits) + (prefix >> 24)); + if (prefix != 0) { + for (unsigned p = prefix & 0xffffff; p != 0; p >>= 8) + *it++ = static_cast(p & 0xff); + } + return base_iterator(out, write_digits(it)); + } auto data = write_int_data(num_digits, prefix, specs); - using iterator = remove_reference_t; - return write_padded(out, specs, data.size, [=](iterator it) { - if (prefix.size() != 0) - it = copy_str(prefix.begin(), prefix.end(), it); - it = std::fill_n(it, data.padding, static_cast('0')); - return f(it); - }); + return write_padded( + out, specs, data.size, [=](reserve_iterator it) { + for (unsigned p = prefix & 0xffffff; p != 0; p >>= 8) + *it++ = static_cast(p & 0xff); + it = detail::fill_n(it, data.padding, static_cast('0')); + return write_digits(it); + }); } -template -OutputIt write(OutputIt out, basic_string_view s, - const basic_format_specs& specs) { - auto data = s.data(); - auto size = s.size(); - if (specs.precision >= 0 && to_unsigned(specs.precision) < size) - size = code_point_index(s, to_unsigned(specs.precision)); - auto width = specs.width != 0 - ? count_code_points(basic_string_view(data, size)) - : 0; - using iterator = remove_reference_t; - return write_padded(out, specs, size, width, [=](iterator it) { - return copy_str(data, data + size, it); - }); -} +template class digit_grouping { + private: + thousands_sep_result sep_; -// The handle_int_type_spec handler that writes an integer. -template struct int_writer { - OutputIt out; - locale_ref locale; - const basic_format_specs& specs; - UInt abs_value; - char prefix[4]; - unsigned prefix_size; - - using iterator = - remove_reference_t(), 0))>; - - string_view get_prefix() const { return string_view(prefix, prefix_size); } - - template - int_writer(OutputIt output, locale_ref loc, Int value, - const basic_format_specs& s) - : out(output), - locale(loc), - specs(s), - abs_value(static_cast(value)), - prefix_size(0) { - static_assert(std::is_same, UInt>::value, ""); - if (is_negative(value)) { - prefix[0] = '-'; - ++prefix_size; - abs_value = 0 - abs_value; - } else if (specs.sign != sign::none && specs.sign != sign::minus) { - prefix[0] = specs.sign == sign::plus ? '+' : ' '; - ++prefix_size; - } - } + struct next_state { + std::string::const_iterator group; + int pos; + }; + next_state initial_state() const { return {sep_.grouping.begin(), 0}; } - void on_dec() { - auto num_digits = count_digits(abs_value); - out = write_int( - out, num_digits, get_prefix(), specs, [this, num_digits](iterator it) { - return format_decimal(it, abs_value, num_digits).end; - }); + // Returns the next digit group separator position. + int next(next_state& state) const { + if (!sep_.thousands_sep) return max_value(); + if (state.group == sep_.grouping.end()) + return state.pos += sep_.grouping.back(); + if (*state.group <= 0 || *state.group == max_value()) + return max_value(); + state.pos += *state.group++; + return state.pos; } - void on_hex() { - if (specs.alt) { - prefix[prefix_size++] = '0'; - prefix[prefix_size++] = specs.type; - } - int num_digits = count_digits<4>(abs_value); - out = write_int(out, num_digits, get_prefix(), specs, - [this, num_digits](iterator it) { - return format_uint<4, Char>(it, abs_value, num_digits, - specs.type != 'x'); - }); + public: + explicit digit_grouping(locale_ref loc, bool localized = true) { + if (localized) + sep_ = thousands_sep(loc); + else + sep_.thousands_sep = Char(); } + explicit digit_grouping(thousands_sep_result sep) : sep_(sep) {} - void on_bin() { - if (specs.alt) { - prefix[prefix_size++] = '0'; - prefix[prefix_size++] = static_cast(specs.type); - } - int num_digits = count_digits<1>(abs_value); - out = write_int(out, num_digits, get_prefix(), specs, - [this, num_digits](iterator it) { - return format_uint<1, Char>(it, abs_value, num_digits); - }); + Char separator() const { return sep_.thousands_sep; } + + int count_separators(int num_digits) const { + int count = 0; + auto state = initial_state(); + while (num_digits > next(state)) ++count; + return count; } - void on_oct() { - int num_digits = count_digits<3>(abs_value); - if (specs.alt && specs.precision <= num_digits && abs_value != 0) { - // Octal prefix '0' is counted as a digit, so only add it if precision - // is not greater than the number of digits. - prefix[prefix_size++] = '0'; - } - out = write_int(out, num_digits, get_prefix(), specs, - [this, num_digits](iterator it) { - return format_uint<3, Char>(it, abs_value, num_digits); - }); - } - - enum { sep_size = 1 }; - - void on_num() { - std::string groups = grouping(locale); - if (groups.empty()) return on_dec(); - auto sep = thousands_sep(locale); - if (!sep) return on_dec(); - int num_digits = count_digits(abs_value); - int size = num_digits, n = num_digits; - std::string::const_iterator group = groups.cbegin(); - while (group != groups.cend() && n > *group && *group > 0 && - *group != max_value()) { - size += sep_size; - n -= *group; - ++group; + // Applies grouping to digits and write the output to out. + template + Out apply(Out out, basic_string_view digits) const { + auto num_digits = static_cast(digits.size()); + auto separators = basic_memory_buffer(); + separators.push_back(0); + auto state = initial_state(); + while (int i = next(state)) { + if (i >= num_digits) break; + separators.push_back(i); } - if (group == groups.cend()) size += sep_size * ((n - 1) / groups.back()); - char digits[40]; - format_decimal(digits, abs_value, num_digits); - basic_memory_buffer buffer; - size += static_cast(prefix_size); - const auto usize = to_unsigned(size); - buffer.resize(usize); - basic_string_view s(&sep, sep_size); - // Index of a decimal digit with the least significant digit having index 0. - int digit_index = 0; - group = groups.cbegin(); - auto p = buffer.data() + size - 1; - for (int i = num_digits - 1; i > 0; --i) { - *p-- = static_cast(digits[i]); - if (*group <= 0 || ++digit_index % *group != 0 || - *group == max_value()) - continue; - if (group + 1 != groups.cend()) { - digit_index = 0; - ++group; + for (int i = 0, sep_index = static_cast(separators.size() - 1); + i < num_digits; ++i) { + if (num_digits - i == separators[sep_index]) { + *out++ = separator(); + --sep_index; } - std::uninitialized_copy(s.data(), s.data() + s.size(), - make_checked(p, s.size())); - p -= s.size(); + *out++ = static_cast(digits[to_unsigned(i)]); } - *p-- = static_cast(*digits); - if (prefix_size != 0) *p = static_cast('-'); - auto data = buffer.data(); - out = write_padded( - out, specs, usize, usize, - [=](iterator it) { return copy_str(data, data + size, it); }); + return out; } +}; - void on_chr() { *out++ = static_cast(abs_value); } +template +auto write_int_localized(OutputIt out, UInt value, unsigned prefix, + const basic_format_specs& specs, + const digit_grouping& grouping) -> OutputIt { + static_assert(std::is_same, UInt>::value, ""); + int num_digits = count_digits(value); + char digits[40]; + format_decimal(digits, value, num_digits); + unsigned size = to_unsigned((prefix != 0 ? 1 : 0) + num_digits + + grouping.count_separators(num_digits)); + return write_padded( + out, specs, size, size, [&](reserve_iterator it) { + if (prefix != 0) *it++ = static_cast(prefix); + return grouping.apply(it, string_view(digits, to_unsigned(num_digits))); + }); +} + +template +auto write_int_localized(OutputIt& out, UInt value, unsigned prefix, + const basic_format_specs& specs, locale_ref loc) + -> bool { + auto grouping = digit_grouping(loc); + out = write_int_localized(out, value, prefix, specs, grouping); + return true; +} + +FMT_CONSTEXPR inline void prefix_append(unsigned& prefix, unsigned value) { + prefix |= prefix != 0 ? value << 8 : value; + prefix += (1u + (value > 0xff ? 1 : 0)) << 24; +} + +template struct write_int_arg { + UInt abs_value; + unsigned prefix; +}; - FMT_NORETURN void on_error() { - FMT_THROW(format_error("invalid type specifier")); +template +FMT_CONSTEXPR auto make_write_int_arg(T value, sign_t sign) + -> write_int_arg> { + auto prefix = 0u; + auto abs_value = static_cast>(value); + if (is_negative(value)) { + prefix = 0x01000000 | '-'; + abs_value = 0 - abs_value; + } else { + constexpr const unsigned prefixes[4] = {0, 0, 0x1000000u | '+', + 0x1000000u | ' '}; + prefix = prefixes[sign]; } -}; + return {abs_value, prefix}; +} + +template +FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, write_int_arg arg, + const basic_format_specs& specs, + locale_ref loc) -> OutputIt { + static_assert(std::is_same>::value, ""); + auto abs_value = arg.abs_value; + auto prefix = arg.prefix; + switch (specs.type) { + case presentation_type::none: + case presentation_type::dec: { + if (specs.localized && + write_int_localized(out, static_cast>(abs_value), + prefix, specs, loc)) { + return out; + } + auto num_digits = count_digits(abs_value); + return write_int( + out, num_digits, prefix, specs, [=](reserve_iterator it) { + return format_decimal(it, abs_value, num_digits).end; + }); + } + case presentation_type::hex_lower: + case presentation_type::hex_upper: { + bool upper = specs.type == presentation_type::hex_upper; + if (specs.alt) + prefix_append(prefix, unsigned(upper ? 'X' : 'x') << 8 | '0'); + int num_digits = count_digits<4>(abs_value); + return write_int( + out, num_digits, prefix, specs, [=](reserve_iterator it) { + return format_uint<4, Char>(it, abs_value, num_digits, upper); + }); + } + case presentation_type::bin_lower: + case presentation_type::bin_upper: { + bool upper = specs.type == presentation_type::bin_upper; + if (specs.alt) + prefix_append(prefix, unsigned(upper ? 'B' : 'b') << 8 | '0'); + int num_digits = count_digits<1>(abs_value); + return write_int(out, num_digits, prefix, specs, + [=](reserve_iterator it) { + return format_uint<1, Char>(it, abs_value, num_digits); + }); + } + case presentation_type::oct: { + int num_digits = count_digits<3>(abs_value); + // Octal prefix '0' is counted as a digit, so only add it if precision + // is not greater than the number of digits. + if (specs.alt && specs.precision <= num_digits && abs_value != 0) + prefix_append(prefix, '0'); + return write_int(out, num_digits, prefix, specs, + [=](reserve_iterator it) { + return format_uint<3, Char>(it, abs_value, num_digits); + }); + } + case presentation_type::chr: + return write_char(out, static_cast(abs_value), specs); + default: + throw_format_error("invalid type specifier"); + } + return out; +} +template +FMT_CONSTEXPR FMT_NOINLINE auto write_int_noinline( + OutputIt out, write_int_arg arg, const basic_format_specs& specs, + locale_ref loc) -> OutputIt { + return write_int(out, arg, specs, loc); +} +template ::value && + !std::is_same::value && + std::is_same>::value)> +FMT_CONSTEXPR FMT_INLINE auto write(OutputIt out, T value, + const basic_format_specs& specs, + locale_ref loc) -> OutputIt { + return write_int_noinline(out, make_write_int_arg(value, specs.sign), specs, + loc); +} +// An inlined version of write used in format string compilation. +template ::value && + !std::is_same::value && + !std::is_same>::value)> +FMT_CONSTEXPR FMT_INLINE auto write(OutputIt out, T value, + const basic_format_specs& specs, + locale_ref loc) -> OutputIt { + return write_int(out, make_write_int_arg(value, specs.sign), specs, loc); +} template -OutputIt write_nonfinite(OutputIt out, bool isinf, - const basic_format_specs& specs, - const float_specs& fspecs) { +FMT_CONSTEXPR auto write(OutputIt out, basic_string_view s, + const basic_format_specs& specs) -> OutputIt { + auto data = s.data(); + auto size = s.size(); + if (specs.precision >= 0 && to_unsigned(specs.precision) < size) + size = code_point_index(s, to_unsigned(specs.precision)); + auto width = + specs.width != 0 ? compute_width(basic_string_view(data, size)) : 0; + return write_padded(out, specs, size, width, + [=](reserve_iterator it) { + return copy_str(data, data + size, it); + }); +} +template +FMT_CONSTEXPR auto write(OutputIt out, + basic_string_view> s, + const basic_format_specs& specs, locale_ref) + -> OutputIt { + check_string_type_spec(specs.type); + return write(out, s, specs); +} +template +FMT_CONSTEXPR auto write(OutputIt out, const Char* s, + const basic_format_specs& specs, locale_ref) + -> OutputIt { + return check_cstring_type_spec(specs.type) + ? write(out, basic_string_view(s), specs, {}) + : write_ptr(out, to_uintptr(s), &specs); +} + +template +FMT_CONSTEXPR20 auto write_nonfinite(OutputIt out, bool isinf, + basic_format_specs specs, + const float_specs& fspecs) -> OutputIt { auto str = isinf ? (fspecs.upper ? "INF" : "inf") : (fspecs.upper ? "NAN" : "nan"); constexpr size_t str_size = 3; auto sign = fspecs.sign; auto size = str_size + (sign ? 1 : 0); - using iterator = remove_reference_t; - return write_padded(out, specs, size, [=](iterator it) { - if (sign) *it++ = static_cast(data::signs[sign]); + // Replace '0'-padding with space for non-finite values. + const bool is_zero_fill = + specs.fill.size() == 1 && *specs.fill.data() == static_cast('0'); + if (is_zero_fill) specs.fill[0] = static_cast(' '); + return write_padded(out, specs, size, [=](reserve_iterator it) { + if (sign) *it++ = detail::sign(sign); return copy_str(str, str + str_size, it); }); } @@ -1738,74 +1704,118 @@ struct big_decimal_fp { int exponent; }; -inline int get_significand_size(const big_decimal_fp& fp) { +constexpr auto get_significand_size(const big_decimal_fp& fp) -> int { return fp.significand_size; } template -inline int get_significand_size(const dragonbox::decimal_fp& fp) { +inline auto get_significand_size(const dragonbox::decimal_fp& fp) -> int { return count_digits(fp.significand); } template -inline OutputIt write_significand(OutputIt out, const char* significand, - int& significand_size) { +constexpr auto write_significand(OutputIt out, const char* significand, + int significand_size) -> OutputIt { return copy_str(significand, significand + significand_size, out); } template -inline OutputIt write_significand(OutputIt out, UInt significand, - int significand_size) { +inline auto write_significand(OutputIt out, UInt significand, + int significand_size) -> OutputIt { return format_decimal(out, significand, significand_size).end; } +template +FMT_CONSTEXPR20 auto write_significand(OutputIt out, T significand, + int significand_size, int exponent, + const Grouping& grouping) -> OutputIt { + if (!grouping.separator()) { + out = write_significand(out, significand, significand_size); + return detail::fill_n(out, exponent, static_cast('0')); + } + auto buffer = memory_buffer(); + write_significand(appender(buffer), significand, significand_size); + detail::fill_n(appender(buffer), exponent, '0'); + return grouping.apply(out, string_view(buffer.data(), buffer.size())); +} template ::value)> -inline Char* write_significand(Char* out, UInt significand, - int significand_size, int integral_size, - Char decimal_point) { +inline auto write_significand(Char* out, UInt significand, int significand_size, + int integral_size, Char decimal_point) -> Char* { if (!decimal_point) return format_decimal(out, significand, significand_size).end; - auto end = format_decimal(out + 1, significand, significand_size).end; - if (integral_size == 1) - out[0] = out[1]; - else - std::copy_n(out + 1, integral_size, out); - out[integral_size] = decimal_point; + out += significand_size + 1; + Char* end = out; + int floating_size = significand_size - integral_size; + for (int i = floating_size / 2; i > 0; --i) { + out -= 2; + copy2(out, digits2(significand % 100)); + significand /= 100; + } + if (floating_size % 2 != 0) { + *--out = static_cast('0' + significand % 10); + significand /= 10; + } + *--out = decimal_point; + format_decimal(out - integral_size, significand, integral_size); return end; } template >::value)> -inline OutputIt write_significand(OutputIt out, UInt significand, - int significand_size, int integral_size, - Char decimal_point) { +inline auto write_significand(OutputIt out, UInt significand, + int significand_size, int integral_size, + Char decimal_point) -> OutputIt { // Buffer is large enough to hold digits (digits10 + 1) and a decimal point. Char buffer[digits10() + 2]; auto end = write_significand(buffer, significand, significand_size, integral_size, decimal_point); - return detail::copy_str(buffer, end, out); + return detail::copy_str_noinline(buffer, end, out); } template -inline OutputIt write_significand(OutputIt out, const char* significand, - int significand_size, int integral_size, - Char decimal_point) { - out = detail::copy_str(significand, significand + integral_size, out); +FMT_CONSTEXPR auto write_significand(OutputIt out, const char* significand, + int significand_size, int integral_size, + Char decimal_point) -> OutputIt { + out = detail::copy_str_noinline(significand, + significand + integral_size, out); if (!decimal_point) return out; *out++ = decimal_point; - return detail::copy_str(significand + integral_size, - significand + significand_size, out); + return detail::copy_str_noinline(significand + integral_size, + significand + significand_size, out); } -template -OutputIt write_float(OutputIt out, const DecimalFP& fp, - const basic_format_specs& specs, float_specs fspecs, - Char decimal_point) { +template +FMT_CONSTEXPR20 auto write_significand(OutputIt out, T significand, + int significand_size, int integral_size, + Char decimal_point, + const Grouping& grouping) -> OutputIt { + if (!grouping.separator()) { + return write_significand(out, significand, significand_size, integral_size, + decimal_point); + } + auto buffer = basic_memory_buffer(); + write_significand(buffer_appender(buffer), significand, + significand_size, integral_size, decimal_point); + grouping.apply( + out, basic_string_view(buffer.data(), to_unsigned(integral_size))); + return detail::copy_str_noinline(buffer.data() + integral_size, + buffer.end(), out); +} + +template > +FMT_CONSTEXPR20 auto do_write_float(OutputIt out, const DecimalFP& fp, + const basic_format_specs& specs, + float_specs fspecs, locale_ref loc) + -> OutputIt { auto significand = fp.significand; int significand_size = get_significand_size(fp); - static const Char zero = static_cast('0'); + constexpr Char zero = static_cast('0'); auto sign = fspecs.sign; size_t size = to_unsigned(significand_size) + (sign ? 1 : 0); - using iterator = remove_reference_t; + using iterator = reserve_iterator; + + Char decimal_point = + fspecs.locale ? detail::decimal_point(loc) : static_cast('.'); int output_exp = fp.exponent + significand_size - 1; auto use_exp_format = [=]() { @@ -1820,7 +1830,8 @@ OutputIt write_float(OutputIt out, const DecimalFP& fp, if (use_exp_format()) { int num_zeros = 0; if (fspecs.showpoint) { - num_zeros = (std::max)(fspecs.precision - significand_size, 0); + num_zeros = fspecs.precision - significand_size; + if (num_zeros < 0) num_zeros = 0; size += to_unsigned(num_zeros); } else if (significand_size == 1) { decimal_point = Char(); @@ -1832,11 +1843,11 @@ OutputIt write_float(OutputIt out, const DecimalFP& fp, size += to_unsigned((decimal_point ? 1 : 0) + 2 + exp_digits); char exp_char = fspecs.upper ? 'E' : 'e'; auto write = [=](iterator it) { - if (sign) *it++ = static_cast(data::signs[sign]); + if (sign) *it++ = detail::sign(sign); // Insert a decimal point after the first digit and add an exponent. it = write_significand(it, significand, significand_size, 1, decimal_point); - if (num_zeros > 0) it = std::fill_n(it, num_zeros, zero); + if (num_zeros > 0) it = detail::fill_n(it, num_zeros, zero); *it++ = static_cast(exp_char); return write_exponent(output_exp, it); }; @@ -1855,25 +1866,29 @@ OutputIt write_float(OutputIt out, const DecimalFP& fp, #endif if (fspecs.showpoint) { if (num_zeros <= 0 && fspecs.format != float_format::fixed) num_zeros = 1; - if (num_zeros > 0) size += to_unsigned(num_zeros); + if (num_zeros > 0) size += to_unsigned(num_zeros) + 1; } + auto grouping = Grouping(loc, fspecs.locale); + size += to_unsigned(grouping.count_separators(significand_size)); return write_padded(out, specs, size, [&](iterator it) { - if (sign) *it++ = static_cast(data::signs[sign]); - it = write_significand(it, significand, significand_size); - it = std::fill_n(it, fp.exponent, zero); + if (sign) *it++ = detail::sign(sign); + it = write_significand(it, significand, significand_size, + fp.exponent, grouping); if (!fspecs.showpoint) return it; *it++ = decimal_point; - return num_zeros > 0 ? std::fill_n(it, num_zeros, zero) : it; + return num_zeros > 0 ? detail::fill_n(it, num_zeros, zero) : it; }); } else if (exp > 0) { // 1234e-2 -> 12.34[0+] int num_zeros = fspecs.showpoint ? fspecs.precision - significand_size : 0; size += 1 + to_unsigned(num_zeros > 0 ? num_zeros : 0); + auto grouping = Grouping(loc, fspecs.locale); + size += to_unsigned(grouping.count_separators(significand_size)); return write_padded(out, specs, size, [&](iterator it) { - if (sign) *it++ = static_cast(data::signs[sign]); + if (sign) *it++ = detail::sign(sign); it = write_significand(it, significand, significand_size, exp, - decimal_point); - return num_zeros > 0 ? std::fill_n(it, num_zeros, zero) : it; + decimal_point, grouping); + return num_zeros > 0 ? detail::fill_n(it, num_zeros, zero) : it; }); } // 1234e-6 -> 0.001234 @@ -1882,37 +1897,109 @@ OutputIt write_float(OutputIt out, const DecimalFP& fp, fspecs.precision < num_zeros) { num_zeros = fspecs.precision; } - size += 2 + to_unsigned(num_zeros); + bool pointy = num_zeros != 0 || significand_size != 0 || fspecs.showpoint; + size += 1 + (pointy ? 1 : 0) + to_unsigned(num_zeros); return write_padded(out, specs, size, [&](iterator it) { - if (sign) *it++ = static_cast(data::signs[sign]); + if (sign) *it++ = detail::sign(sign); *it++ = zero; - if (num_zeros == 0 && significand_size == 0 && !fspecs.showpoint) return it; + if (!pointy) return it; *it++ = decimal_point; - it = std::fill_n(it, num_zeros, zero); + it = detail::fill_n(it, num_zeros, zero); return write_significand(it, significand, significand_size); }); } +template class fallback_digit_grouping { + public: + constexpr fallback_digit_grouping(locale_ref, bool) {} + + constexpr Char separator() const { return Char(); } + + constexpr int count_separators(int) const { return 0; } + + template + constexpr Out apply(Out out, basic_string_view) const { + return out; + } +}; + +template +FMT_CONSTEXPR20 auto write_float(OutputIt out, const DecimalFP& fp, + const basic_format_specs& specs, + float_specs fspecs, locale_ref loc) + -> OutputIt { + if (is_constant_evaluated()) { + return do_write_float>(out, fp, specs, fspecs, + loc); + } else { + return do_write_float(out, fp, specs, fspecs, loc); + } +} + +template ::value)> +FMT_CONSTEXPR20 bool isinf(T value) { + if (is_constant_evaluated()) { +#if defined(__cpp_if_constexpr) + if constexpr (std::numeric_limits::is_iec559) { + auto bits = detail::bit_cast(static_cast(value)); + constexpr auto significand_bits = + dragonbox::float_info::significand_bits; + return (bits & exponent_mask()) && + !(bits & ((uint64_t(1) << significand_bits) - 1)); + } +#endif + } + return std::isinf(value); +} + +template ::value)> +FMT_CONSTEXPR20 bool isfinite(T value) { + if (is_constant_evaluated()) { +#if defined(__cpp_if_constexpr) + if constexpr (std::numeric_limits::is_iec559) { + auto bits = detail::bit_cast(static_cast(value)); + return (bits & exponent_mask()) != exponent_mask(); + } +#endif + } + return std::isfinite(value); +} + +template ::value)> +FMT_INLINE FMT_CONSTEXPR bool signbit(T value) { + if (is_constant_evaluated()) { +#ifdef __cpp_if_constexpr + if constexpr (std::numeric_limits::is_iec559) { + auto bits = detail::bit_cast(static_cast(value)); + return (bits & (uint64_t(1) << (num_bits() - 1))) != 0; + } +#endif + } + return std::signbit(value); +} + template ::value)> -OutputIt write(OutputIt out, T value, basic_format_specs specs, - locale_ref loc = {}) { +FMT_CONSTEXPR20 auto write(OutputIt out, T value, + basic_format_specs specs, locale_ref loc = {}) + -> OutputIt { if (const_check(!is_supported_floating_point(value))) return out; float_specs fspecs = parse_float_type_spec(specs); fspecs.sign = specs.sign; - if (std::signbit(value)) { // value < 0 is false for NaN so use signbit. + if (detail::signbit(value)) { // value < 0 is false for NaN so use signbit. fspecs.sign = sign::minus; value = -value; } else if (fspecs.sign == sign::minus) { fspecs.sign = sign::none; } - if (!std::isfinite(value)) - return write_nonfinite(out, std::isinf(value), specs, fspecs); + if (!detail::isfinite(value)) + return write_nonfinite(out, detail::isinf(value), specs, fspecs); if (specs.align == align::numeric && fspecs.sign) { auto it = reserve(out, 1); - *it++ = static_cast(data::signs[fspecs.sign]); + *it++ = detail::sign(fspecs.sign); out = base_iterator(out, it); fspecs.sign = sign::none; if (specs.width != 0) --specs.width; @@ -1920,30 +2007,35 @@ OutputIt write(OutputIt out, T value, basic_format_specs specs, memory_buffer buffer; if (fspecs.format == float_format::hex) { - if (fspecs.sign) buffer.push_back(data::signs[fspecs.sign]); + if (fspecs.sign) buffer.push_back(detail::sign(fspecs.sign)); snprintf_float(promote_float(value), specs.precision, fspecs, buffer); - return write_bytes(out, {buffer.data(), buffer.size()}, specs); + return write_bytes(out, {buffer.data(), buffer.size()}, + specs); } - int precision = specs.precision >= 0 || !specs.type ? specs.precision : 6; + int precision = specs.precision >= 0 || specs.type == presentation_type::none + ? specs.precision + : 6; if (fspecs.format == float_format::exp) { if (precision == max_value()) - FMT_THROW(format_error("number is too big")); + throw_format_error("number is too big"); else ++precision; } if (const_check(std::is_same())) fspecs.binary32 = true; - fspecs.use_grisu = is_fast_float(); + if (!is_fast_float()) fspecs.fallback = true; int exp = format_float(promote_float(value), precision, fspecs, buffer); fspecs.precision = precision; - Char point = - fspecs.locale ? decimal_point(loc) : static_cast('.'); auto fp = big_decimal_fp{buffer.data(), static_cast(buffer.size()), exp}; - return write_float(out, fp, specs, fspecs, point); + return write_float(out, fp, specs, fspecs, loc); } template ::value)> -OutputIt write(OutputIt out, T value) { +FMT_CONSTEXPR20 auto write(OutputIt out, T value) -> OutputIt { + if (is_constant_evaluated()) { + return write(out, value, basic_format_specs()); + } + if (const_check(!is_supported_floating_point(value))) return out; using floaty = conditional_t::value, double, T>; @@ -1951,90 +2043,53 @@ OutputIt write(OutputIt out, T value) { auto bits = bit_cast(value); auto fspecs = float_specs(); - auto sign_bit = bits & (uint(1) << (num_bits() - 1)); - if (sign_bit != 0) { + if (detail::signbit(value)) { fspecs.sign = sign::minus; value = -value; } - static const auto specs = basic_format_specs(); + constexpr auto specs = basic_format_specs(); uint mask = exponent_mask(); if ((bits & mask) == mask) return write_nonfinite(out, std::isinf(value), specs, fspecs); auto dec = dragonbox::to_decimal(static_cast(value)); - return write_float(out, dec, specs, fspecs, static_cast('.')); + return write_float(out, dec, specs, fspecs, {}); } template ::value && !is_fast_float::value)> -inline OutputIt write(OutputIt out, T value) { +inline auto write(OutputIt out, T value) -> OutputIt { return write(out, value, basic_format_specs()); } template -OutputIt write_char(OutputIt out, Char value, - const basic_format_specs& specs) { - using iterator = remove_reference_t; - return write_padded(out, specs, 1, [=](iterator it) { - *it++ = value; - return it; - }); -} - -template -OutputIt write_ptr(OutputIt out, UIntPtr value, - const basic_format_specs* specs) { - int num_digits = count_digits<4>(value); - auto size = to_unsigned(num_digits) + size_t(2); - using iterator = remove_reference_t; - auto write = [=](iterator it) { - *it++ = static_cast('0'); - *it++ = static_cast('x'); - return format_uint<4, Char>(it, value, num_digits); - }; - return specs ? write_padded(out, *specs, size, write) - : base_iterator(out, write(reserve(out, size))); -} - -template struct is_integral : std::is_integral {}; -template <> struct is_integral : std::true_type {}; -template <> struct is_integral : std::true_type {}; - -template -OutputIt write(OutputIt out, monostate) { +auto write(OutputIt out, monostate, basic_format_specs = {}, + locale_ref = {}) -> OutputIt { FMT_ASSERT(false, ""); return out; } -template ::value)> -OutputIt write(OutputIt out, string_view value) { - auto it = reserve(out, value.size()); - it = copy_str(value.begin(), value.end(), it); - return base_iterator(out, it); -} - template -OutputIt write(OutputIt out, basic_string_view value) { +FMT_CONSTEXPR auto write(OutputIt out, basic_string_view value) + -> OutputIt { auto it = reserve(out, value.size()); - it = std::copy(value.begin(), value.end(), it); + it = copy_str_noinline(value.begin(), value.end(), it); return base_iterator(out, it); } -template -buffer_appender write(buffer_appender out, - basic_string_view value) { - get_container(out).append(value.begin(), value.end()); - return out; +template ::value)> +constexpr auto write(OutputIt out, const T& value) -> OutputIt { + return write(out, to_string_view(value)); } template ::value && !std::is_same::value && !std::is_same::value)> -OutputIt write(OutputIt out, T value) { +FMT_CONSTEXPR auto write(OutputIt out, T value) -> OutputIt { auto abs_value = static_cast>(value); bool negative = is_negative(value); // Don't do -abs_value since it trips unsigned-integer-overflow sanitizer. @@ -2052,336 +2107,130 @@ OutputIt write(OutputIt out, T value) { return base_iterator(out, it); } -template -OutputIt write(OutputIt out, bool value) { - return write(out, string_view(value ? "true" : "false")); +// FMT_ENABLE_IF() condition separated to workaround an MSVC bug. +template < + typename Char, typename OutputIt, typename T, + bool check = + std::is_enum::value && !std::is_same::value && + mapped_type_constant>::value != + type::custom_type, + FMT_ENABLE_IF(check)> +FMT_CONSTEXPR auto write(OutputIt out, T value) -> OutputIt { + return write( + out, static_cast::type>(value)); +} + +template ::value)> +FMT_CONSTEXPR auto write(OutputIt out, T value, + const basic_format_specs& specs = {}, + locale_ref = {}) -> OutputIt { + return specs.type != presentation_type::none && + specs.type != presentation_type::string + ? write(out, value ? 1 : 0, specs, {}) + : write_bytes(out, value ? "true" : "false", specs); } template -OutputIt write(OutputIt out, Char value) { +FMT_CONSTEXPR auto write(OutputIt out, Char value) -> OutputIt { auto it = reserve(out, 1); *it++ = value; return base_iterator(out, it); } template -OutputIt write(OutputIt out, const Char* value) { +FMT_CONSTEXPR_CHAR_TRAITS auto write(OutputIt out, const Char* value) + -> OutputIt { if (!value) { - FMT_THROW(format_error("string pointer is null")); + throw_format_error("string pointer is null"); } else { - auto length = std::char_traits::length(value); - out = write(out, basic_string_view(value, length)); + out = write(out, basic_string_view(value)); } return out; } -template -OutputIt write(OutputIt out, const void* value) { - return write_ptr(out, to_uintptr(value), nullptr); +template ::value)> +auto write(OutputIt out, const T* value, + const basic_format_specs& specs = {}, locale_ref = {}) + -> OutputIt { + check_pointer_type_spec(specs.type, error_handler()); + return write_ptr(out, to_uintptr(value), &specs); } -template -auto write(OutputIt out, const T& value) -> typename std::enable_if< - mapped_type_constant>::value == - type::custom_type, - OutputIt>::type { - using context_type = basic_format_context; +// A write overload that handles implicit conversions. +template > +FMT_CONSTEXPR auto write(OutputIt out, const T& value) -> enable_if_t< + std::is_class::value && !is_string::value && + !std::is_same::value && + !std::is_same().map(value))>::value, + OutputIt> { + return write(out, arg_mapper().map(value)); +} + +template > +FMT_CONSTEXPR auto write(OutputIt out, const T& value) + -> enable_if_t::value == type::custom_type, + OutputIt> { using formatter_type = - conditional_t::value, - typename context_type::template formatter_type, + conditional_t::value, + typename Context::template formatter_type, fallback_formatter>; - context_type ctx(out, {}, {}); + auto ctx = Context(out, {}, {}); return formatter_type().format(value, ctx); } // An argument visitor that formats the argument and writes it via the output // iterator. It's a class and not a generic lambda for compatibility with C++11. -template struct default_arg_formatter { - using context = basic_format_context; +template struct default_arg_formatter { + using iterator = buffer_appender; + using context = buffer_context; - OutputIt out; + iterator out; basic_format_args args; locale_ref loc; - template OutputIt operator()(T value) { + template auto operator()(T value) -> iterator { return write(out, value); } - - OutputIt operator()(typename basic_format_arg::handle handle) { + auto operator()(typename basic_format_arg::handle h) -> iterator { basic_format_parse_context parse_ctx({}); - basic_format_context format_ctx(out, args, loc); - handle.format(parse_ctx, format_ctx); + context format_ctx(out, args, loc); + h.format(parse_ctx, format_ctx); return format_ctx.out(); } }; -template -class arg_formatter_base { - public: - using iterator = OutputIt; - using char_type = Char; - using format_specs = basic_format_specs; +template struct arg_formatter { + using iterator = buffer_appender; + using context = buffer_context; - private: - iterator out_; - locale_ref locale_; - format_specs* specs_; + iterator out; + const basic_format_specs& specs; + locale_ref locale; - // Attempts to reserve space for n extra characters in the output range. - // Returns a pointer to the reserved range or a reference to out_. - auto reserve(size_t n) -> decltype(detail::reserve(out_, n)) { - return detail::reserve(out_, n); + template + FMT_CONSTEXPR FMT_INLINE auto operator()(T value) -> iterator { + return detail::write(out, value, specs, locale); } - - using reserve_iterator = remove_reference_t(), 0))>; - - template void write_int(T value, const format_specs& spec) { - using uint_type = uint32_or_64_or_128_t; - int_writer w(out_, locale_, value, spec); - handle_int_type_spec(spec.type, w); - out_ = w.out; - } - - void write(char value) { - auto&& it = reserve(1); - *it++ = value; - } - - template ::value)> - void write(Ch value) { - out_ = detail::write(out_, value); - } - - void write(string_view value) { - auto&& it = reserve(value.size()); - it = copy_str(value.begin(), value.end(), it); - } - void write(wstring_view value) { - static_assert(std::is_same::value, ""); - auto&& it = reserve(value.size()); - it = std::copy(value.begin(), value.end(), it); - } - - template - void write(const Ch* s, size_t size, const format_specs& specs) { - auto width = specs.width != 0 - ? count_code_points(basic_string_view(s, size)) - : 0; - out_ = write_padded(out_, specs, size, width, [=](reserve_iterator it) { - return copy_str(s, s + size, it); - }); - } - - template - void write(basic_string_view s, const format_specs& specs = {}) { - out_ = detail::write(out_, s, specs); - } - - void write_pointer(const void* p) { - out_ = write_ptr(out_, to_uintptr(p), specs_); - } - - struct char_spec_handler : ErrorHandler { - arg_formatter_base& formatter; - Char value; - - char_spec_handler(arg_formatter_base& f, Char val) - : formatter(f), value(val) {} - - void on_int() { - // char is only formatted as int if there are specs. - formatter.write_int(static_cast(value), *formatter.specs_); - } - void on_char() { - if (formatter.specs_) - formatter.out_ = write_char(formatter.out_, value, *formatter.specs_); - else - formatter.write(value); - } - }; - - struct cstring_spec_handler : error_handler { - arg_formatter_base& formatter; - const Char* value; - - cstring_spec_handler(arg_formatter_base& f, const Char* val) - : formatter(f), value(val) {} - - void on_string() { formatter.write(value); } - void on_pointer() { formatter.write_pointer(value); } - }; - - protected: - iterator out() { return out_; } - format_specs* specs() { return specs_; } - - void write(bool value) { - if (specs_) - write(string_view(value ? "true" : "false"), *specs_); - else - out_ = detail::write(out_, value); - } - - void write(const Char* value) { - if (!value) { - FMT_THROW(format_error("string pointer is null")); - } else { - auto length = std::char_traits::length(value); - basic_string_view sv(value, length); - specs_ ? write(sv, *specs_) : write(sv); - } - } - - public: - arg_formatter_base(OutputIt out, format_specs* s, locale_ref loc) - : out_(out), locale_(loc), specs_(s) {} - - iterator operator()(monostate) { - FMT_ASSERT(false, "invalid argument type"); - return out_; - } - - template ::value)> - FMT_INLINE iterator operator()(T value) { - if (specs_) - write_int(value, *specs_); - else - out_ = detail::write(out_, value); - return out_; - } - - iterator operator()(Char value) { - handle_char_specs(specs_, - char_spec_handler(*this, static_cast(value))); - return out_; - } - - iterator operator()(bool value) { - if (specs_ && specs_->type) return (*this)(value ? 1 : 0); - write(value != 0); - return out_; - } - - template ::value)> - iterator operator()(T value) { - auto specs = specs_ ? *specs_ : format_specs(); - if (const_check(is_supported_floating_point(value))) - out_ = detail::write(out_, value, specs, locale_); - else - FMT_ASSERT(false, "unsupported float argument type"); - return out_; - } - - iterator operator()(const Char* value) { - if (!specs_) return write(value), out_; - handle_cstring_type_spec(specs_->type, cstring_spec_handler(*this, value)); - return out_; - } - - iterator operator()(basic_string_view value) { - if (specs_) { - check_string_type_spec(specs_->type, error_handler()); - write(value, *specs_); - } else { - write(value); - } - return out_; - } - - iterator operator()(const void* value) { - if (specs_) check_pointer_type_spec(specs_->type, error_handler()); - write_pointer(value); - return out_; + auto operator()(typename basic_format_arg::handle) -> iterator { + // User-defined types are handled separately because they require access + // to the parse context. + return out; } }; -/** The default argument formatter. */ -template -class arg_formatter : public arg_formatter_base { - private: - using char_type = Char; - using base = arg_formatter_base; - using context_type = basic_format_context; +template struct custom_formatter { + basic_format_parse_context& parse_ctx; + buffer_context& ctx; - context_type& ctx_; - basic_format_parse_context* parse_ctx_; - const Char* ptr_; - - public: - using iterator = typename base::iterator; - using format_specs = typename base::format_specs; - - /** - \rst - Constructs an argument formatter object. - *ctx* is a reference to the formatting context, - *specs* contains format specifier information for standard argument types. - \endrst - */ - explicit arg_formatter( - context_type& ctx, - basic_format_parse_context* parse_ctx = nullptr, - format_specs* specs = nullptr, const Char* ptr = nullptr) - : base(ctx.out(), specs, ctx.locale()), - ctx_(ctx), - parse_ctx_(parse_ctx), - ptr_(ptr) {} - - using base::operator(); - - /** Formats an argument of a user-defined type. */ - iterator operator()(typename basic_format_arg::handle handle) { - if (ptr_) advance_to(*parse_ctx_, ptr_); - handle.format(*parse_ctx_, ctx_); - return ctx_.out(); + void operator()( + typename basic_format_arg>::handle h) const { + h.format(parse_ctx, ctx); } -}; - -template FMT_CONSTEXPR bool is_name_start(Char c) { - return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || '_' == c; -} - -// Parses the range [begin, end) as an unsigned integer. This function assumes -// that the range is non-empty and the first character is a digit. -template -FMT_CONSTEXPR int parse_nonnegative_int(const Char*& begin, const Char* end, - ErrorHandler&& eh) { - FMT_ASSERT(begin != end && '0' <= *begin && *begin <= '9', ""); - unsigned value = 0; - // Convert to unsigned to prevent a warning. - constexpr unsigned max_int = max_value(); - unsigned big = max_int / 10; - do { - // Check for overflow. - if (value > big) { - value = max_int + 1; - break; - } - value = value * 10 + unsigned(*begin - '0'); - ++begin; - } while (begin != end && '0' <= *begin && *begin <= '9'); - if (value > max_int) eh.on_error("number is too big"); - return static_cast(value); -} - -template class custom_formatter { - private: - using char_type = typename Context::char_type; - - basic_format_parse_context& parse_ctx_; - Context& ctx_; - - public: - explicit custom_formatter(basic_format_parse_context& parse_ctx, - Context& ctx) - : parse_ctx_(parse_ctx), ctx_(ctx) {} - - void operator()(typename basic_format_arg::handle h) const { - h.format(parse_ctx_, ctx_); - } - template void operator()(T) const {} }; @@ -2396,13 +2245,13 @@ template class width_checker { explicit FMT_CONSTEXPR width_checker(ErrorHandler& eh) : handler_(eh) {} template ::value)> - FMT_CONSTEXPR unsigned long long operator()(T value) { + FMT_CONSTEXPR auto operator()(T value) -> unsigned long long { if (is_negative(value)) handler_.on_error("negative width"); return static_cast(value); } template ::value)> - FMT_CONSTEXPR unsigned long long operator()(T) { + FMT_CONSTEXPR auto operator()(T) -> unsigned long long { handler_.on_error("width is not integer"); return 0; } @@ -2416,13 +2265,13 @@ template class precision_checker { explicit FMT_CONSTEXPR precision_checker(ErrorHandler& eh) : handler_(eh) {} template ::value)> - FMT_CONSTEXPR unsigned long long operator()(T value) { + FMT_CONSTEXPR auto operator()(T value) -> unsigned long long { if (is_negative(value)) handler_.on_error("negative precision"); return static_cast(value); } template ::value)> - FMT_CONSTEXPR unsigned long long operator()(T) { + FMT_CONSTEXPR auto operator()(T) -> unsigned long long { handler_.on_error("precision is not integer"); return 0; } @@ -2431,779 +2280,94 @@ template class precision_checker { ErrorHandler& handler_; }; -// A format specifier handler that sets fields in basic_format_specs. -template class specs_setter { - public: - explicit FMT_CONSTEXPR specs_setter(basic_format_specs& specs) - : specs_(specs) {} - - FMT_CONSTEXPR specs_setter(const specs_setter& other) - : specs_(other.specs_) {} - - FMT_CONSTEXPR void on_align(align_t align) { specs_.align = align; } - FMT_CONSTEXPR void on_fill(basic_string_view fill) { - specs_.fill = fill; - } - FMT_CONSTEXPR void on_plus() { specs_.sign = sign::plus; } - FMT_CONSTEXPR void on_minus() { specs_.sign = sign::minus; } - FMT_CONSTEXPR void on_space() { specs_.sign = sign::space; } - FMT_CONSTEXPR void on_hash() { specs_.alt = true; } - - FMT_CONSTEXPR void on_zero() { - specs_.align = align::numeric; - specs_.fill[0] = Char('0'); - } - - FMT_CONSTEXPR void on_width(int width) { specs_.width = width; } - FMT_CONSTEXPR void on_precision(int precision) { - specs_.precision = precision; - } - FMT_CONSTEXPR void end_precision() {} - - FMT_CONSTEXPR void on_type(Char type) { - specs_.type = static_cast(type); - } - - protected: - basic_format_specs& specs_; -}; - -template class numeric_specs_checker { - public: - FMT_CONSTEXPR numeric_specs_checker(ErrorHandler& eh, detail::type arg_type) - : error_handler_(eh), arg_type_(arg_type) {} - - FMT_CONSTEXPR void require_numeric_argument() { - if (!is_arithmetic_type(arg_type_)) - error_handler_.on_error("format specifier requires numeric argument"); - } - - FMT_CONSTEXPR void check_sign() { - require_numeric_argument(); - if (is_integral_type(arg_type_) && arg_type_ != type::int_type && - arg_type_ != type::long_long_type && arg_type_ != type::char_type) { - error_handler_.on_error("format specifier requires signed argument"); - } - } - - FMT_CONSTEXPR void check_precision() { - if (is_integral_type(arg_type_) || arg_type_ == type::pointer_type) - error_handler_.on_error("precision not allowed for this argument type"); - } - - private: - ErrorHandler& error_handler_; - detail::type arg_type_; -}; - -// A format specifier handler that checks if specifiers are consistent with the -// argument type. -template class specs_checker : public Handler { - private: - numeric_specs_checker checker_; - - // Suppress an MSVC warning about using this in initializer list. - FMT_CONSTEXPR Handler& error_handler() { return *this; } - - public: - FMT_CONSTEXPR specs_checker(const Handler& handler, detail::type arg_type) - : Handler(handler), checker_(error_handler(), arg_type) {} - - FMT_CONSTEXPR specs_checker(const specs_checker& other) - : Handler(other), checker_(error_handler(), other.arg_type_) {} - - FMT_CONSTEXPR void on_align(align_t align) { - if (align == align::numeric) checker_.require_numeric_argument(); - Handler::on_align(align); - } - - FMT_CONSTEXPR void on_plus() { - checker_.check_sign(); - Handler::on_plus(); - } - - FMT_CONSTEXPR void on_minus() { - checker_.check_sign(); - Handler::on_minus(); - } - - FMT_CONSTEXPR void on_space() { - checker_.check_sign(); - Handler::on_space(); - } - - FMT_CONSTEXPR void on_hash() { - checker_.require_numeric_argument(); - Handler::on_hash(); - } - - FMT_CONSTEXPR void on_zero() { - checker_.require_numeric_argument(); - Handler::on_zero(); - } - - FMT_CONSTEXPR void end_precision() { checker_.check_precision(); } -}; - template