From 590a62d1b253c2348859d6b66464c2070f186c52 Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Mon, 3 Jan 2022 04:53:10 +0000 Subject: [PATCH] Add a script to run clang-tidy on the entire MLIR codebase --- mlir/utils/clang-tidy/README.md | 50 +++++++++++++ mlir/utils/clang-tidy/apply-clang-tidy.sh | 116 ++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 mlir/utils/clang-tidy/README.md create mode 100755 mlir/utils/clang-tidy/apply-clang-tidy.sh diff --git a/mlir/utils/clang-tidy/README.md b/mlir/utils/clang-tidy/README.md new file mode 100644 index 0000000..f8ddcbe --- /dev/null +++ b/mlir/utils/clang-tidy/README.md @@ -0,0 +1,50 @@ +### Apply clang-tidy fixes on the repo + +This script runs clang-tidy on every C++ source file in MLIR and commit +the results of the checks individually. Be aware that it'll take over +10h to process the entire codebase. + +The advised way to use this is to build clang-tidy (in release mode) and +have another build directory for MLIR. Here is a sample invocation from +the root of the repo: + +```bash +{ time \ + CLANG_TIDY=build-clang/bin/clang-tidy \ + TIMING_TIDY=time \ + ./mlir/utils/apply-clang-tidy.sh build mlir ~/clang-tidy-fails/ +; } 2>&1 | tee ~/clang-tidy.log +``` + +- `build-clang/` contains the result of a build of clang-tidy, configured + and built somehow with: +```bash +$ cmake ../llvm \ + -DLLVM_ENABLE_PROJECTS="clang;mlir;clang-tools-extra" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -G Ninja +$ ninja clang-tidy +``` +- `build/` must be a directory with MLIR onfigured. It is highly advised to + use `ccache` as well, as this directory will be used to rerun + `ninja check-mlir` after every single clang-tidy fix. +```bash +$ cmake ../llvm \ + -DLLVM_ENABLE_PROJECTS="mlir" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_TARGETS_TO_BUILD="Native;NVPTX;AMDGPU" \ + -DLLVM_CCACHE_BUILD=ON \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DLLVM_ENABLE_LLD=ON \ + -DLLVM_BUILD_EXAMPLES=OFF \ + -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ + -G Ninja +``` +- `mlir/` is the directory where to find the files, it can be replaced by a + subfolder or the path to a single file. +- `mkdir -p ~/clang-tidy-fails/` will be a directory containing the patches + that clang-tidy produces but also fail the build. + diff --git a/mlir/utils/clang-tidy/apply-clang-tidy.sh b/mlir/utils/clang-tidy/apply-clang-tidy.sh new file mode 100755 index 0000000..8266de8 --- /dev/null +++ b/mlir/utils/clang-tidy/apply-clang-tidy.sh @@ -0,0 +1,116 @@ +#!/bin/bash -u + +if [[ $# -lt 2 || $# -gt 4 ]]; then + echo "Usage: $0 [rejects dir] [checks]" + echo " - has to be a LLVM build directory (you should use CCACHE!)." + echo " - is the path that contains the .cpp files to update." + echo " - [rejects dir] is a directory where rejected patch (build failure) will be stored." + echo " - [checks] is an optional space-separated list of check to use instead of auto-detecting" + echo " Also define the env var CLANG_TIDY the path to use for the clang-tidy binary (default to 'clang-tidy' in the PATH)" + echo " Also define the env var TIMING_TIDY to 'time' to prefix clang-tidy execution with it" + echo "" + echo "This tool will execute clang-tidy on every .cpp file in the provided path and" + echo "rerun the tests. On success, a commit is added to the repo for each individual" + echo "pair ." + exit 1 +fi +BUILD_DIR=$1 +SRCS=$2 +REJECT_DIR=${3:-} +PRESET_CHECKS=${4:-} +SRC_DIR=$PWD +if [[ -v CLANG_TIDY ]] && [[ ! -z "$CLANG_TIDY" ]] ; then + CLANG_TIDY=$(realpath $CLANG_TIDY) + if [[ ! -f "$CLANG_TIDY" ]]; then + echo "Invalid path '$CLANG_TIDY'" + exit 1 + fi +else + CLANG_TIDY=clang-tidy +fi +TIMING_TIDY=${TIMING_TIDY:-} +echo "Using: '$CLANG_TIDY" + +if [[ ! -z "$REJECT_DIR" ]] && [[ ! -d "$REJECT_DIR" ]]; then + echo "Expects 'rejects dir' to be a directory, got '$REJECT_DIR'" + exit 1 +fi + +ensure_clean_build() { + git reset --hard HEAD + time ninja -C $BUILD_DIR check-mlir-build-only > ${REJECT_DIR}/ninja.clean.log 2>&1 + if [[ $? != 0 ]] ; then + echo "-- Build failed on clean state, cleaning TableGen files and retry" + # Reinitialize the TableGen generated file to have a clean state + find $BUILD_DIR/tools/mlir/ | grep '\.inc' | while read file ; do rm $file ; done + time ninja -C $BUILD_DIR check-mlir-build-only > ${REJECT_DIR}/ninja.clean.log 2>&1 + if [[ $? != 0 ]] ; then + echo "check-mlir-build-only failed on clean state! (see ninja.clean.log)" + git status + exit 1 + fi + fi +} + +tmpfile=$(mktemp /tmp/mhlo-temp-checks.XXXXXX) +find $SRCS | grep ".cpp$" | sort | while read file ; do + echo "================================" + echo "======= Processing $file =======" + date + echo "================================" + CHECKS= + if [[ ! -z "$PRESET_CHECKS" ]]; then + CHECKS="$PRESET_CHECKS" + else + CHECKS=$($CLANG_TIDY $file -p $BUILD_DIR --list-checks \ + | grep -v "Enabled checks:" | grep -v "^$" \ + | while read check ; do echo -n "${check} " ; done;) + fi + echo "-----------------------------------" + echo "-- Reset state before applying all checks on file $file" + ensure_clean_build + + echo "-----------------------------------" + echo "-- Apply all checks on file $file" + echo "$TIMING_TIDY $CLANG_TIDY -p $BUILD_DIR $file -fix" + $TIMING_TIDY $CLANG_TIDY -p $BUILD_DIR $file -fix \ + | grep "warning:.*\]$" | sed -r 's#.*\[(.*)]$#\1#' | sort -u > $tmpfile + git clang-format -f + if [[ $(git diff --stat) == '' ]]; then + echo 'Nothing was applied, skip' + continue + fi + echo "-----------------------------------" + echo "-- Got some diff, run one check at a time now" + cat $tmpfile | while read check ; do + echo "-----------------------------------" + echo "-- Reset state before applying check $check on file $file" + ensure_clean_build + + echo "-----------------------------------" + echo "-- Apply check $check on file $file" + echo "$TIMING_TIDY $CLANG_TIDY -p $BUILD_DIR $file --checks="-*,$check" -fix" + { $TIMING_TIDY $CLANG_TIDY -p $BUILD_DIR $file --checks="-*,$check" -fix ; } 2>&1 + git clang-format -f + if [[ $(git diff --stat) == '' ]]; then + echo 'Nothing was applied, skip' + continue + fi + echo "-----------------------------------" + echo "-- Test check $check on file $file" + # Clang-tidy sometimes update files in the build directory, erase the .inc file generate by tablegen + # to force them to be regenerated now. + find $BUILD_DIR/tools/mlir/ | grep '\.inc' | while read file ; do rm $file ; done + ninja -C $BUILD_DIR check-mlir > ${REJECT_DIR}/ninja.${check}.$(basename $file).log 2>&1 + if [[ $? != 0 ]] ; then + echo "check-mlir failed! (see ninja.${check}.${file}.log)" + [[ ! -z "$REJECT_DIR" ]] && git diff > "${REJECT_DIR}/${check}_$(basename ${file}).reject.diff" + continue + fi + echo "-----------------------------------" + echo "-- Success, commit changes for check $check on file $file" + git clang-format -f + + git commit -a -m "Apply clang-tidy fixes for $check in $(basename $file) (NFC)" + done +done -- 2.7.4