From 83d660afee1b2f5b331cea7be632baf9cec25149 Mon Sep 17 00:00:00 2001 From: Hyunggi Lee Date: Tue, 9 Mar 2021 14:29:17 +0900 Subject: [PATCH] Use ccache.tar file with --pkg-ccache option 1. first build. create _ccache.tar $ gbs build -A armv7l --ccache after build is done _ccache.tar file will be created BUILD-ROOTS/scratch.armv7l.0/home/abuild/rpmbuild/OTHER/_ccache.tar backup _ccache.tar file to your storage 2. rebuild with _ccache.tar $ gbs build -A armv7l --ccache --pkg-ccache /path/to/_ccache.tar test build chromium-efl build env: laptop NT371B5M - Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz - SAMSUNG MZNLN256 256G SSD first clean build gbs build -A armv7l --ccache build time: 29806s rebuild gbs build -A armv7l --ccache --overwrite --pkg-ccache ~/_ccache.tar build time: 3294s rebuild (with qemu-accel (ccache, ninja) gbs build -A armv7l --ccache --overwrite --pkg-ccache ~/_ccache.tar build time: 2209s related upstream patch https://github.com/openSUSE/obs-build/pull/580 https://github.com/openSUSE/obs-build/pull/578 https://github.com/openSUSE/obs-build/pull/577 https://github.com/openSUSE/obs-build/pull/576 https://github.com/openSUSE/obs-build/pull/571 https://github.com/openSUSE/obs-build/pull/560 https://github.com/openSUSE/obs-build/pull/559 Change-Id: Ib5e6f60eddcca6f4ba67e209f9068f11a125d86e Signed-off-by: Hyunggi Lee --- build | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 5 deletions(-) mode change 100755 => 100644 build diff --git a/build b/build old mode 100755 new mode 100644 index f98e1b7..374ebc9 --- a/build +++ b/build @@ -67,6 +67,7 @@ DO_CHECKS=true SHORT_CIRCUIT=false NO_TOPDIR_CLEANUP=false CLEAN_BUILD= +REMOVE_CCACHE= KEEP_PACKS= RECIPEFILES=() SRCDIR= @@ -99,6 +100,8 @@ CCACHE= DLNOSIGNATURE= CACHE_DIR=/var/cache/build USEHIGHERDEPS= +PKG_CCACHE= +CCACHE_SETUP_START_TIME= # This is for insserv export YAST_IS_RUNNING=instsys @@ -232,6 +235,10 @@ Known Parameters: --ccache Use ccache to speed up rebuilds + --pkg-ccache /path/to/ccache.tar.gz + path to archive of ccache directory content. + Its contents will be extracted to /.ccache + --icecream N Use N parallel build jobs with icecream @@ -425,9 +432,9 @@ toshellscript() { setupccache() { if test -n "$CCACHE" ; then + CCACHE_SETUP_START_TIME=`date +%s` if mkdir -p "$BUILD_ROOT"/var/lib/build/ccache/bin; then - for i in $(ls "$BUILD_ROOT"/usr/bin | grep -E '^(cc|gcc|[cg][+][+])([-]?[234][.]?[0-9])*$'); do -# ln -sf /usr/bin/ccache $BUILD_ROOT/var/lib/build/ccache/bin/$i + for i in $(ls $BUILD_ROOT/usr/bin | grep -E '^(cc|gcc|[cg][+][+]|clang|clang[+][+])([-]?[.0-9])*$'); do rm -f "$BUILD_ROOT"/var/lib/build/ccache/bin/$i test -e "$BUILD_ROOT"/usr/bin/$i || continue echo '#! /bin/sh' > "$BUILD_ROOT"/var/lib/build/ccache/bin/$i @@ -439,11 +446,33 @@ setupccache() { done fi mkdir -p "$BUILD_ROOT/.ccache" - chown -R "$ABUILD_UID:$ABUILD_GID" "$BUILD_ROOT/.ccache" - echo "export CCACHE_DIR=/.ccache" > "$BUILD_ROOT"/etc/profile.d/build_ccache.sh + # overwrite /etc/profile.d/build_ccache.sh + echo 'export CCACHE_DIR=/.ccache' > "$BUILD_ROOT"/etc/profile.d/build_ccache.sh echo 'export PATH=/var/lib/build/ccache/bin:$PATH' >> "$BUILD_ROOT"/etc/profile.d/build_ccache.sh + + # Note + # Disable compression setting temporary + # Until ccache is applied to qemu-accel, it is very slow when using the function, so it will be applied later. + # + # https://ccache.dev/releasenotes.html#_ccache_4_0 + # Changed compression algorithm from Deflate (zlib) to Zstandard, enabled by default. + enable_compression=false + + local cconf="$BUILD_ROOT"/etc/ccache.conf + if test -e $cconf; then + grep -q "^compression*" $cconf && sed -i "s:compression.*:compression = $enable_compression:g" $cconf || echo "compression = $enable_compression" >> $cconf + grep -q "^cache_dir*" $cconf && sed -i "s:cache_dir.*:cache_dir = /.ccache:g" $cconf || echo "cache_dir = /.ccache" >> $cconf else - rm -f "$BUILD_ROOT"/var/lib/build/ccache/bin/{gcc,g++,cc,c++} + echo "compression = $enable_compression" >> $cconf + echo "cache_dir = /.ccache" >> $cconf + fi + local ccachetar="$BUILD_ROOT/.build.oldpackages/_ccache.tar" + test -e $ccachetar && tar -xf $ccachetar -C "$BUILD_ROOT/.ccache/" + rm -f $ccachetar + chown -R "$ABUILD_UID:$ABUILD_GID" "$BUILD_ROOT/.ccache" + else + CCACHE_SETUP_START_TIME= + rm -f "$BUILD_ROOT"/var/lib/build/ccache/bin/{gcc,g++,cc,c++,clang,clang++} fi } @@ -765,6 +794,7 @@ while test -n "$1"; do ;; -clean) CLEAN_BUILD='--clean' + REMOVE_CCACHE="$CLEAN_BUILD" ;; -kill) KILL=true @@ -891,6 +921,12 @@ while test -n "$1"; do -ccache) CCACHE=true ;; + -pkg-ccache) + needarg + PKG_CCACHE="$ARG" + shift + CCACHE=true + ;; -statistics) DO_STATISTICS=1 ;; @@ -1204,6 +1240,11 @@ for RECIPEFILE in "${RECIPEFILES[@]}" ; do mount -n -tdevpts -omode=0620,gid=5 none "$BUILD_ROOT"/dev/pts copy_oldpackages + # chroot based builds + # rsync as source and dest could be same + test -n "$PKG_CCACHE" -a -e "$PKG_CCACHE" && \ + mkdir -p "$BUILD_ROOT/.build.oldpackages/" && \ + rsync -v "$PKG_CCACHE" "$BUILD_ROOT/.build.oldpackages/_ccache.tar" fi # hack to process preinstallimages early @@ -1465,6 +1506,17 @@ if test -n "$RPMS" -a "$DO_CHECKS" != false ; then recipe_run_rpmlint fi +if test -n "$CCACHE" ; then + if test -n "$REMOVE_CCACHE" ; then + echo "... cleaning ccache" + test_cmd="ccache -h | grep -c evict-older-than" + clean_cmd="ccache --evict-older-than $(($(date +%s) - $CCACHE_SETUP_START_TIME))s" + chroot $BUILD_ROOT su -c "$test_cmd && $clean_cmd" - $BUILD_USER + fi + echo "... saving ccache" + tar ${REMOVE_CCACHE:+--remove-files} -cf "$BUILD_ROOT/$TOPDIR/OTHER/_ccache.tar" -C "$BUILD_ROOT/.ccache/" . +fi + if test \( -n "$RPMS" -o -n "$DEBS" \) -a -n "$CREATE_BASELIBS"; then create_baselibs fi -- 2.7.4