From 9c97ca1b99626869bf0507bf02dcfca266550e27 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Mon, 19 Nov 2018 18:43:31 +0000 Subject: [PATCH] Add docker configurations used by the buildbots. These are the scripts I use to create the docker images for the build bots and run them. llvm-svn: 347244 --- libcxx/utils/docker/build_docker_image.sh | 109 +++++++++++++++++ libcxx/utils/docker/debian9/Dockerfile | 113 ++++++++++++++++++ libcxx/utils/docker/scripts/build_gcc.sh | 91 +++++++++++++++ libcxx/utils/docker/scripts/build_install_llvm.sh | 114 ++++++++++++++++++ libcxx/utils/docker/scripts/checkout_git.sh | 130 +++++++++++++++++++++ .../utils/docker/scripts/install_clang_packages.sh | 64 ++++++++++ libcxx/utils/docker/scripts/run_buildbot.sh | 55 +++++++++ 7 files changed, 676 insertions(+) create mode 100755 libcxx/utils/docker/build_docker_image.sh create mode 100644 libcxx/utils/docker/debian9/Dockerfile create mode 100755 libcxx/utils/docker/scripts/build_gcc.sh create mode 100755 libcxx/utils/docker/scripts/build_install_llvm.sh create mode 100755 libcxx/utils/docker/scripts/checkout_git.sh create mode 100755 libcxx/utils/docker/scripts/install_clang_packages.sh create mode 100755 libcxx/utils/docker/scripts/run_buildbot.sh diff --git a/libcxx/utils/docker/build_docker_image.sh b/libcxx/utils/docker/build_docker_image.sh new file mode 100755 index 0000000..0d2d6d3 --- /dev/null +++ b/libcxx/utils/docker/build_docker_image.sh @@ -0,0 +1,109 @@ +#!/bin/bash +#===- libcxx/utils/docker/build_docker_image.sh ----------------------------===// +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +#===----------------------------------------------------------------------===// +set -e + +IMAGE_SOURCE="" +DOCKER_REPOSITORY="" +DOCKER_TAG="" + +function show_usage() { + cat << EOF +Usage: build_docker_image.sh [options] [-- [cmake_args]...] + +Available options: + General: + -h|--help show this help message + Docker-specific: + -s|--source image source dir (i.e. debian8, nvidia-cuda, etc) + -d|--docker-repository docker repository for the image + -t|--docker-tag docker tag for the image + +Required options: --source and --docker-repository. + +For example, running: +$ build_docker_image.sh -s debian9 -d mydocker/debian9-clang -t latest +will produce two docker images: + mydocker/debian9-clang-build:latest - an intermediate image used to compile + clang. + mydocker/clang-debian9:latest - a small image with preinstalled clang. +Please note that this example produces a not very useful installation, since it +doesn't override CMake defaults, which produces a Debug and non-boostrapped +version of clang. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + show_usage + exit 0 + ;; + -s|--source) + shift + IMAGE_SOURCE="$1" + shift + ;; + -d|--docker-repository) + shift + DOCKER_REPOSITORY="$1" + shift + ;; + -t|--docker-tag) + shift + DOCKER_TAG="$1" + shift + ;; + *) + echo "Unknown argument $1" + exit 1 + ;; + esac +done + + +command -v docker >/dev/null || + { + echo "Docker binary cannot be found. Please install Docker to use this script." + exit 1 + } + +if [ "$IMAGE_SOURCE" == "" ]; then + echo "Required argument missing: --source" + exit 1 +fi + +if [ "$DOCKER_REPOSITORY" == "" ]; then + echo "Required argument missing: --docker-repository" + exit 1 +fi + +SOURCE_DIR=$(dirname $0) +if [ ! -d "$SOURCE_DIR/$IMAGE_SOURCE" ]; then + echo "No sources for '$IMAGE_SOURCE' were found in $SOURCE_DIR" + exit 1 +fi + +BUILD_DIR=$(mktemp -d) +trap "rm -rf $BUILD_DIR" EXIT +echo "Using a temporary directory for the build: $BUILD_DIR" + +cp -r "$SOURCE_DIR/$IMAGE_SOURCE" "$BUILD_DIR/$IMAGE_SOURCE" +cp -r "$SOURCE_DIR/scripts" "$BUILD_DIR/scripts" + + +if [ "$DOCKER_TAG" != "" ]; then + DOCKER_TAG=":$DOCKER_TAG" +fi + +echo "Building ${DOCKER_REPOSITORY}${DOCKER_TAG} from $IMAGE_SOURCE" +docker build -t "${DOCKER_REPOSITORY}${DOCKER_TAG}" \ + -f "$BUILD_DIR/$IMAGE_SOURCE/Dockerfile" \ + "$BUILD_DIR" +echo "Done" diff --git a/libcxx/utils/docker/debian9/Dockerfile b/libcxx/utils/docker/debian9/Dockerfile new file mode 100644 index 0000000..560cbd1 --- /dev/null +++ b/libcxx/utils/docker/debian9/Dockerfile @@ -0,0 +1,113 @@ +#===- libcxx/utils/docker/debian9/Dockerfile -------------------------===// +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +#===----------------------------------------------------------------------===// + +# Setup the base builder image with the packages we'll need to build GCC and Clang from source. +FROM launcher.gcr.io/google/debian9:latest as builder-base +LABEL maintainer "libc++ Developers" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ + build-essential \ + wget \ + subversion \ + unzip \ + automake \ + python \ + cmake \ + ninja-build \ + curl \ + git \ + gcc-multilib \ + g++-multilib \ + libc6-dev \ + bison \ + flex \ + libtool \ + autoconf \ + binutils-dev \ + binutils-gold \ + software-properties-common && \ + update-alternatives --install "/usr/bin/ld" "ld" "/usr/bin/ld.gold" 20 && \ + update-alternatives --install "/usr/bin/ld" "ld" "/usr/bin/ld.bfd" 10 + +# Build GCC 4.9 for testing our C++11 against +FROM builder-base as gcc-49-builder +LABEL maintainer "libc++ Developers" + +ADD scripts/build_gcc.sh /tmp/build_gcc.sh + +RUN git clone --depth=1 --branch gcc-4_9_4-release git://gcc.gnu.org/git/gcc.git /tmp/gcc-4.9.4 +RUN cd /tmp/gcc-4.9.4/ && ./contrib/download_prerequisites +RUN /tmp/build_gcc.sh --source /tmp/gcc-4.9.4 --to /opt/gcc-4.9.4 + +# Build GCC ToT for testing in all dialects. +FROM builder-base as gcc-tot-builder +LABEL maintainer "libc++ Developers" + +ADD scripts/build_gcc.sh /tmp/build_gcc.sh + +RUN git clone --depth=1 git://gcc.gnu.org/git/gcc.git /tmp/gcc-tot +RUN cd /tmp/gcc-tot && ./contrib/download_prerequisites +RUN /tmp/build_gcc.sh --source /tmp/gcc-tot --to /opt/gcc-tot + +# Build LLVM 4.0 which is used to test against a "legacy" compiler. +FROM builder-base as llvm-4-builder +LABEL maintainer "libc++ Developers" + +ADD scripts/checkout_git.sh /tmp/checkout_git.sh +ADD scripts/build_install_llvm.sh /tmp/build_install_llvm.sh + +RUN /tmp/checkout_git.sh --to /tmp/llvm-4.0 -p clang -p compiler-rt --branch release_40 +RUN /tmp/build_install_llvm.sh \ + --install /opt/llvm-4.0 \ + --source /tmp/llvm-4.0 \ + --build /tmp/build-llvm-4.0 \ + -i install-clang -i install-clang-headers \ + -i install-compiler-rt \ + -- \ + -DCMAKE_BUILD_TYPE=RELEASE \ + -DLLVM_ENABLE_ASSERTIONS=ON + +# Stage 2. Produce a minimal release image with build results. +FROM launcher.gcr.io/google/debian9:latest +LABEL maintainer "libc++ Developers" + +# Copy over the GCC and Clang installations +COPY --from=gcc-49-builder /opt/gcc-4.9.4 /opt/gcc-4.9.4 +COPY --from=gcc-tot-builder /opt/gcc-tot /opt/gcc-tot +COPY --from=llvm-4-builder /opt/llvm-4.0 /opt/llvm-4.0 + +RUN ln -s /opt/gcc-4.9.4/bin/gcc /usr/local/bin/gcc-4.9 && \ + ln -s /opt/gcc-4.9.4/bin/g++ /usr/local/bin/g++-4.9 + +RUN apt-get update && \ + apt-get install -y \ + ca-certificates \ + gnupg \ + build-essential \ + apt-transport-https \ + curl \ + software-properties-common + +RUN apt-get install -y --no-install-recommends \ + systemd \ + sysvinit-utils \ + cmake \ + subversion \ + git \ + ninja-build \ + gcc-multilib \ + g++-multilib \ + python \ + buildbot-slave + +ADD scripts /libcxx-scripts/ +RUN /libcxx-scripts/install_clang_packages.sh diff --git a/libcxx/utils/docker/scripts/build_gcc.sh b/libcxx/utils/docker/scripts/build_gcc.sh new file mode 100755 index 0000000..85feb16 --- /dev/null +++ b/libcxx/utils/docker/scripts/build_gcc.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +#===- libcxx/utils/docker/scripts/build-gcc.sh ----------------------------===// +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +#===-----------------------------------------------------------------------===// + +set -e + + +function show_usage() { + cat << EOF +Usage: build-gcc.sh [options] + +Run autoconf with the specified arguments. Used inside docker container. + +Available options: + -h|--help show this help message + --source the source path from which to run the configuration. + --to destination directory where to install the targets. +Required options: --to, at least one --install-target. + +All options after '--' are passed to CMake invocation. +EOF +} + +GCC_INSTALL_DIR="" +GCC_SOURCE_DIR="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --to) + shift + GCC_INSTALL_DIR="$1" + shift + ;; + --source) + shift + GCC_SOURCE_DIR="$1" + shift + ;; + -h|--help) + show_usage + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + esac +done + +if [ "$GCC_INSTALL_DIR" == "" ]; then + echo "No install directory. Please specify the --to argument." + exit 1 +fi + +if [ "$GCC_SOURCE_DIR" == "" ]; then + echo "No source directory. Please specify the --source argument." + exit 1 +fi + +GCC_NAME=`basename $GCC_SOURCE_DIR` +GCC_BUILD_DIR="/tmp/gcc-build-root/build-$GCC_NAME" + +mkdir -p "$GCC_INSTALL_DIR" +mkdir -p "$GCC_BUILD_DIR" +pushd "$GCC_BUILD_DIR" + +# Run the build as specified in the build arguments. +echo "Running configuration" +$GCC_SOURCE_DIR/configure --prefix=$GCC_INSTALL_DIR \ + --disable-bootstrap --disable-libgomp --disable-libitm \ + --disable-libvtv --disable-libcilkrts --disable-libmpx \ + --disable-liboffloadmic --disable-libcc1 --enable-languages=c,c++ + +NPROC=`nproc` +echo "Running build with $NPROC threads" +make -j$NPROC + +echo "Installing to $GCC_INSTALL_DIR" +make install -j$NPROC + +popd + +# Cleanup. +rm -rf "$GCC_BUILD_DIR" + +echo "Done" \ No newline at end of file diff --git a/libcxx/utils/docker/scripts/build_install_llvm.sh b/libcxx/utils/docker/scripts/build_install_llvm.sh new file mode 100755 index 0000000..6f19a96 --- /dev/null +++ b/libcxx/utils/docker/scripts/build_install_llvm.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +#===- llvm/utils/docker/scripts/build_install_llvm.sh ---------------------===// +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +#===-----------------------------------------------------------------------===// + +set -e + +function show_usage() { + cat << EOF +Usage: build_install_llvm.sh [options] -- [cmake-args] + +Run cmake with the specified arguments. Used inside docker container. +Passes additional -DCMAKE_INSTALL_PREFIX and puts the build results into +the directory specified by --to option. + +Available options: + -h|--help show this help message + -i|--install-target name of a cmake install target to build and include in + the resulting archive. Can be specified multiple times. + --install destination directory where to install the targets. + --source location of the source tree. + --build location to use as the build directory. +Required options: --to, --source, --build, and at least one --install-target. + +All options after '--' are passed to CMake invocation. +EOF +} + +CMAKE_ARGS="" +CMAKE_INSTALL_TARGETS="" +CLANG_INSTALL_DIR="" +CLANG_SOURCE_DIR="" +CLANG_BUILD_DIR="" + +while [[ $# -gt 0 ]]; do + case "$1" in + -i|--install-target) + shift + CMAKE_INSTALL_TARGETS="$CMAKE_INSTALL_TARGETS $1" + shift + ;; + --source) + shift + CLANG_SOURCE_DIR="$1" + shift + ;; + --build) + shift + CLANG_BUILD_DIR="$1" + shift + ;; + --install) + shift + CLANG_INSTALL_DIR="$1" + shift + ;; + --) + shift + CMAKE_ARGS="$*" + shift $# + ;; + -h|--help) + show_usage + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + esac +done + +if [ "$CLANG_SOURCE_DIR" == "" ]; then + echo "No source directory. Please pass --source." + exit 1 +fi + +if [ "$CLANG_BUILD_DIR" == "" ]; then + echo "No build directory. Please pass --build" + exit 1 +fi + +if [ "$CMAKE_INSTALL_TARGETS" == "" ]; then + echo "No install targets. Please pass one or more --install-target." + exit 1 +fi + +if [ "$CLANG_INSTALL_DIR" == "" ]; then + echo "No install directory. Please specify the --to argument." + exit 1 +fi + +echo "Building in $CLANG_BUILD_DIR" +mkdir -p "$CLANG_BUILD_DIR" +pushd "$CLANG_BUILD_DIR" + +# Run the build as specified in the build arguments. +echo "Running build" +cmake -GNinja \ + -DCMAKE_INSTALL_PREFIX="$CLANG_INSTALL_DIR" \ + $CMAKE_ARGS \ + "$CLANG_SOURCE_DIR" +ninja $CMAKE_INSTALL_TARGETS + +popd + +# Cleanup. +rm -rf "$CLANG_BUILD_DIR" + +echo "Done" diff --git a/libcxx/utils/docker/scripts/checkout_git.sh b/libcxx/utils/docker/scripts/checkout_git.sh new file mode 100755 index 0000000..2227002 --- /dev/null +++ b/libcxx/utils/docker/scripts/checkout_git.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +#===- llvm/utils/docker/scripts/checkout.sh ---------------------===// +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +#===-----------------------------------------------------------------------===// + +set -e + +function show_usage() { + cat << EOF +Usage: checkout.sh [options] + +Checkout svn sources into /tmp/clang-build/src. Used inside a docker container. + +Available options: + -h|--help show this help message + -b|--branch svn branch to checkout, i.e. 'trunk', + 'branches/release_40' + (default: 'trunk') + -p|--llvm-project name of an svn project to checkout. + For clang, please use 'clang', not 'cfe'. + Project 'llvm' is always included and ignored, if + specified. + Can be specified multiple times. +EOF +} + +LLVM_BRANCH="" +# We always checkout llvm +LLVM_PROJECTS="llvm" +SOURCE_DIR="" + +function contains_project() { + local TARGET_PROJ="$1" + local PROJ + for PROJ in $LLVM_PROJECTS; do + if [ "$PROJ" == "$TARGET_PROJ" ]; then + return 0 + fi + done + return 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --to) + shift + SOURCE_DIR="$1" + shift + ;; + -b|--branch) + shift + LLVM_BRANCH="$1" + shift + ;; + -p|--llvm-project) + shift + PROJ="$1" + shift + + if [ "$PROJ" == "cfe" ]; then + PROJ="clang" + fi + + if ! contains_project "$PROJ" ; then + if [ "$PROJ" == "clang-tools-extra" ] && [ ! contains_project "clang" ]; then + echo "Project 'clang-tools-extra' specified before 'clang'. Adding 'clang' to a list of projects first." + LLVM_PROJECTS="$LLVM_PROJECTS clang" + fi + LLVM_PROJECTS="$LLVM_PROJECTS $PROJ" + else + echo "Project '$PROJ' is already enabled, ignoring extra occurrences." + fi + ;; + -h|--help) + show_usage + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + esac +done + +if [ "$SOURCE_DIR" == "" ]; then + echo "Must specify checkout directory using --to" + exit 1 +fi + +if [ "$LLVM_BRANCH" == "" ]; then + GIT_BRANCH_ARG="" +else + GIT_BRANCH_ARG="--branch $LLVM_BRANCH" +fi + +if [ "$LLVM_SVN_REV" != "" ]; then + SVN_REV_ARG="-r$LLVM_SVN_REV" + echo "Checking out svn revision r$LLVM_SVN_REV." +else + SVN_REV_ARG="" + echo "Checking out latest svn revision." +fi + +# Get the sources from svn. +echo "Checking out sources from git" + +for LLVM_PROJECT in $LLVM_PROJECTS; do + if [ "$LLVM_PROJECT" == "llvm" ]; then + CHECKOUT_DIR="$SOURCE_DIR" + elif [ "$LLVM_PROJECT" == "libcxx" ] || [ "$LLVM_PROJECT" == "libcxxabi" ] || [ "$LLVM_PROJECT" == "compiler-rt" ]; then + CHECKOUT_DIR="$SOURCE_DIR/projects/$LLVM_PROJECT" + elif [ "$LLVM_PROJECT" == "clang" ]; then + CHECKOUT_DIR="$SOURCE_DIR/tools/clang" + elif [ "$LLVM_PROJECT" == "clang-tools-extra" ]; then + CHECKOUT_DIR="$SOURCE_DIR/tools/clang/tools/extra" + else + CHECKOUT_DIR="$SOURCE_DIR/$LLVM_PROJECT" + fi + + echo "Checking out https://git.llvm.org/git/$LLVM_PROJECT to $CHECKOUT_DIR" + git clone --depth=1 $GIT_BRANCH_ARG \ + "https://git.llvm.org/git/$LLVM_PROJECT.git" \ + "$CHECKOUT_DIR" +done + +echo "Done" diff --git a/libcxx/utils/docker/scripts/install_clang_packages.sh b/libcxx/utils/docker/scripts/install_clang_packages.sh new file mode 100755 index 0000000..fabee0e --- /dev/null +++ b/libcxx/utils/docker/scripts/install_clang_packages.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +#===- libcxx/utils/docker/scripts/install_clang_package.sh -----------------===// +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +#===-----------------------------------------------------------------------===// + +set -e + +function show_usage() { + cat << EOF +Usage: install_clang_package.sh [options] + +Install +Available options: + -h|--help show this help message + --version the numeric version of the package to use. +EOF +} + +VERSION="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) + shift + VERSION="$1" + shift + ;; + -h|--help) + show_usage + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + esac +done + + + +curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - +add-apt-repository -s "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs) main" +apt-get update +apt-get install -y --no-install-recommends clang + +echo "Testing clang version..." +clang --version + +echo "Testing clang++ version..." +clang++ --version + +# Figure out the libc++ and libc++abi package versions that we want. +if [ "$VERSION" == "" ]; then + VERSION="$(apt-cache search 'libc\+\+-[0-9]-dev' | awk '{print $1}' | awk -F- '{print $2}')" + echo "Installing version '$VERSION'" +fi + +apt-get install -y --no-install-recommends "libc++-$VERSION-dev" "libc++abi-$VERSION-dev" + +echo "Done" diff --git a/libcxx/utils/docker/scripts/run_buildbot.sh b/libcxx/utils/docker/scripts/run_buildbot.sh new file mode 100755 index 0000000..10cc09b --- /dev/null +++ b/libcxx/utils/docker/scripts/run_buildbot.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -x + +BOT_DIR=/b +BOT_NAME=$1 +BOT_PASS=$2 + +mkdir -p $BOT_DIR + +#curl "https://repo.stackdriver.com/stack-install.sh" | bash -s -- --write-gcm + +apt-get update -y +apt-get upgrade -y + +systemctl set-property buildslave.service TasksMax=100000 + +buildslave stop $BOT_DIR + +chown buildbot:buildbot $BOT_DIR + +echo "Connecting as $BOT_NAME" +buildslave create-slave --allow-shutdown=signal $BOT_DIR lab.llvm.org:9990 $BOT_NAME $BOT_PASS + +echo "Eric Fiselier " > $BOT_DIR/info/admin + +{ + uname -a | head -n1 + cmake --version | head -n1 + g++ --version | head -n1 + ld --version | head -n1 + date + lscpu +} > $BOT_DIR/info/host + +echo "SLAVE_RUNNER=/usr/bin/buildslave +SLAVE_ENABLED[1]=\"1\" +SLAVE_NAME[1]=\"buildslave1\" +SLAVE_USER[1]=\"buildbot\" +SLAVE_BASEDIR[1]=\"$BOT_DIR\" +SLAVE_OPTIONS[1]=\"\" +SLAVE_PREFIXCMD[1]=\"\"" > /etc/default/buildslave + +chown -R buildbot:buildbot $BOT_DIR +systemctl daemon-reload +service buildslave restart + +sleep 30 +cat $BOT_DIR/twistd.log +grep "slave is ready" $BOT_DIR/twistd.log || shutdown now + +# GCE can restart instance after 24h in the middle of the build. +# Gracefully restart before that happen. +sleep 72000 +while pkill -SIGHUP buildslave; do sleep 5; done; +shutdown now \ No newline at end of file -- 2.7.4