From 7b9e9f2857149aa480a5ef5d4dac346cf99245fc Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Mon, 7 Nov 2016 17:40:28 +0000 Subject: [PATCH] Add some facilities to work with a git monorepo (experimental setup) Summary: Some changes are made to cmake, especially the addition of a new LLVM_ENABLE_PROJECTS option that makes the build system aware of the monorepo directory structure. Also a new script is added in llvm/utils/git-svn/. When present in the $PATH, it enables a `git llvm` command. It is providing at this point only the ability to push from the git monorepo: `git llvm push`. It is intended to evolves with more features, for instance I plan on features like `git llvm show r284955` to help working with sequential revision numbers. The push feature is taken from Justin Lebar's script available here: https://github.com/jlebar/llvm-repo-tools/ Reviewers: jlebar Subscribers: mgorny, modocache, llvm-commits Differential Revision: https://reviews.llvm.org/D26334 llvm-svn: 286123 --- libcxxabi/CMakeLists.txt | 2 + llvm/CMakeLists.txt | 19 +++ llvm/docs/GettingStarted.rst | 54 ++++++++ llvm/utils/git-svn/git-llvm | 296 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 371 insertions(+) create mode 100755 llvm/utils/git-svn/git-llvm diff --git a/libcxxabi/CMakeLists.txt b/libcxxabi/CMakeLists.txt index 73ec5c5..d0e1f55 100644 --- a/libcxxabi/CMakeLists.txt +++ b/libcxxabi/CMakeLists.txt @@ -144,6 +144,7 @@ find_path( ${LIBCXXABI_LIBCXX_PATH}/include ${CMAKE_BINARY_DIR}/${LIBCXXABI_LIBCXX_INCLUDES} ${LLVM_MAIN_SRC_DIR}/projects/libcxx/include + ${LLVM_MAIN_SRC_DIR}/../libcxx/include ${LLVM_INCLUDE_DIR}/c++/v1 ) @@ -156,6 +157,7 @@ find_path( PATHS ${LIBCXXABI_LIBCXX_PATH} ${LIBCXXABI_LIBCXX_INCLUDES}/../ ${LLVM_MAIN_SRC_DIR}/projects/libcxx/ + ${LLVM_MAIN_SRC_DIR}/../libcxx/ NO_DEFAULT_PATH ) diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt index 7a70ebc..ab43a4a 100644 --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -93,6 +93,25 @@ if(CMAKE_HOST_APPLE AND APPLE) endif() endif() +# Git Mono-Repo handling: automatically set the LLVM_EXTERNAL_${project}_SOURCE_DIR +set(LLVM_ALL_PROJECTS "clang;libcxx;libcxxabi;lldb;compiler-rt;lld;polly") +set(LLVM_ENABLE_PROJECTS "" CACHE STRING + "Semicolon-separated list of projects to build (${LLVM_ALL_PROJECTS}), or \"all\".") +if( LLVM_ENABLE_PROJECTS STREQUAL "all" ) + set( LLVM_ENABLE_PROJECTS ${LLVM_ALL_PROJECTS}) +endif() +foreach(proj ${LLVM_ENABLE_PROJECTS}) + set(PROJ_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../${proj}") + if(NOT EXISTS "${PROJ_DIR}" OR NOT IS_DIRECTORY "${PROJ_DIR}") + message(FATAL_ERROR "LLVM_ENABLE_PROJECTS requests ${proj} but directory not found: ${PROJ_DIR}") + endif() + string(TOUPPER "${proj}" upper_proj) + set(LLVM_EXTERNAL_${upper_proj}_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../${proj}") + if (proj STREQUAL "clang") + set(LLVM_EXTERNAL_CLANG_TOOLS_EXTRA_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../clang-tools-extra") + endif() +endforeach() + # The following only works with the Ninja generator in CMake >= 3.0. set(LLVM_PARALLEL_COMPILE_JOBS "" CACHE STRING "Define the maximum number of concurrent compilation jobs.") diff --git a/llvm/docs/GettingStarted.rst b/llvm/docs/GettingStarted.rst index 59fbc2e..8fd2a3c 100644 --- a/llvm/docs/GettingStarted.rst +++ b/llvm/docs/GettingStarted.rst @@ -680,6 +680,60 @@ about files with uncommitted changes. The fix is to rebuild the metadata: Please, refer to the Git-SVN manual (``man git-svn``) for more information. +For developers to work with a git monorepo +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + + This set-up is using unofficial mirror hosted on GitHub, use with caution. + +To set up a clone of all the llvm projects using a unified repository: + +.. code-block:: console + + % export TOP_LEVEL_DIR=`pwd` + % git clone https://github.com/llvm-project/llvm-project/ + % cd llvm-project + % git config branch.master.rebase true + +You can configure various build directory from this clone, starting with a build +of LLVM alone: + +.. code-block:: console + + % cd $TOP_LEVEL_DIR + % mkdir llvm-build && cd llvm-build + % cmake -GNinja ../llvm-project/llvm + +Or lldb: + +.. code-block:: console + + % cd $TOP_LEVEL_DIR + % mkdir lldb-build && cd lldb-build + % cmake -GNinja ../llvm-project/llvm -DLLVM_ENABLE_PROJECTS=lldb + +Or a combination of multiple projects: + +.. code-block:: console + + % cd $TOP_LEVEL_DIR + % mkdir clang-build && cd clang-build + % cmake -GNinja ../llvm-project/llvm -DLLVM_ENABLE_PROJECTS="clang;libcxx;compiler-rt" + +A helper script is provided in `llvm/utils/git-svn/git-llvm`. After you add it +to your path, you can push committed changes upstream with `git llvm push`. + +.. code-block:: console + + % export PATH=$PATH:$TOP_LEVEL_DIR/llvm-project/llvm/utils/git-svn/ + % git llvm push + +While this is using SVN under the hood, it does not require any interaction from +you with git-svn. +After a few minutes, `git pull` should get back the changes as they were +commited. + Local LLVM Configuration ------------------------ diff --git a/llvm/utils/git-svn/git-llvm b/llvm/utils/git-svn/git-llvm new file mode 100755 index 0000000..353fd12 --- /dev/null +++ b/llvm/utils/git-svn/git-llvm @@ -0,0 +1,296 @@ +#!/usr/bin/env python +# +# ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========# +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +# ==------------------------------------------------------------------------==# + +r""" +git-llvm integration +==================== + +This file provides integration for git. + +For further details, run: + + git llvm -h + +Requires Python 2.7 +""" + +from __future__ import print_function +import argparse +import collections +import contextlib +import errno +import os +import re +import subprocess +import sys +import tempfile +import time + + +desc = ''' +TODO +''' + +# It's *almost* a straightforward mapping from the monorepo to svn... +GIT_TO_SVN_DIR = { + d: (d + '/trunk') + for d in [ + 'clang-tools-extra', + 'compiler-rt', + 'dragonegg', + 'klee', + 'libclc', + 'libcxx', + 'libcxxabi', + 'lld', + 'lldb', + 'llvm', + 'polly', + ] +} +GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'}) + +VERBOSE = False +QUIET = False + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def log(*args, **kwargs): + if QUIET: + return + print(*args, **kwargs) + + +def log_verbose(*args, **kwargs): + if not VERBOSE: + return + print(*args, **kwargs) + + +def die(msg): + eprint(msg) + sys.exit(1) + + +def shell(cmd, strip=True, cwd=None, stdin=None): + log_verbose('Running: %s' % ' '.join(cmd)) + + start = time.time() + p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, stdin=subprocess.PIPE) + stdout, stderr = p.communicate(input=stdin) + elapsed = time.time() - start + + if elapsed >= .1: + log_verbose('Warning: command took %0.1fs' % elapsed) + + if p.returncode == 0: + if stderr: + eprint('`%s` printed to stderr:' % ' '.join(args)) + eprint(stderr.rstrip()) + if strip: + stdout = stdout.rstrip('\r\n') + return stdout + eprint('`%s` returned %s' % (' '.join(args), p.returncode)) + if stderr: + eprint(stderr.rstrip()) + sys.exit(2) + + +def git(*cmd, **kwargs): + return shell(['git'] + list(cmd), kwargs.get('strip', True)) + + +def svn(cwd, *cmd, **kwargs): + # TODO: Better way to do default arg when we have *cmd? + return shell(['svn'] + list(cmd), cwd=cwd, stdin=kwargs.get('stdin', None)) + + +def get_default_rev_range(): + # Get the branch tracked by the current branch, as set by + # git branch --set-upstream-to See http://serverfault.com/a/352236/38694. + cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD') + upstream_branch = git('for-each-ref', '--format=%(upstream:short)', + cur_branch) + if not upstream_branch: + upstream_branch = 'origin/master' + + # Get the newest common ancestor between HEAD and our upstream branch. + upstream_rev = git('merge-base', 'HEAD', upstream_branch) + return '%s..' % upstream_rev + + +def get_revs_to_push(rev_range): + if not rev_range: + rev_range = get_default_rev_range() + # Use git show rather than some plumbing command to figure out which revs + # are in rev_range because it handles single revs (HEAD^) and ranges + # (foo..bar) like we want. + revs = git('show', '--reverse', '--quiet', + '--pretty=%h', rev_range).split('\n') + # filter empty entries + revs = [r for r in revs if r != ''] + if not revs: + die('Nothing to push: No revs in range %s.' % rev_range) + return revs + + +def clean_and_update_svn(svn_repo): + svn(svn_repo, 'revert', '-R', '.') + + # Unfortunately it appears there's no svn equivalent for git clean, so we + # have to do it ourselves. + for line in svn(svn_repo, 'status').split('\n'): + if not line.startswith('?'): + continue + sp = line.split(' ', 1) + if len(sp) != 2: + raise RuntimeError('Unexpected svn status line: %s' % line) + os.remove(os.path.join(svn_repo, sp[1].strip())) + svn(svn_repo, 'update') + + +def svn_init(svn_root): + if not os.path.exists(svn_root): + print('Creating svn staging directory: (%s)' % (svn_root)) + os.makedirs(svn_root) + print('This is a one-time initialization, please be patient for a few ' + ' minutes...') + svn(svn_root, 'checkout', '--depth=immediates', + 'https://llvm.org/svn/llvm-project/', '.') + svn_dirs = list(GIT_TO_SVN_DIR.values()) + svn(svn_root, 'update', *svn_dirs) + print('svn staging area ready in \'%s\'' % (svn_root)) + if not os.path.isdir(svn_root): + die('Can\'t initialize svn staging dir (%s)' % (svn_root)) + + +def svn_push(args): + '''Push changes back to SVN: this is extracted from Justin Lebar's script + available here: https://github.com/jlebar/llvm-repo-tools/ + ''' + # Get the git root + git_root = git('rev-parse', '--show-toplevel') + if not os.path.isdir(git_root): + die('Can\'t find git root dir') + + # Push from the root of the git repo + os.chdir(git_root) + + # We need a staging area for SVN, let's hide it in the .git directory. + svn_root = os.path.join(git_root, '.git', 'llvm-upstream-svn') + svn_init(svn_root) + + rev_range = args.rev_range + dry_run = args.dry_run + revs = get_revs_to_push(rev_range) + log('Pushing %d commit%s:\n%s' % + (len(revs), 's' if len(revs) != 1 + else '', '\n'.join(' ' + git('show', '--oneline', '--quiet', c) + for c in revs))) + for r in revs: + clean_and_update_svn(svn_root) + push(svn_root, r, dry_run) + + +def first_dirname(d): + while True: + (head, tail) = os.path.split(d) + if not head or head == '/': + return tail + d = head + + +def push(svn_repo, rev, dry_run): + files = git('diff-tree', '--no-commit-id', '--name-only', '-r', + rev).split('\n') + subrepos = {first_dirname(f) for f in files} + if not subrepos: + raise RuntimeError('Empty diff for rev %s?' % rev) + + status = svn(svn_repo, 'status') + if status: + die('Can\'t push git rev %s because svn status is not empty:\n%s' % + (rev, status)) + + for sr in subrepos: + diff = git('show', '--binary', rev, '--', sr, strip=False) + svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr]) + # git is the only thing that can handle its own patches... + log_verbose('Apply patch: %s' % diff) + shell(['git', 'apply', '-p2', '-'], cwd=svn_sr_path, stdin=diff) + + status_lines = svn(svn_repo, 'status').split('\n') + + # Check that patch didn't dump any .orig or .rej files. + for line in status_lines: + if line.endswith('.rej') or line.endswith('.orig'): + raise RuntimeError('patch left a .orig/.rej file in the svn repo: ' + '%s.' % line) + for l in (l for l in status_lines if l.startswith('?')): + svn(svn_repo, 'add', l[1:].strip()) + for l in (l for l in status_lines if l.startswith('!')): + svn(svn_repo, 'remove', l[1:].strip()) + + # Now we're ready to commit. + commit_msg = git('show', '--pretty=%B', '--quiet', rev) + if not dry_run: + log(svn(svn_repo, 'commit', '-m', commit_msg)) + log('Committed %s to svn.' % rev) + else: + log('Would have committed %s to svn, if this weren\'t a dry run.' + % rev) + + +if __name__ == '__main__': + argv = sys.argv[1:] + p = argparse.ArgumentParser( + prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter, + description=desc) + subparsers = p.add_subparsers(title='subcommands', + description='valid subcommands', + help='additional help') + p.add_argument('-q', '--quiet', action='count', default=0, + help='print less information') + p.add_argument('-v', '--verbose', action='count', default=0, + help='print more information') + + parser_push = subparsers.add_parser( + 'push', help='push changes back to the LLVM SVN repository') + parser_push.add_argument( + '-n', + '--dry-run', + dest='dry_run', + action='store_true', + help='Do everything other than commit to svn. Leaves junk in the svn ' + 'repo, so probably will not work well if you try to commit more ' + 'than one rev.') + parser_push.add_argument( + 'rev_range', + metavar='GIT_REVS', + type=str, + nargs='?', + help='revs to push (default: everything not in the branch\'s ' + 'upstream, or not in origin/master if the branch lacks ' + 'an explicit upstream)') + parser_push.set_defaults(func=svn_push) + args = p.parse_args(argv) + VERBOSE = args.verbose + QUIET = args.quiet + + if QUIET and VERBOSE: + die('Error -v and -q cannot be used together: make your mind!') + + # Dispatch to the right subcommand + args.func(args) -- 2.7.4