From: Chaoren Lin Date: Tue, 3 Feb 2015 01:50:39 +0000 (+0000) Subject: Added remote-build.py and wired it into Xcode for Linux and MacOSX+Linux builds. X-Git-Tag: llvmorg-3.7.0-rc1~13311 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=56f981bfceaeeee00ad4097d29723cfb05e59bdf;p=platform%2Fupstream%2Fllvm.git Added remote-build.py and wired it into Xcode for Linux and MacOSX+Linux builds. llvm-svn: 227908 --- diff --git a/lldb/.gitignore b/lldb/.gitignore index 4b15010..5953089 100644 --- a/lldb/.gitignore +++ b/lldb/.gitignore @@ -23,10 +23,15 @@ .sw? # OS X specific files. .DS_store +DerivedData/ + +# Remote build configuration files. +.remote-build.conf build/ llvm-build/ *xcuserdata +xcshareddata/ test/20* # We should ignore Xcode-style embedding of llvm/ at lldb root dir. diff --git a/lldb/lldb.xcodeproj/project.pbxproj b/lldb/lldb.xcodeproj/project.pbxproj index 4ed9c84..aab09d6 100644 --- a/lldb/lldb.xcodeproj/project.pbxproj +++ b/lldb/lldb.xcodeproj/project.pbxproj @@ -7,6 +7,18 @@ objects = { /* Begin PBXAggregateTarget section */ + 235AFBBB199BC6FD00897A4B /* MacOSX and Linux */ = { + isa = PBXAggregateTarget; + buildConfigurationList = 235AFBBC199BC6FD00897A4B /* Build configuration list for PBXAggregateTarget "MacOSX and Linux" */; + buildPhases = ( + ); + dependencies = ( + 235AFBC4199BC70B00897A4B /* PBXTargetDependency */, + 235AFBC2199BC70700897A4B /* PBXTargetDependency */, + ); + name = "MacOSX and Linux"; + productName = "MacOSX and Linux"; + }; 26CEF3A914FD58BF007286B2 /* desktop_no_xpc */ = { isa = PBXAggregateTarget; buildConfigurationList = 26CEF3AD14FD58BF007286B2 /* Build configuration list for PBXAggregateTarget "desktop_no_xpc" */; @@ -890,6 +902,20 @@ /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ + 235AFBC1199BC70700897A4B /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 08FB7793FE84155DC02AAC07 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 26F5C26910F3D9A4009D5894; + remoteInfo = "lldb-tool"; + }; + 235AFBC3199BC70B00897A4B /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 08FB7793FE84155DC02AAC07 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 235AFBB5199BC6AD00897A4B; + remoteInfo = Linux; + }; 262CFC7111A450CB00946C6C /* PBXContainerItemProxy */ = { isa = PBXContainerItemProxy; containerPortal = 265E9BE1115C2BAA00D0DCCB /* debugserver.xcodeproj */; @@ -5145,6 +5171,20 @@ /* End PBXHeadersBuildPhase section */ /* Begin PBXLegacyTarget section */ + 235AFBB5199BC6AD00897A4B /* Linux */ = { + isa = PBXLegacyTarget; + buildArgumentsString = "-u scripts/Python/remote-build.py -x $(ACTION)"; + buildConfigurationList = 235AFBBA199BC6AD00897A4B /* Build configuration list for PBXLegacyTarget "Linux" */; + buildPhases = ( + ); + buildToolPath = python; + buildWorkingDirectory = .; + dependencies = ( + ); + name = Linux; + passBuildSettingsInEnvironment = 1; + productName = Linux; + }; 2687EAC51508110B00DD8C2E /* install-headers */ = { isa = PBXLegacyTarget; buildArgumentsString = "$(ACTION)"; @@ -5370,6 +5410,8 @@ EDC6D49814E5C19B001B75F8 /* launcherXPCService */, EDE274E214EDCE1F005B0F75 /* launcherRootXPCService */, 2687EAC51508110B00DD8C2E /* install-headers */, + 235AFBB5199BC6AD00897A4B /* Linux */, + 235AFBBB199BC6FD00897A4B /* MacOSX and Linux */, 2690CD161A6DC0D000E717C8 /* lldb-mi */, ); }; @@ -6221,6 +6263,16 @@ /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ + 235AFBC2199BC70700897A4B /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 26F5C26910F3D9A4009D5894 /* lldb-tool */; + targetProxy = 235AFBC1199BC70700897A4B /* PBXContainerItemProxy */; + }; + 235AFBC4199BC70B00897A4B /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 235AFBB5199BC6AD00897A4B /* Linux */; + targetProxy = 235AFBC3199BC70B00897A4B /* PBXContainerItemProxy */; + }; 262CFC7211A450CB00946C6C /* PBXTargetDependency */ = { isa = PBXTargetDependency; name = debugserver; @@ -6459,6 +6511,162 @@ }; name = Release; }; + 235AFBB6199BC6AD00897A4B /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_UNREACHABLE_CODE = YES; + DEBUGGING_SYMBOLS = YES; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_DYNAMIC_NO_PIC = NO; + GCC_GENERATE_DEBUGGING_SYMBOLS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_SYMBOLS_PRIVATE_EXTERN = NO; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + MACOSX_DEPLOYMENT_TARGET = 10.9; + MTL_ENABLE_DEBUG_INFO = YES; + OTHER_CFLAGS = ""; + OTHER_LDFLAGS = ""; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + }; + name = Debug; + }; + 235AFBB7199BC6AD00897A4B /* DebugClang */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_UNREACHABLE_CODE = YES; + DEBUGGING_SYMBOLS = YES; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_DYNAMIC_NO_PIC = NO; + GCC_GENERATE_DEBUGGING_SYMBOLS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_SYMBOLS_PRIVATE_EXTERN = NO; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + MACOSX_DEPLOYMENT_TARGET = 10.9; + MTL_ENABLE_DEBUG_INFO = YES; + OTHER_CFLAGS = ""; + OTHER_LDFLAGS = ""; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + }; + name = DebugClang; + }; + 235AFBB8199BC6AD00897A4B /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_UNREACHABLE_CODE = YES; + COPY_PHASE_STRIP = YES; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + MACOSX_DEPLOYMENT_TARGET = 10.9; + MTL_ENABLE_DEBUG_INFO = NO; + OTHER_CFLAGS = ""; + OTHER_LDFLAGS = ""; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + }; + name = Release; + }; + 235AFBB9199BC6AD00897A4B /* BuildAndIntegration */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_UNREACHABLE_CODE = YES; + COPY_PHASE_STRIP = YES; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + MACOSX_DEPLOYMENT_TARGET = 10.9; + MTL_ENABLE_DEBUG_INFO = NO; + OTHER_CFLAGS = ""; + OTHER_LDFLAGS = ""; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + }; + name = BuildAndIntegration; + }; + 235AFBBD199BC6FD00897A4B /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + 235AFBBE199BC6FD00897A4B /* DebugClang */ = { + isa = XCBuildConfiguration; + buildSettings = { + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = DebugClang; + }; + 235AFBBF199BC6FD00897A4B /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; + 235AFBC0199BC6FD00897A4B /* BuildAndIntegration */ = { + isa = XCBuildConfiguration; + buildSettings = { + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = BuildAndIntegration; + }; 26579F6A126A25920007C5CB /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { @@ -8358,6 +8566,28 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = BuildAndIntegration; }; + 235AFBBA199BC6AD00897A4B /* Build configuration list for PBXLegacyTarget "Linux" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 235AFBB6199BC6AD00897A4B /* Debug */, + 235AFBB7199BC6AD00897A4B /* DebugClang */, + 235AFBB8199BC6AD00897A4B /* Release */, + 235AFBB9199BC6AD00897A4B /* BuildAndIntegration */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = BuildAndIntegration; + }; + 235AFBBC199BC6FD00897A4B /* Build configuration list for PBXAggregateTarget "MacOSX and Linux" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 235AFBBD199BC6FD00897A4B /* Debug */, + 235AFBBE199BC6FD00897A4B /* DebugClang */, + 235AFBBF199BC6FD00897A4B /* Release */, + 235AFBC0199BC6FD00897A4B /* BuildAndIntegration */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = BuildAndIntegration; + }; 26579F6D126A25BF0007C5CB /* Build configuration list for PBXNativeTarget "darwin-debug" */ = { isa = XCConfigurationList; buildConfigurations = ( diff --git a/lldb/scripts/Python/remote-build.py b/lldb/scripts/Python/remote-build.py new file mode 100755 index 0000000..72986a0 --- /dev/null +++ b/lldb/scripts/Python/remote-build.py @@ -0,0 +1,300 @@ +#!/usr/bin/python + +from __future__ import print_function + +import argparse +import getpass +import os +import os.path +import re +import select +import sys +import subprocess + +_COMMON_SYNC_OPTS = "-avzh --delete" +_COMMON_EXCLUDE_OPTS = "--exclude=DerivedData --exclude=.svn --exclude=.git --exclude=llvm-build/Release+Asserts" + +def normalize_configuration(config_text): + if not config_text: + return "debug" + + config_lower = config_text.lower() + if config_lower in ["debug", "release"]: + return config_lower + else: + raise Exception("unknown configuration specified: %s" % config_text) + +def parse_args(): + DEFAULT_REMOTE_ROOT_DIR = "/mnt/ssd/work/macosx.sync" + DEFAULT_REMOTE_HOSTNAME = "tfiala2.mtv.corp.google.com" + OPTIONS_FILENAME = ".remote-build.conf" + DEFAULT_SSH_PORT = "22" + + parser = argparse.ArgumentParser(fromfile_prefix_chars='@') + + parser.add_argument( + "--configuration", "-c", + help="specify configuration (Debug, Release)", + default=normalize_configuration(os.environ.get('CONFIGURATION', 'Debug'))) + parser.add_argument( + "--debug", "-d", + action="store_true", + help="help debug the remote-build script by adding extra logging") + parser.add_argument( + "--local-lldb-dir", "-l", metavar="DIR", + help="specify local lldb directory (Xcode layout assumed for llvm/clang)", + default=os.getcwd()) + parser.add_argument( + "--port", "-p", + help="specify the port ssh should use to connect to the remote side", + default=DEFAULT_SSH_PORT) + parser.add_argument( + "--remote-address", "-r", metavar="REMOTE-ADDR", + help="specify the dns name or ip address of the remote linux system", + default=DEFAULT_REMOTE_HOSTNAME) + parser.add_argument( + "--remote-dir", metavar="DIR", + help="specify the root of the linux source/build dir", + default=DEFAULT_REMOTE_ROOT_DIR) + parser.add_argument( + "--user", "-u", help="specify the user name for the remote system", + default=getpass.getuser()) + parser.add_argument( + "--xcode-action", "-x", help="$(ACTION) from Xcode", nargs='?', default=None) + + command_line_args = sys.argv[1:] + if os.path.exists(OPTIONS_FILENAME): + # Prepend the file so that command line args override the file contents. + command_line_args.insert(0, "@%s" % OPTIONS_FILENAME) + + return parser.parse_args(command_line_args) + + +def maybe_create_remote_root_dir(args): + commandline = [ + "ssh", + "-p", args.port, + "%s@%s" % (args.user, args.remote_address), + "mkdir", + "-p", + args.remote_dir] + print("create remote root dir command:\n{}".format(commandline)) + return subprocess.call(commandline) + + +def init_with_args(args): + # Expand any user directory specs in local-side source dir (on MacOSX). + args.local_lldb_dir = os.path.expanduser(args.local_lldb_dir) + + # Append the configuration type to the remote build dir. + args.configuration = normalize_configuration(args.configuration) + args.remote_build_dir = os.path.join( + args.remote_dir, + "build-%s" % args.configuration) + + # We assume the local lldb directory is really named 'lldb'. + # This is because on the remote end, the local lldb root dir + # is copied over underneath llvm/tools and will be named there + # whatever it is named locally. The remote build will assume + # is is called lldb. + if os.path.basename(args.local_lldb_dir) != 'lldb': + raise Exception( + "local lldb root needs to be called 'lldb' but was {} instead" + .format(os.path.basename(args.local_lldb_dir))) + + args.lldb_dir_relative_regex = re.compile("%s/llvm/tools/lldb/" % args.remote_dir) + args.llvm_dir_relative_regex = re.compile("%s/" % args.remote_dir) + + print("Xcode action:", args.xcode_action) + + # Ensure the remote directory exists. + result = maybe_create_remote_root_dir(args) + if result == 0: + print("using remote root dir: %s" % args.remote_dir) + else: + print("remote root dir doesn't exist and could not be created, " + + "error code:", result) + return False + + return True + +def sync_llvm(args): + commandline = ["rsync"] + commandline.extend(_COMMON_SYNC_OPTS.split()) + commandline.extend(_COMMON_EXCLUDE_OPTS.split()) + commandline.append("--exclude=/llvm/tools/lldb") + commandline.extend(["-e", "ssh -p {}".format(args.port)]) + commandline.extend([ + "%s/llvm" % args.local_lldb_dir, + "%s@%s:%s" % (args.user, args.remote_address, args.remote_dir)]) + if args.debug: + print("going to execute llvm sync: {}".format(commandline)) + return subprocess.call(commandline) + + +def sync_lldb(args): + commandline = ["rsync"] + commandline.extend(_COMMON_SYNC_OPTS.split()) + commandline.extend(_COMMON_EXCLUDE_OPTS.split()) + commandline.append("--exclude=/lldb/llvm") + commandline.extend(["-e", "ssh -p {}".format(args.port)]) + commandline.extend([ + args.local_lldb_dir, + "%s@%s:%s/llvm/tools" % (args.user, args.remote_address, args.remote_dir)]) + if args.debug: + print("going to execute lldb sync: {}".format(commandline)) + return subprocess.call(commandline) + + +def build_cmake_command(args): + # args.remote_build_dir + # args.configuration in ('release', 'debug') + + if args.configuration == 'debug-optimized': + build_type_name = "RelWithDebInfo" + elif args.configuration == 'release': + build_type_name = "Release" + else: + build_type_name = "Debug" + + ld_flags = "\"-lstdc++ -lm\"" + + install_dir = os.path.join( + args.remote_build_dir, "..", "install-{}".format(args.configuration)) + + command_line = [ + "cmake", + "-GNinja", + "-DCMAKE_CXX_COMPILER=clang", + "-DCMAKE_C_COMPILER=clang", + # "-DCMAKE_CXX_FLAGS=%s" % cxx_flags, + "-DCMAKE_SHARED_LINKER_FLAGS=%s" % ld_flags, + "-DCMAKE_EXE_LINKER_FLAGS=%s" % ld_flags, + "-DCMAKE_INSTALL_PREFIX:PATH=%s" % install_dir, + "-DCMAKE_BUILD_TYPE=%s" % build_type_name, + "-Wno-dev", + os.path.join("..", "llvm") + ] + + return command_line + + +def maybe_configure(args): + commandline = [ + "ssh", + "-p", args.port, + "%s@%s" % (args.user, args.remote_address), + "cd", args.remote_dir, "&&", + "mkdir", "-p", args.remote_build_dir, "&&", + "cd", args.remote_build_dir, "&&" + ] + commandline.extend(build_cmake_command(args)) + + if args.debug: + print("configure command: {}".format(commandline)) + + return subprocess.call(commandline) + + +def filter_build_line(args, line): + lldb_relative_line = args.lldb_dir_relative_regex.sub('', line) + if len(lldb_relative_line) != len(line): + # We substituted - return the modified line + return lldb_relative_line + + # No match on lldb path (longer on linux than llvm path). Try + # the llvm path match. + return args.llvm_dir_relative_regex.sub('', line) + + +def run_remote_build_command(args, build_command_list): + commandline = [ + "ssh", + "-p", args.port, + "%s@%s" % (args.user, args.remote_address), + "cd", args.remote_build_dir, "&&"] + commandline.extend(build_command_list) + + if args.debug: + print("running remote build command: {}".format(commandline)) + + proc = subprocess.Popen( + commandline, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + # Filter stdout/stderr output for file path mapping. + # We do this to enable Xcode to see filenames relative to the + # MacOSX-side directory structure. + while True: + reads = [proc.stdout.fileno(), proc.stderr.fileno()] + select_result = select.select(reads, [], []) + + for fd in select_result[0]: + if fd == proc.stdout.fileno(): + line = proc.stdout.readline() + display_line = filter_build_line(args, line.rstrip()) + if display_line and len(display_line) > 0: + print(display_line) + elif fd == proc.stderr.fileno(): + line = proc.stderr.readline() + display_line = filter_build_line(args, line.rstrip()) + if display_line and len(display_line) > 0: + print(display_line, file=sys.stderr) + + proc_retval = proc.poll() + if proc_retval != None: + # Process stopped. Drain output before finishing up. + + # Drain stdout. + while True: + line = proc.stdout.readline() + if line: + display_line = filter_build_line(args, line.rstrip()) + if display_line and len(display_line) > 0: + print(display_line) + else: + break + + # Drain stderr. + while True: + line = proc.stderr.readline() + if line: + display_line = filter_build_line(args, line.rstrip()) + if display_line and len(display_line) > 0: + print(display_line, file=sys.stderr) + else: + break + + return proc_retval + + +def build(args): + return run_remote_build_command(args, ["time", "ninja"]) + + +def clean(args): + return run_remote_build_command(args, ["ninja", "clean"]) + + +if __name__ == "__main__": + # Handle arg parsing. + args = parse_args() + + # Initialize the system. + if not init_with_args(args): + exit(1) + + # Sync over llvm and clang source. + sync_llvm(args) + + # Sync over lldb source. + sync_lldb(args) + + # Configure the remote build if it's not already. + maybe_configure(args) + + if args.xcode_action == 'clean': + exit(clean(args)) + else: + exit(build(args))