From: Jonathan Roelofs Date: Thu, 26 Feb 2015 00:42:17 +0000 (+0000) Subject: Add remote testing support to the lit config. X-Git-Tag: llvmorg-3.7.0-rc1~10821 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=49ff203ec4008517cdb76a3962c1d20183aad3a7;p=platform%2Fupstream%2Fllvm.git Add remote testing support to the lit config. Executors can be specified at configure time by using the -DLIBCXX_EXECUTOR="" option. Examples include: $ cmake -DLIBCXX_EXECUTOR="TimeoutExecutor(30,LocalExecutor())" This runs individual tests with a maximum duration $ cmake -DLIBCXX_EXECUTOR="SSHExecutor('hostname','username')" This runs tests on a remote target, using scp to shuttle binaries to the target, and ssh to invoke commands there. $ cmake -DLIBCXX_EXECUTOR="PrefixExecutor('/path/to/run/script',LocalExecutor())" This assumes the script knows how to copy run the executables passed to it, and allows for the ultimate control. This is useful for running things inside emulators like Valgrind & QEMU. TODO: This doesn't claim to support ShTest tests yet, that will take a bit more thought & finagling (I'm still not sure how to orchestrate copy-in for those cases. I've also punted on what to do about tests that read data files. The testsuite has several tests that need to read *.dat files placed next to them, and currently those aren't copied over when using, say, an SSHExecutor. The affected tests are: libc++ :: std/input.output/file.streams/fstreams/filebuf.virtuals/pbackfail.pass.cpp libc++ :: std/input.output/file.streams/fstreams/filebuf.virtuals/underflow.pass.cpp libc++ :: std/input.output/file.streams/fstreams/ifstream.assign/member_swap.pass.cpp libc++ :: std/input.output/file.streams/fstreams/ifstream.assign/move_assign.pass.cpp libc++ :: std/input.output/file.streams/fstreams/ifstream.assign/nonmember_swap.pass.cpp libc++ :: std/input.output/file.streams/fstreams/ifstream.cons/move.pass.cpp libc++ :: std/input.output/file.streams/fstreams/ifstream.cons/pointer.pass.cpp libc++ :: std/input.output/file.streams/fstreams/ifstream.cons/string.pass.cpp libc++ :: std/input.output/file.streams/fstreams/ifstream.members/close.pass.cpp libc++ :: std/input.output/file.streams/fstreams/ifstream.members/open_pointer.pass.cpp libc++ :: std/input.output/file.streams/fstreams/ifstream.members/open_string.pass.cpp libc++ :: std/input.output/file.streams/fstreams/ifstream.members/rdbuf.pass.cpp libc++ :: std/localization/locales/locale.convenience/conversions/conversions.buffer/pbackfail.pass.cpp libc++ :: std/localization/locales/locale.convenience/conversions/conversions.buffer/underflow.pass.cpp Note: One thing to watch out for when using the SSHExecutor for cross-testing is that you'll also want to specify a TargetInfo object (so that the host's features aren't used for available-features checks and flags setup). http://reviews.llvm.org/D7380 llvm-svn: 230592 --- diff --git a/libcxx/test/CMakeLists.txt b/libcxx/test/CMakeLists.txt index 6153431..8f928b4 100644 --- a/libcxx/test/CMakeLists.txt +++ b/libcxx/test/CMakeLists.txt @@ -46,6 +46,8 @@ if (LIT_EXECUTABLE) pythonize_bool(LIBCXX_ENABLE_MONOTONIC_CLOCK) set(LIBCXX_TARGET_INFO "libcxx.test.target_info.LocalTI" CACHE STRING "TargetInfo to use when setting up test environment.") + set(LIBCXX_EXECUTOR "None" CACHE STRING + "Executor to use when running tests.") set(AUTO_GEN_COMMENT "## Autogenerated by libcxx configuration.\n# Do not edit!") diff --git a/libcxx/test/libcxx/test/config.py b/libcxx/test/libcxx/test/config.py index 162cddc..9fd974b 100644 --- a/libcxx/test/libcxx/test/config.py +++ b/libcxx/test/libcxx/test/config.py @@ -12,7 +12,8 @@ import lit.util # pylint: disable=import-error,no-name-in-module from libcxx.test.format import LibcxxTestFormat from libcxx.compiler import CXXCompiler - +from libcxx.test.executor import * +from libcxx.test.tracing import * def loadSiteConfig(lit_config, config, param_name, env_name): # We haven't loaded the site specific configuration (the user is @@ -78,6 +79,7 @@ class Configuration(object): "parameter '{}' should be true or false".format(name)) def configure(self): + self.configure_executor() self.configure_target_info() self.configure_cxx() self.configure_triple() @@ -110,6 +112,32 @@ class Configuration(object): list(self.config.available_features)) self.lit_config.note('Using environment: %r' % self.env) + def get_test_format(self): + return LibcxxTestFormat( + self.cxx, + self.use_clang_verify, + self.execute_external, + self.executor, + exec_env=self.env) + + def configure_executor(self): + exec_str = self.get_lit_conf('executor', "None") + te = eval(exec_str) + if te: + self.lit_config.note("Using executor: %r" % exec_str) + if self.lit_config.useValgrind: + # We have no way of knowing where in the chain the + # ValgrindExecutor is supposed to go. It is likely + # that the user wants it at the end, but we have no + # way of getting at that easily. + selt.lit_config.fatal("Cannot infer how to create a Valgrind " + " executor.") + else: + te = LocalExecutor() + if self.lit_config.useValgrind: + te = ValgrindExecutor(self.lit_config.valgrindArgs, te) + self.executor = te + def configure_target_info(self): default = "libcxx.test.target_info.LocalTI" info_str = self.get_lit_conf('target_info', default) @@ -119,13 +147,6 @@ class Configuration(object): if info_str != default: self.lit_config.note("inferred target_info as: %r" % info_str) - def get_test_format(self): - return LibcxxTestFormat( - self.cxx, - self.use_clang_verify, - self.execute_external, - exec_env=self.env) - def configure_cxx(self): # Gather various compiler parameters. cxx = self.get_lit_conf('cxx_under_test') @@ -319,7 +340,7 @@ class Configuration(object): # Configure include paths self.cxx.compile_flags += ['-nostdinc++'] self.configure_compile_flags_header_includes() - if sys.platform.startswith('linux'): + if self.target_info.platform() == 'linux': self.cxx.compile_flags += ['-D__STDC_FORMAT_MACROS', '-D__STDC_LIMIT_MACROS', '-D__STDC_CONSTANT_MACROS'] @@ -601,15 +622,16 @@ class Configuration(object): # linux-gnu is needed in the triple to properly identify linuxes # that use GLIBC. Handle redhat and opensuse triples as special # cases and append the missing `-gnu` portion. - if target_triple.endswith('redhat-linux') or \ - target_triple.endswith('suse-linux'): + if (target_triple.endswith('redhat-linux') or + target_triple.endswith('suse-linux')): target_triple += '-gnu' self.config.target_triple = target_triple self.lit_config.note( "inferred target_triple as: %r" % self.config.target_triple) def configure_env(self): - if sys.platform == 'darwin' and not self.use_system_cxx_lib: + if (self.target_info.platform() == 'darwin' and + not self.use_system_cxx_lib): libcxx_library = self.get_lit_conf('libcxx_library') if libcxx_library: cxx_library_root = os.path.dirname(libcxx_library) diff --git a/libcxx/test/libcxx/test/executor.py b/libcxx/test/libcxx/test/executor.py new file mode 100644 index 0000000..0caa274 --- /dev/null +++ b/libcxx/test/libcxx/test/executor.py @@ -0,0 +1,166 @@ +import os + +import tracing + +from lit.util import executeCommand # pylint: disable=import-error + + +class Executor(object): + def run(self, exe_path, cmd, local_cwd, env=None): + """Execute a command. + Be very careful not to change shared state in this function. + Executor objects are shared between python processes in `lit -jN`. + Args: + exe_path: str: Local path to the executable to be run + cmd: [str]: subprocess.call style command + local_cwd: str: Local path to the working directory + env: {str: str}: Environment variables to execute under + Returns: + out, err, exitCode + """ + raise NotImplementedError + + +class LocalExecutor(Executor): + def __init__(self): + super(LocalExecutor, self).__init__() + + def run(self, exe_path, cmd=None, work_dir='.', env=None): + cmd = cmd or [exe_path] + env_cmd = [] + if env: + env_cmd += ['env'] + env_cmd += ['%s=%s' % (k, v) for k, v in env.items()] + if work_dir == '.': + work_dir = os.getcwd() + return executeCommand(env_cmd + cmd, cwd=work_dir) + + +class PrefixExecutor(Executor): + """Prefix an executor with some other command wrapper. + + Most useful for setting ulimits on commands, or running an emulator like + qemu and valgrind. + """ + def __init__(self, commandPrefix, chain): + super(PrefixExecutor, self).__init__() + + self.commandPrefix = commandPrefix + self.chain = chain + + def run(self, exe_path, cmd=None, work_dir='.', env=None): + cmd = cmd or [exe_path] + return self.chain.run(self.commandPrefix + cmd, work_dir, env=env) + + +class PostfixExecutor(Executor): + """Postfix an executor with some args.""" + def __init__(self, commandPostfix, chain): + super(PostfixExecutor, self).__init__() + + self.commandPostfix = commandPostfix + self.chain = chain + + def run(self, exe_path, cmd=None, work_dir='.', env=None): + cmd = cmd or [exe_path] + return self.chain.run(cmd + self.commandPostfix, work_dir, env=env) + + + +class TimeoutExecutor(PrefixExecutor): + """Execute another action under a timeout. + + Deprecated. http://reviews.llvm.org/D6584 adds timeouts to LIT. + """ + def __init__(self, duration, chain): + super(TimeoutExecutor, self).__init__( + ['timeout', duration], chain) + + +class SSHExecutor(Executor): + def __init__(self, host, username=None): + super(SSHExecutor, self).__init__() + + self.user_prefix = username + '@' if username else '' + self.host = host + self.scp_command = 'scp' + self.ssh_command = 'ssh' + + self.local_run = executeCommand + # TODO(jroelofs): switch this on some -super-verbose-debug config flag + if False: + self.local_run = tracing.trace_function( + self.local_run, log_calls=True, log_results=True, + label='ssh_local') + + def remote_temp_dir(self): + return self._remote_temp(True) + + def remote_temp_file(self): + return self._remote_temp(False) + + def _remote_temp(self, is_dir): + # TODO: detect what the target system is, and use the correct + # mktemp command for it. (linux and darwin differ here, and I'm + # sure windows has another way to do it) + + # Not sure how to do suffix on osx yet + dir_arg = '-d' if is_dir else '' + cmd = 'mktemp -q {} /tmp/libcxx.XXXXXXXXXX'.format(dir_arg) + temp_path, err, exitCode = self.__execute_command_remote([cmd]) + temp_path = temp_path.strip() + if exitCode != 0: + raise RuntimeError(err) + return temp_path + + def copy_in(self, local_srcs, remote_dsts): + scp = self.scp_command + remote = self.host + remote = self.user_prefix + remote + + # This could be wrapped up in a tar->scp->untar for performance + # if there are lots of files to be copied/moved + for src, dst in zip(local_srcs, remote_dsts): + cmd = [scp, '-p', src, remote + ':' + dst] + self.local_run(cmd) + + def delete_remote(self, remote): + try: + self.__execute_command_remote(['rm', '-rf', remote]) + except OSError: + # TODO: Log failure to delete? + pass + + def run(self, exe_path, cmd=None, work_dir='.', env=None): + target_exe_path = None + target_cwd = None + try: + target_exe_path = self.remote_temp_file() + target_cwd = self.remote_temp_dir() + if cmd: + # Replace exe_path with target_exe_path. + cmd = [c if c != exe_path else target_exe_path for c in cmd] + else: + cmd = [target_exe_path] + self.copy_in([exe_path], [target_exe_path]) + return self.__execute_command_remote(cmd, target_cwd, env) + except: + raise + finally: + if target_exe_path: + self.delete_remote(target_exe_path) + if target_cwd: + self.delete_remote(target_cwd) + + def __execute_command_remote(self, cmd, remote_work_dir='.', env=None): + remote = self.user_prefix + self.host + ssh_cmd = [self.ssh_command, '-oBatchMode=yes', remote] + if env: + env_cmd = ['env'] + ['%s=%s' % (k, v) for k, v in env.items()] + else: + env_cmd = [] + remote_cmd = ' '.join(env_cmd + cmd) + if remote_work_dir != '.': + remote_cmd = 'cd ' + remote_work_dir + ' && ' + remote_cmd + return self.local_run(ssh_cmd + [remote_cmd]) + diff --git a/libcxx/test/libcxx/test/format.py b/libcxx/test/libcxx/test/format.py index cb36230..0a2b893 100644 --- a/libcxx/test/libcxx/test/format.py +++ b/libcxx/test/libcxx/test/format.py @@ -20,10 +20,12 @@ class LibcxxTestFormat(object): FOO.sh.cpp - A test that uses LIT's ShTest format. """ - def __init__(self, cxx, use_verify_for_fail, execute_external, exec_env): + def __init__(self, cxx, use_verify_for_fail, execute_external, + executor, exec_env): self.cxx = cxx self.use_verify_for_fail = use_verify_for_fail self.execute_external = execute_external + self.executor = executor self.exec_env = dict(exec_env) # TODO: Move this into lit's FileBasedTest @@ -73,6 +75,10 @@ class LibcxxTestFormat(object): # Dispatch the test based on its suffix. if is_sh_test: + if self.executor: + # We can't run ShTest tests with a executor yet. + # For now, bail on trying to run them + return lit.Test.UNSUPPORTED, 'ShTest format not yet supported' return lit.TestRunner._runShTest(test, lit_config, self.execute_external, script, tmpBase, execDir) @@ -104,15 +110,12 @@ class LibcxxTestFormat(object): report += "Compilation failed unexpectedly!" return lit.Test.FAIL, report # Run the test - cmd = [] + local_cwd = os.path.dirname(source_path) + env = None if self.exec_env: - cmd += ['env'] - cmd += ['%s=%s' % (k, v) for k, v in self.exec_env.items()] - if lit_config.useValgrind: - cmd = lit_config.valgrindArgs + cmd - cmd += [exec_path] - out, err, rc = lit.util.executeCommand( - cmd, cwd=os.path.dirname(source_path)) + env = self.exec_env + out, err, rc = self.executor.run(exec_path, [exec_path], + local_cwd, env) if rc != 0: report = libcxx.util.makeReport(cmd, out, err, rc) report = "Compiled With: %s\n%s" % (compile_cmd, report) diff --git a/libcxx/test/libcxx/test/tracing.py b/libcxx/test/libcxx/test/tracing.py new file mode 100644 index 0000000..efef158 --- /dev/null +++ b/libcxx/test/libcxx/test/tracing.py @@ -0,0 +1,34 @@ +import os +import inspect + + +def trace_function(function, log_calls, log_results, label=''): + def wrapper(*args, **kwargs): + kwarg_strs = ['{}={}'.format(k, v) for (k, v) in kwargs] + arg_str = ', '.join([str(a) for a in args] + kwarg_strs) + call_str = '{}({})'.format(function.func_name, arg_str) + + # Perform the call itself, logging before, after, and anything thrown. + try: + if log_calls: + print '{}: Calling {}'.format(label, call_str) + res = function(*args, **kwargs) + if log_results: + print '{}: {} -> {}'.format(label, call_str, res) + return res + except Exception as ex: + if log_results: + print '{}: {} raised {}'.format(label, call_str, type(ex)) + raise ex + + return wrapper + + +def trace_object(obj, log_calls, log_results, label=''): + for name, member in inspect.getmembers(obj): + if inspect.ismethod(member): + # Skip meta-functions, decorate everything else + if not member.func_name.startswith('__'): + setattr(obj, name, trace_function(member, log_calls, + log_results, label)) + return obj diff --git a/libcxx/test/lit.site.cfg.in b/libcxx/test/lit.site.cfg.in index 6faa3b8..5a4445b 100644 --- a/libcxx/test/lit.site.cfg.in +++ b/libcxx/test/lit.site.cfg.in @@ -18,6 +18,7 @@ config.target_triple = "@LIBCXX_TARGET_TRIPLE@" config.sysroot = "@LIBCXX_SYSROOT@" config.gcc_toolchain = "@LIBCXX_GCC_TOOLCHAIN@" config.target_info = "@LIBCXX_TARGET_INFO@" +config.executor = "@LIBCXX_EXECUTOR@" # Let the main config do the real work. lit_config.load_config(config, "@LIBCXX_SOURCE_DIR@/test/lit.cfg")