From 6eef7d05249fb88bf5b31289f4c27d75f50c1ae4 Mon Sep 17 00:00:00 2001 From: Julian Lettner Date: Wed, 27 Feb 2019 19:06:20 +0000 Subject: [PATCH] [Darwin][NFC] Refactor throttling of 64bit sanitizer tests on Darwin Underlying condition for throttling is "has large mmap'd regions" (i.e., shadow memory) and not sanitizers in general (e.g., UBSan does not need to be throttled). Rename parallelism group `darwin-64bit-sanitizer` to `shadow-memory` and apply it unconditionally to all tests which require it. We can then have all the Darwin throttling logic in one place in the commen lit config. Throttle sanitizer_common unit tests. Configuration was previously missing from sanitizer_common/Unit/lit.site.cfg. Reviewed by: kubamracek Differential Revision: https://reviews.llvm.org/D58677 llvm-svn: 355018 --- compiler-rt/test/asan/lit.cfg | 4 +--- compiler-rt/test/fuzzer/lit.cfg | 4 +--- compiler-rt/test/lit.common.cfg | 26 +++++++++++++--------- .../test/sanitizer_common/Unit/lit.site.cfg.in | 3 +++ compiler-rt/test/sanitizer_common/lit.common.cfg | 4 +--- compiler-rt/test/tsan/lit.cfg | 4 +--- compiler-rt/unittests/lit.common.unit.cfg | 8 +++---- compiler-rt/unittests/lit_unittest_cfg_utils.py | 6 ++--- 8 files changed, 30 insertions(+), 29 deletions(-) diff --git a/compiler-rt/test/asan/lit.cfg b/compiler-rt/test/asan/lit.cfg index 471144e..143cd3be 100644 --- a/compiler-rt/test/asan/lit.cfg +++ b/compiler-rt/test/asan/lit.cfg @@ -220,6 +220,4 @@ else: if config.host_os not in ['Linux', 'Darwin', 'FreeBSD', 'SunOS', 'Windows', 'NetBSD']: config.unsupported = True -if config.host_os == 'Darwin': - if config.target_arch in ["x86_64", "x86_64h"]: - config.parallelism_group = "darwin-64bit-sanitizer" +config.parallelism_group = 'shadow-memory' diff --git a/compiler-rt/test/fuzzer/lit.cfg b/compiler-rt/test/fuzzer/lit.cfg index 1bc1d7d..6f2a4dac 100644 --- a/compiler-rt/test/fuzzer/lit.cfg +++ b/compiler-rt/test/fuzzer/lit.cfg @@ -114,6 +114,4 @@ if default_asan_opts_str: config.substitutions.append(('%env_asan_opts=', 'env ASAN_OPTIONS=' + default_asan_opts_str)) -if config.host_os == 'Darwin': - if config.target_arch in ["x86_64", "x86_64h"]: - config.parallelism_group = "darwin-64bit-sanitizer" +config.parallelism_group = 'shadow-memory' diff --git a/compiler-rt/test/lit.common.cfg b/compiler-rt/test/lit.common.cfg index bf3897e..ed556a7 100644 --- a/compiler-rt/test/lit.common.cfg +++ b/compiler-rt/test/lit.common.cfg @@ -402,17 +402,23 @@ llvm_config_cmd.wait() if platform.system() == 'Windows': config.test_retry_attempts = 2 -# Only run up to 3 64-bit sanitized processes simultaneously on Darwin. -# Using more scales badly and hogs the system due to inefficient handling -# of large mmap'd regions (terabytes) by the kernel. -if platform.system() == 'Darwin': - lit_config.parallelism_groups["darwin-64bit-sanitizer"] = 3 +# No throttling on non-Darwin platforms. +lit_config.parallelism_groups['shadow-memory'] = None -# Force sequential execution when running tests on iOS devices. -if config.host_os == 'Darwin' and config.apple_platform != "osx" and not config.apple_platform.endswith("sim"): - lit_config.warning("iOS device test cases being run sequentially") - lit_config.parallelism_groups["ios-device"] = 1 - config.parallelism_group = "ios-device" +if platform.system() == 'Darwin': + ios_device = config.apple_platform != 'osx' and not config.apple_platform.endswith('sim') + # Force sequential execution when running tests on iOS devices. + if ios_device: + lit_config.warning('Forcing sequential execution for iOS device tests') + lit_config.parallelism_groups['ios-device'] = 1 + config.parallelism_group = 'ios-device' + + # Only run up to 3 processes that require shadow memory simultaneously on + # 64-bit Darwin. Using more scales badly and hogs the system due to + # inefficient handling of large mmap'd regions (terabytes) by the kernel. + elif config.target_arch in ['x86_64', 'x86_64h']: + lit_config.warning('Throttling sanitizer tests that require shadow memory on Darwin 64bit') + lit_config.parallelism_groups['shadow-memory'] = 3 # Multiple substitutions are necessary to support multiple shared objects used # at once. diff --git a/compiler-rt/test/sanitizer_common/Unit/lit.site.cfg.in b/compiler-rt/test/sanitizer_common/Unit/lit.site.cfg.in index c62e23c..46855d9 100644 --- a/compiler-rt/test/sanitizer_common/Unit/lit.site.cfg.in +++ b/compiler-rt/test/sanitizer_common/Unit/lit.site.cfg.in @@ -12,3 +12,6 @@ config.name = 'SanitizerCommon-Unit' config.test_exec_root = os.path.join("@COMPILER_RT_BINARY_DIR@", "lib", "sanitizer_common", "tests") config.test_source_root = config.test_exec_root + +if config.host_os == 'Darwin': + config.parallelism_group = config.darwin_sanitizer_parallelism_group_func diff --git a/compiler-rt/test/sanitizer_common/lit.common.cfg b/compiler-rt/test/sanitizer_common/lit.common.cfg index 323c1ef..8457f92 100644 --- a/compiler-rt/test/sanitizer_common/lit.common.cfg +++ b/compiler-rt/test/sanitizer_common/lit.common.cfg @@ -71,6 +71,4 @@ config.suffixes = ['.c', '.cc', '.cpp'] if config.host_os not in ['Linux', 'Darwin', 'NetBSD', 'FreeBSD']: config.unsupported = True -if config.host_os == 'Darwin': - if config.target_arch in ["x86_64", "x86_64h"]: - config.parallelism_group = "darwin-64bit-sanitizer" +config.parallelism_group = 'shadow-memory' diff --git a/compiler-rt/test/tsan/lit.cfg b/compiler-rt/test/tsan/lit.cfg index 76d60cf..8dc5228 100644 --- a/compiler-rt/test/tsan/lit.cfg +++ b/compiler-rt/test/tsan/lit.cfg @@ -86,6 +86,4 @@ if config.host_os not in ['FreeBSD', 'Linux', 'Darwin', 'NetBSD']: if config.android: config.unsupported = True -if config.host_os == 'Darwin': - if config.target_arch in ["x86_64", "x86_64h"]: - config.parallelism_group = "darwin-64bit-sanitizer" +config.parallelism_group = 'shadow-memory' diff --git a/compiler-rt/unittests/lit.common.unit.cfg b/compiler-rt/unittests/lit.common.unit.cfg index 31206e9..fba034a 100644 --- a/compiler-rt/unittests/lit.common.unit.cfg +++ b/compiler-rt/unittests/lit.common.unit.cfg @@ -30,10 +30,10 @@ if 'TEMP' in os.environ: config.environment['TEMP'] = os.environ['TEMP'] if config.host_os == 'Darwin': - # Only run up to 3 64-bit sanitized processes simultaneously on Darwin. - # Using more scales badly and hogs the system due to inefficient handling - # of large mmap'd regions (terabytes) by the kernel. - lit_config.parallelism_groups["darwin-64bit-sanitizer"] = 3 + # Only run up to 3 processes that require shadow memory simultaneously on + # 64-bit Darwin. Using more scales badly and hogs the system due to + # inefficient handling of large mmap'd regions (terabytes) by the kernel. + lit_config.parallelism_groups["shadow-memory"] = 3 # The test config gets pickled and sent to multiprocessing workers, and that # only works for code if it is stored at the top level of some module. diff --git a/compiler-rt/unittests/lit_unittest_cfg_utils.py b/compiler-rt/unittests/lit_unittest_cfg_utils.py index ff7b1ee..721e81b 100644 --- a/compiler-rt/unittests/lit_unittest_cfg_utils.py +++ b/compiler-rt/unittests/lit_unittest_cfg_utils.py @@ -1,4 +1,4 @@ -# Put all 64-bit sanitizer tests in the darwin-64bit-sanitizer parallelism -# group. This will only run three of them concurrently. +# Put all 64-bit tests in the shadow-memory parallelism group. We throttle those +# in our common lit config (lit.common.unit.cfg). def darwin_sanitizer_parallelism_group_func(test): - return "darwin-64bit-sanitizer" if "x86_64" in test.file_path else "" + return "shadow-memory" if "x86_64" in test.file_path else None -- 2.7.4