From 9e306e4531e57a3be410d0f8b5229fe8b67abde9 Mon Sep 17 00:00:00 2001 From: Russ Keldorph Date: Tue, 8 Nov 2016 04:45:52 -0800 Subject: [PATCH] Script to run CoreFx tests against a private CoreCLR Move the increasingly complex logic required for the CI to run CoreFx tests from the CoreCLR repo into a separate script. Also enable automated CoreFx jitstress testing for x86. --- netci.groovy | 123 ++++++---------- tests/scripts/run-corefx-tests.py | 297 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 345 insertions(+), 75 deletions(-) create mode 100644 tests/scripts/run-corefx-tests.py diff --git a/netci.groovy b/netci.groovy index b1ffaf8..c2f6d63 100755 --- a/netci.groovy +++ b/netci.groovy @@ -234,8 +234,8 @@ def static getStressModeDisplayName(def scenario) { } // Generates the string for creating a file that sets environment variables -// that makes it possible to run stress modes. Writes the script to a file called -// SetStressModes.[sh/cmd] +// that makes it possible to run stress modes. Writes the script to the file +// specified by the stepScriptLocation parameter. def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) { def stepScript = '' if (os == 'Windows_NT') { @@ -269,28 +269,6 @@ def static genStressModeScriptStep(def os, def stressModeName, def stressModeVar return stepScript } -// Corefx doesn't have a support to pass stress mode environment variables. This function -// generates commands to set or export environment variables -def static getStressModeEnvSetCmd(def os, def stressModeName) { - def envVars = Constants.jitStressModeScenarios[stressModeName] - def setEnvVars = '' - if (os == 'Windows_NT') { - envVars.each{ VarName, Value -> - if (VarName != '') { - setEnvVars += "set ${VarName}=${Value}\n" - } - } - } - else { - envVars.each{ VarName, Value -> - if (VarName != '') { - setEnvVars += "export ${VarName}=${Value}\n" - } - } - } - return setEnvVars -} - // Calculates the name of the build job based on some typical parameters. // def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly, def isLinuxEmulatorBuild = false) { @@ -852,7 +830,7 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, case 'corefx_jitstressregs8': case 'corefx_jitstressregs0x10': case 'corefx_jitstressregs0x80': - def displayName = 'CoreFx' + getStressModeDisplayName(scenario) + def displayName = 'CoreFx ' + getStressModeDisplayName(scenario) assert (os == 'Windows_NT') || (os in Constants.crossList) Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})", "(?i).*test\\W+${os}\\W+${scenario}.*") @@ -1128,7 +1106,7 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, assert (os == 'Windows_NT') || (os in Constants.crossList) Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})", "(?i).*test\\W+${os}\\W+${scenario}.*") - break + break case 'corefx_baseline': case 'corefx_minopts': case 'corefx_jitstress1': @@ -1602,6 +1580,22 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - ${displayStr})", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*") break + case 'corefx_baseline': + case 'corefx_minopts': + case 'corefx_jitstress1': + case 'corefx_jitstress2': + case 'corefx_jitstressregs1': + case 'corefx_jitstressregs2': + case 'corefx_jitstressregs3': + case 'corefx_jitstressregs4': + case 'corefx_jitstressregs8': + case 'corefx_jitstressregs0x10': + case 'corefx_jitstressregs0x80': + def displayName = 'CoreFx ' + getStressModeDisplayName(scenario) + assert (os == 'Windows_NT') + Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - ${displayName})", + "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*") + break default: println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}"); assert false @@ -1834,12 +1828,8 @@ combinedScenarios.each { scenario -> } break case 'x64': - // Everything implemented - break case 'x86': - if (enableCorefxTesting) { - return - } + // Everything implemented break case 'x86compatjit': case 'x86lb': @@ -2151,24 +2141,26 @@ combinedScenarios.each { scenario -> runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${gcTestArguments}" if (Constants.jitStressModeScenarios.containsKey(scenario)) { + def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat" + buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation) + if (enableCorefxTesting) { - // Sync to corefx repo - // Move coreclr files to a subdirectory, %workspace%/clr. Otherwise, corefx build - // thinks that %workspace% is the project base directory. - buildCommands += "powershell new-item clr -type directory -force" - buildCommands += 'powershell foreach ($x in get-childitem -force) { if (\$x.name -ne \'clr\') { move-item $x clr }}' - buildCommands += "git clone -b $branch --single-branch https://github.com/dotnet/corefx fx" - - buildCommands += getStressModeEnvSetCmd(os, scenario); - - // Run corefx build and testing - buildCommands += "cd fx && call \"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat\" x86 && Build.cmd -Release -- /p:BUILDTOOLS_OVERRIDE_RUNTIME=%WORKSPACE%\\clr\\bin\\Product\\Windows_NT.x64.Checked " + def fxRoot = "%WORKSPACE%\\_\\fx" + buildCommands += "python %WORKSPACE%\\tests\\scripts\\run-corefx-tests.cmd -arch ${arch} -build_type ${configuration} -fx_root ${fxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}" + + // Archive only result xml files since corefx/bin/tests is very large around 10 GB. + // For windows, pull full test results and test drops for x86/x64 + Utilities.addArchival(newJob, "${fxRoot}/bin/tests/**/testResults.xml") + + // Set timeout + setTestJobTimeOut(newJob, scenario) + + if (architecture == 'x64' || !isPR) { + Utilities.addXUnitDotNETResults(newJob, "${fxRoot}/bin/tests/**/testResults.xml") + } } else { - def stepScriptLocation = "%WORKSPACE%\\bin\\tests\\SetStressModes.bat" - buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation) - // Run tests with the - buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}" + buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}" } } else if (architecture == 'x64' || architecture == 'x86') { @@ -2216,19 +2208,6 @@ combinedScenarios.each { scenario -> setTestJobTimeOut(newJob, scenario) } } - else { - // Archive only result xml files since corefx/bin/tests is very large around 10 GB. - // For windows, pull full test results and test drops for x86/x64 - Utilities.addArchival(newJob, "fx/bin/tests/**/testResults.xml") - - // Set timeout - setTestJobTimeOut(newJob, scenario) - - if (architecture == 'x64' || !isPR) { - Utilities.addXUnitDotNETResults(newJob, 'fx/bin/tests/**/testResults.xml') - } - } - break case 'arm': assert (scenario == 'default') @@ -2326,26 +2305,22 @@ combinedScenarios.each { scenario -> assert os == 'Ubuntu' assert architecture == 'x64' assert lowerConfiguration == 'checked' + assert Constants.jitStressModeScenarios.containsKey(scenario) - // Build coreclr and move it to clr directory + // Build coreclr buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}" - buildCommands += "rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr" - // Get corefx - buildCommands += "git clone -b $branch --single-branch https://github.com/dotnet/corefx fx" - - // Set environment variable - def setEnvVar = getStressModeEnvSetCmd(os, scenario) + def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh" + buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName) // Build and text corefx - buildCommands += "rm -rf \$WORKSPACE/fx_home; mkdir \$WORKSPACE/fx_home" - buildCommands += setEnvVar - buildCommands += "cd fx; export HOME=\$WORKSPACE/fx_home; ./build.sh -Release -Outerloop -TestWithLocalLibraries -- /p:BUILDTOOLS_OVERRIDE_RUNTIME=\$WORKSPACE/clr/bin/Product/Linux.x64.Checked" + def fxRoot = "\$WORKSPACE%/_/fx" + buildCommands += "python \$WORKSPACE/tests/scripts/run-corefx-tests.cmd -arch ${arch} -build_type ${configuration} -fx_root ${fxRoot} -fx_branch ${branch} -env_script ${scriptFileName}" // Archive and process test result - Utilities.addArchival(newJob, "fx/bin/tests/**/testResults.xml") + Utilities.addArchival(newJob, "${fxRoot}/bin/tests/**/testResults.xml") setTestJobTimeOut(newJob, scenario) - Utilities.addXUnitDotNETResults(newJob, 'fx/bin/tests/**/testResults.xml') + Utilities.addXUnitDotNETResults(newJob, "${fxRoot}/bin/tests/**/testResults.xml") } break case 'arm64': @@ -2844,10 +2819,8 @@ combinedScenarios.each { scenario -> if (Constants.jitStressModeScenarios.containsKey(scenario)) { def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh" def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName) - if (createScriptCmds != "") { - shell("${createScriptCmds}") - testEnvOpt = "--test-env=" + scriptFileName - } + shell("${createScriptCmds}") + testEnvOpt = "--test-env=" + scriptFileName } if (isGCStressRelatedTesting(scenario)) { diff --git a/tests/scripts/run-corefx-tests.py b/tests/scripts/run-corefx-tests.py new file mode 100644 index 0000000..b595b75 --- /dev/null +++ b/tests/scripts/run-corefx-tests.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python +# +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. +# See the LICENSE file in the project root for more information. +# +########################################################################## +########################################################################## +# +# Module: run-corefx-tests.py +# +# Notes: +# +# Script to clone the CoreFx repo, build, and run its tests. +# +########################################################################## +########################################################################## + +import argparse +import os +import re +import shutil +import subprocess +import sys + + +########################################################################## +# Globals +########################################################################## + +Corefx_url = 'https://github.com/dotnet/corefx.git' + +# This should be factored out of build.sh +Unix_name_map = { + 'Linux': 'Linux', + 'Darwin': 'OSX', + 'FreeBSD': 'FreeBSD', + 'OpenBSD': 'OpenBSD', + 'NetBSD': 'NetBSD', + 'SunOS': 'SunOS' +} + +Is_windows = (os.name == 'nt') + +########################################################################## +# Delete protocol +########################################################################## + +def del_rw(action, name, exc): + os.chmod(name, 0651) + os.remove(name) + +########################################################################## +# Argument Parser +########################################################################## + +description = 'Tool to facilitate running CoreFx tests from the CoreCLR repo' + +parser = argparse.ArgumentParser(description=description) + +parser.add_argument('-arch', dest='arch', default='x64') +parser.add_argument('-build_type', dest='build_type', default='Debug') +parser.add_argument('-clr_root', dest='clr_root', default=None) +parser.add_argument('-fx_root', dest='fx_root', default=None) +parser.add_argument('-fx_branch', dest='fx_branch', default='master') +parser.add_argument('-env_script', dest='env_script', default=None) + + +########################################################################## +# Helper Functions +########################################################################## + +def validate_args(args): + """ Validate all of the arguments parsed. + Args: + args (argparser.ArgumentParser): Args parsed by the argument parser. + Returns: + (arch, build_type, clr_root, fx_root, fx_branch, env_script) + (str, str, str, str, str, str) + Notes: + If the arguments are valid then return them all in a tuple. If not, raise + an exception stating x argument is incorrect. + """ + + arch = args.arch + build_type = args.build_type + clr_root = args.clr_root + fx_root = args.fx_root + fx_branch = args.fx_branch + env_script = args.env_script + + def validate_arg(arg, check): + """ Validate an individual arg + Args: + arg (str|bool): argument to be validated + check (lambda: x-> bool): test that returns either True or False + : based on whether the check passes. + + Returns: + is_valid (bool): Is the argument valid? + """ + + helper = lambda item: item is not None and check(item) + + if not helper(arg): + raise Exception('Argument: %s is not valid.' % (arg)) + + valid_archs = ['x86', 'x64', 'arm', 'arm64'] + valid_build_types = ['Debug', 'Checked', 'Release'] + + arch = next((a for a in valid_archs if a.lower() == arch.lower()), arch) + build_type = next((b for b in valid_build_types if b.lower() == build_type.lower()), build_type) + + validate_arg(arch, lambda item: item in valid_archs) + validate_arg(build_type, lambda item: item in valid_build_types) + validate_arg(fx_branch, lambda item: True) + + if clr_root is None: + clr_root = nth_dirname(os.path.abspath(sys.argv[0]), 3) + else: + clr_root = os.path.normpath(clr_root) + validate_arg(clr_root, lambda item: os.path.isdir(clr_root)) + + if fx_root is None: + fx_root = os.path.join(clr_root, '_', 'fx') + else: + fx_root = os.path.normpath(fx_root) + validate_arg(fx_root, lambda item: os.path.isdir( + os.path.dirname(fx_root))) + + if env_script is not None: + validate_arg(env_script, lambda item: os.path.isfile(env_script)) + env_script = os.path.abspath(env_script) + + args = (arch, build_type, clr_root, fx_root, fx_branch, env_script) + + log('Configuration:') + log(' arch: %s' % arch) + log(' build_type: %s' % build_type) + log(' clr_root: %s' % clr_root) + log(' fx_root: %s' % fx_root) + log(' fx_branch: %s' % fx_branch) + log(' env_script: %s' % env_script) + + return args + + +def nth_dirname(path, n): + """ Find the Nth parent directory of the given path + Args: + path (str): path name containing at least N components + n (int): num of basenames to remove + Returns: + outpath (str): path with the last n components removed + Notes: + If n is 0, path is returned unmodified + """ + + assert n >= 0 + + for i in range(0, n): + path = os.path.dirname(path) + + return path + + +def dotnet_rid_os(dotnet_path): + """ Determine the OS identifier from the RID as reported by dotnet + Args: + dotnet_path (str): path to folder containing dotnet(.exe) + Returns: + rid_os (str): OS component of RID as reported by dotnet + """ + dotnet_info = subprocess.check_output([os.path.join(dotnet_path, 'dotnet'), '--info']) + m = re.search('^\s*RID:\s+([^-]*)-(\S*)\s*$', dotnet_info, re.MULTILINE) + return m.group(1) + + +def log(message): + """ Print logging information + Args: + message (str): message to be printed + """ + + print '[%s]: %s' % (sys.argv[0], message) + + +########################################################################## +# Main +########################################################################## + +def main(args): + global Corefx_url + global Unix_name_map + + arch, build_type, clr_root, fx_root, fx_branch, env_script = validate_args( + args) + + clr_os = 'Windows_NT' if Is_windows else Unix_name_map[os.uname()[0]] + + core_root = os.path.join(clr_root, + 'bin', + 'Product', + '%s.%s.%s' % (clr_os, arch, build_type)) + + # corefx creates both files that are read-only and files that include non-ascii + # characters. Using onerror=del_rw allows us to delete all of the read-only files. + # To delete the files with non-ascii characters, when rmtree fails due to those + # files, we then will call rd on Windows. + + if os.path.exists(fx_root): + if Is_windows: + vbcscompiler_running = True + while vbcscompiler_running: + res = subprocess.check_output(['tasklist']) + if not 'VBCSCompiler.exe' in res: + vbcscompiler_running = False + os.chdir(fx_root) + os.system('git clean -fxd') + os.chdir(clr_root) + shutil.rmtree(fx_root, onerror=del_rw) + + command = 'git clone -b %s --single-branch %s %s' % ( + fx_branch, Corefx_url, fx_root) + + log(command) + + testing = False + + if testing: + os.makedirs(fx_root) + returncode = 0 + else: + returncode = os.system(command) + + if returncode != 0: + sys.exit(returncode) + + cwd = os.getcwd() + log('cd ' + fx_root) + os.chdir(fx_root) + + if Is_windows: + command = '.\\build.cmd' + if env_script is not None: + command = ('cmd /c %s&&' % env_script) + command + else: + # CoreFx build.sh requires HOME to be set, and it isn't by default + # under our CI. + fx_home = os.path.join(fx_root, 'tempHome') + if not os.path.exists(fx_home): + os.makedirs(fx_home) + os.putenv('HOME', fx_home) + log('HOME=' + fx_home) + + command = './build.sh' + if env_script is not None: + command = ('. %s;' % env_script) + command + + if testing: + rid_os = dotnet_rid_os('') + else: + if clr_os == "Windows_NT": + rid_os = "win7" + else: + rid_os = dotnet_rid_os(os.path.join(clr_root, 'Tools', 'dotnetcli')) + + command = ' '.join(( + command, + '-Release', + '-TestNugetRuntimeId=%s-%s' % (rid_os, arch), + '--', + '/p:BUILDTOOLS_OVERRIDE_RUNTIME="%s"' % core_root, + '/p:WithoutCategories=IgnoreForCI' + )) + + if not Is_windows: + command += ' /p:TestWithLocalNativeLibraries=true' + + log(command) + + if testing: + returncode = 0 + else: + returncode = os.system(command) + + sys.exit(returncode) + + +########################################################################## +# setup for Main +########################################################################## + +if __name__ == '__main__': + Args = parser.parse_args(sys.argv[1:]) + + main(Args) -- 2.7.4