From: Matt Mitchell Date: Tue, 20 Oct 2015 18:20:45 +0000 (-0700) Subject: Add x86 builds to CI (most off by default) X-Git-Tag: accepted/tizen/base/20180629.140029~6274^2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=95f3cb2d020ffcfdfa644470a4ce6294d708df6b;p=platform%2Fupstream%2Fcoreclr.git Add x86 builds to CI (most off by default) * Builds are available through trigger phrases * x86 windows_nt on for pushes by default Additionally: * fixes workspace parameter quoting for Linux coreclr tests --- diff --git a/netci.groovy b/netci.groovy index 87e47bc..9229081 100644 --- a/netci.groovy +++ b/netci.groovy @@ -24,9 +24,18 @@ def osGroupMap = ['Ubuntu':'Linux', // Innerloop build OS's def osList = ['Ubuntu', 'OSX', 'Windows_NT', 'FreeBSD', 'CentOS7.1', 'OpenSUSE13.2'] -def static getBuildJobName(def configuration, def os) { - // Massage names a bit - return configuration.toLowerCase() + '_' + os.toLowerCase() +def static getBuildJobName(def configuration, def architecture, def os) { + // If the architecture is x64, do not add that info into the build name. + // Need to change around some systems and other builds to pick up the right builds + // to do that. + + if (architecture == 'x64') { + return configuration.toLowerCase() + '_' + os.toLowerCase() + } + else { + // Massage names a bit + return architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase() + } } // ************************** @@ -36,91 +45,128 @@ def static getBuildJobName(def configuration, def os) { // ************************** // Loop over the options and build up the innerloop build matrix +// Default architecture is x64. Right now that isn't built into the +// build name. We'll need to change that eventually, but for now leave as is. +// x86 builds have the new build name -['Debug', 'Release'].each { configuration -> - osList.each { os -> - // Calculate names - def lowerConfiguration = configuration.toLowerCase() - - // Calculate job name - def jobName = getBuildJobName(configuration, os) - def buildCommand = ''; - - def osGroup = osGroupMap[os] - - // Calculate the build command - if (os == 'Windows_NT') { - // On Windows we build the mscorlibs too. - buildCommand = "build.cmd ${lowerConfiguration} && tests\\runtest.cmd ${lowerConfiguration} && build.cmd ${lowerConfiguration} linuxmscorlib && build.cmd ${lowerConfiguration} freebsdmscorlib && build.cmd ${lowerConfiguration} osxmscorlib" - } - else { - // On other OS's we skipmscorlib but run the pal tests - buildCommand = "./build.sh skipmscorlib verbose ${lowerConfiguration} && src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.x64.${configuration} \${WORKSPACE}/bin/paltestout" - } - - // Create the new job - def newCommitJob = job(Utilities.getFullJobName(project, jobName, false)) { - // Set the label. - label(machineLabelMap[os]) - steps { - if (os == 'Windows_NT') { - // Batch - batchFile(buildCommand) +['x64', 'x86'].each { architecture -> + ['Debug', 'Release'].each { configuration -> + osList.each { os -> + // Calculate names + def lowerConfiguration = configuration.toLowerCase() + + // Calculate job name + def jobName = getBuildJobName(configuration, architecture, os) + def buildCommands = []; + + def osGroup = osGroupMap[os] + + // Calculate the build commands + if (os == 'Windows_NT') { + // On Windows we build the mscorlibs too. + buildCommands += "build.cmd ${lowerConfiguration} ${architecture}" + + // If x64, we run tests + if (architecture == 'x64') { + buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture}" } - else { - // Shell - shell(buildCommand) + + // Build the mscorlib for the other OS's + buildCommands += "build.cmd ${lowerConfiguration} ${architecture} linuxmscorlib" + buildCommands += "build.cmd ${lowerConfiguration} ${architecture} freebsdmscorlib" + buildCommands += "build.cmd ${lowerConfiguration} ${architecture} osxmscorlib" + } + else { + // On other OS's we skipmscorlib but run the pal tests + buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture}" + buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout" + } + + // Create the new job + def newCommitJob = job(Utilities.getFullJobName(project, jobName, false)) { + // Set the label. + label(machineLabelMap[os]) + steps { + if (os == 'Windows_NT') { + buildCommands.each { buildCommand -> + batchFile(buildCommand) + } + } + else { + buildCommands.each { buildCommand -> + shell(buildCommand) + } + } } } - } - // Add commit job options - Utilities.addScm(newCommitJob, project) - Utilities.addStandardNonPRParameters(newCommitJob) - Utilities.addGithubPushTrigger(newCommitJob) - - // Create the new PR job - - def newPRJob = job(Utilities.getFullJobName(project, jobName, true)) { - // Set the label. - label(machineLabelMap[os]) - steps { - if (os == 'Windows_NT') { - // Batch - batchFile(buildCommand) - } - else { - // Shell - shell(buildCommand) + // Add commit job options + Utilities.addScm(newCommitJob, project) + Utilities.addStandardNonPRParameters(newCommitJob) + + // Add a push trigger. Do not enable the push trigger for non-windows x86, + // which do not work at the moment. + if (architecture == 'x64' || osGroup == 'Windows_NT') { + Utilities.addGithubPushTrigger(newCommitJob) + } + + // Create the new PR job + + def newPRJob = job(Utilities.getFullJobName(project, jobName, true)) { + // Set the label. + label(machineLabelMap[os]) + steps { + if (os == 'Windows_NT') { + buildCommands.each { buildCommand -> + batchFile(buildCommand) + } + } + else { + buildCommands.each { buildCommand -> + shell(buildCommand) + } + } } } - } - - // Add a PR trigger. For some OS's, create an explicit trigger - // PR's are run for everything except SuSE - if (os != 'OpenSUSE13.2') { - Utilities.addGithubPRTrigger(newPRJob, "${os} ${configuration} Build") - } - Utilities.addPRTestSCM(newPRJob, project) - Utilities.addStandardPRParameters(newPRJob, project) - - // Add common options: - - [newPRJob, newCommitJob].each { newJob -> - Utilities.addStandardOptions(newJob) - if (osGroup == 'Windows_NT') { - Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml') - Utilities.addArchival(newJob, "bin/Product/**,bin/tests/**", "bin/tests/obj/**") - } else { - // Add .NET results for the - Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml') - Utilities.addArchival(newJob, "bin/Product/**") - Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**") + // Add a PR trigger. For some OS's, create an explicit trigger + // PR's are run for everything except SuSE by default. Add on-demand triggers + // for everything else. This probably deserves a bit of cleanup eventually once we + // have an easy way to do regex trigger phrases, to put in some more structured test phrases. + def triggerPhraseString = '' + if (os == 'OpenSUSE13.2' && architecture == 'x64') { + triggerPhraseString = '@dotnet-bot test suse' + } else if (architecture == 'x86' && osGroup == 'Windows_NT') { + triggerPhraseString = '@dotnet-bot test x86' + } else if (architecture == 'x86' && osGroup == 'Linux') { + triggerPhraseString = '@dotnet-bot test x86 linux' + } else if (architecture == 'x86' && osGroup == 'OSX') { + triggerPhraseString = '@dotnet-bot test x86 osx' + } else if (architecture == 'x86' && osGroup == 'FreeBSD') { + triggerPhraseString = '@dotnet-bot test x86 freebsd' } - } - } -} + Utilities.addGithubPRTrigger(newPRJob, "${os} ${architecture} ${configuration} Build", triggerPhraseString) + Utilities.addPRTestSCM(newPRJob, project) + Utilities.addStandardPRParameters(newPRJob, project) + + // Add common options: + + [newPRJob, newCommitJob].each { newJob -> + Utilities.addStandardOptions(newJob) + + if (osGroup == 'Windows_NT') { + Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml') + Utilities.addArchival(newJob, "bin/Product/**,bin/tests/**", "bin/tests/obj/**") + } else { + // Add .NET results for the + Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml') + Utilities.addArchival(newJob, "bin/Product/**") + Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**") + } + } + } // os + } // configuration +} // architecture // Ubuntu cross compiled arm and arm64 builds // Scheduled for nightly and on-demand PR for now @@ -188,101 +234,103 @@ def static getBuildJobName(def configuration, def os) { } // Create the Linux coreclr test leg for debug and release. -// Put the OS's supported for coreclr cross testing here -['Ubuntu'].each { os -> - [true, false].each { isPR -> - ['Debug', 'Release'].each { configuration -> - - def lowerConfiguration = configuration.toLowerCase() - def osGroup = osGroupMap[os] - def jobName = getBuildJobName(configuration, os) + "_tst" - def inputCoreCLRBuildName = Utilities.getFolderName(project) + '/' + - Utilities.getFullJobName(project, getBuildJobName(configuration, os), isPR) - def inputWindowTestsBuildName = Utilities.getFolderName(project) + '/' + - Utilities.getFullJobName(project, getBuildJobName(configuration, 'windows_nt'), isPR) - - def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) { - // Set the label. - label(machineLabelMap[os]) - - // Add parameters for the inputs +// Architectures +['x64', 'x86'].each { architecture -> + // Put the OS's supported for coreclr cross testing here + ['Ubuntu'].each { os -> + [true, false].each { isPR -> + ['Debug', 'Release'].each { configuration -> - parameters { - stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from') - stringParam('CORECLR_LINUX_BUILD', '', 'Build number to copy CoreCLR linux binaries from') - } + def lowerConfiguration = configuration.toLowerCase() + def osGroup = osGroupMap[os] + def jobName = getBuildJobName(configuration, architecture, os) + "_tst" + def inputCoreCLRBuildName = Utilities.getFolderName(project) + '/' + + Utilities.getFullJobName(project, getBuildJobName(configuration, architecture, os), isPR) + def inputWindowTestsBuildName = Utilities.getFolderName(project) + '/' + + Utilities.getFullJobName(project, getBuildJobName(configuration, architecture, 'windows_nt'), isPR) - steps { - // Set up the copies + def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) { + // Set the label. + label(machineLabelMap[os]) - // Coreclr build we are trying to test + // Add parameters for the inputs - copyArtifacts(inputCoreCLRBuildName) { - excludePatterns('**/testResults.xml', '**/*.ni.dll') - buildSelector { - buildNumber('${CORECLR_LINUX_BUILD}') - } + parameters { + stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from') + stringParam('CORECLR_LINUX_BUILD', '', 'Build number to copy CoreCLR linux binaries from') } - // Coreclr build containing the tests and mscorlib - - copyArtifacts(inputWindowTestsBuildName) { - excludePatterns('**/testResults.xml', '**/*.ni.dll') - buildSelector { - buildNumber('${CORECLR_WINDOWS_BUILD}') + steps { + // Set up the copies + + // Coreclr build we are trying to test + + copyArtifacts(inputCoreCLRBuildName) { + excludePatterns('**/testResults.xml', '**/*.ni.dll') + buildSelector { + buildNumber('${CORECLR_LINUX_BUILD}') + } } - } - - // Corefx native components - copyArtifacts("dotnet_corefx_linux_nativecomp_debug") { - includePatterns('bin/**') - buildSelector { - latestSuccessful(true) + + // Coreclr build containing the tests and mscorlib + + copyArtifacts(inputWindowTestsBuildName) { + excludePatterns('**/testResults.xml', '**/*.ni.dll') + buildSelector { + buildNumber('${CORECLR_WINDOWS_BUILD}') + } } - } - - // Corefx linux binaries - copyArtifacts("dotnet_corefx_linux_nativecomp_debug") { - includePatterns('bin/Linux*/**') - buildSelector { - latestSuccessful(true) + + // Corefx native components + copyArtifacts("dotnet_corefx_linux_nativecomp_debug") { + includePatterns('bin/**') + buildSelector { + latestSuccessful(true) + } + } + + // Corefx linux binaries + copyArtifacts("dotnet_corefx_linux_nativecomp_debug") { + includePatterns('bin/Linux*/**') + buildSelector { + latestSuccessful(true) + } } + + // Execute the shell command + + shell(""" + ./tests/runtest.sh \\ + --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.Debug\" \\ + --testNativeBinDir=\"\${WORKSPACE}/bin/obj/Linux.${architecture}.Debug/tests\" \\ + --coreClrBinDir=\"\${WORKSPACE}/bin/Product/Linux.${architecture}.Debug\" \\ + --mscorlibDir=\"\${WORKSPACE}/bin/Product/Linux.${architecture}.Debug\" \\ + --coreFxBinDir=\"\${WORKSPACE}/bin/Linux.AnyCPU.Debug\" \\ + --coreFxNativeBinDir=\"\${WORKSPACE}/bin/Linux.${architecture}.Debug\"""") } - - // Execute the shell command - - shell(""" -./tests/runtest.sh \\ - --testRootDir='\${WORKSPACE}/bin/tests/Windows_NT.x64.Debug' \\ - --testNativeBinDir='\${WORKSPACE}/bin/obj/Linux.x64.Debug/tests' \\ - --coreClrBinDir='\${WORKSPACE}/bin/Product/Linux.x64.Debug' \\ - --mscorlibDir='\${WORKSPACE}/bin/Product/Linux.x64.Debug' \\ - --coreFxBinDir='\${WORKSPACE}/bin/Linux.AnyCPU.Debug' \\ - --coreFxNativeBinDir='\${WORKSPACE}/bin/Linux.x64.Debug'""") } - } - - if (!isPR) { - // Add rolling job options - Utilities.addScm(newJob, project) - Utilities.addStandardNonPRParameters(newJob) - } - else { - // Add PR job options - Utilities.addPRTestSCM(newJob, project) - Utilities.addStandardPRParameters(newJob, project) - } - Utilities.addStandardOptions(newJob) - Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml') - - // Create a build flow to join together the build and tests required to run this - // test. - // Windows CoreCLR build and Linux CoreCLR build (in parallel) -> - // Linux CoreCLR test - def flowJobName = getBuildJobName(configuration, os) + "_flow" - def fullTestJobName = Utilities.getFolderName(project) + '/' + newJob.name - def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR)) { - buildFlow(""" + + if (!isPR) { + // Add rolling job options + Utilities.addScm(newJob, project) + Utilities.addStandardNonPRParameters(newJob) + } + else { + // Add PR job options + Utilities.addPRTestSCM(newJob, project) + Utilities.addStandardPRParameters(newJob, project) + } + Utilities.addStandardOptions(newJob) + Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml') + + // Create a build flow to join together the build and tests required to run this + // test. + // Windows CoreCLR build and Linux CoreCLR build (in parallel) -> + // Linux CoreCLR test + def flowJobName = getBuildJobName(configuration, architecture, os) + "_flow" + def fullTestJobName = Utilities.getFolderName(project) + '/' + newJob.name + def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR)) { + buildFlow(""" // Grab the checked out git commit hash so that it can be passed to the child // builds. gitCommit = build.environment.get('GIT_COMMIT') @@ -291,31 +339,35 @@ parallel ( { linuxBuildJob = build(params + [GitBranchOrCommit: gitCommit], '${inputCoreCLRBuildName}') }, { windowsBuildJob = build(params + [GitBranchOrCommit: gitCommit], '${inputWindowTestsBuildName}') } ) - + // And then build the test build build(params + [CORECLR_LINUX_BUILD: linuxBuildJob.build.number, CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number, GitBranchOrCommit: gitCommit], '${fullTestJobName}') """) - // Needs a workspace - configure { - def buildNeedsWorkspace = it / 'buildNeedsWorkspace' - buildNeedsWorkspace.setValue('true') + // Needs a workspace + configure { + def buildNeedsWorkspace = it / 'buildNeedsWorkspace' + buildNeedsWorkspace.setValue('true') + } } - } - if (isPR) { - Utilities.addPRTestSCM(newFlowJob, project) - Utilities.addStandardPRParameters(newFlowJob, project) - Utilities.addGithubPRTrigger(newFlowJob, "Linux ${configuration} Build and Test", '@dotnet-bot test linux') - } - else { - Utilities.addScm(newFlowJob, project) - Utilities.addStandardNonPRParameters(newFlowJob) + if (isPR) { + Utilities.addPRTestSCM(newFlowJob, project) + Utilities.addStandardPRParameters(newFlowJob, project) + if (architecture == 'x64') { + Utilities.addGithubPRTrigger(newFlowJob, "Linux ${architecture} ${configuration} Build and Test", '@dotnet-bot test linux') + } else { + Utilities.addGithubPRTrigger(newFlowJob, "Linux ${architecture} ${configuration} Build and Test", '@dotnet-bot test linux x86') + } + } + else { + Utilities.addScm(newFlowJob, project) + Utilities.addStandardNonPRParameters(newFlowJob) + } + + Utilities.addStandardOptions(newFlowJob) } - - Utilities.addStandardOptions(newFlowJob) } } -} - +} \ No newline at end of file