From 193eb5a66309db4be666a1e5c5fb532e7a18116e Mon Sep 17 00:00:00 2001 From: Hyeongseok Oh Date: Mon, 27 Nov 2017 22:30:54 +0900 Subject: [PATCH] [x86/Linux] Enable CI unittest (#14442) * [x86/Linux] Preparing pri2 unittest - Copy test binary from x86 Windows build - Copy CoreFX managed from arm CoreFX managed build - Add archival * Fix to merge test job generation Change approach by expanding existing x64 & arm64 linux infrastructure * [CI][Linux/x86] Fix configuration and architecture in netci.groovy Fix to use configuration instead of lowerConfiguration USe architecture and osGroup * [CI][Linux/x86] Save test native builds in build job Fix to save test native build in build job Saved native builds are used on test job --- netci.groovy | 111 +++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 104 insertions(+), 7 deletions(-) diff --git a/netci.groovy b/netci.groovy index ce51786..cb0beed 100755 --- a/netci.groovy +++ b/netci.groovy @@ -505,6 +505,52 @@ def static appendStressModeScriptStep(def os, def appendScript, def stepScriptLo return stepScript } +def static isNeedDocker(def architecture, def os, def isBuild) { + if (isBuild) { + if (architecture == 'x86' && os == 'Ubuntu') { + return true + } + else if (architecture == 'arm') { + if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') { + return true + } + } + } + else { + if (architecture == 'x86' && os == 'Ubuntu') { + return true + } + } + return false +} + +def static getDockerImageName(def architecture, def os, def isBuild) { + // We must change some docker private images to official later + if (isBuild) { + if (architecture == 'x86' && os == 'Ubuntu') { + return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548" + } + else if (architecture == 'arm') { + if (os == 'Ubuntu') { + return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239" + } + else if (os == 'Ubuntu16.04') { + return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548" + } + else if (os == 'Tizen') { + return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs" + } + } + } + else { + if (architecture == 'x86' && os == 'Ubuntu') { + return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test" + } + } + println("Unknown architecture to use docker: ${architecture} ${os}"); + assert false +} + // Calculates the name of the build job based on some typical parameters. // def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) { @@ -576,7 +622,7 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def switch (architecture) { case 'x64': case 'x86': - if (architecture == 'x86' && os == 'Ubuntu') { + if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') { Utilities.addPeriodicTrigger(job, '@daily') } else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) { @@ -1244,6 +1290,10 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, case 'x86': // editor brace matching: { assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default'))) if (os == 'Ubuntu') { + // Triggers on the non-flow jobs aren't necessary here + if (!isFlowJob) { + break + } // on-demand only for ubuntu x86 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*") @@ -1643,7 +1693,11 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR case 'x86': if (architecture == 'x86' && os == 'Ubuntu') { // build and PAL test - buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}" + def dockerImage = getDockerImageName(architecture, os, true) + buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}" + dockerImage = getDockerImageName(architecture, os, false) + buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout" + Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**") Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml') break } @@ -2080,7 +2134,7 @@ Constants.allScenarios.each { scenario -> // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows. Constants.allScenarios.each { scenario -> [true, false].each { isPR -> - ['arm', 'armlb', 'x64', 'arm64'].each { architecture -> + ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture -> Constants.crossList.each { os -> if (architecture == 'arm64') { if (os != "Ubuntu" && os != "Windows_NT") { @@ -2091,6 +2145,11 @@ Constants.allScenarios.each { scenario -> return } } + else if (architecture == 'x86') { + if (os != "Ubuntu") { + return + } + } def validWindowsNTCrossArches = ["arm", "armlb", "arm64"] @@ -2112,6 +2171,10 @@ Constants.allScenarios.each { scenario -> if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') { return } + else if (architecture == 'x86') { + // Linux/x86 only want default test + if (scenario != 'default') { + return } } @@ -2369,6 +2432,13 @@ Constants.allScenarios.each { scenario -> if (os == 'Ubuntu') { osJobName = 'ubuntu14.04' } + else if (architecture == 'x86') { + if (os == 'Ubuntu') { + // Linux/x86 corefx jobs does not build managed yet + // Clone linux/arm corefx managed packages and overwrite linux/x86 native + osJobName = "linux_arm_cross" + } + } else { osJobName = os.toLowerCase() } @@ -2379,7 +2449,7 @@ Constants.allScenarios.each { scenario -> } } - shell ("mkdir ./bin/CoreFxBinDir") + shell("mkdir ./bin/CoreFxBinDir") // Unpack the corefx binaries shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir") @@ -2389,11 +2459,33 @@ Constants.allScenarios.each { scenario -> shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir") shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun") } + else if (architecture == 'x86') { + shell("mkdir ./bin/CoreFxNative") + + copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") { + includePatterns('bin/build.tar.gz') + targetDirectory('bin/CoreFxNative') + buildSelector { + latestSuccessful(true) + } + } + + shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir") + } // Unzip the tests first. Exit with 0 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0") // Execute the tests + def runDocker = isNeedDocker(architecture, os, false) + def dockerPrefix = "" + def dockerCmd = "" + if (runDocker) { + def dockerImage = getDockerImageName(architecture, os, false) + dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} " + dockerCmd = dockerPrefix + "${dockerImage} " + } + // If we are running a stress mode, we'll set those variables first def testEnvOpt = "" if (isJitStressScenario(scenario)) { @@ -2407,7 +2499,7 @@ Constants.allScenarios.each { scenario -> shell('./init-tools.sh') } - shell("""./tests/runtest.sh \\ + shell("""${dockerCmd}./tests/runtest.sh \\ --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\ --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\ --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\ @@ -2418,10 +2510,15 @@ Constants.allScenarios.each { scenario -> if (isGcReliabilityFramework(scenario)) { // runtest.sh doesn't actually execute the reliability framework - do it here. if (serverGCString != '') { - shell("export COMPlus_gcServer=1") + if (runDocker) { + dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} " + } + else { + shell("export COMPlus_gcServer=1") + } } - shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}") + shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}") } } -- 2.7.4