perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora23':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27 // Setup perflab tests runs
  28 [true, false].each { isPR ->
  29     ['Windows_NT'].each { os ->
  30                 ['x64'].each { architecture ->
  31                         def configuration = 'Release'
  32                         def runType = isPR ? 'private' : 'rolling'
  33                         def benchViewName = isPR ? 'coreclr private %ghprbPullTitle%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  34                         def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}", isPR)) {
  35                                 // Set the label.
  36                                 label('windows_clr_perf')
  37                                 wrappers {
  38                                         credentialsBinding {
  39                                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  40                                         }
  41                                 }
  42
  43                                 steps {
  44                                         // Batch
  45
  46                                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  47                                         batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  48                                         //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  49                                         //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  50                                         batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
  51                                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name " + "\"" + benchViewName + "\"" + " --user " + "\"dotnet-bot@microsoft.com\"\n" +
  52                                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type " + runType)
  53                                         batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  54                                         batchFile("set __TestIntermediateDir=int&&build.cmd release ${architecture}")
  55                                         batchFile("tests\\runtest.cmd release ${architecture} GenerateLayoutOnly")
  56                                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${architecture} -configuration ${configuration} -testBinLoc bin\\tests\\Windows_NT.${architecture}.Release\\performance\\perflab\\Perflab -library -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype " + runType)
  57                                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${architecture} -configuration ${configuration} -testBinLoc bin\\tests\\Windows_NT.${architecture}.Release\\Jit\\Performance\\CodeQuality -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype " + runType)
  58                                 }
  59                         }
  60
  61                         // Save machinedata.json to /artifact/bin/ Jenkins dir
  62                         def archiveSettings = new ArchivalSettings()
  63                         archiveSettings.addFiles('perf-*.xml')
  64                         archiveSettings.addFiles('perf-*.etl')
  65                         Utilities.addArchival(newJob, archiveSettings)
  66
  67                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
  68
  69                         if (isPR) {
  70                                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
  71                                 builder.setGithubContext("${os} CoreCLR Perf Tests")
  72                                 builder.triggerOnlyOnComment()
  73                                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
  74                                 builder.triggerForBranch(branch)
  75                                 builder.emitTrigger(newJob)
  76                         }
  77                         else {
  78                                 // Set a push trigger
  79                                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
  80                                 builder.emitTrigger(newJob)
  81                         }
  82                 }
  83     }
  84 }
  85
  86 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
  87 [true, false].each { isPR ->
  88     ['Ubuntu'].each { os ->
  89         def osGroup = getOSGroup(os)
  90         def architecture = 'x64'
  91         def configuration = 'Release'
  92         def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
  93             steps {
  94                 shell("sudo bash ./tests/scripts/perf-prep.sh --branch=${projectName}")
  95                 shell("sudo ./init-tools.sh")
  96                 shell("""sudo bash ./tests/scripts/run-xunit-perf.sh \\
  97                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
  98                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
  99                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 100                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 101                 --coreFxBinDir=\"\${WORKSPACE}/bin/${osGroup}.AnyCPU.${configuration};\${WORKSPACE}/bin/Unix.AnyCPU.${configuration};\${WORKSPACE}/bin/AnyOS.AnyCPU.${configuration}\" \\
 102                 --coreFxNativeBinDir=\"\${WORKSPACE}/bin/${osGroup}.${architecture}.${configuration}\"""")
 103             }
 104         }
 105
 106         Utilities.setMachineAffinity(newJob, os, 'latest-or-auto') // Just run against Linux VM's for now.
 107
 108         // Save machinedata.json to /artifact/bin/ Jenkins dir
 109         def archiveSettings = new ArchivalSettings()
 110         archiveSettings.addFiles('sandbox/perf-*.xml')
 111         archiveSettings.addFiles('machinedata.json')
 112         Utilities.addArchival(newJob, archiveSettings)
 113
 114         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 115
 116         // For perf, we need to keep the run results longer
 117         newJob.with {
 118             // Enable the log rotator
 119             logRotator {
 120                 artifactDaysToKeep(7)
 121                 daysToKeep(300)
 122                 artifactNumToKeep(25)
 123                 numToKeep(1000)
 124             }
 125         }
 126         if (isPR) {
 127             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 128             builder.setGithubContext("${os} Perf Tests")
 129             builder.triggerOnlyOnComment()
 130             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
 131             builder.triggerForBranch(branch)
 132             builder.emitTrigger(newJob)
 133         }
 134         else {
 135             // Set a push trigger
 136             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 137             builder.emitTrigger(newJob)
 138         }
 139     } // os
 140 } // isPR