perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27 // Setup perflab tests runs
  28 [true, false].each { isPR ->
  29     ['Windows_NT'].each { os ->
  30         ['x64', 'x86', 'x86jit32'].each { arch ->
  31             def architecture = arch
  32             def testEnv = ''
  33
  34             if (arch == 'x86jit32')
  35             {
  36                 architecture = 'x86'
  37                 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\compatjit_x86_testenv.cmd'
  38             }
  39             else if (arch == 'x86')
  40             {
  41                 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
  42             }
  43
  44             def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}_${arch}", isPR)) {
  45                 // Set the label.
  46                 label('windows_clr_perf')
  47                 wrappers {
  48                     credentialsBinding {
  49                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  50                     }
  51                 }
  52
  53             if (isPR)
  54             {
  55                 parameters
  56                 {
  57                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  58                 }
  59             }
  60             def configuration = 'Release'
  61             def runType = isPR ? 'private' : 'rolling'
  62             def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  63
  64                 steps {
  65                     // Batch
  66
  67                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  68                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  69                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  70                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  71                     batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
  72                     "set BENCHVIEWNAME=${benchViewName}\n" +
  73                     "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
  74                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
  75                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  76                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  77                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  78
  79                     if (arch == 'x86jit32')
  80                     {
  81                         // Download package and copy compatjit into Core_Root
  82                         batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
  83                         "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
  84                     }
  85
  86                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  87
  88                     batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} ${testEnv} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
  89                     batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} ${testEnv} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
  90                 }
  91             }
  92
  93             // Save machinedata.json to /artifact/bin/ Jenkins dir
  94             def archiveSettings = new ArchivalSettings()
  95             archiveSettings.addFiles('Perf-*.xml')
  96             archiveSettings.addFiles('Perf-*.etl')
  97             Utilities.addArchival(newJob, archiveSettings)
  98
  99             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 100
 101             newJob.with {
 102                 wrappers {
 103                     timeout {
 104                         absolute(240)
 105                     }
 106                 }
 107             }
 108
 109             if (isPR) {
 110                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 111                 builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
 112                 builder.triggerOnlyOnComment()
 113                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
 114                 builder.triggerForBranch(branch)
 115                 builder.emitTrigger(newJob)
 116             }
 117             else {
 118                 // Set a push trigger
 119                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 120                 builder.emitTrigger(newJob)
 121             }
 122         }
 123     }
 124 }
 125
 126 // Setup throughput perflab tests runs
 127 [true, false].each { isPR ->
 128     ['Windows_NT'].each { os ->
 129         ['x64', 'x86', 'x86jit32'].each { arch ->
 130             def architecture = arch
 131
 132             if (arch == 'x86jit32')
 133             {
 134                 architecture = 'x86'
 135             }
 136
 137             def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) {
 138                 // Set the label.
 139                 label('windows_clr_perf')
 140                 wrappers {
 141                     credentialsBinding {
 142                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 143                     }
 144                 }
 145
 146             if (isPR)
 147             {
 148                 parameters
 149                 {
 150                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 151                 }
 152             }
 153             def configuration = 'Release'
 154             def runType = isPR ? 'private' : 'rolling'
 155             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 156
 157                 steps {
 158                     // Batch
 159
 160                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 161                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 162                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 163                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 164                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 165                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 166                     batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
 167                     "set BENCHVIEWNAME=${benchViewName}\n" +
 168                     "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
 169                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
 170                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 171                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 172                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
 173                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 174                     if (arch == 'x86jit32')
 175                     {
 176                         // Download package and copy compatjit into Core_Root
 177                         batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
 178                         "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
 179                     }
 180                     batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 181                 }
 182             }
 183
 184             // Save machinedata.json to /artifact/bin/ Jenkins dir
 185             def archiveSettings = new ArchivalSettings()
 186             archiveSettings.addFiles('throughput-*.csv')
 187             Utilities.addArchival(newJob, archiveSettings)
 188
 189             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 190
 191             if (isPR) {
 192                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 193                 builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests")
 194                 builder.triggerOnlyOnComment()
 195                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*")
 196                 builder.triggerForBranch(branch)
 197                 builder.emitTrigger(newJob)
 198             }
 199             else {
 200                 // Set a push trigger
 201                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 202                 builder.emitTrigger(newJob)
 203             }
 204         }
 205     }
 206 }
 207
 208 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 209 [true, false].each { isPR ->
 210     ['Ubuntu14.04'].each { os ->
 211         def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
 212
 213             label('linux_clr_perf')
 214                 wrappers {
 215                     credentialsBinding {
 216                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 217                     }
 218                 }
 219
 220             if (isPR)
 221             {
 222                 parameters
 223                 {
 224                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 225                 }
 226             }
 227             def osGroup = getOSGroup(os)
 228             def architecture = 'x64'
 229             def configuration = 'Release'
 230             def runType = isPR ? 'private' : 'rolling'
 231             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 232
 233             steps {
 234                 shell("bash ./tests/scripts/perf-prep.sh")
 235                 shell("./init-tools.sh")
 236                 shell("./build.sh ${architecture} ${configuration}")
 237                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 238                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 239                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 240                 shell("""sudo -E bash ./tests/scripts/run-xunit-perf.sh \\
 241                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 242                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 243                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 244                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 245                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 246                 --runType=\"${runType}\" \\
 247                 --benchViewOS=\"${os}\" \\
 248                 --uploadToBenchview""")
 249             }
 250         }
 251
 252         // Save machinedata.json to /artifact/bin/ Jenkins dir
 253         def archiveSettings = new ArchivalSettings()
 254         archiveSettings.addFiles('sandbox/perf-*.xml')
 255         archiveSettings.addFiles('machinedata.json')
 256         Utilities.addArchival(newJob, archiveSettings)
 257
 258         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 259
 260         // For perf, we need to keep the run results longer
 261         newJob.with {
 262             // Enable the log rotator
 263             logRotator {
 264                 artifactDaysToKeep(7)
 265                 daysToKeep(300)
 266                 artifactNumToKeep(25)
 267                 numToKeep(1000)
 268             }
 269         }
 270         if (isPR) {
 271             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 272             builder.setGithubContext("${os} Perf Tests")
 273             builder.triggerOnlyOnComment()
 274             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
 275             builder.triggerForBranch(branch)
 276             builder.emitTrigger(newJob)
 277         }
 278         else {
 279             // Set a push trigger
 280             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 281             builder.emitTrigger(newJob)
 282         }
 283     } // os
 284 } // isPR
 285
 286 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 287 [true, false].each { isPR ->
 288     ['Ubuntu14.04'].each { os ->
 289         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)) {
 290
 291             label('linux_clr_perf')
 292                 wrappers {
 293                     credentialsBinding {
 294                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 295                     }
 296                 }
 297
 298             if (isPR)
 299             {
 300                 parameters
 301                 {
 302                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 303                 }
 304             }
 305             def osGroup = getOSGroup(os)
 306             def architecture = 'x64'
 307             def configuration = 'Release'
 308             def runType = isPR ? 'private' : 'rolling'
 309             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 310
 311             steps {
 312                 shell("bash ./tests/scripts/perf-prep.sh --throughput")
 313                 shell("./init-tools.sh")
 314                 shell("./build.sh ${architecture} ${configuration}")
 315                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 316                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 317                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 318                 shell("""sudo -E python3.5 ./tests/scripts/run-throughput-perf.py \\
 319                 -arch \"${architecture}\" \\
 320                 -os \"${os}\" \\
 321                 -configuration \"${configuration}\" \\
 322                 -clr_root \"\${WORKSPACE}\" \\
 323                 -assembly_root \"\${WORKSPACE}/_/fx/bin/runtime/netcoreapp-${osGroup}-${configuration}-${architecture}\" \\
 324                 -run_type \"${runType}\" \\
 325                 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 326             }
 327         }
 328
 329         // Save machinedata.json to /artifact/bin/ Jenkins dir
 330         def archiveSettings = new ArchivalSettings()
 331         archiveSettings.addFiles('throughput-*.csv')
 332         archiveSettings.addFiles('machinedata.json')
 333         Utilities.addArchival(newJob, archiveSettings)
 334
 335         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 336
 337         // For perf, we need to keep the run results longer
 338         newJob.with {
 339             // Enable the log rotator
 340             logRotator {
 341                 artifactDaysToKeep(7)
 342                 daysToKeep(300)
 343                 artifactNumToKeep(25)
 344                 numToKeep(1000)
 345             }
 346         }
 347         if (isPR) {
 348             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 349             builder.setGithubContext("${os} Throughput Perf Tests")
 350             builder.triggerOnlyOnComment()
 351             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+throughput.*")
 352             builder.triggerForBranch(branch)
 353             builder.emitTrigger(newJob)
 354         }
 355         else {
 356             // Set a push trigger
 357             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 358             builder.emitTrigger(newJob)
 359         }
 360     } // os
 361 } // isPR