perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27 // Setup perflab tests runs
  28 [true, false].each { isPR ->
  29     ['Windows_NT'].each { os ->
  30         ['x64', 'x86'].each { arch ->
  31             [true, false].each { isSmoketest ->
  32                 def architecture = arch
  33                 def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_smoketest" : "perf_perflab_${os}_${arch}"
  34
  35                 if (arch == 'x86jit32')
  36                 {
  37                     architecture = 'x86'
  38                     testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\compatjit_x86_testenv.cmd'
  39                 }
  40                 else if (arch == 'x86')
  41                 {
  42                     testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
  43                 }
  44
  45                 def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
  46                     // Set the label.
  47                     label('windows_clr_perf')
  48                     wrappers {
  49                         credentialsBinding {
  50                             string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  51                         }
  52                     }
  53
  54                 if (isPR)
  55                 {
  56                     parameters
  57                     {
  58                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  59                     }
  60                 }
  61                 if (isSmoketest)
  62                 {
  63                     parameters
  64                     {
  65                         stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
  66                         stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
  67                     }
  68                 }
  69                 else
  70                 {
  71                     parameters
  72                     {
  73                         stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
  74                         stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
  75                     }
  76                 }
  77                 def configuration = 'Release'
  78                 def runType = isPR ? 'private' : 'rolling'
  79                 def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  80                 def uploadString = isSmoketest ? '' : '-uploadToBenchview'
  81
  82                     steps {
  83                         // Batch
  84
  85                         batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
  86                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  87                         batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  88                         //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  89                         //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  90                         batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
  91                         "set BENCHVIEWNAME=${benchViewName}\n" +
  92                         "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
  93                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
  94                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  95                         batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  96                         batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  97
  98                         if (arch == 'x86jit32')
  99                         {
 100                             // Download package and copy compatjit into Core_Root
 101                             batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
 102                             "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
 103                         }
 104
 105                         batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 106
 107                         // Run with just stopwatch
 108                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType}")
 109                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType}")
 110
 111                         // Run with the full set of counters enabled
 112                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 113                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
 114                     }
 115                 }
 116
 117                 if (isSmoketest)
 118                 {
 119                     Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
 120                 }
 121                 // Save machinedata.json to /artifact/bin/ Jenkins dir
 122                 def archiveSettings = new ArchivalSettings()
 123                 archiveSettings.addFiles('Perf-*.xml')
 124                 archiveSettings.addFiles('Perf-*.etl')
 125                 Utilities.addArchival(newJob, archiveSettings)
 126
 127                 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 128
 129                 newJob.with {
 130                     wrappers {
 131                         timeout {
 132                             absolute(240)
 133                         }
 134                     }
 135                 }
 136
 137                 if (isPR) {
 138                     TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 139                     if (isSmoketest)
 140                     {
 141                         builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests Correctness")
 142                     }
 143                     else
 144                     {
 145                         builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
 146                         builder.triggerOnlyOnComment()
 147                         builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
 148                     }
 149                     builder.triggerForBranch(branch)
 150                     builder.emitTrigger(newJob)
 151                 }
 152                 else {
 153                     // Set a push trigger
 154                     TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 155                     builder.emitTrigger(newJob)
 156                 }
 157             }
 158         }
 159     }
 160 }
 161
 162 // Setup throughput perflab tests runs
 163 [true, false].each { isPR ->
 164     ['Windows_NT'].each { os ->
 165         ['x64', 'x86'].each { arch ->
 166             def architecture = arch
 167
 168             def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) {
 169                 // Set the label.
 170                 label('windows_clr_perf')
 171                 wrappers {
 172                     credentialsBinding {
 173                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 174                     }
 175                 }
 176
 177             if (isPR)
 178             {
 179                 parameters
 180                 {
 181                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 182                 }
 183             }
 184             def configuration = 'Release'
 185             def runType = isPR ? 'private' : 'rolling'
 186             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 187
 188                 steps {
 189                     // Batch
 190
 191                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 192                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 193                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 194                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 195                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 196                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 197                     batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
 198                     "set BENCHVIEWNAME=${benchViewName}\n" +
 199                     "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
 200                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
 201                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 202                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 203                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
 204                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 205                     batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 206                 }
 207             }
 208
 209             // Save machinedata.json to /artifact/bin/ Jenkins dir
 210             def archiveSettings = new ArchivalSettings()
 211             archiveSettings.addFiles('throughput-*.csv')
 212             Utilities.addArchival(newJob, archiveSettings)
 213
 214             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 215
 216             if (isPR) {
 217                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 218                 builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests")
 219                 builder.triggerOnlyOnComment()
 220                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*")
 221                 builder.triggerForBranch(branch)
 222                 builder.emitTrigger(newJob)
 223             }
 224             else {
 225                 // Set a push trigger
 226                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 227                 builder.emitTrigger(newJob)
 228             }
 229         }
 230     }
 231 }
 232
 233 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 234 [true, false].each { isPR ->
 235     ['Ubuntu14.04'].each { os ->
 236         def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
 237
 238             label('linux_clr_perf')
 239                 wrappers {
 240                     credentialsBinding {
 241                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 242                     }
 243                 }
 244
 245             if (isPR)
 246             {
 247                 parameters
 248                 {
 249                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 250                 }
 251             }
 252             def osGroup = getOSGroup(os)
 253             def architecture = 'x64'
 254             def configuration = 'Release'
 255             def runType = isPR ? 'private' : 'rolling'
 256             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 257
 258             steps {
 259                 shell("bash ./tests/scripts/perf-prep.sh")
 260                 shell("./init-tools.sh")
 261                 shell("./build.sh ${architecture} ${configuration}")
 262                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 263                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 264                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 265                 shell("""sudo -E bash ./tests/scripts/run-xunit-perf.sh \\
 266                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 267                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 268                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 269                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 270                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 271                 --runType=\"${runType}\" \\
 272                 --benchViewOS=\"${os}\" \\
 273                 --uploadToBenchview""")
 274             }
 275         }
 276
 277         // Save machinedata.json to /artifact/bin/ Jenkins dir
 278         def archiveSettings = new ArchivalSettings()
 279         archiveSettings.addFiles('sandbox/perf-*.xml')
 280         archiveSettings.addFiles('machinedata.json')
 281         Utilities.addArchival(newJob, archiveSettings)
 282
 283         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 284
 285         // For perf, we need to keep the run results longer
 286         newJob.with {
 287             // Enable the log rotator
 288             logRotator {
 289                 artifactDaysToKeep(7)
 290                 daysToKeep(300)
 291                 artifactNumToKeep(25)
 292                 numToKeep(1000)
 293             }
 294         }
 295         if (isPR) {
 296             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 297             builder.setGithubContext("${os} Perf Tests")
 298             builder.triggerOnlyOnComment()
 299             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
 300             builder.triggerForBranch(branch)
 301             builder.emitTrigger(newJob)
 302         }
 303         else {
 304             // Set a push trigger
 305             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 306             builder.emitTrigger(newJob)
 307         }
 308     } // os
 309 } // isPR
 310
 311 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 312 [true, false].each { isPR ->
 313     ['Ubuntu14.04'].each { os ->
 314         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)) {
 315
 316             label('linux_clr_perf')
 317                 wrappers {
 318                     credentialsBinding {
 319                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 320                     }
 321                 }
 322
 323             if (isPR)
 324             {
 325                 parameters
 326                 {
 327                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 328                 }
 329             }
 330             def osGroup = getOSGroup(os)
 331             def architecture = 'x64'
 332             def configuration = 'Release'
 333             def runType = isPR ? 'private' : 'rolling'
 334             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 335
 336             steps {
 337                 shell("bash ./tests/scripts/perf-prep.sh --throughput")
 338                 shell("./init-tools.sh")
 339                 shell("./build.sh ${architecture} ${configuration}")
 340                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 341                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 342                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 343                 shell("""sudo -E python3.5 ./tests/scripts/run-throughput-perf.py \\
 344                 -arch \"${architecture}\" \\
 345                 -os \"${os}\" \\
 346                 -configuration \"${configuration}\" \\
 347                 -clr_root \"\${WORKSPACE}\" \\
 348                 -assembly_root \"\${WORKSPACE}/_/fx/bin/runtime/netcoreapp-${osGroup}-${configuration}-${architecture}\" \\
 349                 -run_type \"${runType}\" \\
 350                 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 351             }
 352         }
 353
 354         // Save machinedata.json to /artifact/bin/ Jenkins dir
 355         def archiveSettings = new ArchivalSettings()
 356         archiveSettings.addFiles('throughput-*.csv')
 357         archiveSettings.addFiles('machinedata.json')
 358         Utilities.addArchival(newJob, archiveSettings)
 359
 360         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 361
 362         // For perf, we need to keep the run results longer
 363         newJob.with {
 364             // Enable the log rotator
 365             logRotator {
 366                 artifactDaysToKeep(7)
 367                 daysToKeep(300)
 368                 artifactNumToKeep(25)
 369                 numToKeep(1000)
 370             }
 371         }
 372         if (isPR) {
 373             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 374             builder.setGithubContext("${os} Throughput Perf Tests")
 375             builder.triggerOnlyOnComment()
 376             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+throughput.*")
 377             builder.triggerForBranch(branch)
 378             builder.emitTrigger(newJob)
 379         }
 380         else {
 381             // Set a push trigger
 382             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 383             builder.emitTrigger(newJob)
 384         }
 385     } // os
 386 } // isPR