perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27 // Setup perflab tests runs
  28 [true, false].each { isPR ->
  29     ['Windows_NT'].each { os ->
  30         ['x64', 'x86'].each { arch ->
  31             def architecture = arch
  32
  33             def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}_${arch}", isPR)) {
  34                 // Set the label.
  35                 label('windows_clr_perf')
  36                 wrappers {
  37                     credentialsBinding {
  38                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  39                     }
  40                 }
  41
  42             if (isPR)
  43             {
  44                 parameters
  45                 {
  46                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  47                 }
  48             }
  49             def configuration = 'Release'
  50             def runType = isPR ? 'private' : 'rolling'
  51             def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  52
  53                 steps {
  54                     // Batch
  55
  56                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  57                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  58                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  59                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  60                     batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
  61                     "set BENCHVIEWNAME=${benchViewName}\n" +
  62                     "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
  63                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
  64                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  65                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  66                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  67
  68                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  69
  70                     batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
  71                     batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
  72                 }
  73             }
  74
  75             // Save machinedata.json to /artifact/bin/ Jenkins dir
  76             def archiveSettings = new ArchivalSettings()
  77             archiveSettings.addFiles('Perf-*.xml')
  78             archiveSettings.addFiles('Perf-*.etl')
  79             Utilities.addArchival(newJob, archiveSettings)
  80
  81             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
  82
  83             newJob.with {
  84                 wrappers {
  85                     timeout {
  86                         absolute(240)
  87                     }
  88                 }
  89             }
  90
  91             if (isPR) {
  92                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
  93                 builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
  94                 builder.triggerOnlyOnComment()
  95                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
  96                 builder.triggerForBranch(branch)
  97                 builder.emitTrigger(newJob)
  98             }
  99             else {
 100                 // Set a push trigger
 101                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 102                 builder.emitTrigger(newJob)
 103             }
 104         }
 105     }
 106 }
 107
 108 // Setup throughput perflab tests runs
 109 [true, false].each { isPR ->
 110     ['Windows_NT'].each { os ->
 111         ['x64', 'x86'].each { arch ->
 112             def architecture = arch
 113
 114             def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) {
 115                 // Set the label.
 116                 label('windows_clr_perf')
 117                 wrappers {
 118                     credentialsBinding {
 119                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 120                     }
 121                 }
 122
 123             if (isPR)
 124             {
 125                 parameters
 126                 {
 127                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 128                 }
 129             }
 130             def configuration = 'Release'
 131             def runType = isPR ? 'private' : 'rolling'
 132             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 133
 134                 steps {
 135                     // Batch
 136
 137                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 138                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 139                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 140                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 141                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 142                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 143                     batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
 144                     "set BENCHVIEWNAME=${benchViewName}\n" +
 145                     "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
 146                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
 147                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 148                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 149                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
 150                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 151                     batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 152                 }
 153             }
 154
 155             // Save machinedata.json to /artifact/bin/ Jenkins dir
 156             def archiveSettings = new ArchivalSettings()
 157             archiveSettings.addFiles('throughput-*.csv')
 158             Utilities.addArchival(newJob, archiveSettings)
 159
 160             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 161
 162             if (isPR) {
 163                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 164                 builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests")
 165                 builder.triggerOnlyOnComment()
 166                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*")
 167                 builder.triggerForBranch(branch)
 168                 builder.emitTrigger(newJob)
 169             }
 170             else {
 171                 // Set a push trigger
 172                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 173                 builder.emitTrigger(newJob)
 174             }
 175         }
 176     }
 177 }
 178
 179 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 180 [true, false].each { isPR ->
 181     ['Ubuntu14.04'].each { os ->
 182         def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
 183
 184             label('linux_clr_perf')
 185                 wrappers {
 186                     credentialsBinding {
 187                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 188                     }
 189                 }
 190
 191             if (isPR)
 192             {
 193                 parameters
 194                 {
 195                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 196                 }
 197             }
 198             def osGroup = getOSGroup(os)
 199             def architecture = 'x64'
 200             def configuration = 'Release'
 201             def runType = isPR ? 'private' : 'rolling'
 202             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 203
 204             steps {
 205                 shell("bash ./tests/scripts/perf-prep.sh")
 206                 shell("./init-tools.sh")
 207                 shell("./build.sh ${architecture} ${configuration}")
 208                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 209                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 210                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 211                 shell("""sudo -E bash ./tests/scripts/run-xunit-perf.sh \\
 212                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 213                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 214                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 215                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 216                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 217                 --runType=\"${runType}\" \\
 218                 --benchViewOS=\"${os}\" \\
 219                 --uploadToBenchview""")
 220             }
 221         }
 222
 223         // Save machinedata.json to /artifact/bin/ Jenkins dir
 224         def archiveSettings = new ArchivalSettings()
 225         archiveSettings.addFiles('sandbox/perf-*.xml')
 226         archiveSettings.addFiles('machinedata.json')
 227         Utilities.addArchival(newJob, archiveSettings)
 228
 229         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 230
 231         // For perf, we need to keep the run results longer
 232         newJob.with {
 233             // Enable the log rotator
 234             logRotator {
 235                 artifactDaysToKeep(7)
 236                 daysToKeep(300)
 237                 artifactNumToKeep(25)
 238                 numToKeep(1000)
 239             }
 240         }
 241         if (isPR) {
 242             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 243             builder.setGithubContext("${os} Perf Tests")
 244             builder.triggerOnlyOnComment()
 245             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
 246             builder.triggerForBranch(branch)
 247             builder.emitTrigger(newJob)
 248         }
 249         else {
 250             // Set a push trigger
 251             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 252             builder.emitTrigger(newJob)
 253         }
 254     } // os
 255 } // isPR
 256
 257 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 258 [true, false].each { isPR ->
 259     ['Ubuntu14.04'].each { os ->
 260         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)) {
 261
 262             label('linux_clr_perf')
 263                 wrappers {
 264                     credentialsBinding {
 265                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 266                     }
 267                 }
 268
 269             if (isPR)
 270             {
 271                 parameters
 272                 {
 273                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 274                 }
 275             }
 276             def osGroup = getOSGroup(os)
 277             def architecture = 'x64'
 278             def configuration = 'Release'
 279             def runType = isPR ? 'private' : 'rolling'
 280             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 281
 282             steps {
 283                 shell("bash ./tests/scripts/perf-prep.sh --throughput")
 284                 shell("./init-tools.sh")
 285                 shell("./build.sh ${architecture} ${configuration}")
 286                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 287                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 288                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 289                 shell("""sudo -E python3.5 ./tests/scripts/run-throughput-perf.py \\
 290                 -arch \"${architecture}\" \\
 291                 -os \"${os}\" \\
 292                 -configuration \"${configuration}\" \\
 293                 -clr_root \"\${WORKSPACE}\" \\
 294                 -assembly_root \"\${WORKSPACE}/_/fx/bin/runtime/netcoreapp-${osGroup}-${configuration}-${architecture}\" \\
 295                 -run_type \"${runType}\" \\
 296                 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 297             }
 298         }
 299
 300         // Save machinedata.json to /artifact/bin/ Jenkins dir
 301         def archiveSettings = new ArchivalSettings()
 302         archiveSettings.addFiles('throughput-*.csv')
 303         archiveSettings.addFiles('machinedata.json')
 304         Utilities.addArchival(newJob, archiveSettings)
 305
 306         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 307
 308         // For perf, we need to keep the run results longer
 309         newJob.with {
 310             // Enable the log rotator
 311             logRotator {
 312                 artifactDaysToKeep(7)
 313                 daysToKeep(300)
 314                 artifactNumToKeep(25)
 315                 numToKeep(1000)
 316             }
 317         }
 318         if (isPR) {
 319             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 320             builder.setGithubContext("${os} Throughput Perf Tests")
 321             builder.triggerOnlyOnComment()
 322             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+throughput.*")
 323             builder.triggerForBranch(branch)
 324             builder.emitTrigger(newJob)
 325         }
 326         else {
 327             // Set a push trigger
 328             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 329             builder.emitTrigger(newJob)
 330         }
 331     } // os
 332 } // isPR