perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27 // Setup perflab tests runs
  28 [true, false].each { isPR ->
  29     ['Windows_NT'].each { os ->
  30         ['x64', 'x86', 'x86jit32'].each { arch ->
  31             def architecture = arch
  32             def testEnv = ''
  33
  34             if (arch == 'x86jit32')
  35             {
  36                 architecture = 'x86'
  37                 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\compatjit_x86_testenv.cmd'
  38             }
  39             else if (arch == 'x86')
  40             {
  41                 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
  42             }
  43
  44             def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}_${arch}", isPR)) {
  45                 // Set the label.
  46                 label('windows_clr_perf')
  47                 wrappers {
  48                     credentialsBinding {
  49                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  50                     }
  51                 }
  52
  53             if (isPR)
  54             {
  55                 parameters
  56                 {
  57                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  58                 }
  59             }
  60             def configuration = 'Release'
  61             def runType = isPR ? 'private' : 'rolling'
  62             def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  63
  64                 steps {
  65                     // Batch
  66
  67                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  68                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  69                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  70                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  71                     batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
  72                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
  73                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  74                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  75                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  76                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  77
  78                     if (arch == 'x86jit32')
  79                     {
  80                         // Download package and copy compatjit into Core_Root
  81                         batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
  82                         "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
  83                     }
  84
  85                     batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} ${testEnv} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
  86                     batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} ${testEnv} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
  87                 }
  88             }
  89
  90             // Save machinedata.json to /artifact/bin/ Jenkins dir
  91             def archiveSettings = new ArchivalSettings()
  92             archiveSettings.addFiles('Perf-*.xml')
  93             archiveSettings.addFiles('Perf-*.etl')
  94             Utilities.addArchival(newJob, archiveSettings)
  95
  96             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
  97
  98             newJob.with {
  99                 wrappers {
 100                     timeout {
 101                         absolute(240)
 102                     }
 103                 }
 104             }
 105
 106             if (isPR) {
 107                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 108                 builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
 109                 builder.triggerOnlyOnComment()
 110                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
 111                 builder.triggerForBranch(branch)
 112                 builder.emitTrigger(newJob)
 113             }
 114             else {
 115                 // Set a push trigger
 116                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 117                 builder.emitTrigger(newJob)
 118             }
 119         }
 120     }
 121 }
 122
 123 // Setup throughput perflab tests runs
 124 [true, false].each { isPR ->
 125     ['Windows_NT'].each { os ->
 126         ['x64', 'x86', 'x86jit32'].each { arch ->
 127             def architecture = arch
 128
 129             if (arch == 'x86jit32')
 130             {
 131                 architecture = 'x86'
 132             }
 133
 134             def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) {
 135                 // Set the label.
 136                 label('windows_clr_perf')
 137                 wrappers {
 138                     credentialsBinding {
 139                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 140                     }
 141                 }
 142
 143             if (isPR)
 144             {
 145                 parameters
 146                 {
 147                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 148                 }
 149             }
 150             def configuration = 'Release'
 151             def runType = isPR ? 'private' : 'rolling'
 152             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 153
 154                 steps {
 155                     // Batch
 156
 157                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 158                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 159                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 160                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 161                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 162                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 163                     batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
 164                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
 165                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 166                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 167                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
 168                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 169                     if (arch == 'x86jit32')
 170                     {
 171                         // Download package and copy compatjit into Core_Root
 172                         batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
 173                         "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
 174                     }
 175                     batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 176                 }
 177             }
 178
 179             // Save machinedata.json to /artifact/bin/ Jenkins dir
 180             def archiveSettings = new ArchivalSettings()
 181             archiveSettings.addFiles('throughput-*.csv')
 182             Utilities.addArchival(newJob, archiveSettings)
 183
 184             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 185
 186             if (isPR) {
 187                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 188                 builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests")
 189                 builder.triggerOnlyOnComment()
 190                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*")
 191                 builder.triggerForBranch(branch)
 192                 builder.emitTrigger(newJob)
 193             }
 194             else {
 195                 // Set a push trigger
 196                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 197                 builder.emitTrigger(newJob)
 198             }
 199         }
 200     }
 201 }
 202
 203 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 204 [true, false].each { isPR ->
 205     ['Ubuntu14.04'].each { os ->
 206         def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
 207
 208             label('linux_clr_perf')
 209                 wrappers {
 210                     credentialsBinding {
 211                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 212                     }
 213                 }
 214
 215             if (isPR)
 216             {
 217                 parameters
 218                 {
 219                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 220                 }
 221             }
 222             def osGroup = getOSGroup(os)
 223             def architecture = 'x64'
 224             def configuration = 'Release'
 225             def runType = isPR ? 'private' : 'rolling'
 226             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 227
 228             steps {
 229                 shell("bash ./tests/scripts/perf-prep.sh")
 230                 shell("./init-tools.sh")
 231                 shell("./build.sh ${architecture} ${configuration}")
 232                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 233                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 234                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 235                 shell("""sudo -E bash ./tests/scripts/run-xunit-perf.sh \\
 236                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 237                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 238                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 239                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 240                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 241                 --runType=\"${runType}\" \\
 242                 --benchViewOS=\"${os}\" \\
 243                 --uploadToBenchview""")
 244             }
 245         }
 246
 247         // Save machinedata.json to /artifact/bin/ Jenkins dir
 248         def archiveSettings = new ArchivalSettings()
 249         archiveSettings.addFiles('sandbox/perf-*.xml')
 250         archiveSettings.addFiles('machinedata.json')
 251         Utilities.addArchival(newJob, archiveSettings)
 252
 253         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 254
 255         // For perf, we need to keep the run results longer
 256         newJob.with {
 257             // Enable the log rotator
 258             logRotator {
 259                 artifactDaysToKeep(7)
 260                 daysToKeep(300)
 261                 artifactNumToKeep(25)
 262                 numToKeep(1000)
 263             }
 264         }
 265         if (isPR) {
 266             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 267             builder.setGithubContext("${os} Perf Tests")
 268             builder.triggerOnlyOnComment()
 269             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
 270             builder.triggerForBranch(branch)
 271             builder.emitTrigger(newJob)
 272         }
 273         else {
 274             // Set a push trigger
 275             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 276             builder.emitTrigger(newJob)
 277         }
 278     } // os
 279 } // isPR
 280
 281 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 282 [true, false].each { isPR ->
 283     ['Ubuntu14.04'].each { os ->
 284         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)) {
 285
 286             label('linux_clr_perf')
 287                 wrappers {
 288                     credentialsBinding {
 289                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 290                     }
 291                 }
 292
 293             if (isPR)
 294             {
 295                 parameters
 296                 {
 297                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 298                 }
 299             }
 300             def osGroup = getOSGroup(os)
 301             def architecture = 'x64'
 302             def configuration = 'Release'
 303             def runType = isPR ? 'private' : 'rolling'
 304             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 305
 306             steps {
 307                 shell("bash ./tests/scripts/perf-prep.sh --throughput")
 308                 shell("./init-tools.sh")
 309                 shell("./build.sh ${architecture} ${configuration}")
 310                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 311                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 312                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 313                 shell("""sudo -E python3.5 ./tests/scripts/run-throughput-perf.py \\
 314                 -arch \"${architecture}\" \\
 315                 -os \"${os}\" \\
 316                 -configuration \"${configuration}\" \\
 317                 -clr_root \"\${WORKSPACE}\" \\
 318                 -assembly_root \"\${WORKSPACE}/_/fx/bin/runtime/netcoreapp-${osGroup}-${configuration}-${architecture}\" \\
 319                 -run_type \"${runType}\" \\
 320                 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 321             }
 322         }
 323
 324         // Save machinedata.json to /artifact/bin/ Jenkins dir
 325         def archiveSettings = new ArchivalSettings()
 326         archiveSettings.addFiles('throughput-*.csv')
 327         archiveSettings.addFiles('machinedata.json')
 328         Utilities.addArchival(newJob, archiveSettings)
 329
 330         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 331
 332         // For perf, we need to keep the run results longer
 333         newJob.with {
 334             // Enable the log rotator
 335             logRotator {
 336                 artifactDaysToKeep(7)
 337                 daysToKeep(300)
 338                 artifactNumToKeep(25)
 339                 numToKeep(1000)
 340             }
 341         }
 342         if (isPR) {
 343             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 344             builder.setGithubContext("${os} Throughput Perf Tests")
 345             builder.triggerOnlyOnComment()
 346             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+throughput.*")
 347             builder.triggerForBranch(branch)
 348             builder.emitTrigger(newJob)
 349         }
 350         else {
 351             // Set a push trigger
 352             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 353             builder.emitTrigger(newJob)
 354         }
 355     } // os
 356 } // isPR