perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora23':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27 // Setup perflab tests runs
  28 [true, false].each { isPR ->
  29     ['Windows_NT'].each { os ->
  30                 ['x64', 'x86', 'x86jit32'].each { arch ->
  31             def architecture = arch
  32             def testEnv = ''
  33
  34             if (arch == 'x86jit32')
  35             {
  36                 architecture = 'x86'
  37                 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\compatjit_x86_testenv.cmd'
  38             }
  39             else if (arch == 'x86')
  40             {
  41                 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
  42             }
  43
  44                         def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}_${arch}", isPR)) {
  45                                 // Set the label.
  46                                 label('windows_clr_perf')
  47                                 wrappers {
  48                                         credentialsBinding {
  49                                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  50                                         }
  51                                 }
  52
  53                         if (isPR)
  54                         {
  55                                 parameters
  56                                 {
  57                                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  58                                 }
  59                         }
  60                         def configuration = 'Release'
  61                         def runType = isPR ? 'private' : 'rolling'
  62                         def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  63
  64                                 steps {
  65                                         // Batch
  66
  67                                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  68                                         batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  69                                         //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  70                                         //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  71                                         batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
  72                                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
  73                                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  74                                         batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  75                                         batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  76                                         batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  77
  78                     if (arch == 'x86jit32')
  79                     {
  80                         // Download package and copy compatjit into Core_Root
  81                         batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
  82                         "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
  83                     }
  84
  85                                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} ${testEnv} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
  86                                         batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} ${testEnv} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
  87                                 }
  88                         }
  89
  90                         // Save machinedata.json to /artifact/bin/ Jenkins dir
  91                         def archiveSettings = new ArchivalSettings()
  92                         archiveSettings.addFiles('perf-*.xml')
  93                         archiveSettings.addFiles('perf-*.etl')
  94                         Utilities.addArchival(newJob, archiveSettings)
  95
  96                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
  97
  98                         if (isPR) {
  99                                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 100                                 builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
 101                                 builder.triggerOnlyOnComment()
 102                                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
 103                                 builder.triggerForBranch(branch)
 104                                 builder.emitTrigger(newJob)
 105                         }
 106                         else {
 107                                 // Set a push trigger
 108                                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 109                                 builder.emitTrigger(newJob)
 110                         }
 111                 }
 112     }
 113 }
 114
 115 // Setup throughput perflab tests runs
 116 [true, false].each { isPR ->
 117     ['Windows_NT'].each { os ->
 118                 ['x64', 'x86', 'x86jit32'].each { arch ->
 119             def architecture = arch
 120
 121             if (arch == 'x86jit32')
 122             {
 123                 architecture = 'x86'
 124             }
 125
 126                         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) {
 127                                 // Set the label.
 128                                 label('windows_clr_perf')
 129                                 wrappers {
 130                                         credentialsBinding {
 131                                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 132                                         }
 133                                 }
 134
 135                         if (isPR)
 136                         {
 137                                 parameters
 138                                 {
 139                                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 140                                 }
 141                         }
 142                         def configuration = 'Release'
 143                         def runType = isPR ? 'private' : 'rolling'
 144                         def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 145
 146                                 steps {
 147                                         // Batch
 148
 149                                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 150                                         batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 151                                         batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 152                                         batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 153                                         //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 154                                         //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 155                                         batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
 156                                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
 157                                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 158                                         batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 159                                         batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
 160                                         batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 161                     if (arch == 'x86jit32')
 162                     {
 163                         // Download package and copy compatjit into Core_Root
 164                         batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
 165                         "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
 166                     }
 167                                         batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 168                                 }
 169                         }
 170
 171                         // Save machinedata.json to /artifact/bin/ Jenkins dir
 172                         def archiveSettings = new ArchivalSettings()
 173                         archiveSettings.addFiles('throughput-*.csv')
 174                         Utilities.addArchival(newJob, archiveSettings)
 175
 176                         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 177
 178                         if (isPR) {
 179                                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 180                                 builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests")
 181                                 builder.triggerOnlyOnComment()
 182                                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*")
 183                                 builder.triggerForBranch(branch)
 184                                 builder.emitTrigger(newJob)
 185                         }
 186                         else {
 187                                 // Set a push trigger
 188                                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 189                                 builder.emitTrigger(newJob)
 190                         }
 191                 }
 192     }
 193 }
 194
 195 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 196 [true, false].each { isPR ->
 197     ['Ubuntu14.04'].each { os ->
 198         def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
 199
 200                         label('linux_clr_perf')
 201                                 wrappers {
 202                                         credentialsBinding {
 203                                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 204                                         }
 205                                 }
 206
 207                         if (isPR)
 208                         {
 209                                 parameters
 210                                 {
 211                                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 212                                 }
 213                         }
 214                         def osGroup = getOSGroup(os)
 215                         def architecture = 'x64'
 216                         def configuration = 'Release'
 217                         def runType = isPR ? 'private' : 'rolling'
 218                         def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 219
 220             steps {
 221                 shell("bash ./tests/scripts/perf-prep.sh")
 222                 shell("./init-tools.sh")
 223                                 shell("./build.sh ${architecture} ${configuration}")
 224                                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 225                                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 226                                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 227                 shell("""sudo -E bash ./tests/scripts/run-xunit-perf.sh \\
 228                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 229                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 230                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 231                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 232                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 233                                 --runType=\"${runType}\" \\
 234                                 --benchViewOS=\"${os}\" \\
 235                                 --uploadToBenchview""")
 236             }
 237         }
 238
 239         // Save machinedata.json to /artifact/bin/ Jenkins dir
 240         def archiveSettings = new ArchivalSettings()
 241         archiveSettings.addFiles('sandbox/perf-*.xml')
 242         archiveSettings.addFiles('machinedata.json')
 243         Utilities.addArchival(newJob, archiveSettings)
 244
 245         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 246
 247         // For perf, we need to keep the run results longer
 248         newJob.with {
 249             // Enable the log rotator
 250             logRotator {
 251                 artifactDaysToKeep(7)
 252                 daysToKeep(300)
 253                 artifactNumToKeep(25)
 254                 numToKeep(1000)
 255             }
 256         }
 257         if (isPR) {
 258             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 259             builder.setGithubContext("${os} Perf Tests")
 260             builder.triggerOnlyOnComment()
 261             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
 262             builder.triggerForBranch(branch)
 263             builder.emitTrigger(newJob)
 264         }
 265         else {
 266             // Set a push trigger
 267             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 268             builder.emitTrigger(newJob)
 269         }
 270     } // os
 271 } // isPR
 272
 273 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 274 [true, false].each { isPR ->
 275     ['Ubuntu14.04'].each { os ->
 276         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)) {
 277
 278                         label('linux_clr_perf')
 279                                 wrappers {
 280                                         credentialsBinding {
 281                                                 string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 282                                         }
 283                                 }
 284
 285                         if (isPR)
 286                         {
 287                                 parameters
 288                                 {
 289                                         stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 290                                 }
 291                         }
 292                         def osGroup = getOSGroup(os)
 293                         def architecture = 'x64'
 294                         def configuration = 'Release'
 295                         def runType = isPR ? 'private' : 'rolling'
 296                         def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 297
 298             steps {
 299                 shell("bash ./tests/scripts/perf-prep.sh --throughput")
 300                 shell("./init-tools.sh")
 301                                 shell("./build.sh ${architecture} ${configuration}")
 302                                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 303                                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 304                                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 305                 shell("""sudo -E python3.5 ./tests/scripts/run-throughput-perf.py \\
 306                 -arch \"${architecture}\" \\
 307                 -os \"${os}\" \\
 308                 -configuration \"${configuration}\" \\
 309                 -clr_root \"\${WORKSPACE}\" \\
 310                 -assembly_root \"\${WORKSPACE}/_/fx/bin/runtime/netcoreapp-${osGroup}-${configuration}-${architecture}\" \\
 311                                 -run_type \"${runType}\" \\
 312                                 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 313             }
 314         }
 315
 316         // Save machinedata.json to /artifact/bin/ Jenkins dir
 317         def archiveSettings = new ArchivalSettings()
 318         archiveSettings.addFiles('throughput-*.csv')
 319         archiveSettings.addFiles('machinedata.json')
 320         Utilities.addArchival(newJob, archiveSettings)
 321
 322         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 323
 324         // For perf, we need to keep the run results longer
 325         newJob.with {
 326             // Enable the log rotator
 327             logRotator {
 328                 artifactDaysToKeep(7)
 329                 daysToKeep(300)
 330                 artifactNumToKeep(25)
 331                 numToKeep(1000)
 332             }
 333         }
 334         if (isPR) {
 335             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 336             builder.setGithubContext("${os} Throughput Perf Tests")
 337             builder.triggerOnlyOnComment()
 338             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+throughput.*")
 339             builder.triggerForBranch(branch)
 340             builder.emitTrigger(newJob)
 341         }
 342         else {
 343             // Set a push trigger
 344             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 345             builder.emitTrigger(newJob)
 346         }
 347     } // os
 348 } // isPR