perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'OSX':'OSX',
  16         'Windows_NT':'Windows_NT',
  17         'FreeBSD':'FreeBSD',
  18         'CentOS7.1': 'Linux',
  19         'OpenSUSE13.2': 'Linux',
  20         'OpenSUSE42.1': 'Linux',
  21         'LinuxARMEmulator': 'Linux']
  22     def osGroup = osGroupMap.get(os, null)
  23     assert osGroup != null : "Could not find os group for ${os}"
  24     return osGroupMap[os]
  25 }
  26 // Setup perflab tests runs
  27 [true, false].each { isPR ->
  28     ['Windows_NT'].each { os ->
  29         ['x64', 'x86', 'x86jit32'].each { arch ->
  30             def architecture = arch
  31             def testEnv = ''
  32
  33             if (arch == 'x86jit32')
  34             {
  35                 architecture = 'x86'
  36                 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\compatjit_x86_testenv.cmd'
  37             }
  38             else if (arch == 'x86')
  39             {
  40                 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
  41             }
  42
  43             def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}_${arch}", isPR)) {
  44                 // Set the label.
  45                 label('windows_clr_perf')
  46                 wrappers {
  47                     credentialsBinding {
  48                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  49                     }
  50                 }
  51
  52             if (isPR)
  53             {
  54                 parameters
  55                 {
  56                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  57                 }
  58             }
  59             def configuration = 'Release'
  60             def runType = isPR ? 'private' : 'rolling'
  61             def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  62
  63                 steps {
  64                     // Batch
  65
  66                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  67                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  68                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  69                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  70                     batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
  71                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
  72                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  73                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  74                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  75                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  76
  77                     if (arch == 'x86jit32')
  78                     {
  79                         // Download package and copy compatjit into Core_Root
  80                         batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
  81                         "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
  82                     }
  83
  84                     batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} ${testEnv} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
  85                     batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} ${testEnv} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
  86                 }
  87             }
  88
  89             // Save machinedata.json to /artifact/bin/ Jenkins dir
  90             def archiveSettings = new ArchivalSettings()
  91             archiveSettings.addFiles('Perf-*.xml')
  92             archiveSettings.addFiles('Perf-*.etl')
  93             Utilities.addArchival(newJob, archiveSettings)
  94
  95             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
  96
  97             newJob.with {
  98                 wrappers {
  99                     timeout {
 100                         absolute(240)
 101                     }
 102                 }
 103             }
 104
 105             if (isPR) {
 106                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 107                 builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
 108                 builder.triggerOnlyOnComment()
 109                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
 110                 builder.triggerForBranch(branch)
 111                 builder.emitTrigger(newJob)
 112             }
 113             else {
 114                 // Set a push trigger
 115                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 116                 builder.emitTrigger(newJob)
 117             }
 118         }
 119     }
 120 }
 121
 122 // Setup throughput perflab tests runs
 123 [true, false].each { isPR ->
 124     ['Windows_NT'].each { os ->
 125         ['x64', 'x86', 'x86jit32'].each { arch ->
 126             def architecture = arch
 127
 128             if (arch == 'x86jit32')
 129             {
 130                 architecture = 'x86'
 131             }
 132
 133             def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) {
 134                 // Set the label.
 135                 label('windows_clr_perf')
 136                 wrappers {
 137                     credentialsBinding {
 138                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 139                     }
 140                 }
 141
 142             if (isPR)
 143             {
 144                 parameters
 145                 {
 146                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 147                 }
 148             }
 149             def configuration = 'Release'
 150             def runType = isPR ? 'private' : 'rolling'
 151             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 152
 153                 steps {
 154                     // Batch
 155
 156                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 157                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 158                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 159                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 160                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 161                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 162                     batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
 163                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
 164                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 165                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 166                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
 167                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 168                     if (arch == 'x86jit32')
 169                     {
 170                         // Download package and copy compatjit into Core_Root
 171                         batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
 172                         "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
 173                     }
 174                     batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 175                 }
 176             }
 177
 178             // Save machinedata.json to /artifact/bin/ Jenkins dir
 179             def archiveSettings = new ArchivalSettings()
 180             archiveSettings.addFiles('throughput-*.csv')
 181             Utilities.addArchival(newJob, archiveSettings)
 182
 183             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 184
 185             if (isPR) {
 186                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 187                 builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests")
 188                 builder.triggerOnlyOnComment()
 189                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*")
 190                 builder.triggerForBranch(branch)
 191                 builder.emitTrigger(newJob)
 192             }
 193             else {
 194                 // Set a push trigger
 195                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 196                 builder.emitTrigger(newJob)
 197             }
 198         }
 199     }
 200 }
 201
 202 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 203 [true, false].each { isPR ->
 204     ['Ubuntu14.04'].each { os ->
 205         def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
 206
 207             label('linux_clr_perf')
 208                 wrappers {
 209                     credentialsBinding {
 210                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 211                     }
 212                 }
 213
 214             if (isPR)
 215             {
 216                 parameters
 217                 {
 218                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 219                 }
 220             }
 221             def osGroup = getOSGroup(os)
 222             def architecture = 'x64'
 223             def configuration = 'Release'
 224             def runType = isPR ? 'private' : 'rolling'
 225             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 226
 227             steps {
 228                 shell("bash ./tests/scripts/perf-prep.sh")
 229                 shell("./init-tools.sh")
 230                 shell("./build.sh ${architecture} ${configuration}")
 231                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 232                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 233                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 234                 shell("""sudo -E bash ./tests/scripts/run-xunit-perf.sh \\
 235                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 236                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 237                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 238                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 239                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 240                 --runType=\"${runType}\" \\
 241                 --benchViewOS=\"${os}\" \\
 242                 --uploadToBenchview""")
 243             }
 244         }
 245
 246         // Save machinedata.json to /artifact/bin/ Jenkins dir
 247         def archiveSettings = new ArchivalSettings()
 248         archiveSettings.addFiles('sandbox/perf-*.xml')
 249         archiveSettings.addFiles('machinedata.json')
 250         Utilities.addArchival(newJob, archiveSettings)
 251
 252         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 253
 254         // For perf, we need to keep the run results longer
 255         newJob.with {
 256             // Enable the log rotator
 257             logRotator {
 258                 artifactDaysToKeep(7)
 259                 daysToKeep(300)
 260                 artifactNumToKeep(25)
 261                 numToKeep(1000)
 262             }
 263         }
 264         if (isPR) {
 265             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 266             builder.setGithubContext("${os} Perf Tests")
 267             builder.triggerOnlyOnComment()
 268             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
 269             builder.triggerForBranch(branch)
 270             builder.emitTrigger(newJob)
 271         }
 272         else {
 273             // Set a push trigger
 274             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 275             builder.emitTrigger(newJob)
 276         }
 277     } // os
 278 } // isPR
 279
 280 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 281 [true, false].each { isPR ->
 282     ['Ubuntu14.04'].each { os ->
 283         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)) {
 284
 285             label('linux_clr_perf')
 286                 wrappers {
 287                     credentialsBinding {
 288                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 289                     }
 290                 }
 291
 292             if (isPR)
 293             {
 294                 parameters
 295                 {
 296                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 297                 }
 298             }
 299             def osGroup = getOSGroup(os)
 300             def architecture = 'x64'
 301             def configuration = 'Release'
 302             def runType = isPR ? 'private' : 'rolling'
 303             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 304
 305             steps {
 306                 shell("bash ./tests/scripts/perf-prep.sh --throughput")
 307                 shell("./init-tools.sh")
 308                 shell("./build.sh ${architecture} ${configuration}")
 309                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 310                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 311                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 312                 shell("""sudo -E python3.5 ./tests/scripts/run-throughput-perf.py \\
 313                 -arch \"${architecture}\" \\
 314                 -os \"${os}\" \\
 315                 -configuration \"${configuration}\" \\
 316                 -clr_root \"\${WORKSPACE}\" \\
 317                 -assembly_root \"\${WORKSPACE}/_/fx/bin/runtime/netcoreapp-${osGroup}-${configuration}-${architecture}\" \\
 318                 -run_type \"${runType}\" \\
 319                 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 320             }
 321         }
 322
 323         // Save machinedata.json to /artifact/bin/ Jenkins dir
 324         def archiveSettings = new ArchivalSettings()
 325         archiveSettings.addFiles('throughput-*.csv')
 326         archiveSettings.addFiles('machinedata.json')
 327         Utilities.addArchival(newJob, archiveSettings)
 328
 329         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 330
 331         // For perf, we need to keep the run results longer
 332         newJob.with {
 333             // Enable the log rotator
 334             logRotator {
 335                 artifactDaysToKeep(7)
 336                 daysToKeep(300)
 337                 artifactNumToKeep(25)
 338                 numToKeep(1000)
 339             }
 340         }
 341         if (isPR) {
 342             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 343             builder.setGithubContext("${os} Throughput Perf Tests")
 344             builder.triggerOnlyOnComment()
 345             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+throughput.*")
 346             builder.triggerForBranch(branch)
 347             builder.emitTrigger(newJob)
 348         }
 349         else {
 350             // Set a push trigger
 351             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 352             builder.emitTrigger(newJob)
 353         }
 354     } // os
 355 } // isPR