perf.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*;
   4
   5 def project = GithubProject
   6 def branch = GithubBranchName
   7 def projectName = Utilities.getFolderName(project)
   8 def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
   9
  10 def static getOSGroup(def os) {
  11     def osGroupMap = ['Ubuntu14.04':'Linux',
  12         'RHEL7.2': 'Linux',
  13         'Ubuntu16.04': 'Linux',
  14         'Debian8.4':'Linux',
  15         'Fedora24':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux',
  21         'OpenSUSE42.1': 'Linux',
  22         'LinuxARMEmulator': 'Linux']
  23     def osGroup = osGroupMap.get(os, null)
  24     assert osGroup != null : "Could not find os group for ${os}"
  25     return osGroupMap[os]
  26 }
  27 // Setup perflab tests runs
  28 [true, false].each { isPR ->
  29     ['Windows_NT'].each { os ->
  30         ['x64', 'x86', 'x86jit32'].each { arch ->
  31             def architecture = arch
  32             def testEnv = ''
  33
  34             if (arch == 'x86jit32')
  35             {
  36                 architecture = 'x86'
  37                 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\compatjit_x86_testenv.cmd'
  38             }
  39             else if (arch == 'x86')
  40             {
  41                 testEnv = '-testEnv %WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd'
  42             }
  43
  44             def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}_${arch}", isPR)) {
  45                 // Set the label.
  46                 label('windows_clr_perf')
  47                 wrappers {
  48                     credentialsBinding {
  49                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
  50                     }
  51                 }
  52
  53             if (isPR)
  54             {
  55                 parameters
  56                 {
  57                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
  58                 }
  59             }
  60             def configuration = 'Release'
  61             def runType = isPR ? 'private' : 'rolling'
  62             def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
  63
  64                 steps {
  65                     // Batch
  66
  67                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
  68                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
  69                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
  70                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
  71                     batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
  72                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
  73                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
  74                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
  75                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
  76
  77                     if (arch == 'x86jit32')
  78                     {
  79                         // Download package and copy compatjit into Core_Root
  80                         batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
  81                         "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
  82                     }
  83
  84                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
  85
  86                     batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} ${testEnv} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
  87                     batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} ${testEnv} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -uploadToBenchview \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -runtype ${runType}")
  88                 }
  89             }
  90
  91             // Save machinedata.json to /artifact/bin/ Jenkins dir
  92             def archiveSettings = new ArchivalSettings()
  93             archiveSettings.addFiles('Perf-*.xml')
  94             archiveSettings.addFiles('Perf-*.etl')
  95             Utilities.addArchival(newJob, archiveSettings)
  96
  97             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
  98
  99             newJob.with {
 100                 wrappers {
 101                     timeout {
 102                         absolute(240)
 103                     }
 104                 }
 105             }
 106
 107             if (isPR) {
 108                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 109                 builder.setGithubContext("${os} ${arch} CoreCLR Perf Tests")
 110                 builder.triggerOnlyOnComment()
 111                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+perf.*")
 112                 builder.triggerForBranch(branch)
 113                 builder.emitTrigger(newJob)
 114             }
 115             else {
 116                 // Set a push trigger
 117                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 118                 builder.emitTrigger(newJob)
 119             }
 120         }
 121     }
 122 }
 123
 124 // Setup throughput perflab tests runs
 125 [true, false].each { isPR ->
 126     ['Windows_NT'].each { os ->
 127         ['x64', 'x86', 'x86jit32'].each { arch ->
 128             def architecture = arch
 129
 130             if (arch == 'x86jit32')
 131             {
 132                 architecture = 'x86'
 133             }
 134
 135             def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) {
 136                 // Set the label.
 137                 label('windows_clr_perf')
 138                 wrappers {
 139                     credentialsBinding {
 140                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 141                     }
 142                 }
 143
 144             if (isPR)
 145             {
 146                 parameters
 147                 {
 148                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 149                 }
 150             }
 151             def configuration = 'Release'
 152             def runType = isPR ? 'private' : 'rolling'
 153             def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 154
 155                 steps {
 156                     // Batch
 157
 158                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
 159                     batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
 160                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 161                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
 162                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
 163                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
 164                     batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
 165                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
 166                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
 167                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
 168                     batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
 169                     batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 170                     if (arch == 'x86jit32')
 171                     {
 172                         // Download package and copy compatjit into Core_Root
 173                         batchFile("C:\\Tools\\nuget.exe install runtime.win7-${architecture}.Microsoft.NETCore.Jit -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion\n" +
 174                         "xcopy \"%WORKSPACE%\\runtime.win7-x86.Microsoft.NETCore.Jit\\runtimes\\win7-x86\\native\\compatjit.dll\" \"%WORKSPACE%\\bin\\Product\\${os}.${architecture}.${configuration}\" /Y")
 175                     }
 176                     batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
 177                 }
 178             }
 179
 180             // Save machinedata.json to /artifact/bin/ Jenkins dir
 181             def archiveSettings = new ArchivalSettings()
 182             archiveSettings.addFiles('throughput-*.csv')
 183             Utilities.addArchival(newJob, archiveSettings)
 184
 185             Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 186
 187             if (isPR) {
 188                 TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 189                 builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests")
 190                 builder.triggerOnlyOnComment()
 191                 builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*")
 192                 builder.triggerForBranch(branch)
 193                 builder.emitTrigger(newJob)
 194             }
 195             else {
 196                 // Set a push trigger
 197                 TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 198                 builder.emitTrigger(newJob)
 199             }
 200         }
 201     }
 202 }
 203
 204 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 205 [true, false].each { isPR ->
 206     ['Ubuntu14.04'].each { os ->
 207         def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
 208
 209             label('linux_clr_perf')
 210                 wrappers {
 211                     credentialsBinding {
 212                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 213                     }
 214                 }
 215
 216             if (isPR)
 217             {
 218                 parameters
 219                 {
 220                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 221                 }
 222             }
 223             def osGroup = getOSGroup(os)
 224             def architecture = 'x64'
 225             def configuration = 'Release'
 226             def runType = isPR ? 'private' : 'rolling'
 227             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 228
 229             steps {
 230                 shell("bash ./tests/scripts/perf-prep.sh")
 231                 shell("./init-tools.sh")
 232                 shell("./build.sh ${architecture} ${configuration}")
 233                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 234                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 235                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 236                 shell("""sudo -E bash ./tests/scripts/run-xunit-perf.sh \\
 237                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
 238                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
 239                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 240                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
 241                 --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
 242                 --runType=\"${runType}\" \\
 243                 --benchViewOS=\"${os}\" \\
 244                 --uploadToBenchview""")
 245             }
 246         }
 247
 248         // Save machinedata.json to /artifact/bin/ Jenkins dir
 249         def archiveSettings = new ArchivalSettings()
 250         archiveSettings.addFiles('sandbox/perf-*.xml')
 251         archiveSettings.addFiles('machinedata.json')
 252         Utilities.addArchival(newJob, archiveSettings)
 253
 254         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 255
 256         // For perf, we need to keep the run results longer
 257         newJob.with {
 258             // Enable the log rotator
 259             logRotator {
 260                 artifactDaysToKeep(7)
 261                 daysToKeep(300)
 262                 artifactNumToKeep(25)
 263                 numToKeep(1000)
 264             }
 265         }
 266         if (isPR) {
 267             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 268             builder.setGithubContext("${os} Perf Tests")
 269             builder.triggerOnlyOnComment()
 270             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
 271             builder.triggerForBranch(branch)
 272             builder.emitTrigger(newJob)
 273         }
 274         else {
 275             // Set a push trigger
 276             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 277             builder.emitTrigger(newJob)
 278         }
 279     } // os
 280 } // isPR
 281
 282 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 283 [true, false].each { isPR ->
 284     ['Ubuntu14.04'].each { os ->
 285         def newJob = job(Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)) {
 286
 287             label('linux_clr_perf')
 288                 wrappers {
 289                     credentialsBinding {
 290                         string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
 291                     }
 292                 }
 293
 294             if (isPR)
 295             {
 296                 parameters
 297                 {
 298                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
 299                 }
 300             }
 301             def osGroup = getOSGroup(os)
 302             def architecture = 'x64'
 303             def configuration = 'Release'
 304             def runType = isPR ? 'private' : 'rolling'
 305             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 306
 307             steps {
 308                 shell("bash ./tests/scripts/perf-prep.sh --throughput")
 309                 shell("./init-tools.sh")
 310                 shell("./build.sh ${architecture} ${configuration}")
 311                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
 312                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
 313                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
 314                 shell("""sudo -E python3.5 ./tests/scripts/run-throughput-perf.py \\
 315                 -arch \"${architecture}\" \\
 316                 -os \"${os}\" \\
 317                 -configuration \"${configuration}\" \\
 318                 -clr_root \"\${WORKSPACE}\" \\
 319                 -assembly_root \"\${WORKSPACE}/_/fx/bin/runtime/netcoreapp-${osGroup}-${configuration}-${architecture}\" \\
 320                 -run_type \"${runType}\" \\
 321                 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
 322             }
 323         }
 324
 325         // Save machinedata.json to /artifact/bin/ Jenkins dir
 326         def archiveSettings = new ArchivalSettings()
 327         archiveSettings.addFiles('throughput-*.csv')
 328         archiveSettings.addFiles('machinedata.json')
 329         Utilities.addArchival(newJob, archiveSettings)
 330
 331         Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 332
 333         // For perf, we need to keep the run results longer
 334         newJob.with {
 335             // Enable the log rotator
 336             logRotator {
 337                 artifactDaysToKeep(7)
 338                 daysToKeep(300)
 339                 artifactNumToKeep(25)
 340                 numToKeep(1000)
 341             }
 342         }
 343         if (isPR) {
 344             TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
 345             builder.setGithubContext("${os} Throughput Perf Tests")
 346             builder.triggerOnlyOnComment()
 347             builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+throughput.*")
 348             builder.triggerForBranch(branch)
 349             builder.emitTrigger(newJob)
 350         }
 351         else {
 352             // Set a push trigger
 353             TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
 354             builder.emitTrigger(newJob)
 355         }
 356     } // os
 357 } // isPR