netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs
  12 folder('jitstress')
  13
  14 def static getOSGroup(def os) {
  15     def osGroupMap = ['Ubuntu':'Linux',
  16         'RHEL7.2': 'Linux',
  17         'Ubuntu16.04': 'Linux',
  18         'Ubuntu16.10': 'Linux',
  19         'Debian8.4':'Linux',
  20         'Fedora23':'Linux',
  21         'OSX':'OSX',
  22         'Windows_NT':'Windows_NT',
  23         'FreeBSD':'FreeBSD',
  24         'CentOS7.1': 'Linux',
  25         'OpenSUSE13.2': 'Linux',
  26         'OpenSUSE42.1': 'Linux',
  27         'LinuxARMEmulator': 'Linux']
  28     def osGroup = osGroupMap.get(os, null)
  29     assert osGroup != null : "Could not find os group for ${os}"
  30     return osGroupMap[os]
  31 }
  32
  33 // We use this class (vs variables) so that the static functions can access data here.
  34 class Constants {
  35     // Innerloop build OS's
  36     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  37     // test execution in the build flow runs.  It generates the exact same build
  38     // as Windows_NT but without the tests.
  39     def static osList = ['Ubuntu', 'Debian8.4', 'OSX', 'Windows_NT', 'Windows_NT_BuildOnly', 'FreeBSD', 'CentOS7.1', 'OpenSUSE13.2', 'OpenSUSE42.1', 'RHEL7.2', 'LinuxARMEmulator', 'Ubuntu16.04', 'Ubuntu16.10', 'Fedora23']
  40     def static crossList = ['Ubuntu', 'OSX', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'OpenSUSE13.2']
  41     // This is a set of JIT stress modes combined with the set of variables that
  42     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  43     // the values are the environment variables
  44     def static jitStressModeScenarios = ['minopts' : ['COMPlus_JITMinOpts' : '1'], 'forcerelocs' : ['COMPlus_ForceRelocs' : '1'],
  45                'jitstress1' : ['COMPlus_JitStress' : '1'], 'jitstress2' : ['COMPlus_JitStress' : '2'],
  46                'jitstressregs1' : ['COMPlus_JitStressRegs' : '1'], 'jitstressregs2' : ['COMPlus_JitStressRegs' : '2'],
  47                'jitstressregs3' : ['COMPlus_JitStressRegs' : '3'], 'jitstressregs4' : ['COMPlus_JitStressRegs' : '4'],
  48                'jitstressregs8' : ['COMPlus_JitStressRegs' : '8'], 'jitstressregs0x10' : ['COMPlus_JitStressRegs' : '0x10'],
  49                'jitstressregs0x80' : ['COMPlus_JitStressRegs' : '0x80'],
  50                'jitstress2_jitstressregs1'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  51                'jitstress2_jitstressregs2'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  52                'jitstress2_jitstressregs3'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  53                'jitstress2_jitstressregs4'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  54                'jitstress2_jitstressregs8'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  55                'jitstress2_jitstressregs0x10' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  56                'jitstress2_jitstressregs0x80' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  57                'corefx_baseline' : [ : ], // corefx baseline
  58                'corefx_minopts' : ['COMPlus_JITMinOpts' : '1'],
  59                'corefx_jitstress1' : ['COMPlus_JitStress' : '1'],
  60                'corefx_jitstress2' : ['COMPlus_JitStress' : '2'],
  61                'corefx_jitstressregs1' : ['COMPlus_JitStressRegs' : '1'], 'corefx_jitstressregs2' : ['COMPlus_JitStressRegs' : '2'],
  62                'corefx_jitstressregs3' : ['COMPlus_JitStressRegs' : '3'], 'corefx_jitstressregs4' : ['COMPlus_JitStressRegs' : '4'],
  63                'corefx_jitstressregs8' : ['COMPlus_JitStressRegs' : '8'], 'corefx_jitstressregs0x10' : ['COMPlus_JitStressRegs' : '0x10'],
  64                'corefx_jitstressregs0x80' : ['COMPlus_JitStressRegs' : '0x80'],
  65                'gcstress0x3' : ['COMPlus_GCStress'  : '0x3'], 'gcstress0xc' : ['COMPlus_GCStress'  : '0xC'],
  66                'zapdisable' : ['COMPlus_ZapDisable'  : '0xC'],
  67                'heapverify1' : ['COMPlus_HeapVerify'  : '1'],
  68                'gcstress0xc_zapdisable' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_ZapDisable'  : '1'],
  69                'gcstress0xc_zapdisable_jitstress2' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_ZapDisable'  : '1', 'COMPlus_JitStress'  : '2'],
  70                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_ZapDisable'  : '1', 'COMPlus_HeapVerify'  : '1'],
  71                'gcstress0xc_jitstress1' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_JitStress'  : '1'],
  72                'gcstress0xc_jitstress2' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_JitStress'  : '2'],
  73                'gcstress0xc_minopts_heapverify1' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_JITMinOpts'  : '1', 'COMPlus_HeapVerify'  : '1']
  74                ]
  75     // This is a set of r2r jit stress scenarios
  76     def static r2rJitStressScenarios = ['r2r_jitstress1', 'r2r_jitstress2', 'r2r_jitstressregs1', 'r2r_jitstressregs2', 'r2r_jitstressregs3',
  77                                         'r2r_jitstressregs4', 'r2r_jitstressregs8', 'r2r_jitstressregsx10', 'r2r_jitstressregsx80',
  78                                         'r2r_jitminopts', 'r2r_jitforcerelocs']
  79     // This is the basic set of scenarios
  80     def static basicScenarios = ['default', 'pri1', 'ilrt', 'r2r', 'pri1r2r', 'gcstress15_pri1r2r', 'longgc', 'coverage', 'formatting', 'gcsimulator'] + r2rJitStressScenarios
  81     def static configurationList = ['Debug', 'Checked', 'Release']
  82     // This is the set of architectures
  83     def static architectureList = ['arm', 'arm64', 'x64', 'x86ryujit', 'x86lb']
  84 }
  85
  86 def static setMachineAffinity(def job, def os, def architecture) {
  87     if (architecture == 'arm64' && os == 'Windows_NT') {
  88         // For cross compilation
  89         job.with {
  90             label('arm64')
  91         }
  92     } else if ((architecture == 'arm' || architecture == 'arm64') && os == 'Ubuntu') {
  93         Utilities.setMachineAffinity(job, os, 'arm-cross-latest');
  94     } else {
  95         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
  96     }
  97 }
  98
  99 def static isJITStressJob(def scenario) {
 100     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 101            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 102 }
 103
 104 def static isGCStressRelatedTesting(def scenario) {
 105     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 106     // Detect it and make it a GCStress related.
 107     if (scenario == 'gcstress15_pri1r2r')
 108     {
 109         return true;
 110     }
 111
 112     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 113     def scenarioName = scenario.toLowerCase()
 114     def isGCStressTesting = false
 115     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 116         if (k in gcStressTestEnvVars) {
 117             isGCStressTesting = true;
 118         }
 119     }
 120     return isGCStressTesting
 121 }
 122
 123 def static isCorefxTesting(def scenario) {
 124     def corefx_prefix = 'corefx_'
 125     if (scenario.length() < corefx_prefix.length()) {
 126         return false
 127     }
 128     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 129 }
 130
 131 def static isR2R(def scenario) {
 132     return (scenario == 'r2r' || scenario == 'pri1r2r')
 133 }
 134
 135 def static isCoverage(def scenario) {
 136     return (scenario == 'coverage')
 137 }
 138
 139 def static isLongGc(def scenario) {
 140     return (scenario == 'longgc' || scenario == 'gcsimulator')
 141 }
 142
 143 def static setTestJobTimeOut(newJob, scenario) {
 144     if (isGCStressRelatedTesting(scenario)) {
 145         Utilities.setJobTimeout(newJob, 4320)
 146     }
 147     else if (isCorefxTesting(scenario)) {
 148         Utilities.setJobTimeout(newJob, 360)
 149     }
 150     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 151         Utilities.setJobTimeout(newJob, 240)
 152     }
 153     else if (isR2R(scenario)) {
 154         Utilities.setJobTimeout(newJob, 240)
 155     }
 156     else if (isCoverage(scenario)) {
 157         Utilities.setJobTimeout(newJob, 1440)
 158     }
 159     else if (isLongGc(scenario)) {
 160         Utilities.setJobTimeout(newJob, 1440)
 161     }
 162     // Non-test jobs use the default timeout value.
 163 }
 164
 165 def static getStressModeDisplayName(def scenario) {
 166     def displayStr = ''
 167     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 168         def prefixLength = 'COMPlus_'.length()
 169         if (k.length() >= prefixLength) {
 170             def modeName = k.substring(prefixLength, k.length())
 171             displayStr += ' ' + modeName + '=' + v
 172         }
 173     }
 174     return displayStr
 175 }
 176
 177 // Generates the string for creating a file that sets environment variables
 178 // that makes it possible to run stress modes.  Writes the script to a file called
 179 // SetStressModes.[sh/cmd]
 180 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 181     def stepScript = ''
 182     if (os == 'Windows_NT') {
 183         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 184         stepScript += "del ${stepScriptLocation}\r\n"
 185
 186         // Timeout in ms, default is 10 minutes. For stress
 187         // modes up this to 30 minutes
 188         def timeout = 1800000
 189
 190         // Set the Timeout
 191         stepScript += "set __TestTimeout=${timeout}\r\n"
 192         stressModeVars.each{ k, v ->
 193             // Write out what we are writing to the script file
 194             stepScript += "echo Setting ${k}=${v}\r\n"
 195             // Write out the set itself to the script file`
 196             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 197         }
 198     }
 199     else {
 200         stepScript += "echo Setting variables for ${stressModeName}\n"
 201         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 202         stressModeVars.each{ k, v ->
 203             // Write out what we are writing to the script file
 204             stepScript += "echo Setting ${k}=${v}\n"
 205             // Write out the set itself to the script file`
 206             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 207         }
 208         stepScript += "chmod +x ${stepScriptLocation}\n"
 209     }
 210     return stepScript
 211 }
 212
 213 // Corefx doesn't have a support to pass stress mode environment variables. This function
 214 // generates commands to set or export environment variables
 215 def static getStressModeEnvSetCmd(def os, def stressModeName) {
 216     def envVars = Constants.jitStressModeScenarios[stressModeName]
 217     def setEnvVars = ''
 218     if (os == 'Windows_NT') {
 219         envVars.each{ VarName, Value   ->
 220             if (VarName != '') {
 221                 setEnvVars += "set ${VarName}=${Value}\n"
 222             }
 223         }
 224     }
 225     else {
 226         envVars.each{ VarName, Value   ->
 227             if (VarName != '') {
 228                 setEnvVars += "export ${VarName}=${Value}\n"
 229             }
 230         }
 231     }
 232     return setEnvVars
 233 }
 234
 235 // Calculates the name of the build job based on some typical parameters.
 236 //
 237 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly, def isLinuxEmulatorBuild = false) {
 238     // If the architecture is x64, do not add that info into the build name.
 239     // Need to change around some systems and other builds to pick up the right builds
 240     // to do that.
 241
 242     def suffix = scenario != 'default' ? "_${scenario}" : '';
 243     if (isBuildOnly) {
 244         suffix += '_bld'
 245     }
 246     def baseName = ''
 247     switch (architecture) {
 248         case 'x64':
 249             if (scenario == 'default') {
 250                 // For now we leave x64 off of the name for compatibility with other jobs
 251                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 252             }
 253             else if (scenario == 'formatting') {
 254                 // we don't care about the configuration for the formatting job. It runs all configs
 255                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 256             }
 257             else {
 258                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 259             }
 260             break
 261         case 'arm64':
 262         case 'arm':
 263             // These are cross builds
 264             if (isLinuxEmulatorBuild == false) {
 265                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 266             }
 267             else {
 268                 baseName = architecture.toLowerCase() + '_emulator_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 269             }
 270             break
 271         case 'x86ryujit':
 272             baseName = 'x86_ryujit_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 273             break
 274         case 'x86lb':
 275             baseName = 'x86_lb_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 276             break
 277         default:
 278             println("Unknown architecture: ${architecture}");
 279             assert false
 280             break
 281     }
 282
 283     return baseName + suffix
 284 }
 285
 286 static void addEmailPublisher(def job, def recipient) {
 287     job.with {
 288         publishers {
 289             extendedEmail(recipient, '$DEFAULT_SUBJECT', '$DEFAULT_CONTENT') {
 290                 trigger('Aborted', '$PROJECT_DEFAULT_SUBJECT', '$PROJECT_DEFAULT_CONTENT', null, true, true, true, true)
 291                 trigger('Failure', '$PROJECT_DEFAULT_SUBJECT', '$PROJECT_DEFAULT_CONTENT', null, true, true, true, true)
 292             }
 293         }
 294     }
 295 }
 296
 297 // **************************
 298 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 299 // of coreclr builds over linux/osx/freebsd/windows and debug/release/checked.  In addition, the windows
 300 // builds will do a couple extra steps.
 301 // **************************
 302
 303 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 304 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 305 // it's just used for internal builds
 306 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 307 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def isLinuxEmulatorBuild) {
 308     if (isWindowsBuildOnlyJob) {
 309         return
 310     }
 311
 312     def bidailyCrossList = ['RHEL7.2', 'Debian8.4', 'OpenSUSE13.2']
 313     // Non pull request builds.
 314     if (!isPR) {
 315         // Check scenario.
 316         switch (scenario) {
 317             case 'default':
 318                 switch (architecture) {
 319                     case 'x64':
 320                     case 'x86ryujit':
 321                     case 'x86lb':
 322                         if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 323                             Utilities.addGithubPushTrigger(job)
 324                         }
 325                         break
 326                     case 'arm':
 327                         Utilities.addGithubPushTrigger(job)
 328                         break
 329                     case 'arm64':
 330                         if (os == 'Windows_NT') {
 331                             Utilities.addGithubPushTrigger(job)
 332                             addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 333                         }
 334                         break
 335                     default:
 336                         println("Unknown architecture: ${architecture}");
 337                         assert false
 338                         break
 339                 }
 340                 break
 341             case 'pri1':
 342                 // Pri one gets a push trigger, and only for release
 343                 if (architecture == 'x64') {
 344                     if (configuration == 'Release') {
 345                         // We expect release jobs to be Windows, or in the cross list
 346                         assert (os == 'Windows_NT') || (os in Constants.crossList)
 347                         if (!os in bidailyCrossList) {
 348                             if (isFlowJob || os == 'Windows_NT') {
 349                                 Utilities.addGithubPushTrigger(job)
 350                             }
 351                         }
 352                         else {
 353                             if (isFlowJob) {
 354                                 Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 355                             }
 356                         }
 357                     }
 358                 }
 359                 break
 360             case 'r2r':
 361                 //r2r jobs that aren't pri1 can only be triggered by phrase
 362                 break
 363             case 'pri1r2r':
 364                 assert !(os in bidailyCrossList)
 365                 //pri1 r2r gets a push trigger for checked/release
 366                 if (configuration == 'Checked' || configuration == 'Release') {
 367                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 368                     if (architecture == 'x64' && os != 'OSX') {
 369                         //Flow jobs should be Windows, Ubuntu, OSX, or CentOS
 370                         if (isFlowJob || os == 'Windows_NT') {
 371                             Utilities.addGithubPushTrigger(job)
 372                         }
 373                     // OSX pri1r2r jobs should only run every 12 hours, not daily.
 374                     } else if (architecture == 'x64' && os == 'OSX'){
 375                         if (isFlowJob) {
 376                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 377                         }
 378                     }
 379                     // For x86, only add per-commit jobs for Windows
 380                     else if (architecture == 'x86ryujit' || architecture == 'x86lb') {
 381                         if (os == 'Windows_NT') {
 382                             Utilities.addGithubPushTrigger(job)
 383                         }
 384                     }
 385                     // arm64 pri1r2r jobs should only run every 12 hours.
 386                     else if (architecture == 'arm64') {
 387                         if (os == 'Windows_NT') {
 388                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 389                             addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 390                         }
 391                     }
 392                 }
 393                 break
 394             case 'r2r_jitstress1':
 395             case 'r2r_jitstress2':
 396             case 'r2r_jitstressregs1':
 397             case 'r2r_jitstressregs2':
 398             case 'r2r_jitstressregs3':
 399             case 'r2r_jitstressregs4':
 400             case 'r2r_jitstressregs8':
 401             case 'r2r_jitstressregsx10':
 402             case 'r2r_jitstressregsx80':
 403             case 'r2r_jitminopts':
 404             case 'r2r_jitforcerelocs':
 405             case 'gcstress15_pri1r2r':
 406                 assert !(os in bidailyCrossList)
 407
 408                 // GCStress=C is currently not supported on OS X
 409                 if (os == 'OSX' && isGCStressRelatedTesting(scenario)) {
 410                     break
 411                 }
 412
 413                 //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 414                 if (configuration == 'Checked' || configuration == 'Release') {
 415                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 416                     if (architecture == 'x64') {
 417                         //Flow jobs should be Windows, Ubuntu, OSX, or CentOS
 418                         if (isFlowJob || os == 'Windows_NT') {
 419                             // Add a weekly periodic trigger
 420                             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 421                         }
 422                     }
 423                     // For x86, only add per-commit jobs for Windows
 424                     else if (architecture == 'x86ryujit' || architecture == 'x86lb') {
 425                         if (os == 'Windows_NT') {
 426                             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 427                         }
 428                     }
 429                 }
 430                 break
 431             case 'longgc':
 432                 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX')
 433                 assert configuration == 'Release'
 434                 assert architecture == 'x64'
 435                 Utilities.addPeriodicTrigger(job, '@daily')
 436                 addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 437                 break
 438             case 'gcsimulator':
 439                 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX')
 440                 assert configuration == 'Release'
 441                 assert architecture == 'x64'
 442                 Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 443                 addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 444                 break
 445             case 'ilrt':
 446                 assert !(os in bidailyCrossList)
 447                 // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 448                 if (architecture == 'x64' && configuration == 'Release') {
 449                     // We don't expect to see a job generated except in these scenarios
 450                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 451                     if (isFlowJob || os == 'Windows_NT') {
 452                         Utilities.addPeriodicTrigger(job, '@daily')
 453                     }
 454                 }
 455                 break
 456             case 'coverage':
 457                 assert (os == 'Ubuntu' || os == 'Windows_NT')
 458                 assert configuration == 'Release'
 459                 assert architecture == 'x64'
 460                 Utilities.addPeriodicTrigger(job, '@weekly')
 461                 break
 462             case 'formatting':
 463                 assert (os == 'Windows_NT' || os == "Ubuntu")
 464                 assert architecture == 'x64'
 465                 Utilities.addGithubPushTrigger(job)
 466                 break
 467             case 'jitstressregs1':
 468             case 'jitstressregs2':
 469             case 'jitstressregs3':
 470             case 'jitstressregs4':
 471             case 'jitstressregs8':
 472             case 'jitstressregs0x10':
 473             case 'jitstressregs0x80':
 474             case 'minopts':
 475             case 'forcerelocs':
 476             case 'jitstress1':
 477             case 'jitstress2':
 478             case 'jitstress2_jitstressregs1':
 479             case 'jitstress2_jitstressregs2':
 480             case 'jitstress2_jitstressregs3':
 481             case 'jitstress2_jitstressregs4':
 482             case 'jitstress2_jitstressregs8':
 483             case 'jitstress2_jitstressregs0x10':
 484             case 'jitstress2_jitstressregs0x80':
 485             case 'corefx_baseline':
 486             case 'corefx_minopts':
 487             case 'corefx_jitstress1':
 488             case 'corefx_jitstress2':
 489             case 'corefx_jitstressregs1':
 490             case 'corefx_jitstressregs2':
 491             case 'corefx_jitstressregs3':
 492             case 'corefx_jitstressregs4':
 493             case 'corefx_jitstressregs8':
 494             case 'corefx_jitstressregs0x10':
 495             case 'corefx_jitstressregs0x80':
 496             case 'zapdisable':
 497                 if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 498                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 499                     Utilities.addPeriodicTrigger(job, '@daily')
 500                 }
 501                 break
 502             case 'heapverify1':
 503             case 'gcstress0x3':
 504                 if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 505                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 506                     if (architecture == 'arm64') {
 507                         assert (os == 'Windows_NT')
 508                         Utilities.addPeriodicTrigger(job, '@daily')
 509                         addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 510                     }
 511                     else {
 512                         Utilities.addPeriodicTrigger(job, '@weekly')
 513                     }
 514                 }
 515                 break
 516             case 'gcstress0xc':
 517             case 'gcstress0xc_zapdisable':
 518             case 'gcstress0xc_zapdisable_jitstress2':
 519             case 'gcstress0xc_zapdisable_heapverify1':
 520             case 'gcstress0xc_jitstress1':
 521             case 'gcstress0xc_jitstress2':
 522             case 'gcstress0xc_minopts_heapverify1':
 523                 // GCStress=C is currently not supported on OS X
 524                 if (os != 'CentOS7.1' && os != 'OSX' && !(os in bidailyCrossList)) {
 525                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 526                     if (architecture == 'arm64') {
 527                         assert (os == 'Windows_NT')
 528                         // TODO: Enable a periodic trigger after tests are updated.
 529                         // Utilities.addPeriodicTrigger(job, '@daily')
 530                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 531                     }
 532                     else {
 533                         Utilities.addPeriodicTrigger(job, '@weekly')
 534                     }
 535                 }
 536                 break
 537             default:
 538                 println("Unknown scenario: ${scenario}");
 539                 assert false
 540                 break
 541         }
 542         return
 543     }
 544     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 545     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 546     def osGroup = getOSGroup(os)
 547     switch (architecture) {
 548         case 'x64':
 549             if (scenario == 'coverage') {
 550                 assert configuration == 'Release'
 551                 if (os == 'Ubuntu') {
 552                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 553                 }
 554                 break
 555             }
 556
 557             if (scenario == 'formatting') {
 558                 assert configuration == 'Checked'
 559                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 560                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 561                 }
 562                 break
 563             }
 564
 565             switch (os) {
 566                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 567                 case 'OpenSUSE13.2':
 568                 case 'Debian8.4':
 569                 case 'RHEL7.2':
 570                     if (scenario == 'default') {
 571                         assert !isFlowJob
 572                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 573                     }
 574                     else if (scenario == 'pri1' && isFlowJob) {
 575                         assert (configuration == 'Release')
 576                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 577                     }
 578                     break
 579                 case 'Fedora23':
 580                 case 'Ubuntu16.04':
 581                 case 'Ubuntu16.10':
 582                 case 'OpenSUSE42.1':
 583                     assert !isFlowJob
 584                     assert scenario == 'default'
 585                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 586                     break
 587                 case 'Ubuntu':
 588                 case 'OSX':
 589                     // Triggers on the non-flow jobs aren't necessary here
 590                     // Corefx testing uses non-flow jobs.
 591                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 592                         break
 593                     }
 594                     switch (scenario) {
 595                         case 'default':
 596                             // Ubuntu uses checked for default PR tests
 597                             if (configuration == 'Checked') {
 598                                 // Default trigger
 599                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 600                             }
 601                             break
 602                         case 'pri1':
 603                             if (configuration == 'Release') {
 604                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 605                             }
 606                             break
 607                         case 'ilrt':
 608                             if (configuration == 'Release') {
 609                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 610                             }
 611                             break
 612                         case 'r2r':
 613                             if (configuration == 'Release' || configuration == 'Checked') {
 614                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 615                             }
 616                             break
 617                         case 'pri1r2r':
 618                             if (configuration == 'Release' || configuration == 'Checked') {
 619                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 620                             }
 621                             break
 622                         case 'gcstress15_pri1r2r':
 623                             if (configuration == 'Release' || configuration == 'Checked') {
 624                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 625                             }
 626                             break
 627                         case 'r2r_jitstress1':
 628                             if (configuration == 'Release' || configuration == 'Checked') {
 629                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 630                             }
 631                             break
 632                         case 'r2r_jitstress2':
 633                             if (configuration == 'Release' || configuration == 'Checked') {
 634                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 635                             }
 636                             break
 637                         case 'r2r_jitstressregs1':
 638                             if (configuration == 'Release' || configuration == 'Checked') {
 639                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 640                             }
 641                             break
 642                         case 'r2r_jitstressregs2':
 643                             if (configuration == 'Release' || configuration == 'Checked') {
 644                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 645                             }
 646                             break
 647                         case 'r2r_jitstressregs3':
 648                             if (configuration == 'Release' || configuration == 'Checked') {
 649                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 650                             }
 651                             break
 652                         case 'r2r_jitstressregs4':
 653                             if (configuration == 'Release' || configuration == 'Checked') {
 654                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 655                             }
 656                             break
 657                         case 'r2r_jitstressregs8':
 658                             if (configuration == 'Release' || configuration == 'Checked') {
 659                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 660                             }
 661                             break
 662                         case 'r2r_jitstressregsx10':
 663                             if (configuration == 'Release' || configuration == 'Checked') {
 664                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 665                             }
 666                             break
 667                         case 'r2r_jitstressregsx80':
 668                             if (configuration == 'Release' || configuration == 'Checked') {
 669                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 670                             }
 671                             break
 672                         case 'r2r_jitminopts':
 673                             if (configuration == 'Release' || configuration == 'Checked') {
 674                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} JITMinOpts R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 675                             }
 676                             break
 677                         case 'r2r_jitforcerelocs':
 678                             if (configuration == 'Release' || configuration == 'Checked') {
 679                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ForceRelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 680                             }
 681                             break
 682                         case 'longgc':
 683                             if (configuration == 'Release') {
 684                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 685                             }
 686                             break
 687                         case 'gcsimulator':
 688                             if (configuration == 'Release') {
 689                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 690                             }
 691                             break
 692                         case 'minopts':
 693                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 694                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - MinOpts)",
 695                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 696                             break
 697                         case 'jitstress1':
 698                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 699                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=1)",
 700                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 701                             break
 702                         case 'jitstress2':
 703                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 704                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=2)",
 705                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 706                             break
 707                         case 'forcerelocs':
 708                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 709                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ForceRelocs)",
 710                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 711                         case 'jitstressregs1':
 712                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 713                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=1)",
 714                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 715                             break
 716                         case 'jitstressregs2':
 717                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 718                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=2)",
 719                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 720                             break
 721                         case 'jitstressregs3':
 722                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 723                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=3)",
 724                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 725                             break
 726                         case 'jitstressregs4':
 727                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 728                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=4)",
 729                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 730                             break
 731                         case 'jitstressregs8':
 732                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 733                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=8)",
 734                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 735                             break
 736                         case 'jitstressregs0x10':
 737                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 738                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x10)",
 739                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 740                             break
 741                         case 'jitstressregs0x80':
 742                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 743                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x80)",
 744                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 745                             break
 746                         case 'jitstress2_jitstressregs1':
 747                         case 'jitstress2_jitstressregs2':
 748                         case 'jitstress2_jitstressregs3':
 749                         case 'jitstress2_jitstressregs4':
 750                         case 'jitstress2_jitstressregs8':
 751                         case 'jitstress2_jitstressregs0x10':
 752                         case 'jitstress2_jitstressregs0x80':
 753                         case 'gcstress0x3':
 754                         case 'gcstress0xc':
 755                         case 'zapdisable':
 756                         case 'heapverify1':
 757                         case 'gcstress0xc_zapdisable':
 758                         case 'gcstress0xc_zapdisable_jitstress2':
 759                         case 'gcstress0xc_zapdisable_heapverify1':
 760                         case 'gcstress0xc_jitstress1':
 761                         case 'gcstress0xc_jitstress2':
 762                         case 'gcstress0xc_minopts_heapverify1':
 763                             def displayStr = getStressModeDisplayName(scenario)
 764                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 765                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 766                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 767                             break
 768                         case 'corefx_baseline':
 769                         case 'corefx_minopts':
 770                         case 'corefx_jitstress1':
 771                         case 'corefx_jitstress2':
 772                         case 'corefx_jitstressregs1':
 773                         case 'corefx_jitstressregs2':
 774                         case 'corefx_jitstressregs3':
 775                         case 'corefx_jitstressregs4':
 776                         case 'corefx_jitstressregs8':
 777                         case 'corefx_jitstressregs0x10':
 778                         case 'corefx_jitstressregs0x80':
 779                             def displayName = 'CoreFx' + getStressModeDisplayName(scenario)
 780                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 781                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 782                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 783                             break
 784                         default:
 785                             println("Unknown scenario: ${scenario}");
 786                             assert false
 787                             break
 788                     }
 789                     break
 790                 case 'CentOS7.1':
 791                     switch (scenario) {
 792                         case 'pri1':
 793                             if (configuration == 'Release') {
 794                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 795                             }
 796                             break
 797                         case 'r2r':
 798                             if (configuration == 'Checked' || configuration == 'Release') {
 799                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 800                             }
 801                             break
 802                         case 'pri1r2r':
 803                             if (configuration == 'Checked' || configuration == 'Release') {
 804                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 805                             }
 806                             break
 807                         case 'gcstress15_pri1r2r':
 808                             if (configuration == 'Release' || configuration == 'Checked') {
 809                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 810                             }
 811                             break
 812                         case 'r2r_jitstress1':
 813                             if (configuration == 'Release' || configuration == 'Checked') {
 814                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 815                             }
 816                             break
 817                         case 'r2r_jitstress2':
 818                             if (configuration == 'Release' || configuration == 'Checked') {
 819                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 820                             }
 821                             break
 822                         case 'r2r_jitstressregs1':
 823                             if (configuration == 'Release' || configuration == 'Checked') {
 824                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 825                             }
 826                             break
 827                         case 'r2r_jitstressregs2':
 828                             if (configuration == 'Release' || configuration == 'Checked') {
 829                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 830                             }
 831                             break
 832                         case 'r2r_jitstressregs3':
 833                             if (configuration == 'Release' || configuration == 'Checked') {
 834                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 835                             }
 836                             break
 837                         case 'r2r_jitstressregs4':
 838                             if (configuration == 'Release' || configuration == 'Checked') {
 839                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 840                             }
 841                             break
 842                         case 'r2r_jitstressregs8':
 843                             if (configuration == 'Release' || configuration == 'Checked') {
 844                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 845                             }
 846                             break
 847                         case 'r2r_jitstressregsx10':
 848                             if (configuration == 'Release' || configuration == 'Checked') {
 849                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 850                             }
 851                             break
 852                         case 'r2r_jitstressregsx80':
 853                             if (configuration == 'Release' || configuration == 'Checked') {
 854                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 855                             }
 856                             break
 857                         case 'r2r_jitminopts':
 858                             if (configuration == 'Release' || configuration == 'Checked') {
 859                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} JITMinOpts R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 860                             }
 861                             break
 862                         case 'r2r_jitforcerelocs':
 863                             if (configuration == 'Release' || configuration == 'Checked') {
 864                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ForceRelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 865                             }
 866                             break
 867                         default:
 868                             break
 869                     }
 870                 case 'Windows_NT':
 871                     switch (scenario) {
 872                         case 'default':
 873                             // Default trigger
 874                             if (configuration == 'Debug') {
 875                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 876                             }
 877                             break
 878                         case 'pri1':
 879                             // Default trigger
 880                             if (configuration == 'Release') {
 881                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 882                             }
 883                             break
 884                         case 'ilrt':
 885                             if (configuration == 'Release') {
 886                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 887                             }
 888                             break
 889                         case 'r2r':
 890                             if (configuration == 'Checked' || configuration == 'Release') {
 891                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 892                             }
 893                             break
 894                         case 'pri1r2r':
 895                             if (configuration == 'Checked' || configuration == 'Release') {
 896                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 897                             }
 898                             break
 899                         case 'gcstress15_pri1r2r':
 900                             if (configuration == 'Release' || configuration == 'Checked') {
 901                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 902                             }
 903                             break
 904                         case 'r2r_jitstress1':
 905                             if (configuration == 'Release' || configuration == 'Checked') {
 906                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 907                             }
 908                             break
 909                         case 'r2r_jitstress2':
 910                             if (configuration == 'Release' || configuration == 'Checked') {
 911                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 912                             }
 913                             break
 914                         case 'r2r_jitstressregs1':
 915                             if (configuration == 'Release' || configuration == 'Checked') {
 916                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 917                             }
 918                             break
 919                         case 'r2r_jitstressregs2':
 920                             if (configuration == 'Release' || configuration == 'Checked') {
 921                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 922                             }
 923                             break
 924                         case 'r2r_jitstressregs3':
 925                             if (configuration == 'Release' || configuration == 'Checked') {
 926                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 927                             }
 928                             break
 929                         case 'r2r_jitstressregs4':
 930                             if (configuration == 'Release' || configuration == 'Checked') {
 931                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 932                             }
 933                             break
 934                         case 'r2r_jitstressregs8':
 935                             if (configuration == 'Release' || configuration == 'Checked') {
 936                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 937                             }
 938                             break
 939                         case 'r2r_jitstressregsx10':
 940                             if (configuration == 'Release' || configuration == 'Checked') {
 941                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 942                             }
 943                             break
 944                         case 'r2r_jitstressregsx80':
 945                             if (configuration == 'Release' || configuration == 'Checked') {
 946                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 947                             }
 948                             break
 949                         case 'r2r_jitminopts':
 950                             if (configuration == 'Release' || configuration == 'Checked') {
 951                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} JITMinOpts R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 952                             }
 953                             break
 954                         case 'r2r_jitforcerelocs':
 955                             if (configuration == 'Release' || configuration == 'Checked') {
 956                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ForceRelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 957                             }
 958                             break
 959                         case 'longgc':
 960                             if (configuration == 'Release') {
 961                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 962                             }
 963                             break
 964                         case 'gcsimulator':
 965                             if (configuration == 'Release') {
 966                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 967                             }
 968                             break
 969                         case 'minopts':
 970                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 971                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - MinOpts)",
 972                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 973                             break
 974                         case 'forcerelocs':
 975                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 976                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ForceRelocs)",
 977                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 978                             break
 979                         case 'jitstress1':
 980                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 981                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=1)",
 982                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 983                             break
 984                         case 'jitstress2':
 985                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 986                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=2)",
 987                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 988                             break
 989                         case 'jitstressregs1':
 990                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 991                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=1)",
 992                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 993                             break
 994                         case 'jitstressregs2':
 995                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 996                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=2)",
 997                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 998                             break
 999                         case 'jitstressregs3':
1000                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1001                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=3)",
1002                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1003                             break
1004                         case 'jitstressregs4':
1005                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1006                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=4)",
1007                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1008                             break
1009                         case 'jitstressregs8':
1010                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1011                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=8)",
1012                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1013                             break
1014                         case 'jitstressregs0x10':
1015                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1016                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x10)",
1017                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1018                             break
1019                         case 'jitstressregs0x80':
1020                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1021                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x80)",
1022                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1023                             break
1024                         case 'jitstress2_jitstressregs1':
1025                         case 'jitstress2_jitstressregs2':
1026                         case 'jitstress2_jitstressregs3':
1027                         case 'jitstress2_jitstressregs4':
1028                         case 'jitstress2_jitstressregs8':
1029                         case 'jitstress2_jitstressregs0x10':
1030                         case 'jitstress2_jitstressregs0x80':
1031                         case 'gcstress0x3':
1032                         case 'gcstress0xc':
1033                         case 'zapdisable':
1034                         case 'heapverify1':
1035                         case 'gcstress0xc_zapdisable':
1036                         case 'gcstress0xc_zapdisable_jitstress2':
1037                         case 'gcstress0xc_zapdisable_heapverify1':
1038                         case 'gcstress0xc_jitstress1':
1039                         case 'gcstress0xc_jitstress2':
1040                         case 'gcstress0xc_minopts_heapverify1':
1041                             def displayStr = getStressModeDisplayName(scenario)
1042                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1043                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1044                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1045                             break
1046                         case 'corefx_baseline':
1047                         case 'corefx_minopts':
1048                         case 'corefx_jitstress1':
1049                         case 'corefx_jitstress2':
1050                         case 'corefx_jitstressregs1':
1051                         case 'corefx_jitstressregs2':
1052                         case 'corefx_jitstressregs3':
1053                         case 'corefx_jitstressregs4':
1054                         case 'corefx_jitstressregs8':
1055                         case 'corefx_jitstressregs0x10':
1056                         case 'corefx_jitstressregs0x80':
1057                             def displayName = 'CoreFx ' + getStressModeDisplayName(scenario)
1058                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1059                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1060                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1061                             break
1062                         default:
1063                             println("Unknown scenario: ${scenario}");
1064                             assert false
1065                             break
1066                     }
1067                     break
1068                 case 'FreeBSD':
1069                     assert scenario == 'default'
1070                     if (configuration == 'Checked') {
1071                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1072                     }
1073                     break
1074                 default:
1075                     println("Unknown os: ${os}");
1076                     assert false
1077                     break
1078             }
1079             break
1080         case 'arm':
1081             assert scenario == 'default'
1082             switch (os) {
1083                 case 'Ubuntu':
1084                     if (isLinuxEmulatorBuild == false) {
1085                         // Removing the regex will cause this to run on each PR.
1086                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+Linux\\W+arm\\W+cross\\W+${configuration}.*")
1087                     }
1088                     else {
1089                         Utilities.addGithubPRTriggerForBranch(job, branch, "Linux ARM Emulator Cross ${configuration} Build")
1090                     }
1091                     break
1092                 case 'Windows_NT':
1093                     if (configuration == 'Debug' || configuration == 'Release')
1094                     {
1095                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1096                     }
1097                     break
1098                 default:
1099                     println("NYI os: ${os}");
1100                     assert false
1101                     break
1102             }
1103             break
1104         case 'arm64':
1105             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1106
1107             // Set up a private trigger
1108             def contextString = "${os} ${architecture} Cross ${configuration}"
1109             if (scenario != 'default')
1110                 contextString += " ${scenario}"
1111             contextString += " Build"
1112             // Debug builds only.
1113             if (configuration != 'Debug') {
1114                contextString += " and Test"
1115             }
1116
1117             def arm64Users = ['erozenfeld', 'kyulee1', 'pgavlin', 'russellhadley', 'swaroop-sridhar', 'JosephTremoulet', 'jashook', 'RussKeldorph', 'gkhanna79', 'briansull', 'cmckinsey', 'jkotas', 'ramarag', 'markwilkie', 'rahku', 'tzwlai', 'weshaggard']
1118             switch (os) {
1119                 case 'Windows_NT':
1120                     switch (scenario) {
1121                         case 'default':
1122                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1123                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1124                             break
1125                         case 'pri1r2r':
1126                         case 'gcstress0x3':
1127                         case 'gcstress0xc':
1128                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1129                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1130                             break
1131                     }
1132                     break
1133                 default:
1134                     println("NYI os: ${os}");
1135                     assert false
1136                     break
1137             }
1138             break
1139         case 'x86ryujit':
1140             def arch = 'x86'
1141             def jit = 'ryujit'
1142             switch (scenario) {
1143                 case 'default':
1144                     // Default trigger
1145                     if (os == 'Windows_NT') {
1146                         if (configuration == 'Checked') {
1147                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test")
1148                         }
1149                     }
1150                     else {
1151                         // default trigger
1152                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build", "(?i).*test\\W+${arch}\\W+${osGroup}.\\W+${jit}.*")
1153                     }
1154                     break
1155                 case 'pri1':
1156                     // Default trigger
1157                     if (configuration == 'Release') {
1158                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Priority 1 Build and Test")
1159                     }
1160                     break
1161                 case 'ilrt':
1162                     if (configuration == 'Release') {
1163                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1164                     }
1165                     break
1166                 case 'r2r':
1167                     if (configuration == 'Checked' || configuration == 'Release') {
1168                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1169                     }
1170                     break
1171                 case 'pri1r2r':
1172                     if (configuration == 'Checked' || configuration == 'Release') {
1173                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1174                     }
1175                     break
1176                 case 'gcstress15_pri1r2r':
1177                     if (configuration == 'Release' || configuration == 'Checked') {
1178                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1179                     }
1180                     break
1181                 case 'r2r_jitstress1':
1182                     if (configuration == 'Release' || configuration == 'Checked') {
1183                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1184                     }
1185                     break
1186                 case 'r2r_jitstress2':
1187                     if (configuration == 'Release' || configuration == 'Checked') {
1188                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1189                     }
1190                     break
1191                 case 'r2r_jitstressregs1':
1192                     if (configuration == 'Release' || configuration == 'Checked') {
1193                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1194                     }
1195                     break
1196                 case 'r2r_jitstressregs2':
1197                     if (configuration == 'Release' || configuration == 'Checked') {
1198                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1199                     }
1200                     break
1201                 case 'r2r_jitstressregs3':
1202                     if (configuration == 'Release' || configuration == 'Checked') {
1203                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1204                     }
1205                     break
1206                 case 'r2r_jitstressregs4':
1207                     if (configuration == 'Release' || configuration == 'Checked') {
1208                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1209                     }
1210                     break
1211                 case 'r2r_jitstressregs8':
1212                     if (configuration == 'Release' || configuration == 'Checked') {
1213                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1214                     }
1215                     break
1216                 case 'r2r_jitstressregsx10':
1217                     if (configuration == 'Release' || configuration == 'Checked') {
1218                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1219                     }
1220                     break
1221                 case 'r2r_jitstressregsx80':
1222                     if (configuration == 'Release' || configuration == 'Checked') {
1223                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1224                     }
1225                     break
1226                 case 'r2r_jitminopts':
1227                     if (configuration == 'Release' || configuration == 'Checked') {
1228                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} JITMinOpts R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1229                     }
1230                     break
1231                 case 'r2r_jitforcerelocs':
1232                     if (configuration == 'Release' || configuration == 'Checked') {
1233                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} ForceRelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1234                     }
1235                     break
1236                 case 'longgc':
1237                     if (configuration == 'Release') {
1238                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1239                     }
1240                     break
1241                 case 'gcsimulator':
1242                     if (configuration == 'Release') {
1243                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1244                     }
1245                     break
1246                 case 'minopts':
1247                     assert (os == 'Windows_NT')
1248                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - MinOpts)",
1249                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1250                     break
1251                 case 'forcerelocs':
1252                     assert (os == 'Windows_NT')
1253                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - ForceRelocs)",
1254                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1255                     break
1256                 case 'jitstress1':
1257                     assert (os == 'Windows_NT')
1258                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStress=1)",
1259                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1260                     break
1261                 case 'jitstress2':
1262                     assert (os == 'Windows_NT')
1263                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStress=2)",
1264                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1265                     break
1266                 case 'jitstressregs1':
1267                     assert (os == 'Windows_NT')
1268                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=1)",
1269                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1270                     break
1271                 case 'jitstressregs2':
1272                     assert (os == 'Windows_NT')
1273                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=2)",
1274                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1275                     break
1276                 case 'jitstressregs3':
1277                     assert (os == 'Windows_NT')
1278                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=3)",
1279                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1280                     break
1281                 case 'jitstressregs4':
1282                     assert (os == 'Windows_NT')
1283                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=4)",
1284                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1285                     break
1286                 case 'jitstressregs8':
1287                     assert (os == 'Windows_NT')
1288                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=8)",
1289                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1290                     break
1291                 case 'jitstressregs0x10':
1292                     assert (os == 'Windows_NT')
1293                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=0x10)",
1294                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1295                     break
1296                 case 'jitstressregs0x80':
1297                     assert (os == 'Windows_NT')
1298                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=0x80)",
1299                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1300                     break
1301                 case 'jitstress2_jitstressregs1':
1302                 case 'jitstress2_jitstressregs2':
1303                 case 'jitstress2_jitstressregs3':
1304                 case 'jitstress2_jitstressregs4':
1305                 case 'jitstress2_jitstressregs8':
1306                 case 'jitstress2_jitstressregs0x10':
1307                 case 'jitstress2_jitstressregs0x80':
1308                 case 'gcstress0x3':
1309                 case 'gcstress0xc':
1310                 case 'zapdisable':
1311                 case 'heapverify1':
1312                 case 'gcstress0xc_zapdisable':
1313                 case 'gcstress0xc_zapdisable_jitstress2':
1314                 case 'gcstress0xc_zapdisable_heapverify1':
1315                 case 'gcstress0xc_jitstress1':
1316                 case 'gcstress0xc_jitstress2':
1317                 case 'gcstress0xc_minopts_heapverify1':
1318                     def displayStr = getStressModeDisplayName(scenario)
1319                     assert (os == 'Windows_NT')
1320                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - ${displayStr})",
1321                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1322                     break
1323                 default:
1324                     println("Unknown scenario: ${arch} ${jit} ${scenario}");
1325                     assert false
1326                     break
1327             }
1328             break
1329         case 'x86lb':
1330             assert (scenario == 'default' || scenario == 'r2r' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r' || scenario == 'longgc' || scenario == 'gcsimulator' ||
1331                     Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
1332             // For windows, x86 runs by default
1333             def arch = 'x86'
1334             def jit = 'ryujit'
1335             if (architecture == 'x86lb') {
1336                 jit = 'legacy_backend'
1337             }
1338
1339             if (scenario == 'default') {
1340                 if (os == 'Windows_NT') {
1341                     if (configuration == 'Checked') {
1342                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test")
1343                     }
1344                 }
1345                 else {
1346                     // default trigger
1347                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build", "(?i).*test\\W+${arch}\\W+${osGroup}.\\W+${jit}.*")
1348                 }
1349             }
1350             else if (scenario == 'r2r') {
1351                 if (os == 'Windows_NT') {
1352                     if (configuration == 'Release') {
1353                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1354                     }
1355                 }
1356             }
1357             else if (scenario == 'pri1r2r') {
1358                 if (os == 'Windows_NT') {
1359                     if (configuration == 'Release') {
1360                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1361                     }
1362                 }
1363             }
1364             else if (scenario == 'gcstress15_pri1r2r'){
1365                 if (os == 'Windows_NT'){
1366                     if (configuration == 'Release'){
1367                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1368                     }
1369                 }
1370             }
1371             else if (scenario == 'r2r_jitstress1'){
1372                 if (configuration == 'Release' || configuration == 'Checked') {
1373                     if (os == 'Windows_NT'){
1374                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1375                     }
1376                 }
1377             }
1378             else if (scenario == 'r2r_jitstress2'){
1379                 if (configuration == 'Release' || configuration == 'Checked') {
1380                     if (os == 'Windows_NT'){
1381                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1382                     }
1383                 }
1384             }
1385             else if (scenario == 'r2r_jitstressregs1'){
1386                 if (configuration == 'Release' || configuration == 'Checked') {
1387                     if (os == 'Windows_NT'){
1388                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1389                     }
1390                 }
1391             }
1392             else if (scenario == 'r2r_jitstressregs2'){
1393                 if (configuration == 'Release' || configuration == 'Checked') {
1394                     if (os == 'Windows_NT'){
1395                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1396                     }
1397                 }
1398             }
1399             else if (scenario == 'r2r_jitstressregs3'){
1400                 if (configuration == 'Release' || configuration == 'Checked') {
1401                     if (os == 'Windows_NT'){
1402                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1403                     }
1404                 }
1405             }
1406             else if (scenario == 'r2r_jitstressregs4'){
1407                 if (configuration == 'Release' || configuration == 'Checked') {
1408                     if (os == 'Windows_NT'){
1409                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1410                     }
1411                 }
1412             }
1413             else if (scenario == 'r2r_jitstressregs8'){
1414                 if (configuration == 'Release' || configuration == 'Checked') {
1415                     if (os == 'Windows_NT'){
1416                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1417                     }
1418                 }
1419             }
1420             else if (scenario == 'r2r_jitstressregsx10'){
1421                 if (configuration == 'Release' || configuration == 'Checked') {
1422                     if (os == 'Windows_NT'){
1423                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1424                     }
1425                 }
1426             }
1427             else if (scenario == 'r2r_jitstressregsx80'){
1428                 if (configuration == 'Release' || configuration == 'Checked') {
1429                     if (os == 'Windows_NT'){
1430                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1431                     }
1432                 }
1433             }
1434             else if (scenario == 'r2r_jitminopts'){
1435                 if (configuration == 'Release' || configuration == 'Checked') {
1436                     if (os == 'Windows_NT'){
1437                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitminopts R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1438                     }
1439                 }
1440             }
1441             else if (scenario == 'r2r_jitforcerelocs'){
1442                 if (configuration == 'Release' || configuration == 'Checked') {
1443                     if (os == 'Windows_NT'){
1444                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitforcerelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1445                     }
1446                 }
1447             }
1448             else if (scenario == 'longgc') {
1449                 if (os == 'Windows_NT'){
1450                     if (configuration == 'Release'){
1451                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1452                     }
1453                 }
1454             }
1455             else if (scenario == 'gcsimulator') {
1456                 if (os == 'Windows_NT') {
1457                     if (configuration == 'Release') {
1458                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1459                     }
1460                 }
1461             }
1462             break
1463         default:
1464             println("Unknown architecture: ${architecture}");
1465             assert false
1466             break
1467     }
1468 }
1469
1470 // Additional scenario which can alter behavior
1471
1472 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1473 combinedScenarios.each { scenario ->
1474     [true, false].each { isPR ->
1475         Constants.architectureList.each { architecture ->
1476             Constants.configurationList.each { configuration ->
1477                 Constants.osList.each { os ->
1478                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1479                     // and reset the os to Windows_NT
1480                     def isBuildOnly = false
1481                     if (os == 'Windows_NT_BuildOnly') {
1482                         isBuildOnly = true
1483                         os = 'Windows_NT'
1484                     }
1485
1486                     // WinArm32 is only built for Debug and Release
1487                     if (os == 'Windows_NT' && architecture == 'arm')
1488                     {
1489                         if (configuration == 'Checked')
1490                         {
1491                             return
1492                         }
1493                     }
1494                     // If the OS is LinuxARMEmulator and arch is arm, set the isLinuxEmulatorBuild
1495                     // flag to true and reset the os to Ubuntu
1496                     // The isLinuxEmulatorBuild flag will be used at a later time to execute the right
1497                     // set of build commands
1498                     // The tuples (LinuxARMEmulator, other architectures) are not handled and control returns
1499                     def isLinuxEmulatorBuild = false
1500                     if (os == 'LinuxARMEmulator' && architecture == 'arm') {
1501                         // Cross Builds only in Debug and Release modes allowed
1502                         if ( configuration == 'Checked' ) {
1503                             return
1504                         }
1505
1506                         isLinuxEmulatorBuild = true
1507                         os = 'Ubuntu'
1508                     } else if (os == 'LinuxARMEmulator') {
1509                         return
1510                     }
1511
1512                     // Skip totally unimplemented (in CI) configurations.
1513                     switch (architecture) {
1514                         case 'arm64':
1515                             // Windows only
1516                             if (os != 'Windows_NT' || isBuildOnly) {
1517                                 return
1518                             }
1519                             break
1520                         case 'arm':
1521                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1522                                 return
1523                             }
1524                             break
1525                         case 'x86ryujit':
1526                         case 'x86lb':
1527                             // Skip non-windows
1528                             if (os != 'Windows_NT') {
1529                                 return
1530                             }
1531                             break
1532                         case 'x64':
1533                             // Everything implemented
1534                             break
1535                         default:
1536                             println("Unknown architecture: ${architecture}")
1537                             assert false
1538                             break
1539                     }
1540                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1541                     // with checked builds
1542                     def enableCorefxTesting = false
1543                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1544                         if (configuration != 'Checked') {
1545                             return
1546                         }
1547
1548                         enableCorefxTesting = isCorefxTesting(scenario)
1549
1550                         // Since these are just execution time differences,
1551                         // skip platforms that don't execute the tests here (Windows_NT only)
1552                         def isEnabledOS = os == 'Windows_NT' || (os == 'Ubuntu' && enableCorefxTesting)
1553                         if (!isEnabledOS || isBuildOnly) {
1554                             return
1555                         }
1556
1557                         switch (architecture) {
1558                             case 'arm64':
1559                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1560                                     return
1561                                 }
1562                                 break
1563                             case 'x64':
1564                                 // Everything implemented
1565                                 break
1566                             case 'x86ryujit':
1567                                 if (enableCorefxTesting) {
1568                                     return
1569                                 }
1570                                 break
1571                             default:
1572                                 return
1573                         }
1574                     }
1575                     else {
1576                         // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
1577                         // and configuration is not Checked, bail out.
1578                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1579                             return;
1580                         }
1581
1582                         // Skip scenarios
1583                         switch (scenario) {
1584                             case 'pri1':
1585                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1586                                 // the default scenario build
1587                                 if (os != 'Windows_NT') {
1588                                     return
1589                                 }
1590                                 // Only x64 for now
1591                                 if (architecture != 'x64') {
1592                                     return
1593                                 }
1594                                 break
1595                             case 'ilrt':
1596                                 // The ilrt build isn't necessary except for Windows_NT.  Non-Windows NT uses
1597                                 // the default scenario build
1598                                 if (os != 'Windows_NT') {
1599                                     return
1600                                 }
1601                                 // Only x64 for now
1602                                 if (architecture != 'x64') {
1603                                     return
1604                                 }
1605                                 // Release only
1606                                 if (configuration != 'Release') {
1607                                     return
1608                                 }
1609                                 break
1610                             case 'r2r':
1611                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1612                                 // the default scenario build
1613                                 if (os != 'Windows_NT') {
1614                                     return
1615                                 }
1616                                 if (architecture != 'x64') {
1617                                     return
1618                                 }
1619                                 break
1620                             case 'pri1r2r':
1621                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1622                                 // the default scenario build
1623                                 if (os != 'Windows_NT') {
1624                                     return
1625                                 }
1626                                 if (architecture != 'x64') {
1627                                     if (architecture != 'arm64' || configuration == 'Debug') {
1628                                         return
1629                                     }
1630                                 }
1631                                 break
1632                             case 'gcstress15_pri1r2r':
1633                             case 'r2r_jitstress1':
1634                             case 'r2r_jitstress2':
1635                             case 'r2r_jitstressregs1':
1636                             case 'r2r_jitstressregs2':
1637                             case 'r2r_jitstressregs3':
1638                             case 'r2r_jitstressregs4':
1639                             case 'r2r_jitstressregs8':
1640                             case 'r2r_jitstressregsx10':
1641                             case 'r2r_jitstressregsx80':
1642                             case 'r2r_jitminopts':
1643                             case 'r2r_jitforcerelocs':
1644                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
1645                                 // the default scenario build
1646                                 if (os != 'Windows_NT') {
1647                                     return
1648                                 }
1649                                 if (architecture != 'x64') {
1650                                     return
1651                                 }
1652                                 break
1653                             case 'longgc':
1654                             case 'gcsimulator':
1655                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX') {
1656                                     return
1657                                 }
1658                                 if (architecture != 'x64') {
1659                                     return
1660                                 }
1661                                 if (configuration != 'Release') {
1662                                     return
1663                                 }
1664                                 break
1665                             // We need Windows x64 Release bits for the code coverage build
1666                             case 'coverage':
1667                                 if (os != 'Windows_NT') {
1668                                     return
1669                                 }
1670                                 if (architecture != 'x64') {
1671                                     return
1672                                 }
1673                                 if (configuration != 'Release') {
1674                                     return
1675                                 }
1676                                 break
1677                             // We only run Windows and Ubuntu x64 Checked for formatting right now
1678                             case 'formatting':
1679                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
1680                                     return
1681                                 }
1682                                 if (architecture != 'x64') {
1683                                     return
1684                                 }
1685                                 if (configuration != 'Checked') {
1686                                     return
1687                                 }
1688                                 if (isBuildOnly) {
1689                                     return
1690                                 }
1691                                 break
1692                             case 'default':
1693                                 // Nothing skipped
1694                                 break
1695                             default:
1696                                 println("Unknown scenario: ${scenario}")
1697                                 assert false
1698                                 break
1699                         }
1700                     }
1701
1702                     // Calculate names
1703                     def lowerConfiguration = configuration.toLowerCase()
1704                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly, isLinuxEmulatorBuild)
1705                     def folderName = isJITStressJob(scenario) ? 'jitstress' : '';
1706
1707                     // Create the new job
1708                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
1709
1710                     setMachineAffinity(newJob, os, architecture)
1711
1712                     // Add all the standard options
1713                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
1714                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly, isLinuxEmulatorBuild)
1715
1716                     def buildCommands = [];
1717                     def osGroup = getOSGroup(os)
1718
1719                     // Calculate the build steps, archival, and xunit results
1720                     switch (os) {
1721                         case 'Windows_NT':
1722                             switch (architecture) {
1723                                 case 'x64':
1724                                 case 'x86ryujit':
1725                                 case 'x86lb':
1726                                     def arch = architecture
1727                                     def buildOpts = ''
1728                                     if (architecture == 'x86ryujit') {
1729                                         arch = 'x86'
1730                                         buildOpts = 'altjitcrossgen'
1731                                     }
1732                                     else if (architecture == 'x86lb') {
1733                                         arch = 'x86'
1734                                     }
1735
1736                                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1737                                             scenario == 'default' ||
1738                                             scenario == 'r2r' ||
1739                                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1740                                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1741                                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1742                                     }
1743
1744                                     // For Pri 1 tests, we must shorten the output test binary path names.
1745                                     // if __TestIntermediateDir is already set, build-test.cmd will
1746                                     // output test binaries to that directory. If it is not set, the
1747                                     // binaries are sent to a default directory whose name is about
1748                                     // 35 characters long.
1749
1750                                     else if (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage') {
1751                                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1752                                     }
1753                                     else if (scenario == 'ilrt') {
1754                                         // First do the build with skiptests and then build the tests with ilasm roundtrip
1755                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1756                                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch} -ilasmroundtrip"
1757                                     }
1758                                     else if (isLongGc(scenario)) {
1759                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1760                                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1761                                     }
1762                                     else if (scenario == 'formatting') {
1763                                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1764                                         break
1765                                     }
1766                                     else {
1767                                         println("Unknown scenario: ${scenario}")
1768                                         assert false
1769                                     }
1770
1771                                     // If we are running a stress mode, we should write out the set of key
1772                                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1773
1774                                     if (!isBuildOnly) {
1775                                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1776                                         def crossgenStr = ''
1777                                         def runcrossgentestsStr = ''
1778                                         def runjitstressStr = ''
1779                                         def runjitstressregsStr = ''
1780                                         def runjitmioptsStr = ''
1781                                         def runjitforcerelocsStr = ''
1782                                         def gcstressStr = ''
1783                                         def runtestArguments = ''
1784                                         def gcTestArguments = ''
1785
1786                                         if (scenario == 'r2r' ||
1787                                             scenario == 'pri1r2r' ||
1788                                             scenario == 'gcstress15_pri1r2r' ||
1789                                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1790                                                 crossgenStr = 'crossgen'
1791                                                 runcrossgentestsStr = 'runcrossgentests'
1792
1793                                                 if (scenario == 'r2r_jitstress1'){
1794                                                     runjitstressStr = 'jitstress 1'
1795                                                 }
1796                                                 else if (scenario == 'r2r_jitstress2') {
1797                                                     runjitstressStr = 'jitstress 2'
1798                                                 }
1799                                                 else if (scenario == 'r2r_jitstressregs1'){
1800                                                     runjitstressregsStr = 'jitstressregs 1'
1801                                                 }
1802                                                 else if (scenario == 'r2r_jitstressregs2') {
1803                                                     runjitstressregsStr = 'jitstressregs 2'
1804                                                 }
1805                                                 else if (scenario == 'r2r_jitstressregs3') {
1806                                                     runjitstressregsStr = 'jitstressregs 3'
1807                                                 }
1808                                                 else if (scenario == 'r2r_jitstressregs4') {
1809                                                     runjitstressregsStr = 'jitstressregs 4'
1810                                                 }
1811                                                 else if (scenario == 'r2r_jitstressregs8') {
1812                                                     runjitstressregsStr = 'jitstressregs 8'
1813                                                 }
1814                                                 else if (scenario == 'r2r_jitstressregsx10') {
1815                                                     runjitstressregsStr = 'jitstressregs x10'
1816                                                 }
1817                                                 else if (scenario == 'r2r_jitstressregsx80') {
1818                                                     runjitstressregsStr = 'jitstressregs x80'
1819                                                 }
1820                                                 else if (scenario == 'r2r_jitminopts') {
1821                                                     runjitmioptsStr = 'jitminopts'
1822                                                 }
1823                                                 else if (scenario == 'r2r_jitforcerelocs') {
1824                                                     runjitforcerelocsStr = 'jitforcerelocs'
1825                                                 }
1826                                         }
1827                                         if (scenario == 'gcstress15_pri1r2r')
1828                                         {
1829                                            gcstressStr = 'gcstresslevel 0xF'
1830                                         }
1831
1832                                         if (isLongGc(scenario)) {
1833                                             gcTestArguments = "${scenario} sequential"
1834                                         }
1835
1836                                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${gcTestArguments}"
1837                                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1838                                             if (enableCorefxTesting) {
1839                                                 // Sync to corefx repo
1840                                                 // Move coreclr files to a subdirectory, %workspace%/clr. Otherwise, corefx build
1841                                                 // thinks that %workspace% is the project base directory.
1842                                                 buildCommands += "powershell new-item clr -type directory -force"
1843                                                 buildCommands += 'powershell foreach ($x in get-childitem -force) { if (\$x.name -ne \'clr\') { move-item $x clr }}'
1844                                                 buildCommands += "git clone https://github.com/dotnet/corefx fx"
1845
1846                                                 buildCommands += getStressModeEnvSetCmd(os, scenario);
1847
1848                                                 // Run corefx build and testing
1849                                                 buildCommands += "cd fx && call \"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat\" x86 && Build.cmd -Release -- /p:BUILDTOOLS_OVERRIDE_RUNTIME=%WORKSPACE%\\clr\\bin\\Product\\Windows_NT.x64.Checked "
1850                                             }
1851                                             else {
1852                                                 def stepScriptLocation = "%WORKSPACE%\\bin\\tests\\SetStressModes.bat"
1853
1854                                                 if (architecture == 'x86ryujit'){
1855                                                     def x86Vars = ['COMPLUS_AltJit' : '*', 'COMPLUS_AltJitName' : 'protojit.dll', 'COMPLUS_NoGuiOnAssert' : '1']
1856                                                     buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario] + x86Vars, stepScriptLocation)
1857                                                 }
1858                                                 else {
1859                                                     buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1860                                                 }
1861
1862                                                 // Run tests with the
1863                                                 buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1864                                             }
1865                                         }
1866                                         else if (architecture == 'x64') {
1867                                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1868                                         }
1869                                         else if (architecture == 'x86ryujit') {
1870                                             def testEnvLocation = "%WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd"
1871
1872                                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv ${testEnvLocation}"
1873                                         }
1874                                         else if (architecture == 'x86lb') {
1875                                             buildCommands += "tests\\runtest.cmd ${runtestArguments} Exclude0 x86_legacy_backend_issues.targets"
1876                                         }
1877                                     }
1878
1879                                     if (!enableCorefxTesting) {
1880                                         // Run the rest of the build
1881                                         // Build the mscorlib for the other OS's
1882                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1883                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1884                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1885
1886                                         // Zip up the tests directory so that we don't use so much space/time copying
1887                                         // 10s of thousands of files around.
1888                                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1889
1890                                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1891                                             // For windows, pull full test results and test drops for x86/x64.
1892                                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1893                                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip")
1894                                         }
1895
1896                                         if (!isBuildOnly) {
1897                                             if (architecture == 'x64' || !isPR) {
1898                                                 Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml')
1899                                             }
1900                                             setTestJobTimeOut(newJob, scenario)
1901                                         }
1902                                     }
1903                                     else {
1904                                         // Archive only result xml files since corefx/bin/tests is very large around 10 GB.
1905                                         // For windows, pull full test results and test drops for x86/x64
1906                                         Utilities.addArchival(newJob, "fx/bin/tests/**/testResults.xml")
1907
1908                                         // Set timeout
1909                                         setTestJobTimeOut(newJob, scenario)
1910
1911                                         if (architecture == 'x64' || !isPR) {
1912                                             Utilities.addXUnitDotNETResults(newJob, 'fx/bin/tests/**/testResults.xml')
1913                                         }
1914                                     }
1915
1916                                     break
1917                                 case 'arm':
1918                                     assert (scenario == 'default')
1919
1920                                     // Set time out
1921                                     setTestJobTimeOut(newJob, scenario)
1922
1923                                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1924
1925                                     // Add archival.
1926                                     Utilities.addArchival(newJob, "bin/Product/**")
1927                                     break
1928                                 case 'arm64':
1929                                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1930                                     // Set time out
1931                                     setTestJobTimeOut(newJob, scenario)
1932
1933                                     // Debug runs take too long to run. So build job only.
1934                                     if (lowerConfiguration == "debug") {
1935                                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1936                                     }
1937                                     else {
1938                                        buildCommands += "set __TestIntermediateDir=int&&build.cmd skiptests ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1939                                        // Test build and run are launched together.
1940                                        buildCommands += "Z:\\arm64\\common\\scripts\\arm64PostLauncher.cmd %WORKSPACE% ${architecture} ${lowerConfiguration} ${scenario}"
1941                                        Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1942                                     }
1943
1944                                     // Add archival.
1945                                     Utilities.addArchival(newJob, "bin/Product/**")
1946                                     break
1947                                 default:
1948                                     println("Unknown architecture: ${architecture}");
1949                                     assert false
1950                                     break
1951                             }
1952                             break
1953                         case 'Ubuntu':
1954                         case 'Ubuntu16.04':
1955                         case 'Ubuntu16.10':
1956                         case 'Debian8.4':
1957                         case 'OSX':
1958                         case 'FreeBSD':
1959                         case 'CentOS7.1':
1960                         case 'RHEL7.2':
1961                         case 'OpenSUSE13.2':
1962                         case 'OpenSUSE42.1':
1963                         case 'Fedora23':
1964                             switch (architecture) {
1965                                 case 'x64':
1966                                 case 'x86ryujit':
1967                                 case 'x86lb':
1968                                     def arch = architecture
1969                                     if (architecture == 'x86ryujit' || architecture == 'x86lb') {
1970                                         arch = 'x86'
1971                                     }
1972
1973                                     if (scenario == 'formatting') {
1974                                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${arch}"
1975                                         break
1976                                     }
1977
1978                                     if (!enableCorefxTesting) {
1979                                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1980                                         // only on supported OS platforms.
1981                                         if ((os == 'FreeBSD') || (os == 'OpenSUSE13.2'))
1982                                         {
1983                                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${arch}"
1984                                         }
1985                                         else
1986                                         {
1987                                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1988                                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1989                                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${arch}"
1990                                         }
1991                                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${arch}.${configuration} \${WORKSPACE}/bin/paltestout"
1992
1993                                         // Set time out
1994                                         setTestJobTimeOut(newJob, scenario)
1995                                         // Basic archiving of the build
1996                                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so")
1997                                         // And pal tests
1998                                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1999                                     }
2000                                     else {
2001                                         // Corefx stress testing
2002                                         assert os == 'Ubuntu'
2003                                         assert architecture == 'x64'
2004                                         assert lowerConfiguration == 'checked'
2005
2006                                         // Build coreclr and move it to clr directory
2007                                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
2008                                         buildCommands += "rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr"
2009
2010                                         // Get corefx
2011                                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
2012
2013                                         // Set environment variable
2014                                         def setEnvVar = getStressModeEnvSetCmd(os, scenario)
2015
2016                                         // Build and text corefx
2017                                         buildCommands += "rm -rf \$WORKSPACE/fx_home; mkdir \$WORKSPACE/fx_home"
2018                                         buildCommands += setEnvVar
2019                                         buildCommands += "cd fx; export HOME=\$WORKSPACE/fx_home; ./build.sh -Release -Outerloop -TestWithLocalLibraries -- /p:BUILDTOOLS_OVERRIDE_RUNTIME=\$WORKSPACE/clr/bin/Product/Linux.x64.Checked"
2020
2021                                         // Archive and process test result
2022                                         Utilities.addArchival(newJob, "fx/bin/tests/**/testResults.xml")
2023                                         setTestJobTimeOut(newJob, scenario)
2024                                         Utilities.addXUnitDotNETResults(newJob, 'fx/bin/tests/**/testResults.xml')
2025                                     }
2026                                     break
2027                                 case 'arm64':
2028                                     // We don't run the cross build except on Ubuntu
2029                                     assert os == 'Ubuntu'
2030
2031                                     buildCommands += """echo \"Using rootfs in /opt/aarch64-linux-gnu-root\"
2032                                         ROOTFS_DIR=/opt/aarch64-linux-gnu-root ./build.sh skipmscorlib arm64 cross verbose ${lowerConfiguration}"""
2033
2034                                     // Basic archiving of the build, no pal tests
2035                                     Utilities.addArchival(newJob, "bin/Product/**")
2036                                     break
2037                                 case 'arm':
2038                                     // All builds for ARM architecture are run on Ubuntu currently
2039                                     assert os == 'Ubuntu'
2040                                     if (isLinuxEmulatorBuild == false) {
2041                                         buildCommands += """echo \"Using rootfs in /opt/arm-liux-genueabihf-root\"
2042                                             ROOTFS_DIR=/opt/arm-linux-genueabihf-root ./build.sh skipmscorlib arm cross verbose ${lowerConfiguration}"""
2043
2044                                         // Basic archiving of the build, no pal tests
2045                                         Utilities.addArchival(newJob, "bin/Product/**")
2046                                         break
2047                                     }
2048                                     else {
2049                                         // Make sure the build configuration is either of debug or release
2050                                         assert ( lowerConfiguration == 'debug' ) || ( lowerConfiguration == 'release' )
2051
2052                                         // Setup variables to hold emulator folder path and the rootfs mount path
2053                                         def armemul_path = '/opt/linux-arm-emulator'
2054                                         def armrootfs_mountpath = '/opt/linux-arm-emulator-root'
2055
2056                                         // Unzip the Windows test binaries first. Exit with 0
2057                                         buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2058
2059                                         // Unpack the corefx binaries
2060                                         buildCommands += "tar -xf ./bin/build.tar.gz"
2061
2062                                         // Call the ARM emulator build script to cross build and test using the ARM emulator rootfs
2063                                         buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2064                                         --emulatorPath=${armemul_path} \\
2065                                         --mountPath=${armrootfs_mountpath} \\
2066                                         --buildConfig=${lowerConfiguration} \\
2067                                         --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2068                                         --coreFxNativeBinDir=./bin/Linux.arm-softfp.${configuration} \\
2069                                         --coreFxBinDir=\"./bin/Linux.AnyCPU.${configuration};./bin/Unix.AnyCPU.${configuration};./bin/AnyOS.AnyCPU.${configuration}\" \\
2070                                         --testDirFile=./tests/testsRunningInsideARM.txt"""
2071
2072
2073                                         // Basic archiving of the build
2074                                         Utilities.addArchival(newJob, "bin/Product/**")
2075                                         break
2076                                     }
2077                                 default:
2078                                     println("Unknown architecture: ${architecture}");
2079                                     assert false
2080                                     break
2081                             }
2082                             break
2083                         default:
2084                             println("Unknown os: ${os}");
2085                             assert false
2086                             break
2087                     }
2088
2089                     newJob.with {
2090                         steps {
2091                             if (os == 'Windows_NT') {
2092                                 buildCommands.each { buildCommand ->
2093                                     batchFile(buildCommand)
2094                                 }
2095                             }
2096                             else {
2097                                 // Setup corefx and Windows test binaries for Linux ARM Emulator Build
2098                                 if (isLinuxEmulatorBuild) {
2099                                     // Define the Windows Tests and Corefx build job names
2100                                     def WindowTestsName = projectFolder + '/' +
2101                                                           Utilities.getFullJobName(project,
2102                                                                                    getJobName(lowerConfiguration,
2103                                                                                               'x64' ,
2104                                                                                               'windows_nt',
2105                                                                                               'default',
2106                                                                                               true),
2107                                                                                    false)
2108                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2109                                                        Utilities.getFolderName(branch)
2110
2111                                     // Copy the Windows test binaries and the Corefx build binaries
2112                                     copyArtifacts(WindowTestsName) {
2113                                         excludePatterns('**/testResults.xml', '**/*.ni.dll')
2114                                         buildSelector {
2115                                             latestSuccessful(true)
2116                                         }
2117                                     }
2118                                     copyArtifacts("${corefxFolder}/linuxarmemulator_softfp_cross_${lowerConfiguration}") {
2119                                         includePatterns('bin/build.tar.gz')
2120                                         buildSelector {
2121                                             latestSuccessful(true)
2122                                         }
2123                                     }
2124                                 }
2125
2126                                 buildCommands.each { buildCommand ->
2127                                     shell(buildCommand)
2128                                 }
2129                             }
2130                         }
2131                     }
2132
2133                 } // os
2134             } // configuration
2135         } // architecture
2136     } // isPR
2137 } // scenario
2138
2139
2140 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2141 combinedScenarios.each { scenario ->
2142     [true, false].each { isPR ->
2143         // Architectures.  x64 only at this point
2144         ['x64'].each { architecture ->
2145             // Put the OS's supported for coreclr cross testing here
2146             Constants.crossList.each { os ->
2147                 Constants.configurationList.each { configuration ->
2148
2149                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2150                         if (configuration != 'Checked') {
2151                             return
2152                         }
2153                         if (isCorefxTesting(scenario)) {
2154                             return
2155                         }
2156                         //Skip stress modes for these scenarios
2157                         if (os == 'RHEL7.2' || os == 'Debian8.4' || os == 'OpenSUSE13.2') {
2158                             return
2159                         }
2160                     }
2161                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2162                     // and configuration is not Checked, bail out.
2163                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2164                         return;
2165                     }
2166                     // For CentOS, we only want Checked/Release pri1 builds.
2167                     else if (os == 'CentOS7.1') {
2168                         if (scenario != 'pri1' &&
2169                             scenario != 'r2r' &&
2170                             scenario != 'pri1r2r' &&
2171                             scenario != 'gcstress15_pri1r2r' &&
2172                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2173                             return
2174                         }
2175                         if (configuration != 'Checked' && configuration != 'Release') {
2176                             return
2177                         }
2178                     }
2179                     // For RedHat, Debian, and OpenSUSE, we only do Release pri1 builds.
2180                     else if (os == 'RHEL7.2' || os == 'Debian8.4' || os == 'OpenSUSE13.2') {
2181                         if (scenario != 'pri1') {
2182                             return
2183                         }
2184                         if (configuration != 'Release') {
2185                             return
2186                         }
2187                     }
2188                     else {
2189                         // Skip scenarios
2190                         switch (scenario) {
2191                             case 'pri1':
2192                                 // Nothing skipped
2193                                 break
2194                             case 'ilrt':
2195                                 // Release only
2196                                 if (configuration != 'Release') {
2197                                     return
2198                                 }
2199                                 break
2200                             case 'r2r':
2201                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2202                                 if (configuration != 'Checked' && configuration != 'Release') {
2203                                     return
2204                                 }
2205                                 break
2206                             case 'pri1r2r':
2207                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2208                                 if (configuration != 'Checked' && configuration != 'Release') {
2209                                     return
2210                                 }
2211                                 break
2212                             case 'gcstress15_pri1r2r':
2213                             case 'r2r_jitstress1':
2214                             case 'r2r_jitstress2':
2215                             case 'r2r_jitstressregs1':
2216                             case 'r2r_jitstressregs2':
2217                             case 'r2r_jitstressregs3':
2218                             case 'r2r_jitstressregs4':
2219                             case 'r2r_jitstressregs8':
2220                             case 'r2r_jitstressregsx10':
2221                             case 'r2r_jitstressregsx80':
2222                             case 'r2r_jitminopts':
2223                             case 'r2r_jitforcerelocs':
2224                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2225                                 if (configuration != 'Checked' && configuration != 'Release') {
2226                                     return
2227                                 }
2228                                 break
2229                             case 'longgc':
2230                             case 'gcsimulator':
2231                                 // Long GC tests take a long time on non-Release builds
2232                                 if (configuration != 'Release') {
2233                                     return
2234                                 }
2235                                 break
2236                             case 'coverage':
2237                                 //We only want Ubuntu Release for coverage
2238                                 if (os != 'Ubuntu') {
2239                                     return
2240                                 }
2241                                 if (configuration != 'Release') {
2242                                     return
2243                                 }
2244                             case 'formatting':
2245                                 return
2246                             case 'default':
2247                                 // Nothing skipped
2248                                 break
2249                             default:
2250                                 println("Unknown scenario: ${scenario}")
2251                                 assert false
2252                                 break
2253                         }
2254                     }
2255
2256                     def lowerConfiguration = configuration.toLowerCase()
2257                     def osGroup = getOSGroup(os)
2258                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2259
2260                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2261                     def inputScenario = 'default'
2262                     if (scenario == 'coverage') {
2263                         inputScenario = 'coverage'
2264                     }
2265                     def inputCoreCLRBuildName = projectFolder + '/' +
2266                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2267                     // If this is a stress scenario, there isn't any difference in the build job
2268                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2269                     // from the default scenario
2270                     def testBuildScenario = scenario
2271                     if (testBuildScenario == 'coverage' || testBuildScenario == 'pri1r2r'|| testBuildScenario == 'gcstress15_pri1r2r') {
2272                         testBuildScenario = 'pri1'
2273                     }
2274                     else if ( testBuildScenario == 'r2r' || isLongGc(testBuildScenario)) {
2275                         testBuildScenario = 'default'
2276                     }
2277                     def inputWindowTestsBuildName = ''
2278                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2279                         inputWindowTestsBuildName = projectFolder + '/' +
2280                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', 'default', true), isPR)
2281                     }
2282                     else {
2283                         inputWindowTestsBuildName = projectFolder + '/' +
2284                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', testBuildScenario, true), isPR)
2285                     }
2286                     // Enable Server GC for Ubuntu PR builds
2287                     def serverGCString = ''
2288
2289                     // Whether or not this test run should be run sequentially instead
2290                     // of in parallel. Only used for long GC tests.
2291                     def sequentialString = ''
2292
2293                     // Whether or not this test run should run a specific playlist.
2294                     // Only used for long GC tests.
2295
2296                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2297                     // for running long GC and GCSimulator tests, respectively. We don't use them
2298                     // here because using a playlist file produces much more readable output on the CI machines
2299                     // and reduces running time.
2300                     def playlistString = ''
2301
2302                     if (os == 'Ubuntu' && isPR){
2303                         serverGCString = '--useServerGC'
2304                     }
2305
2306                     // pass --crossgen to runtest.sh for crossgen builds
2307                     def crossgenStr = ''
2308                     def runcrossgentestsStr = ''
2309                     def runjitstressStr = ''
2310                     def runjitstressregsStr = ''
2311                     def runjitmioptsStr = ''
2312                     def runjitforcerelocsStr = ''
2313                     def gcstressStr = ''
2314
2315                     if (scenario == 'r2r' ||
2316                         scenario == 'pri1r2r' ||
2317                         scenario == 'gcstress15_pri1r2r' ||
2318                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2319                             crossgenStr = '--crossgen'
2320                             runcrossgentestsStr = '--runcrossgentests'
2321
2322                             if (scenario == 'r2r_jitstress1'){
2323                                 runjitstressStr = '--jitstress=1'
2324                             }
2325                             else if (scenario == 'r2r_jitstress2') {
2326                                 runjitstressStr = '--jitstress=2'
2327                             }
2328                             else if (scenario == 'r2r_jitstressregs1'){
2329                                 runjitstressregsStr = '--jitstressregs=1'
2330                             }
2331                             else if (scenario == 'r2r_jitstressregs2') {
2332                                 runjitstressregsStr = '--jitstressregs=2'
2333                             }
2334                             else if (scenario == 'r2r_jitstressregs3') {
2335                                 runjitstressregsStr = '--jitstressregs=3'
2336                             }
2337                             else if (scenario == 'r2r_jitstressregs4') {
2338                                 runjitstressregsStr = '--jitstressregs=4'
2339                             }
2340                             else if (scenario == 'r2r_jitstressregs8') {
2341                                 runjitstressregsStr = '--jitstressregs=8'
2342                             }
2343                             else if (scenario == 'r2r_jitstressregsx10') {
2344                                 runjitstressregsStr = '--jitstressregs=x10'
2345                             }
2346                             else if (scenario == 'r2r_jitstressregsx80') {
2347                                 runjitstressregsStr = '--jitstressregs=x80'
2348                             }
2349                             else if (scenario == 'r2r_jitminopts') {
2350                                 runjitmioptsStr = '--jitminopts'
2351                             }
2352                             else if (scenario == 'r2r_jitforcerelocs') {
2353                                 runjitforcerelocsStr = '--jitforcerelocs'
2354                             }
2355                     }
2356                     if  (scenario == 'gcstress15_pri1r2r')
2357                     {
2358                         gcstressStr = '--gcstresslevel=0xF'
2359                     }
2360
2361                     if (isLongGc(scenario)) {
2362                         // Long GC tests behave very poorly when they are not
2363                         // the only test running (many of them allocate until OOM).
2364                         sequentialString = '--sequential'
2365
2366                         // The Long GC playlist contains all of the tests that are
2367                         // going to be run. The GCSimulator playlist contains all of
2368                         // the GC simulator tests.
2369                         if (scenario == 'longgc') {
2370                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2371                         }
2372                         else if (scenario == 'gcsimulator') {
2373                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2374                         }
2375                     }
2376
2377                     def folder = isJITStressJob(scenario) ? 'jitstress' : ''
2378                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2379                         // Add parameters for the inputs
2380
2381                         parameters {
2382                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2383                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2384                         }
2385
2386                         steps {
2387                             // Set up the copies
2388
2389                             // Coreclr build containing the tests and mscorlib
2390
2391                             copyArtifacts(inputWindowTestsBuildName) {
2392                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2393                                 buildSelector {
2394                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2395                                 }
2396                             }
2397
2398                             if (scenario == 'coverage') {
2399
2400                                 // Move coreclr to clr directory
2401                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2402
2403                                 // Build coreclr
2404                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2405
2406                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2407                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2408                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2409                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2410                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2411                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2412                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2413                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2414
2415                                 // Run PAL tests
2416                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2417
2418                                 // Remove obj files for PAL tests so they're not included in coverage results
2419                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2420
2421                                 // Unzip the tests first.  Exit with 0
2422                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2423
2424                                 // Get corefx
2425                                 shell("git clone https://github.com/dotnet/corefx fx")
2426
2427                                 // Build Linux corefx
2428                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2429                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2430
2431                                 def testEnvOpt = ""
2432                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2433                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2434                                 shell("${createScriptCmds}")
2435                                 testEnvOpt = "--test-env=" + scriptFileName
2436
2437                                 // Run corefx tests
2438                                 shell("""./fx/run-test.sh \\
2439                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2440                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2441                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2442                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2443                 --configurationGroup Release""")
2444
2445
2446                                 // Run coreclr tests w/ workstation GC
2447                                 shell("""./clr/tests/runtest.sh \\
2448                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2449                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2450                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2451                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2452                 --coreFxBinDir=\"\$(pwd)/fx/bin/${osGroup}.AnyCPU.Release;\$(pwd)/fx/bin/Unix.AnyCPU.Release;\$(pwd)/fx/bin/AnyOS.AnyCPU.Release\" \\
2453                 --coreFxNativeBinDir=\"\$(pwd)/fx/bin/${osGroup}.${architecture}.Release\" \\
2454                 --crossgen --runcrossgentests""")
2455
2456                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2457                                 shell("""./clr/tests/runtest.sh \\
2458                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2459                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2460                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2461                 --useServerGC ${testEnvOpt}""")
2462
2463                                  // Run long-running coreclr GC tests & produce coverage reports
2464                                 shell("""./clr/tests/runtest.sh \\
2465                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2466                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2467                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2468                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2469                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2470                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2471                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2472
2473                             }
2474                             else {
2475
2476                                 // Coreclr build we are trying to test
2477
2478                                 copyArtifacts(inputCoreCLRBuildName) {
2479                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2480                                     buildSelector {
2481                                         buildNumber('${CORECLR_BUILD}')
2482                                     }
2483                                 }
2484
2485                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2486
2487                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2488                                 def osJobName = (os == 'Ubuntu') ? 'ubuntu14.04' : os.toLowerCase()
2489                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2490                                     includePatterns('bin/build.tar.gz')
2491                                     buildSelector {
2492                                         latestSuccessful(true)
2493                                     }
2494                                 }
2495
2496                                 // Unpack the corefx binaries
2497                                 shell("tar -xf ./bin/build.tar.gz")
2498
2499                                 // Unzip the tests first.  Exit with 0
2500                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2501
2502                                 // Execute the tests
2503                                 // If we are running a stress mode, we'll set those variables first
2504                                 def testEnvOpt = ""
2505                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2506                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2507                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2508                                     if (createScriptCmds != "") {
2509                                         shell("${createScriptCmds}")
2510                                         testEnvOpt = "--test-env=" + scriptFileName
2511                                     }
2512                                 }
2513
2514                                 if (isGCStressRelatedTesting(scenario)) {
2515                                     shell('./init-tools.sh')
2516                                 }
2517
2518                                 shell("""./tests/runtest.sh \\
2519                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2520                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2521                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2522                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2523                 --coreFxBinDir=\"\${WORKSPACE}/bin/${osGroup}.AnyCPU.Release;\${WORKSPACE}/bin/Unix.AnyCPU.Release;\${WORKSPACE}/bin/AnyOS.AnyCPU.Release\" \\
2524                 --coreFxNativeBinDir=\"\${WORKSPACE}/bin/${osGroup}.${architecture}.Release\" \\
2525                 --limitedDumpGeneration \\
2526                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${sequentialString} ${playlistString}""")
2527                             }
2528                         }
2529                     }
2530
2531                     if (scenario == 'coverage') {
2532                         // Publish coverage reports
2533                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2534                         addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2535                     }
2536
2537                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2538                     if (os in ['Ubuntu']) {
2539                         SummaryBuilder summaries = new SummaryBuilder()
2540                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2541                         summaries.emit(newJob)
2542                     }
2543
2544                     setMachineAffinity(newJob, os, architecture)
2545                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2546                     // Set timeouts to 240.
2547                     setTestJobTimeOut(newJob, scenario)
2548                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2549
2550                     // Create a build flow to join together the build and tests required to run this
2551                     // test.
2552                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2553                     // Linux CoreCLR test
2554                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2555                     def fullTestJobName = projectFolder + '/' + newJob.name
2556                     // Add a reference to the input jobs for report purposes
2557                     JobReport.Report.addReference(inputCoreCLRBuildName)
2558                     JobReport.Report.addReference(inputWindowTestsBuildName)
2559                     JobReport.Report.addReference(fullTestJobName)
2560                     def newFlowJob;
2561
2562                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2563                     // so that coverage data can be preserved.
2564                     if (scenario == 'coverage') {
2565                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2566                         buildFlow("""
2567 // Build the input Windows job
2568 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2569
2570 // And then build the test build
2571 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2572 """)
2573                         }
2574                     // Normal jobs copy a Windows build & a non-Windows build
2575                     } else {
2576                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2577                         buildFlow("""
2578 // Build the input jobs in parallel
2579 parallel (
2580     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2581     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2582 )
2583
2584 // And then build the test build
2585 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2586                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2587 """)
2588                         }
2589                     }
2590
2591                     setMachineAffinity(newFlowJob, os, architecture)
2592                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2593                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false, false)
2594                 } // configuration
2595             } // os
2596         } // architecture
2597     } // isPR
2598 } // scenario
2599
2600 JobReport.Report.generateJobReport(out)
2601
2602 // Make the call to generate the help job
2603 Utilities.createHelperJob(this, project, branch,
2604     "Welcome to the ${project} Repository",  // This is prepended to the help message
2605     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.