netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs
  12 folder('jitstress')
  13
  14 def static getOSGroup(def os) {
  15     def osGroupMap = ['Ubuntu':'Linux',
  16         'RHEL7.2': 'Linux',
  17         'Ubuntu16.04': 'Linux',
  18         'Debian8.4':'Linux',
  19         'Fedora23':'Linux',
  20         'OSX':'OSX',
  21         'Windows_NT':'Windows_NT',
  22         'FreeBSD':'FreeBSD',
  23         'CentOS7.1': 'Linux',
  24         'OpenSUSE13.2': 'Linux',
  25         'OpenSUSE42.1': 'Linux',
  26         'LinuxARMEmulator': 'Linux']
  27     def osGroup = osGroupMap.get(os, null)
  28     assert osGroup != null : "Could not find os group for ${os}"
  29     return osGroupMap[os]
  30 }
  31
  32 // We use this class (vs variables) so that the static functions can access data here.
  33 class Constants {
  34     // Innerloop build OS's
  35     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  36     // test execution in the build flow runs.  It generates the exact same build
  37     // as Windows_NT but without the tests.
  38     def static osList = ['Ubuntu', 'Debian8.4', 'OSX', 'Windows_NT', 'Windows_NT_BuildOnly', 'FreeBSD', 'CentOS7.1', 'OpenSUSE13.2', 'OpenSUSE42.1', 'RHEL7.2', 'LinuxARMEmulator', 'Ubuntu16.04', 'Fedora23']
  39     def static crossList = ['Ubuntu', 'OSX', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'OpenSUSE13.2']
  40     // This is a set of JIT stress modes combined with the set of variables that
  41     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  42     // the values are the environment variables
  43     def static jitStressModeScenarios = ['minopts' : ['COMPlus_JITMinOpts' : '1'], 'forcerelocs' : ['COMPlus_ForceRelocs' : '1'],
  44                'jitstress1' : ['COMPlus_JitStress' : '1'], 'jitstress2' : ['COMPlus_JitStress' : '2'],
  45                'jitstressregs1' : ['COMPlus_JitStressRegs' : '1'], 'jitstressregs2' : ['COMPlus_JitStressRegs' : '2'],
  46                'jitstressregs3' : ['COMPlus_JitStressRegs' : '3'], 'jitstressregs4' : ['COMPlus_JitStressRegs' : '4'],
  47                'jitstressregs8' : ['COMPlus_JitStressRegs' : '8'], 'jitstressregs0x10' : ['COMPlus_JitStressRegs' : '0x10'],
  48                'jitstressregs0x80' : ['COMPlus_JitStressRegs' : '0x80'],
  49                'jitstress2_jitstressregs1'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  50                'jitstress2_jitstressregs2'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  51                'jitstress2_jitstressregs3'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  52                'jitstress2_jitstressregs4'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  53                'jitstress2_jitstressregs8'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  54                'jitstress2_jitstressregs0x10' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  55                'jitstress2_jitstressregs0x80' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  56                'corefx_baseline' : [ : ], // corefx baseline
  57                'corefx_minopts' : ['COMPlus_JITMinOpts' : '1'],
  58                'corefx_jitstress1' : ['COMPlus_JitStress' : '1'],
  59                'corefx_jitstress2' : ['COMPlus_JitStress' : '2'],
  60                'corefx_jitstressregs1' : ['COMPlus_JitStressRegs' : '1'], 'corefx_jitstressregs2' : ['COMPlus_JitStressRegs' : '2'],
  61                'corefx_jitstressregs3' : ['COMPlus_JitStressRegs' : '3'], 'corefx_jitstressregs4' : ['COMPlus_JitStressRegs' : '4'],
  62                'corefx_jitstressregs8' : ['COMPlus_JitStressRegs' : '8'], 'corefx_jitstressregs0x10' : ['COMPlus_JitStressRegs' : '0x10'],
  63                'corefx_jitstressregs0x80' : ['COMPlus_JitStressRegs' : '0x80'],
  64                'gcstress0x3' : ['COMPlus_GCStress'  : '0x3'], 'gcstress0xc' : ['COMPlus_GCStress'  : '0xC'],
  65                'zapdisable' : ['COMPlus_ZapDisable'  : '0xC'],
  66                'heapverify1' : ['COMPlus_HeapVerify'  : '1'],
  67                'gcstress0xc_zapdisable' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_ZapDisable'  : '1'],
  68                'gcstress0xc_zapdisable_jitstress2' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_ZapDisable'  : '1', 'COMPlus_JitStress'  : '2'],
  69                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_ZapDisable'  : '1', 'COMPlus_HeapVerify'  : '1'],
  70                'gcstress0xc_jitstress1' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_JitStress'  : '1'],
  71                'gcstress0xc_jitstress2' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_JitStress'  : '2'],
  72                'gcstress0xc_minopts_heapverify1' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_JITMinOpts'  : '1', 'COMPlus_HeapVerify'  : '1']
  73                ]
  74     // This is a set of r2r jit stress scenarios
  75     def static r2rJitStressScenarios = ['r2r_jitstress1', 'r2r_jitstress2', 'r2r_jitstressregs1', 'r2r_jitstressregs2', 'r2r_jitstressregs3',
  76                                         'r2r_jitstressregs4', 'r2r_jitstressregs8', 'r2r_jitstressregsx10', 'r2r_jitstressregsx80',
  77                                         'r2r_jitminopts', 'r2r_jitforcerelocs']
  78     // This is the basic set of scenarios
  79     def static basicScenarios = ['default', 'pri1', 'ilrt', 'r2r', 'pri1r2r', 'gcstress15_pri1r2r', 'longgc', 'coverage', 'gcsimulator'] + r2rJitStressScenarios
  80     def static configurationList = ['Debug', 'Checked', 'Release']
  81     // This is the set of architectures
  82     def static architectureList = ['arm', 'arm64', 'x64', 'x86ryujit', 'x86lb']
  83 }
  84
  85 def static setMachineAffinity(def job, def os, def architecture) {
  86     if (architecture == 'arm64' && os == 'Windows_NT') {
  87         // For cross compilation
  88         job.with {
  89             label('arm64')
  90         }
  91     } else if ((architecture == 'arm' || architecture == 'arm64') && os == 'Ubuntu') {
  92         Utilities.setMachineAffinity(job, os, 'arm-cross-latest');
  93     } else {
  94         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
  95     }
  96 }
  97
  98 def static isJITStressJob(def scenario) {
  99     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 100            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 101 }
 102
 103 def static isGCStressRelatedTesting(def scenario) {
 104     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 105     // Detect it and make it a GCStress related.
 106     if (scenario == 'gcstress15_pri1r2r')
 107     {
 108         return true;
 109     }
 110
 111     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 112     def scenarioName = scenario.toLowerCase()
 113     def isGCStressTesting = false
 114     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 115         if (k in gcStressTestEnvVars) {
 116             isGCStressTesting = true;
 117         }
 118     }
 119     return isGCStressTesting
 120 }
 121
 122 def static isCorefxTesting(def scenario) {
 123     def corefx_prefix = 'corefx_'
 124     if (scenario.length() < corefx_prefix.length()) {
 125         return false
 126     }
 127     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 128 }
 129
 130 def static isR2R(def scenario) {
 131     return (scenario == 'r2r' || scenario == 'pri1r2r')
 132 }
 133
 134 def static isCoverage(def scenario) {
 135     return (scenario == 'coverage')
 136 }
 137
 138 def static isLongGc(def scenario) {
 139     return (scenario == 'longgc' || scenario == 'gcsimulator')
 140 }
 141
 142 def static setTestJobTimeOut(newJob, scenario) {
 143     if (isGCStressRelatedTesting(scenario)) {
 144         Utilities.setJobTimeout(newJob, 4320)
 145     }
 146     else if (isCorefxTesting(scenario)) {
 147         Utilities.setJobTimeout(newJob, 360)
 148     }
 149     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 150         Utilities.setJobTimeout(newJob, 240)
 151     }
 152     else if (isR2R(scenario)) {
 153         Utilities.setJobTimeout(newJob, 240)
 154     }
 155     else if (isCoverage(scenario)) {
 156         Utilities.setJobTimeout(newJob, 1440)
 157     }
 158     else if (isLongGc(scenario)) {
 159         Utilities.setJobTimeout(newJob, 1440)
 160     }
 161     // Non-test jobs use the default timeout value.
 162 }
 163
 164 def static getStressModeDisplayName(def scenario) {
 165     def displayStr = ''
 166     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 167         def prefixLength = 'COMPlus_'.length()
 168         if (k.length() >= prefixLength) {
 169             def modeName = k.substring(prefixLength, k.length())
 170             displayStr += ' ' + modeName + '=' + v
 171         }
 172     }
 173     return displayStr
 174 }
 175
 176 // Generates the string for creating a file that sets environment variables
 177 // that makes it possible to run stress modes.  Writes the script to a file called
 178 // SetStressModes.[sh/cmd]
 179 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 180     def stepScript = ''
 181     if (os == 'Windows_NT') {
 182         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 183         stepScript += "del ${stepScriptLocation}\r\n"
 184
 185         // Timeout in ms, default is 10 minutes. For stress
 186         // modes up this to 30 minutes
 187         def timeout = 1800000
 188
 189         // Set the Timeout
 190         stepScript += "set __TestTimeout=${timeout}\r\n"
 191         stressModeVars.each{ k, v ->
 192             // Write out what we are writing to the script file
 193             stepScript += "echo Setting ${k}=${v}\r\n"
 194             // Write out the set itself to the script file`
 195             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 196         }
 197     }
 198     else {
 199         stepScript += "echo Setting variables for ${stressModeName}\n"
 200         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 201         stressModeVars.each{ k, v ->
 202             // Write out what we are writing to the script file
 203             stepScript += "echo Setting ${k}=${v}\n"
 204             // Write out the set itself to the script file`
 205             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 206         }
 207         stepScript += "chmod +x ${stepScriptLocation}\n"
 208     }
 209     return stepScript
 210 }
 211
 212 // Corefx doesn't have a support to pass stress mode environment variables. This function
 213 // generates commands to set or export environment variables
 214 def static getStressModeEnvSetCmd(def os, def stressModeName) {
 215     def envVars = Constants.jitStressModeScenarios[stressModeName]
 216     def setEnvVars = ''
 217     if (os == 'Windows_NT') {
 218         envVars.each{ VarName, Value   ->
 219             if (VarName != '') {
 220                 setEnvVars += "set ${VarName}=${Value}\n"
 221             }
 222         }
 223     }
 224     else {
 225         envVars.each{ VarName, Value   ->
 226             if (VarName != '') {
 227                 setEnvVars += "export ${VarName}=${Value}\n"
 228             }
 229         }
 230     }
 231     return setEnvVars
 232 }
 233
 234 // Calculates the name of the build job based on some typical parameters.
 235 //
 236 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly, def isLinuxEmulatorBuild = false) {
 237     // If the architecture is x64, do not add that info into the build name.
 238     // Need to change around some systems and other builds to pick up the right builds
 239     // to do that.
 240
 241     def suffix = scenario != 'default' ? "_${scenario}" : '';
 242     if (isBuildOnly) {
 243         suffix += '_bld'
 244     }
 245     def baseName = ''
 246     switch (architecture) {
 247         case 'x64':
 248             if (scenario == 'default') {
 249                 // For now we leave x64 off of the name for compatibility with other jobs
 250                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 251             }
 252             else {
 253                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 254             }
 255             break
 256         case 'arm64':
 257         case 'arm':
 258             // These are cross builds
 259             if (isLinuxEmulatorBuild == false) {
 260                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 261             }
 262             else {
 263                 baseName = architecture.toLowerCase() + '_emulator_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 264             }
 265             break
 266         case 'x86ryujit':
 267             baseName = 'x86_ryujit_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 268             break
 269         case 'x86lb':
 270             baseName = 'x86_lb_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 271             break
 272         default:
 273             println("Unknown architecture: ${architecture}");
 274             assert false
 275             break
 276     }
 277
 278     return baseName + suffix
 279 }
 280
 281 static void addEmailPublisher(def job, def recipient) {
 282     job.with {
 283         publishers {
 284             extendedEmail(recipient, '$DEFAULT_SUBJECT', '$DEFAULT_CONTENT') {
 285                 trigger('Aborted', '$PROJECT_DEFAULT_SUBJECT', '$PROJECT_DEFAULT_CONTENT', null, true, true, true, true)
 286                 trigger('Failure', '$PROJECT_DEFAULT_SUBJECT', '$PROJECT_DEFAULT_CONTENT', null, true, true, true, true)
 287             }
 288         }
 289     }
 290 }
 291
 292 // **************************
 293 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 294 // of coreclr builds over linux/osx/freebsd/windows and debug/release/checked.  In addition, the windows
 295 // builds will do a couple extra steps.
 296 // **************************
 297
 298 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 299 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 300 // it's just used for internal builds
 301 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 302 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def isLinuxEmulatorBuild) {
 303     if (isWindowsBuildOnlyJob) {
 304         return
 305     }
 306
 307     def bidailyCrossList = ['RHEL7.2', 'Debian8.4', 'OpenSUSE13.2']
 308     // Non pull request builds.
 309     if (!isPR) {
 310         // Check scenario.
 311         switch (scenario) {
 312             case 'default':
 313                 switch (architecture) {
 314                     case 'x64':
 315                     case 'x86ryujit':
 316                     case 'x86lb':
 317                         if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 318                             Utilities.addGithubPushTrigger(job)
 319                         }
 320                         break
 321                     case 'arm':
 322                         Utilities.addGithubPushTrigger(job)
 323                         break
 324                     case 'arm64':
 325                         if (os == 'Windows_NT') {
 326                             Utilities.addGithubPushTrigger(job)
 327                             addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 328                         }
 329                         break
 330                     default:
 331                         println("Unknown architecture: ${architecture}");
 332                         assert false
 333                         break
 334                 }
 335                 break
 336             case 'pri1':
 337                 // Pri one gets a push trigger, and only for release
 338                 if (architecture == 'x64') {
 339                     if (configuration == 'Release') {
 340                         // We expect release jobs to be Windows, or in the cross list
 341                         assert (os == 'Windows_NT') || (os in Constants.crossList)
 342                         if (!os in bidailyCrossList) {
 343                             if (isFlowJob || os == 'Windows_NT') {
 344                                 Utilities.addGithubPushTrigger(job)
 345                             }
 346                         }
 347                         else {
 348                             if (isFlowJob) {
 349                                 Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 350                             }
 351                         }
 352                     }
 353                 }
 354                 break
 355             case 'r2r':
 356                 //r2r jobs that aren't pri1 can only be triggered by phrase
 357                 break
 358             case 'pri1r2r':
 359                 assert !(os in bidailyCrossList)
 360                 //pri1 r2r gets a push trigger for checked/release
 361                 if (configuration == 'Checked' || configuration == 'Release') {
 362                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 363                     if (architecture == 'x64' && os != 'OSX') {
 364                         //Flow jobs should be Windows, Ubuntu, OSX, or CentOS
 365                         if (isFlowJob || os == 'Windows_NT') {
 366                             Utilities.addGithubPushTrigger(job)
 367                         }
 368                     // OSX pri1r2r jobs should only run every 12 hours, not daily.
 369                     } else if (architecture == 'x64' && os == 'OSX'){
 370                         if (isFlowJob) {
 371                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 372                         }
 373                     }
 374                     // For x86, only add per-commit jobs for Windows
 375                     else if (architecture == 'x86ryujit' || architecture == 'x86lb') {
 376                         if (os == 'Windows_NT') {
 377                             Utilities.addGithubPushTrigger(job)
 378                         }
 379                     }
 380                     // arm64 pri1r2r jobs should only run every 12 hours.
 381                     else if (architecture == 'arm64') {
 382                         if (os == 'Windows_NT') {
 383                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 384                             addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 385                         }
 386                     }
 387                 }
 388                 break
 389             case 'r2r_jitstress1':
 390             case 'r2r_jitstress2':
 391             case 'r2r_jitstressregs1':
 392             case 'r2r_jitstressregs2':
 393             case 'r2r_jitstressregs3':
 394             case 'r2r_jitstressregs4':
 395             case 'r2r_jitstressregs8':
 396             case 'r2r_jitstressregsx10':
 397             case 'r2r_jitstressregsx80':
 398             case 'r2r_jitminopts':
 399             case 'r2r_jitforcerelocs':
 400             case 'gcstress15_pri1r2r':
 401                 assert !(os in bidailyCrossList)
 402
 403                 // GCStress=C is currently not supported on OS X
 404                 if (os == 'OSX' && isGCStressRelatedTesting(scenario)) {
 405                     break
 406                 }
 407
 408                 //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 409                 if (configuration == 'Checked' || configuration == 'Release') {
 410                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 411                     if (architecture == 'x64') {
 412                         //Flow jobs should be Windows, Ubuntu, OSX, or CentOS
 413                         if (isFlowJob || os == 'Windows_NT') {
 414                             // Add a weekly periodic trigger
 415                             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 416                         }
 417                     }
 418                     // For x86, only add per-commit jobs for Windows
 419                     else if (architecture == 'x86ryujit' || architecture == 'x86lb') {
 420                         if (os == 'Windows_NT') {
 421                             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 422                         }
 423                     }
 424                 }
 425                 break
 426             case 'longgc':
 427                 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX')
 428                 assert configuration == 'Release'
 429                 assert architecture == 'x64'
 430                 Utilities.addPeriodicTrigger(job, '@daily')
 431                 addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 432                 break
 433             case 'gcsimulator':
 434                 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX')
 435                 assert configuration == 'Release'
 436                 assert architecture == 'x64'
 437                 Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 438                 addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 439                 break
 440             case 'ilrt':
 441                 assert !(os in bidailyCrossList)
 442                 // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 443                 if (architecture == 'x64' && configuration == 'Release') {
 444                     // We don't expect to see a job generated except in these scenarios
 445                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 446                     if (isFlowJob || os == 'Windows_NT') {
 447                         Utilities.addPeriodicTrigger(job, '@daily')
 448                     }
 449                 }
 450                 break
 451             case 'coverage':
 452                 assert (os == 'Ubuntu' || os == 'Windows_NT')
 453                 assert configuration == 'Release'
 454                 assert architecture == 'x64'
 455                 Utilities.addPeriodicTrigger(job, '@weekly')
 456                 break
 457             case 'jitstressregs1':
 458             case 'jitstressregs2':
 459             case 'jitstressregs3':
 460             case 'jitstressregs4':
 461             case 'jitstressregs8':
 462             case 'jitstressregs0x10':
 463             case 'jitstressregs0x80':
 464             case 'minopts':
 465             case 'forcerelocs':
 466             case 'jitstress1':
 467             case 'jitstress2':
 468             case 'jitstress2_jitstressregs1':
 469             case 'jitstress2_jitstressregs2':
 470             case 'jitstress2_jitstressregs3':
 471             case 'jitstress2_jitstressregs4':
 472             case 'jitstress2_jitstressregs8':
 473             case 'jitstress2_jitstressregs0x10':
 474             case 'jitstress2_jitstressregs0x80':
 475             case 'corefx_baseline':
 476             case 'corefx_minopts':
 477             case 'corefx_jitstress1':
 478             case 'corefx_jitstress2':
 479             case 'corefx_jitstressregs1':
 480             case 'corefx_jitstressregs2':
 481             case 'corefx_jitstressregs3':
 482             case 'corefx_jitstressregs4':
 483             case 'corefx_jitstressregs8':
 484             case 'corefx_jitstressregs0x10':
 485             case 'corefx_jitstressregs0x80':
 486             case 'zapdisable':
 487                 if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 488                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 489                     Utilities.addPeriodicTrigger(job, '@daily')
 490                 }
 491                 break
 492             case 'heapverify1':
 493             case 'gcstress0x3':
 494                 if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 495                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 496                     if (architecture == 'arm64') {
 497                         assert (os == 'Windows_NT')
 498                         Utilities.addPeriodicTrigger(job, '@daily')
 499                         addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 500                     }
 501                     else {
 502                         Utilities.addPeriodicTrigger(job, '@weekly')
 503                     }
 504                 }
 505                 break
 506             case 'gcstress0xc':
 507             case 'gcstress0xc_zapdisable':
 508             case 'gcstress0xc_zapdisable_jitstress2':
 509             case 'gcstress0xc_zapdisable_heapverify1':
 510             case 'gcstress0xc_jitstress1':
 511             case 'gcstress0xc_jitstress2':
 512             case 'gcstress0xc_minopts_heapverify1':
 513                 // GCStress=C is currently not supported on OS X
 514                 if (os != 'CentOS7.1' && os != 'OSX' && !(os in bidailyCrossList)) {
 515                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 516                     if (architecture == 'arm64') {
 517                         assert (os == 'Windows_NT')
 518                         // TODO: Enable a periodic trigger after tests are updated.
 519                         // Utilities.addPeriodicTrigger(job, '@daily')
 520                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 521                     }
 522                     else {
 523                         Utilities.addPeriodicTrigger(job, '@weekly')
 524                     }
 525                 }
 526                 break
 527             default:
 528                 println("Unknown scenario: ${scenario}");
 529                 assert false
 530                 break
 531         }
 532         return
 533     }
 534     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 535     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 536     def osGroup = getOSGroup(os)
 537     switch (architecture) {
 538         case 'x64':
 539             if (scenario == 'coverage') {
 540                 assert configuration == 'Release'
 541                 if (os == 'Ubuntu') {
 542                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 543                 }
 544                 break
 545             }
 546             switch (os) {
 547                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 548                 case 'OpenSUSE13.2':
 549                 case 'Debian8.4':
 550                 case 'RHEL7.2':
 551                     if (scenario == 'default') {
 552                         assert !isFlowJob
 553                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 554                     }
 555                     else if (scenario == 'pri1' && isFlowJob) {
 556                         assert (configuration == 'Release')
 557                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 558                     }
 559                     break
 560                 case 'Fedora23':
 561                 case 'Ubuntu16.04':
 562                 case 'OpenSUSE42.1':
 563                     assert !isFlowJob
 564                     assert scenario == 'default'
 565                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 566                     break
 567                 case 'Ubuntu':
 568                 case 'OSX':
 569                     // Triggers on the non-flow jobs aren't necessary here
 570                     // Corefx testing uses non-flow jobs.
 571                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 572                         break
 573                     }
 574                     switch (scenario) {
 575                         case 'default':
 576                             // Ubuntu uses checked for default PR tests
 577                             if (configuration == 'Checked') {
 578                                 // Default trigger
 579                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 580                             }
 581                             break
 582                         case 'pri1':
 583                             if (configuration == 'Release') {
 584                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 585                             }
 586                             break
 587                         case 'ilrt':
 588                             if (configuration == 'Release') {
 589                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 590                             }
 591                             break
 592                         case 'r2r':
 593                             if (configuration == 'Release' || configuration == 'Checked') {
 594                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 595                             }
 596                             break
 597                         case 'pri1r2r':
 598                             if (configuration == 'Release' || configuration == 'Checked') {
 599                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 600                             }
 601                             break
 602                         case 'gcstress15_pri1r2r':
 603                             if (configuration == 'Release' || configuration == 'Checked') {
 604                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 605                             }
 606                             break
 607                         case 'r2r_jitstress1':
 608                             if (configuration == 'Release' || configuration == 'Checked') {
 609                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 610                             }
 611                             break
 612                         case 'r2r_jitstress2':
 613                             if (configuration == 'Release' || configuration == 'Checked') {
 614                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 615                             }
 616                             break
 617                         case 'r2r_jitstressregs1':
 618                             if (configuration == 'Release' || configuration == 'Checked') {
 619                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 620                             }
 621                             break
 622                         case 'r2r_jitstressregs2':
 623                             if (configuration == 'Release' || configuration == 'Checked') {
 624                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 625                             }
 626                             break
 627                         case 'r2r_jitstressregs3':
 628                             if (configuration == 'Release' || configuration == 'Checked') {
 629                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 630                             }
 631                             break
 632                         case 'r2r_jitstressregs4':
 633                             if (configuration == 'Release' || configuration == 'Checked') {
 634                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 635                             }
 636                             break
 637                         case 'r2r_jitstressregs8':
 638                             if (configuration == 'Release' || configuration == 'Checked') {
 639                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 640                             }
 641                             break
 642                         case 'r2r_jitstressregsx10':
 643                             if (configuration == 'Release' || configuration == 'Checked') {
 644                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 645                             }
 646                             break
 647                         case 'r2r_jitstressregsx80':
 648                             if (configuration == 'Release' || configuration == 'Checked') {
 649                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 650                             }
 651                             break
 652                         case 'r2r_jitminopts':
 653                             if (configuration == 'Release' || configuration == 'Checked') {
 654                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} JITMinOpts R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 655                             }
 656                             break
 657                         case 'r2r_jitforcerelocs':
 658                             if (configuration == 'Release' || configuration == 'Checked') {
 659                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ForceRelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 660                             }
 661                             break
 662                         case 'longgc':
 663                             if (configuration == 'Release') {
 664                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 665                             }
 666                             break
 667                         case 'gcsimulator':
 668                             if (configuration == 'Release') {
 669                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 670                             }
 671                             break
 672                         case 'minopts':
 673                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 674                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - MinOpts)",
 675                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 676                             break
 677                         case 'jitstress1':
 678                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 679                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=1)",
 680                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 681                             break
 682                         case 'jitstress2':
 683                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 684                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=2)",
 685                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 686                             break
 687                         case 'forcerelocs':
 688                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 689                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ForceRelocs)",
 690                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 691                         case 'jitstressregs1':
 692                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 693                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=1)",
 694                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 695                             break
 696                         case 'jitstressregs2':
 697                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 698                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=2)",
 699                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 700                             break
 701                         case 'jitstressregs3':
 702                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 703                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=3)",
 704                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 705                             break
 706                         case 'jitstressregs4':
 707                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 708                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=4)",
 709                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 710                             break
 711                         case 'jitstressregs8':
 712                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 713                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=8)",
 714                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 715                             break
 716                         case 'jitstressregs0x10':
 717                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 718                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x10)",
 719                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 720                             break
 721                         case 'jitstressregs0x80':
 722                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 723                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x80)",
 724                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 725                             break
 726                         case 'jitstress2_jitstressregs1':
 727                         case 'jitstress2_jitstressregs2':
 728                         case 'jitstress2_jitstressregs3':
 729                         case 'jitstress2_jitstressregs4':
 730                         case 'jitstress2_jitstressregs8':
 731                         case 'jitstress2_jitstressregs0x10':
 732                         case 'jitstress2_jitstressregs0x80':
 733                         case 'gcstress0x3':
 734                         case 'gcstress0xc':
 735                         case 'zapdisable':
 736                         case 'heapverify1':
 737                         case 'gcstress0xc_zapdisable':
 738                         case 'gcstress0xc_zapdisable_jitstress2':
 739                         case 'gcstress0xc_zapdisable_heapverify1':
 740                         case 'gcstress0xc_jitstress1':
 741                         case 'gcstress0xc_jitstress2':
 742                         case 'gcstress0xc_minopts_heapverify1':
 743                             def displayStr = getStressModeDisplayName(scenario)
 744                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 745                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 746                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 747                             break
 748                         case 'corefx_baseline':
 749                         case 'corefx_minopts':
 750                         case 'corefx_jitstress1':
 751                         case 'corefx_jitstress2':
 752                         case 'corefx_jitstressregs1':
 753                         case 'corefx_jitstressregs2':
 754                         case 'corefx_jitstressregs3':
 755                         case 'corefx_jitstressregs4':
 756                         case 'corefx_jitstressregs8':
 757                         case 'corefx_jitstressregs0x10':
 758                         case 'corefx_jitstressregs0x80':
 759                             def displayName = 'CoreFx' + getStressModeDisplayName(scenario)
 760                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 761                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 762                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 763                             break
 764                         default:
 765                             println("Unknown scenario: ${scenario}");
 766                             assert false
 767                             break
 768                     }
 769                     break
 770                 case 'CentOS7.1':
 771                     switch (scenario) {
 772                         case 'pri1':
 773                             if (configuration == 'Release') {
 774                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 775                             }
 776                             break
 777                         case 'r2r':
 778                             if (configuration == 'Checked' || configuration == 'Release') {
 779                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 780                             }
 781                             break
 782                         case 'pri1r2r':
 783                             if (configuration == 'Checked' || configuration == 'Release') {
 784                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 785                             }
 786                             break
 787                         case 'gcstress15_pri1r2r':
 788                             if (configuration == 'Release' || configuration == 'Checked') {
 789                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 790                             }
 791                             break
 792                         case 'r2r_jitstress1':
 793                             if (configuration == 'Release' || configuration == 'Checked') {
 794                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 795                             }
 796                             break
 797                         case 'r2r_jitstress2':
 798                             if (configuration == 'Release' || configuration == 'Checked') {
 799                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 800                             }
 801                             break
 802                         case 'r2r_jitstressregs1':
 803                             if (configuration == 'Release' || configuration == 'Checked') {
 804                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 805                             }
 806                             break
 807                         case 'r2r_jitstressregs2':
 808                             if (configuration == 'Release' || configuration == 'Checked') {
 809                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 810                             }
 811                             break
 812                         case 'r2r_jitstressregs3':
 813                             if (configuration == 'Release' || configuration == 'Checked') {
 814                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 815                             }
 816                             break
 817                         case 'r2r_jitstressregs4':
 818                             if (configuration == 'Release' || configuration == 'Checked') {
 819                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 820                             }
 821                             break
 822                         case 'r2r_jitstressregs8':
 823                             if (configuration == 'Release' || configuration == 'Checked') {
 824                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 825                             }
 826                             break
 827                         case 'r2r_jitstressregsx10':
 828                             if (configuration == 'Release' || configuration == 'Checked') {
 829                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 830                             }
 831                             break
 832                         case 'r2r_jitstressregsx80':
 833                             if (configuration == 'Release' || configuration == 'Checked') {
 834                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 835                             }
 836                             break
 837                         case 'r2r_jitminopts':
 838                             if (configuration == 'Release' || configuration == 'Checked') {
 839                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} JITMinOpts R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 840                             }
 841                             break
 842                         case 'r2r_jitforcerelocs':
 843                             if (configuration == 'Release' || configuration == 'Checked') {
 844                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ForceRelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 845                             }
 846                             break
 847                         default:
 848                             break
 849                     }
 850                 case 'Windows_NT':
 851                     switch (scenario) {
 852                         case 'default':
 853                             // Default trigger
 854                             if (configuration == 'Debug') {
 855                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 856                             }
 857                             break
 858                         case 'pri1':
 859                             // Default trigger
 860                             if (configuration == 'Release') {
 861                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 862                             }
 863                             break
 864                         case 'ilrt':
 865                             if (configuration == 'Release') {
 866                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 867                             }
 868                             break
 869                         case 'r2r':
 870                             if (configuration == 'Checked' || configuration == 'Release') {
 871                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 872                             }
 873                             break
 874                         case 'pri1r2r':
 875                             if (configuration == 'Checked' || configuration == 'Release') {
 876                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 877                             }
 878                             break
 879                         case 'gcstress15_pri1r2r':
 880                             if (configuration == 'Release' || configuration == 'Checked') {
 881                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 882                             }
 883                             break
 884                         case 'r2r_jitstress1':
 885                             if (configuration == 'Release' || configuration == 'Checked') {
 886                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 887                             }
 888                             break
 889                         case 'r2r_jitstress2':
 890                             if (configuration == 'Release' || configuration == 'Checked') {
 891                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 892                             }
 893                             break
 894                         case 'r2r_jitstressregs1':
 895                             if (configuration == 'Release' || configuration == 'Checked') {
 896                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 897                             }
 898                             break
 899                         case 'r2r_jitstressregs2':
 900                             if (configuration == 'Release' || configuration == 'Checked') {
 901                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 902                             }
 903                             break
 904                         case 'r2r_jitstressregs3':
 905                             if (configuration == 'Release' || configuration == 'Checked') {
 906                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 907                             }
 908                             break
 909                         case 'r2r_jitstressregs4':
 910                             if (configuration == 'Release' || configuration == 'Checked') {
 911                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 912                             }
 913                             break
 914                         case 'r2r_jitstressregs8':
 915                             if (configuration == 'Release' || configuration == 'Checked') {
 916                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 917                             }
 918                             break
 919                         case 'r2r_jitstressregsx10':
 920                             if (configuration == 'Release' || configuration == 'Checked') {
 921                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 922                             }
 923                             break
 924                         case 'r2r_jitstressregsx80':
 925                             if (configuration == 'Release' || configuration == 'Checked') {
 926                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 927                             }
 928                             break
 929                         case 'r2r_jitminopts':
 930                             if (configuration == 'Release' || configuration == 'Checked') {
 931                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} JITMinOpts R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 932                             }
 933                             break
 934                         case 'r2r_jitforcerelocs':
 935                             if (configuration == 'Release' || configuration == 'Checked') {
 936                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ForceRelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 937                             }
 938                             break
 939                         case 'longgc':
 940                             if (configuration == 'Release') {
 941                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 942                             }
 943                             break
 944                         case 'gcsimulator':
 945                             if (configuration == 'Release') {
 946                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 947                             }
 948                             break
 949                         case 'minopts':
 950                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 951                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - MinOpts)",
 952                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 953                             break
 954                         case 'forcerelocs':
 955                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 956                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ForceRelocs)",
 957                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 958                             break
 959                         case 'jitstress1':
 960                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 961                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=1)",
 962                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 963                             break
 964                         case 'jitstress2':
 965                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 966                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=2)",
 967                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 968                             break
 969                         case 'jitstressregs1':
 970                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 971                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=1)",
 972                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 973                             break
 974                         case 'jitstressregs2':
 975                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 976                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=2)",
 977                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 978                             break
 979                         case 'jitstressregs3':
 980                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 981                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=3)",
 982                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 983                             break
 984                         case 'jitstressregs4':
 985                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 986                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=4)",
 987                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 988                             break
 989                         case 'jitstressregs8':
 990                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 991                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=8)",
 992                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 993                             break
 994                         case 'jitstressregs0x10':
 995                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 996                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x10)",
 997                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 998                             break
 999                         case 'jitstressregs0x80':
1000                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1001                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x80)",
1002                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1003                             break
1004                         case 'jitstress2_jitstressregs1':
1005                         case 'jitstress2_jitstressregs2':
1006                         case 'jitstress2_jitstressregs3':
1007                         case 'jitstress2_jitstressregs4':
1008                         case 'jitstress2_jitstressregs8':
1009                         case 'jitstress2_jitstressregs0x10':
1010                         case 'jitstress2_jitstressregs0x80':
1011                         case 'gcstress0x3':
1012                         case 'gcstress0xc':
1013                         case 'zapdisable':
1014                         case 'heapverify1':
1015                         case 'gcstress0xc_zapdisable':
1016                         case 'gcstress0xc_zapdisable_jitstress2':
1017                         case 'gcstress0xc_zapdisable_heapverify1':
1018                         case 'gcstress0xc_jitstress1':
1019                         case 'gcstress0xc_jitstress2':
1020                         case 'gcstress0xc_minopts_heapverify1':
1021                             def displayStr = getStressModeDisplayName(scenario)
1022                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1023                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1024                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1025                             break
1026                         case 'corefx_baseline':
1027                         case 'corefx_minopts':
1028                         case 'corefx_jitstress1':
1029                         case 'corefx_jitstress2':
1030                         case 'corefx_jitstressregs1':
1031                         case 'corefx_jitstressregs2':
1032                         case 'corefx_jitstressregs3':
1033                         case 'corefx_jitstressregs4':
1034                         case 'corefx_jitstressregs8':
1035                         case 'corefx_jitstressregs0x10':
1036                         case 'corefx_jitstressregs0x80':
1037                             def displayName = 'CoreFx ' + getStressModeDisplayName(scenario)
1038                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1039                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1040                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1041                             break
1042                         default:
1043                             println("Unknown scenario: ${scenario}");
1044                             assert false
1045                             break
1046                     }
1047                     break
1048                 case 'FreeBSD':
1049                     assert scenario == 'default'
1050                     if (configuration == 'Checked') {
1051                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1052                     }
1053                     break
1054                 default:
1055                     println("Unknown os: ${os}");
1056                     assert false
1057                     break
1058             }
1059             break
1060         case 'arm':
1061             assert scenario == 'default'
1062             switch (os) {
1063                 case 'Ubuntu':
1064                     if (isLinuxEmulatorBuild == false) {
1065                         // Removing the regex will cause this to run on each PR.
1066                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+Linux\\W+arm\\W+cross\\W+${configuration}.*")
1067                     }
1068                     else {
1069                         Utilities.addGithubPRTriggerForBranch(job, branch, "Linux ARM Emulator Cross ${configuration} Build")
1070                     }
1071                     break
1072                 default:
1073                     println("NYI os: ${os}");
1074                     assert false
1075                     break
1076             }
1077             break
1078         case 'arm64':
1079             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1080
1081             // Set up a private trigger
1082             def contextString = "${os} ${architecture} Cross ${configuration}"
1083             if (scenario != 'default')
1084                 contextString += " ${scenario}"
1085             contextString += " Build"
1086             // Debug builds only.
1087             if (configuration != 'Debug') {
1088                contextString += " and Test"
1089             }
1090
1091             def arm64Users = ['erozenfeld', 'kyulee1', 'pgavlin', 'russellhadley', 'swaroop-sridhar', 'JosephTremoulet', 'jashook', 'RussKeldorph', 'gkhanna79', 'briansull', 'cmckinsey', 'jkotas', 'ramarag', 'markwilkie', 'rahku', 'tzwlai', 'weshaggard']
1092             switch (os) {
1093                 case 'Windows_NT':
1094                     switch (scenario) {
1095                         case 'default':
1096                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1097                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1098                             break
1099                         case 'pri1r2r':
1100                         case 'gcstress0x3':
1101                         case 'gcstress0xc':
1102                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1103                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1104                             break
1105                     }
1106                     break
1107                 default:
1108                     println("NYI os: ${os}");
1109                     assert false
1110                     break
1111             }
1112             break
1113         case 'x86ryujit':
1114         case 'x86lb':
1115             assert (scenario == 'default' || scenario == 'r2r' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r' || scenario == 'longgc' || scenario == 'gcsimulator' ||
1116                     Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
1117             // For windows, x86 runs by default
1118             def arch = 'x86'
1119             def jit = 'ryujit'
1120             if (architecture == 'x86lb') {
1121                 jit = 'legacy_backend'
1122             }
1123
1124             if (scenario == 'default') {
1125                 if (os == 'Windows_NT') {
1126                     if (configuration == 'Checked') {
1127                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test")
1128                     }
1129                 }
1130                 else {
1131                     // default trigger
1132                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build", "(?i).*test\\W+${arch}\\W+${osGroup}.\\W+${jit}*")
1133                 }
1134             }
1135             else if (scenario == 'r2r') {
1136                 if (os == 'Windows_NT') {
1137                     if (configuration == 'Release') {
1138                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1139                     }
1140                 }
1141             }
1142             else if (scenario == 'pri1r2r') {
1143                 if (os == 'Windows_NT') {
1144                     if (configuration == 'Release') {
1145                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1146                     }
1147                 }
1148             }
1149             else if (scenario == 'gcstress15_pri1r2r'){
1150                 if (os == 'Windows_NT'){
1151                     if (configuration == 'Release'){
1152                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1153                     }
1154                 }
1155             }
1156             else if (scenario == 'r2r_jitstress1'){
1157                 if (configuration == 'Release' || configuration == 'Checked') {
1158                     if (os == 'Windows_NT'){
1159                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1160                     }
1161                 }
1162             }
1163             else if (scenario == 'r2r_jitstress2'){
1164                 if (configuration == 'Release' || configuration == 'Checked') {
1165                     if (os == 'Windows_NT'){
1166                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1167                     }
1168                 }
1169             }
1170             else if (scenario == 'r2r_jitstressregs1'){
1171                 if (configuration == 'Release' || configuration == 'Checked') {
1172                     if (os == 'Windows_NT'){
1173                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1174                     }
1175                 }
1176             }
1177             else if (scenario == 'r2r_jitstressregs2'){
1178                 if (configuration == 'Release' || configuration == 'Checked') {
1179                     if (os == 'Windows_NT'){
1180                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1181                     }
1182                 }
1183             }
1184             else if (scenario == 'r2r_jitstressregs3'){
1185                 if (configuration == 'Release' || configuration == 'Checked') {
1186                     if (os == 'Windows_NT'){
1187                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1188                     }
1189                 }
1190             }
1191             else if (scenario == 'r2r_jitstressregs4'){
1192                 if (configuration == 'Release' || configuration == 'Checked') {
1193                     if (os == 'Windows_NT'){
1194                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1195                     }
1196                 }
1197             }
1198             else if (scenario == 'r2r_jitstressregs8'){
1199                 if (configuration == 'Release' || configuration == 'Checked') {
1200                     if (os == 'Windows_NT'){
1201                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1202                     }
1203                 }
1204             }
1205             else if (scenario == 'r2r_jitstressregsx10'){
1206                 if (configuration == 'Release' || configuration == 'Checked') {
1207                     if (os == 'Windows_NT'){
1208                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1209                     }
1210                 }
1211             }
1212             else if (scenario == 'r2r_jitstressregsx80'){
1213                 if (configuration == 'Release' || configuration == 'Checked') {
1214                     if (os == 'Windows_NT'){
1215                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1216                     }
1217                 }
1218             }
1219             else if (scenario == 'r2r_jitminopts'){
1220                 if (configuration == 'Release' || configuration == 'Checked') {
1221                     if (os == 'Windows_NT'){
1222                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitminopts R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1223                     }
1224                 }
1225             }
1226             else if (scenario == 'r2r_jitforcerelocs'){
1227                 if (configuration == 'Release' || configuration == 'Checked') {
1228                     if (os == 'Windows_NT'){
1229                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitforcerelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1230                     }
1231                 }
1232             }
1233             else if (scenario == 'longgc') {
1234                 if (os == 'Windows_NT'){
1235                     if (configuration == 'Release'){
1236                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1237                     }
1238                 }
1239             }
1240             else if (scenario == 'gcsimulator') {
1241                 if (os == 'Windows_NT') {
1242                     if (configuration == 'Release') {
1243                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1244                     }
1245                 }
1246             }
1247             break
1248         default:
1249             println("Unknown architecture: ${architecture}");
1250             assert false
1251             break
1252     }
1253 }
1254
1255 // Additional scenario which can alter behavior
1256
1257 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1258 combinedScenarios.each { scenario ->
1259     [true, false].each { isPR ->
1260         Constants.architectureList.each { architecture ->
1261             Constants.configurationList.each { configuration ->
1262                 Constants.osList.each { os ->
1263                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1264                     // and reset the os to Windows_NT
1265                     def isBuildOnly = false
1266                     if (os == 'Windows_NT_BuildOnly') {
1267                         isBuildOnly = true
1268                         os = 'Windows_NT'
1269                     }
1270
1271                     // If the OS is LinuxARMEmulator and arch is arm, set the isLinuxEmulatorBuild
1272                     // flag to true and reset the os to Ubuntu
1273                     // The isLinuxEmulatorBuild flag will be used at a later time to execute the right
1274                     // set of build commands
1275                     // The tuples (LinuxARMEmulator, other architectures) are not handled and control returns
1276                     def isLinuxEmulatorBuild = false
1277                     if (os == 'LinuxARMEmulator' && architecture == 'arm') {
1278                         // Cross Builds only in Debug and Release modes allowed
1279                         if ( configuration == 'Checked' ) {
1280                             return
1281                         }
1282
1283                         isLinuxEmulatorBuild = true
1284                         os = 'Ubuntu'
1285                     } else if (os == 'LinuxARMEmulator') {
1286                         return
1287                     }
1288
1289                     // Skip totally unimplemented (in CI) configurations.
1290                     switch (architecture) {
1291                         case 'arm64':
1292                             // Windows only
1293                             if (os != 'Windows_NT' || isBuildOnly) {
1294                                 return
1295                             }
1296                             break
1297                         case 'arm':
1298                             // Only Ubuntu cross implemented
1299                             if (os != 'Ubuntu') {
1300                                 return
1301                             }
1302                             break
1303                         case 'x86ryujit':
1304                         case 'x86lb':
1305                             // Skip non-windows
1306                             if (os != 'Windows_NT') {
1307                                 return
1308                             }
1309                             break
1310                         case 'x64':
1311                             // Everything implemented
1312                             break
1313                         default:
1314                             println("Unknown architecture: ${architecture}")
1315                             assert false
1316                             break
1317                     }
1318                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1319                     // with checked builds
1320                     def enableCorefxTesting = false
1321                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1322                         if (configuration != 'Checked') {
1323                             return
1324                         }
1325
1326                         enableCorefxTesting = isCorefxTesting(scenario)
1327
1328                         // Since these are just execution time differences,
1329                         // skip platforms that don't execute the tests here (Windows_NT only)
1330                         def isEnabledOS = os == 'Windows_NT' || (os == 'Ubuntu' && enableCorefxTesting)
1331                         if (!isEnabledOS || isBuildOnly) {
1332                             return
1333                         }
1334
1335                         switch (architecture) {
1336                             case 'arm64':
1337                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1338                                     return
1339                                 }
1340                                 break
1341                             case 'x64':
1342                                 // Everything implemented
1343                                 break
1344                             default:
1345                                 return
1346                         }
1347                     }
1348                     else {
1349                         // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
1350                         // and configuration is not Checked, bail out.
1351                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1352                             return;
1353                         }
1354
1355                         // Skip scenarios
1356                         switch (scenario) {
1357                             case 'pri1':
1358                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1359                                 // the default scenario build
1360                                 if (os != 'Windows_NT') {
1361                                     return
1362                                 }
1363                                 // Only x64 for now
1364                                 if (architecture != 'x64') {
1365                                     return
1366                                 }
1367                                 break
1368                             case 'ilrt':
1369                                 // The ilrt build isn't necessary except for Windows_NT.  Non-Windows NT uses
1370                                 // the default scenario build
1371                                 if (os != 'Windows_NT') {
1372                                     return
1373                                 }
1374                                 // Only x64 for now
1375                                 if (architecture != 'x64') {
1376                                     return
1377                                 }
1378                                 // Release only
1379                                 if (configuration != 'Release') {
1380                                     return
1381                                 }
1382                                 break
1383                             case 'r2r':
1384                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1385                                 // the default scenario build
1386                                 if (os != 'Windows_NT') {
1387                                     return
1388                                 }
1389                                 if (architecture != 'x64') {
1390                                     return
1391                                 }
1392                                 break
1393                             case 'pri1r2r':
1394                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1395                                 // the default scenario build
1396                                 if (os != 'Windows_NT') {
1397                                     return
1398                                 }
1399                                 if (architecture != 'x64') {
1400                                     if (architecture != 'arm64' || configuration == 'Debug') {
1401                                         return
1402                                     }
1403                                 }
1404                                 break
1405                             case 'gcstress15_pri1r2r':
1406                             case 'r2r_jitstress1':
1407                             case 'r2r_jitstress2':
1408                             case 'r2r_jitstressregs1':
1409                             case 'r2r_jitstressregs2':
1410                             case 'r2r_jitstressregs3':
1411                             case 'r2r_jitstressregs4':
1412                             case 'r2r_jitstressregs8':
1413                             case 'r2r_jitstressregsx10':
1414                             case 'r2r_jitstressregsx80':
1415                             case 'r2r_jitminopts':
1416                             case 'r2r_jitforcerelocs':
1417                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
1418                                 // the default scenario build
1419                                 if (os != 'Windows_NT') {
1420                                     return
1421                                 }
1422                                 if (architecture != 'x64') {
1423                                     return
1424                                 }
1425                                 break
1426                             case 'longgc':
1427                             case 'gcsimulator':
1428                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX') {
1429                                     return
1430                                 }
1431                                 if (architecture != 'x64') {
1432                                     return
1433                                 }
1434                                 if (configuration != 'Release') {
1435                                     return
1436                                 }
1437                                 break
1438                             // We need Windows x64 Release bits for the code coverage build
1439                             case 'coverage':
1440                                 if (os != 'Windows_NT') {
1441                                     return
1442                                 }
1443                                 if (architecture != 'x64') {
1444                                     return
1445                                 }
1446                                 if (configuration != 'Release') {
1447                                     return
1448                                 }
1449                                 break
1450                             case 'default':
1451                                 // Nothing skipped
1452                                 break
1453                             default:
1454                                 println("Unknown scenario: ${scenario}")
1455                                 assert false
1456                                 break
1457                         }
1458                     }
1459
1460                     // Calculate names
1461                     def lowerConfiguration = configuration.toLowerCase()
1462                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly, isLinuxEmulatorBuild)
1463                     def folderName = isJITStressJob(scenario) ? 'jitstress' : '';
1464
1465                     // Create the new job
1466                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
1467
1468                     setMachineAffinity(newJob, os, architecture)
1469
1470                     // Add all the standard options
1471                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
1472                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly, isLinuxEmulatorBuild)
1473
1474                     def buildCommands = [];
1475                     def osGroup = getOSGroup(os)
1476
1477                     // Calculate the build steps, archival, and xunit results
1478                     switch (os) {
1479                         case 'Windows_NT':
1480                             switch (architecture) {
1481                                 case 'x64':
1482                                 case 'x86ryujit':
1483                                 case 'x86lb':
1484                                     def arch = architecture
1485                                     if (architecture == 'x86ryujit' || architecture == 'x86lb') {
1486                                         arch = 'x86'
1487                                     }
1488
1489                                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1490                                             scenario == 'default' ||
1491                                             scenario == 'r2r' ||
1492                                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1493                                         buildOpts = enableCorefxTesting ? 'skiptests' : ''
1494                                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1495                                     }
1496
1497                                     // For Pri 1 tests, we must shorten the output test binary path names.
1498                                     // if __TestIntermediateDir is already set, build-test.cmd will
1499                                     // output test binaries to that directory. If it is not set, the
1500                                     // binaries are sent to a default directory whose name is about
1501                                     // 35 characters long.
1502
1503                                     else if (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage') {
1504                                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} -priority=1"
1505                                     }
1506                                     else if (scenario == 'ilrt') {
1507                                         // First do the build with skiptests and then build the tests with ilasm roundtrip
1508                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} skiptests"
1509                                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch} -ilasmroundtrip"
1510                                     }
1511                                     else if (isLongGc(scenario)) {
1512                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} skiptests"
1513                                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1514                                     }
1515                                     else {
1516                                         println("Unknown scenario: ${scenario}")
1517                                         assert false
1518                                     }
1519
1520                                     // If we are running a stress mode, we should write out the set of key
1521                                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1522
1523                                     if (!isBuildOnly) {
1524                                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1525                                         def crossgenStr = ''
1526                                         def runcrossgentestsStr = ''
1527                                         def runjitstressStr = ''
1528                                         def runjitstressregsStr = ''
1529                                         def runjitmioptsStr = ''
1530                                         def runjitforcerelocsStr = ''
1531                                         def gcstressStr = ''
1532                                         def runtestArguments = ''
1533                                         def gcTestArguments = ''
1534
1535                                         if (scenario == 'r2r' ||
1536                                             scenario == 'pri1r2r' ||
1537                                             scenario == 'gcstress15_pri1r2r' ||
1538                                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1539                                                 crossgenStr = 'crossgen'
1540                                                 runcrossgentestsStr = 'runcrossgentests'
1541
1542                                                 if (scenario == 'r2r_jitstress1'){
1543                                                     runjitstressStr = 'jitstress 1'
1544                                                 }
1545                                                 else if (scenario == 'r2r_jitstress2') {
1546                                                     runjitstressStr = 'jitstress 2'
1547                                                 }
1548                                                 else if (scenario == 'r2r_jitstressregs1'){
1549                                                     runjitstressregsStr = 'jitstressregs 1'
1550                                                 }
1551                                                 else if (scenario == 'r2r_jitstressregs2') {
1552                                                     runjitstressregsStr = 'jitstressregs 2'
1553                                                 }
1554                                                 else if (scenario == 'r2r_jitstressregs3') {
1555                                                     runjitstressregsStr = 'jitstressregs 3'
1556                                                 }
1557                                                 else if (scenario == 'r2r_jitstressregs4') {
1558                                                     runjitstressregsStr = 'jitstressregs 4'
1559                                                 }
1560                                                 else if (scenario == 'r2r_jitstressregs8') {
1561                                                     runjitstressregsStr = 'jitstressregs 8'
1562                                                 }
1563                                                 else if (scenario == 'r2r_jitstressregsx10') {
1564                                                     runjitstressregsStr = 'jitstressregs x10'
1565                                                 }
1566                                                 else if (scenario == 'r2r_jitstressregsx80') {
1567                                                     runjitstressregsStr = 'jitstressregs x80'
1568                                                 }
1569                                                 else if (scenario == 'r2r_jitminopts') {
1570                                                     runjitmioptsStr = 'jitminopts'
1571                                                 }
1572                                                 else if (scenario == 'r2r_jitforcerelocs') {
1573                                                     runjitforcerelocsStr = 'jitforcerelocs'
1574                                                 }
1575                                         }
1576                                         if (scenario == 'gcstress15_pri1r2r')
1577                                         {
1578                                            gcstressStr = 'gcstresslevel 0xF'
1579                                         }
1580
1581                                         if (isLongGc(scenario)) {
1582                                             gcTestArguments = "${scenario} sequential"
1583                                         }
1584
1585                                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${gcTestArguments}"
1586                                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1587                                             if (enableCorefxTesting) {
1588                                                 // Sync to corefx repo
1589                                                 // Move coreclr files to a subdirectory, %workspace%/clr. Otherwise, corefx build
1590                                                 // thinks that %workspace% is the project base directory.
1591                                                 buildCommands += "powershell new-item clr -type directory -force"
1592                                                 buildCommands += 'powershell foreach ($x in get-childitem -force) { if (\$x.name -ne \'clr\') { move-item $x clr }}'
1593                                                 buildCommands += "git clone https://github.com/dotnet/corefx fx"
1594
1595                                                 buildCommands += getStressModeEnvSetCmd(os, scenario);
1596
1597                                                 // Run corefx build and testing
1598                                                 buildCommands += "cd fx && call \"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat\" x86 && Build.cmd -Release -- /p:BUILDTOOLS_OVERRIDE_RUNTIME=%WORKSPACE%\\clr\\bin\\Product\\Windows_NT.x64.Checked "
1599                                             }
1600                                             else {
1601                                                 def stepScriptLocation = "%WORKSPACE%\\bin\\tests\\SetStressModes.bat"
1602                                                 buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1603
1604                                                 // Run tests with the
1605
1606                                                 buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1607                                             }
1608                                         }
1609                                         else if (architecture == 'x64') {
1610                                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1611                                         }
1612                                         else if (architecture == 'x86ryujit') {
1613                                             def testEnvLocation = "%WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd"
1614
1615                                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv ${testEnvLocation}"
1616                                         }
1617                                         else if (architecture == 'x86lb') {
1618                                             buildCommands += "tests\\runtest.cmd ${runtestArguments} Exclude0 x86_legacy_backend_issues.targets"
1619                                         }
1620                                     }
1621
1622                                     if (!enableCorefxTesting) {
1623                                         // Run the rest of the build
1624                                         // Build the mscorlib for the other OS's
1625                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1626                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1627                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1628
1629                                         // Zip up the tests directory so that we don't use so much space/time copying
1630                                         // 10s of thousands of files around.
1631                                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1632
1633                                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1634                                             // For windows, pull full test results and test drops for x86/x64.
1635                                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1636                                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip")
1637                                         }
1638
1639                                         if (!isBuildOnly) {
1640                                             if (architecture == 'x64' || !isPR) {
1641                                                 Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml')
1642                                             }
1643                                             setTestJobTimeOut(newJob, scenario)
1644                                         }
1645                                     }
1646                                     else {
1647                                         // Archive only result xml files since corefx/bin/tests is very large around 10 GB.
1648                                         // For windows, pull full test results and test drops for x86/x64
1649                                         Utilities.addArchival(newJob, "fx/bin/tests/**/testResults.xml")
1650
1651                                         // Set timeout
1652                                         setTestJobTimeOut(newJob, scenario)
1653
1654                                         if (architecture == 'x64' || !isPR) {
1655                                             Utilities.addXUnitDotNETResults(newJob, 'fx/bin/tests/**/testResults.xml')
1656                                         }
1657                                     }
1658
1659                                     break
1660                                 case 'arm64':
1661                                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1662                                     // Set time out
1663                                     setTestJobTimeOut(newJob, scenario)
1664
1665                                     // Debug runs take too long to run. So build job only.
1666                                     if (lowerConfiguration == "debug") {
1667                                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1668                                     }
1669                                     else {
1670                                        buildCommands += "set __TestIntermediateDir=int&&build.cmd skiptests ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1671                                        // Test build and run are launched together.
1672                                        buildCommands += "Z:\\arm64\\common\\scripts\\arm64PostLauncher.cmd %WORKSPACE% ${architecture} ${lowerConfiguration} ${scenario}"
1673                                        Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1674                                     }
1675
1676                                     // Add archival.
1677                                     Utilities.addArchival(newJob, "bin/Product/**")
1678                                     break
1679                                 default:
1680                                     println("Unknown architecture: ${architecture}");
1681                                     assert false
1682                                     break
1683                             }
1684                             break
1685                         case 'Ubuntu':
1686                         case 'Ubuntu16.04':
1687                         case 'Debian8.4':
1688                         case 'OSX':
1689                         case 'FreeBSD':
1690                         case 'CentOS7.1':
1691                         case 'RHEL7.2':
1692                         case 'OpenSUSE13.2':
1693                         case 'OpenSUSE42.1':
1694                         case 'Fedora23':
1695                             switch (architecture) {
1696                                 case 'x64':
1697                                 case 'x86ryujit':
1698                                 case 'x86lb':
1699                                     def arch = architecture
1700                                     if (architecture == 'x86ryujit' || architecture == 'x86lb') {
1701                                         arch = 'x86'
1702                                     }
1703
1704                                     if (!enableCorefxTesting) {
1705                                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1706                                         // only on supported OS platforms.
1707                                         if ((os == 'FreeBSD') || (os == 'OpenSUSE13.2'))
1708                                         {
1709                                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${arch}"
1710                                         }
1711                                         else
1712                                         {
1713                                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1714                                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1715                                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${arch}"
1716                                         }
1717                                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${arch}.${configuration} \${WORKSPACE}/bin/paltestout"
1718
1719                                         // Set time out
1720                                         setTestJobTimeOut(newJob, scenario)
1721                                         // Basic archiving of the build
1722                                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so")
1723                                         // And pal tests
1724                                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1725                                     }
1726                                     else {
1727                                         // Corefx stress testing
1728                                         assert os == 'Ubuntu'
1729                                         assert architecture == 'x64'
1730                                         assert lowerConfiguration == 'checked'
1731
1732                                         // Build coreclr and move it to clr directory
1733                                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1734                                         buildCommands += "rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr"
1735
1736                                         // Get corefx
1737                                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1738
1739                                         // Set environment variable
1740                                         def setEnvVar = getStressModeEnvSetCmd(os, scenario)
1741
1742                                         // Build and text corefx
1743                                         buildCommands += "rm -rf \$WORKSPACE/fx_home; mkdir \$WORKSPACE/fx_home"
1744                                         buildCommands += setEnvVar
1745                                         buildCommands += "cd fx; export HOME=\$WORKSPACE/fx_home; ./build.sh -Release -Outerloop -TestWithLocalLibraries -- /p:BUILDTOOLS_OVERRIDE_RUNTIME=\$WORKSPACE/clr/bin/Product/Linux.x64.Checked"
1746
1747                                         // Archive and process test result
1748                                         Utilities.addArchival(newJob, "fx/bin/tests/**/testResults.xml")
1749                                         setTestJobTimeOut(newJob, scenario)
1750                                         Utilities.addXUnitDotNETResults(newJob, 'fx/bin/tests/**/testResults.xml')
1751                                     }
1752                                     break
1753                                 case 'arm64':
1754                                     // We don't run the cross build except on Ubuntu
1755                                     assert os == 'Ubuntu'
1756
1757                                     buildCommands += """echo \"Using rootfs in /opt/aarch64-linux-gnu-root\"
1758                                         ROOTFS_DIR=/opt/aarch64-linux-gnu-root ./build.sh skipmscorlib arm64 cross verbose ${lowerConfiguration}"""
1759
1760                                     // Basic archiving of the build, no pal tests
1761                                     Utilities.addArchival(newJob, "bin/Product/**")
1762                                     break
1763                                 case 'arm':
1764                                     // All builds for ARM architecture are run on Ubuntu currently
1765                                     assert os == 'Ubuntu'
1766                                     if (isLinuxEmulatorBuild == false) {
1767                                         buildCommands += """echo \"Using rootfs in /opt/arm-liux-genueabihf-root\"
1768                                             ROOTFS_DIR=/opt/arm-linux-genueabihf-root ./build.sh skipmscorlib arm cross verbose ${lowerConfiguration}"""
1769
1770                                         // Basic archiving of the build, no pal tests
1771                                         Utilities.addArchival(newJob, "bin/Product/**")
1772                                         break
1773                                     }
1774                                     else {
1775                                         // Make sure the build configuration is either of debug or release
1776                                         assert ( lowerConfiguration == 'debug' ) || ( lowerConfiguration == 'release' )
1777
1778                                         // Setup variables to hold emulator folder path and the rootfs mount path
1779                                         def armemul_path = '/opt/linux-arm-emulator'
1780                                         def armrootfs_mountpath = '/opt/linux-arm-emulator-root'
1781
1782                                         // Unzip the Windows test binaries first. Exit with 0
1783                                         buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1784
1785                                         // Unpack the corefx binaries
1786                                         buildCommands += "tar -xf ./bin/build.tar.gz"
1787
1788                                         // Call the ARM emulator build script to cross build and test using the ARM emulator rootfs
1789                                         buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1790                                         --emulatorPath=${armemul_path} \\
1791                                         --mountPath=${armrootfs_mountpath} \\
1792                                         --buildConfig=${lowerConfiguration} \\
1793                                         --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1794                                         --coreFxNativeBinDir=./bin/Linux.arm-softfp.${configuration} \\
1795                                         --coreFxBinDir=\"./bin/Linux.AnyCPU.${configuration};./bin/Unix.AnyCPU.${configuration};./bin/AnyOS.AnyCPU.${configuration}\" \\
1796                                         --testDirFile=./tests/testsRunningInsideARM.txt"""
1797
1798
1799                                         // Basic archiving of the build
1800                                         Utilities.addArchival(newJob, "bin/Product/**")
1801                                         break
1802                                     }
1803                                 default:
1804                                     println("Unknown architecture: ${architecture}");
1805                                     assert false
1806                                     break
1807                             }
1808                             break
1809                         default:
1810                             println("Unknown os: ${os}");
1811                             assert false
1812                             break
1813                     }
1814
1815                     newJob.with {
1816                         steps {
1817                             if (os == 'Windows_NT') {
1818                                 buildCommands.each { buildCommand ->
1819                                     batchFile(buildCommand)
1820                                 }
1821                             }
1822                             else {
1823                                 // Setup corefx and Windows test binaries for Linux ARM Emulator Build
1824                                 if (isLinuxEmulatorBuild) {
1825                                     // Define the Windows Tests and Corefx build job names
1826                                     def WindowTestsName = projectFolder + '/' +
1827                                                           Utilities.getFullJobName(project,
1828                                                                                    getJobName(lowerConfiguration,
1829                                                                                               'x64' ,
1830                                                                                               'windows_nt',
1831                                                                                               'default',
1832                                                                                               true),
1833                                                                                    false)
1834                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
1835                                                        Utilities.getFolderName(branch)
1836
1837                                     // Copy the Windows test binaries and the Corefx build binaries
1838                                     copyArtifacts(WindowTestsName) {
1839                                         excludePatterns('**/testResults.xml', '**/*.ni.dll')
1840                                         buildSelector {
1841                                             latestSuccessful(true)
1842                                         }
1843                                     }
1844                                     copyArtifacts("${corefxFolder}/linuxarmemulator_cross_${lowerConfiguration}") {
1845                                         includePatterns('bin/build.tar.gz')
1846                                         buildSelector {
1847                                             latestSuccessful(true)
1848                                         }
1849                                     }
1850                                 }
1851
1852                                 buildCommands.each { buildCommand ->
1853                                     shell(buildCommand)
1854                                 }
1855                             }
1856                         }
1857                     }
1858
1859                 } // os
1860             } // configuration
1861         } // architecture
1862     } // isPR
1863 } // scenario
1864
1865
1866 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
1867 combinedScenarios.each { scenario ->
1868     [true, false].each { isPR ->
1869         // Architectures.  x64 only at this point
1870         ['x64'].each { architecture ->
1871             // Put the OS's supported for coreclr cross testing here
1872             Constants.crossList.each { os ->
1873                 Constants.configurationList.each { configuration ->
1874
1875                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1876                         if (configuration != 'Checked') {
1877                             return
1878                         }
1879                         if (isCorefxTesting(scenario)) {
1880                             return
1881                         }
1882                         //Skip stress modes for these scenarios
1883                         if (os == 'RHEL7.2' || os == 'Debian8.4' || os == 'OpenSUSE13.2') {
1884                             return
1885                         }
1886                     }
1887                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
1888                     // and configuration is not Checked, bail out.
1889                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1890                         return;
1891                     }
1892                     // For CentOS, we only want Checked/Release pri1 builds.
1893                     else if (os == 'CentOS7.1') {
1894                         if (scenario != 'pri1' &&
1895                             scenario != 'r2r' &&
1896                             scenario != 'pri1r2r' &&
1897                             scenario != 'gcstress15_pri1r2r' &&
1898                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
1899                             return
1900                         }
1901                         if (configuration != 'Checked' && configuration != 'Release') {
1902                             return
1903                         }
1904                     }
1905                     // For RedHat, Debian, and OpenSUSE, we only do Release pri1 builds.
1906                     else if (os == 'RHEL7.2' || os == 'Debian8.4' || os == 'OpenSUSE13.2') {
1907                         if (scenario != 'pri1') {
1908                             return
1909                         }
1910                         if (configuration != 'Release') {
1911                             return
1912                         }
1913                     }
1914                     else {
1915                         // Skip scenarios
1916                         switch (scenario) {
1917                             case 'pri1':
1918                                 // Nothing skipped
1919                                 break
1920                             case 'ilrt':
1921                                 // Release only
1922                                 if (configuration != 'Release') {
1923                                     return
1924                                 }
1925                                 break
1926                             case 'r2r':
1927                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
1928                                 if (configuration != 'Checked' && configuration != 'Release') {
1929                                     return
1930                                 }
1931                                 break
1932                             case 'pri1r2r':
1933                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
1934                                 if (configuration != 'Checked' && configuration != 'Release') {
1935                                     return
1936                                 }
1937                                 break
1938                             case 'gcstress15_pri1r2r':
1939                             case 'r2r_jitstress1':
1940                             case 'r2r_jitstress2':
1941                             case 'r2r_jitstressregs1':
1942                             case 'r2r_jitstressregs2':
1943                             case 'r2r_jitstressregs3':
1944                             case 'r2r_jitstressregs4':
1945                             case 'r2r_jitstressregs8':
1946                             case 'r2r_jitstressregsx10':
1947                             case 'r2r_jitstressregsx80':
1948                             case 'r2r_jitminopts':
1949                             case 'r2r_jitforcerelocs':
1950                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
1951                                 if (configuration != 'Checked' && configuration != 'Release') {
1952                                     return
1953                                 }
1954                                 break
1955                             case 'longgc':
1956                             case 'gcsimulator':
1957                                 // Long GC tests take a long time on non-Release builds
1958                                 if (configuration != 'Release') {
1959                                     return
1960                                 }
1961                                 break
1962                             case 'coverage':
1963                                 //We only want Ubuntu Release for coverage
1964                                 if (os != 'Ubuntu') {
1965                                     return
1966                                 }
1967                                 if (configuration != 'Release') {
1968                                     return
1969                                 }
1970                             case 'default':
1971                                 // Nothing skipped
1972                                 break
1973                             default:
1974                                 println("Unknown scenario: ${scenario}")
1975                                 assert false
1976                                 break
1977                         }
1978                     }
1979
1980                     def lowerConfiguration = configuration.toLowerCase()
1981                     def osGroup = getOSGroup(os)
1982                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
1983
1984                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
1985                     def inputScenario = 'default'
1986                     if (scenario == 'coverage') {
1987                         inputScenario = 'coverage'
1988                     }
1989                     def inputCoreCLRBuildName = projectFolder + '/' +
1990                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
1991                     // If this is a stress scenario, there isn't any difference in the build job
1992                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
1993                     // from the default scenario
1994                     def testBuildScenario = scenario
1995                     if (testBuildScenario == 'coverage' || testBuildScenario == 'pri1r2r'|| testBuildScenario == 'gcstress15_pri1r2r') {
1996                         testBuildScenario = 'pri1'
1997                     }
1998                     else if ( testBuildScenario == 'r2r' || isLongGc(testBuildScenario)) {
1999                         testBuildScenario = 'default'
2000                     }
2001                     def inputWindowTestsBuildName = ''
2002                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2003                         inputWindowTestsBuildName = projectFolder + '/' +
2004                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', 'default', true), isPR)
2005                     }
2006                     else {
2007                         inputWindowTestsBuildName = projectFolder + '/' +
2008                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', testBuildScenario, true), isPR)
2009                     }
2010                     // Enable Server GC for Ubuntu PR builds
2011                     def serverGCString = ''
2012
2013                     // Whether or not this test run should be run sequentially instead
2014                     // of in parallel. Only used for long GC tests.
2015                     def sequentialString = ''
2016
2017                     // Whether or not this test run should run a specific playlist.
2018                     // Only used for long GC tests.
2019
2020                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2021                     // for running long GC and GCSimulator tests, respectively. We don't use them
2022                     // here because using a playlist file produces much more readable output on the CI machines
2023                     // and reduces running time.
2024                     def playlistString = ''
2025
2026                     if (os == 'Ubuntu' && isPR){
2027                         serverGCString = '--useServerGC'
2028                     }
2029
2030                     // pass --crossgen to runtest.sh for crossgen builds
2031                     def crossgenStr = ''
2032                     def runcrossgentestsStr = ''
2033                     def runjitstressStr = ''
2034                     def runjitstressregsStr = ''
2035                     def runjitmioptsStr = ''
2036                     def runjitforcerelocsStr = ''
2037                     def gcstressStr = ''
2038
2039                     if (scenario == 'r2r' ||
2040                         scenario == 'pri1r2r' ||
2041                         scenario == 'gcstress15_pri1r2r' ||
2042                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2043                             crossgenStr = '--crossgen'
2044                             runcrossgentestsStr = '--runcrossgentests'
2045
2046                             if (scenario == 'r2r_jitstress1'){
2047                                 runjitstressStr = '--jitstress=1'
2048                             }
2049                             else if (scenario == 'r2r_jitstress2') {
2050                                 runjitstressStr = '--jitstress=2'
2051                             }
2052                             else if (scenario == 'r2r_jitstressregs1'){
2053                                 runjitstressregsStr = '--jitstressregs=1'
2054                             }
2055                             else if (scenario == 'r2r_jitstressregs2') {
2056                                 runjitstressregsStr = '--jitstressregs=2'
2057                             }
2058                             else if (scenario == 'r2r_jitstressregs3') {
2059                                 runjitstressregsStr = '--jitstressregs=3'
2060                             }
2061                             else if (scenario == 'r2r_jitstressregs4') {
2062                                 runjitstressregsStr = '--jitstressregs=4'
2063                             }
2064                             else if (scenario == 'r2r_jitstressregs8') {
2065                                 runjitstressregsStr = '--jitstressregs=8'
2066                             }
2067                             else if (scenario == 'r2r_jitstressregsx10') {
2068                                 runjitstressregsStr = '--jitstressregs=x10'
2069                             }
2070                             else if (scenario == 'r2r_jitstressregsx80') {
2071                                 runjitstressregsStr = '--jitstressregs=x80'
2072                             }
2073                             else if (scenario == 'r2r_jitminopts') {
2074                                 runjitmioptsStr = '--jitminopts'
2075                             }
2076                             else if (scenario == 'r2r_jitforcerelocs') {
2077                                 runjitforcerelocsStr = '--jitforcerelocs'
2078                             }
2079                     }
2080                     if  (scenario == 'gcstress15_pri1r2r')
2081                     {
2082                         gcstressStr = '--gcstresslevel=0xF'
2083                     }
2084
2085                     if (isLongGc(scenario)) {
2086                         // Long GC tests behave very poorly when they are not
2087                         // the only test running (many of them allocate until OOM).
2088                         sequentialString = '--sequential'
2089
2090                         // The Long GC playlist contains all of the tests that are
2091                         // going to be run. The GCSimulator playlist contains all of
2092                         // the GC simulator tests.
2093                         if (scenario == 'longgc') {
2094                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2095                         }
2096                         else if (scenario == 'gcsimulator') {
2097                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2098                         }
2099                     }
2100
2101                     def folder = isJITStressJob(scenario) ? 'jitstress' : ''
2102                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2103                         // Add parameters for the inputs
2104
2105                         parameters {
2106                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2107                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2108                         }
2109
2110                         steps {
2111                             // Set up the copies
2112
2113                             // Coreclr build containing the tests and mscorlib
2114
2115                             copyArtifacts(inputWindowTestsBuildName) {
2116                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2117                                 buildSelector {
2118                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2119                                 }
2120                             }
2121
2122                             if (scenario == 'coverage') {
2123
2124                                 // Move coreclr to clr directory
2125                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2126
2127                                 // Build coreclr
2128                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2129
2130                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2131                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2132                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2133                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2134                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2135                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2136                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2137                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2138
2139                                 // Run PAL tests
2140                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2141
2142                                 // Remove obj files for PAL tests so they're not included in coverage results
2143                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2144
2145                                 // Unzip the tests first.  Exit with 0
2146                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2147
2148                                 // Get corefx
2149                                 shell("git clone https://github.com/dotnet/corefx fx")
2150
2151                                 // Build Linux corefx
2152                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2153                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2154
2155                                 def testEnvOpt = ""
2156                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2157                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2158                                 shell("${createScriptCmds}")
2159                                 testEnvOpt = "--test-env=" + scriptFileName
2160
2161                                 // Run corefx tests
2162                                 shell("""./fx/run-test.sh \\
2163                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2164                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2165                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2166                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2167                 --configurationGroup Release""")
2168
2169
2170                                 // Run coreclr tests w/ workstation GC
2171                                 shell("""./clr/tests/runtest.sh \\
2172                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2173                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2174                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2175                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2176                 --coreFxBinDir=\"\$(pwd)/fx/bin/${osGroup}.AnyCPU.Release;\$(pwd)/fx/bin/Unix.AnyCPU.Release;\$(pwd)/fx/bin/AnyOS.AnyCPU.Release\" \\
2177                 --coreFxNativeBinDir=\"\$(pwd)/fx/bin/${osGroup}.${architecture}.Release\" \\
2178                 --crossgen --runcrossgentests""")
2179
2180                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2181                                 shell("""./clr/tests/runtest.sh \\
2182                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2183                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2184                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2185                 --useServerGC ${testEnvOpt}""")
2186
2187                                  // Run long-running coreclr GC tests & produce coverage reports
2188                                 shell("""./clr/tests/runtest.sh \\
2189                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2190                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2191                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2192                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2193                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2194                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2195                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2196
2197                             }
2198                             else {
2199
2200                                 // Coreclr build we are trying to test
2201
2202                                 copyArtifacts(inputCoreCLRBuildName) {
2203                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2204                                     buildSelector {
2205                                         buildNumber('${CORECLR_BUILD}')
2206                                     }
2207                                 }
2208
2209                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2210
2211                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2212                                 def osJobName = (os == 'Ubuntu') ? 'ubuntu14.04' : os.toLowerCase()
2213                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2214                                     includePatterns('bin/build.tar.gz')
2215                                     buildSelector {
2216                                         latestSuccessful(true)
2217                                     }
2218                                 }
2219
2220                                 // Unpack the corefx binaries
2221                                 shell("tar -xf ./bin/build.tar.gz")
2222
2223                                 // Unzip the tests first.  Exit with 0
2224                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2225
2226                                 // Execute the tests
2227                                 // If we are running a stress mode, we'll set those variables first
2228                                 def testEnvOpt = ""
2229                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2230                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2231                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2232                                     if (createScriptCmds != "") {
2233                                         shell("${createScriptCmds}")
2234                                         testEnvOpt = "--test-env=" + scriptFileName
2235                                     }
2236                                 }
2237
2238                                 if (isGCStressRelatedTesting(scenario)) {
2239                                     shell('./init-tools.sh')
2240                                 }
2241
2242                                 shell("""./tests/runtest.sh \\
2243                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2244                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2245                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2246                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2247                 --coreFxBinDir=\"\${WORKSPACE}/bin/${osGroup}.AnyCPU.Release;\${WORKSPACE}/bin/Unix.AnyCPU.Release;\${WORKSPACE}/bin/AnyOS.AnyCPU.Release\" \\
2248                 --coreFxNativeBinDir=\"\${WORKSPACE}/bin/${osGroup}.${architecture}.Release\" \\
2249                 --limitedDumpGeneration \\
2250                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${sequentialString} ${playlistString}""")
2251                             }
2252                         }
2253                     }
2254
2255                     if (scenario == 'coverage') {
2256                         // Publish coverage reports
2257                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2258                         addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2259                     }
2260
2261                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2262                     if (os in ['Ubuntu']) {
2263                         SummaryBuilder summaries = new SummaryBuilder()
2264                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2265                         summaries.emit(newJob)
2266                     }
2267
2268                     setMachineAffinity(newJob, os, architecture)
2269                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2270                     // Set timeouts to 240.
2271                     setTestJobTimeOut(newJob, scenario)
2272                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2273
2274                     // Create a build flow to join together the build and tests required to run this
2275                     // test.
2276                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2277                     // Linux CoreCLR test
2278                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2279                     def fullTestJobName = projectFolder + '/' + newJob.name
2280                     // Add a reference to the input jobs for report purposes
2281                     JobReport.Report.addReference(inputCoreCLRBuildName)
2282                     JobReport.Report.addReference(inputWindowTestsBuildName)
2283                     JobReport.Report.addReference(fullTestJobName)
2284                     def newFlowJob;
2285
2286                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2287                     // so that coverage data can be preserved.
2288                     if (scenario == 'coverage') {
2289                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2290                         buildFlow("""
2291 // Build the input Windows job
2292 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2293
2294 // And then build the test build
2295 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2296 """)
2297                         }
2298                     // Normal jobs copy a Windows build & a non-Windows build
2299                     } else {
2300                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2301                         buildFlow("""
2302 // Build the input jobs in parallel
2303 parallel (
2304     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2305     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2306 )
2307
2308 // And then build the test build
2309 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2310                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2311 """)
2312                         }
2313                     }
2314
2315                     setMachineAffinity(newFlowJob, os, architecture)
2316                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2317                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false, false)
2318                 } // configuration
2319             } // os
2320         } // architecture
2321     } // isPR
2322 } // scenario
2323
2324 JobReport.Report.generateJobReport(out)