netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs
  12 folder('jitstress')
  13
  14 def static getOSGroup(def os) {
  15     def osGroupMap = ['Ubuntu':'Linux',
  16         'RHEL7.2': 'Linux',
  17         'Ubuntu16.04': 'Linux',
  18         'Debian8.4':'Linux',
  19         'Fedora23':'Linux',
  20         'OSX':'OSX',
  21         'Windows_NT':'Windows_NT',
  22         'FreeBSD':'FreeBSD',
  23         'CentOS7.1': 'Linux',
  24         'OpenSUSE13.2': 'Linux',
  25         'OpenSUSE42.1': 'Linux',
  26         'LinuxARMEmulator': 'Linux']
  27     def osGroup = osGroupMap.get(os, null)
  28     assert osGroup != null : "Could not find os group for ${os}"
  29     return osGroupMap[os]
  30 }
  31
  32 // We use this class (vs variables) so that the static functions can access data here.
  33 class Constants {
  34     // Innerloop build OS's
  35     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  36     // test execution in the build flow runs.  It generates the exact same build
  37     // as Windows_NT but without the tests.
  38     def static osList = ['Ubuntu', 'Debian8.4', 'OSX', 'Windows_NT', 'Windows_NT_BuildOnly', 'FreeBSD', 'CentOS7.1', 'OpenSUSE13.2', 'OpenSUSE42.1', 'RHEL7.2', 'LinuxARMEmulator', 'Ubuntu16.04', 'Fedora23']
  39     def static crossList = ['Ubuntu', 'OSX', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'OpenSUSE13.2']
  40     // This is a set of JIT stress modes combined with the set of variables that
  41     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  42     // the values are the environment variables
  43     def static jitStressModeScenarios = ['minopts' : ['COMPlus_JITMinOpts' : '1'], 'forcerelocs' : ['COMPlus_ForceRelocs' : '1'],
  44                'jitstress1' : ['COMPlus_JitStress' : '1'], 'jitstress2' : ['COMPlus_JitStress' : '2'],
  45                'jitstressregs1' : ['COMPlus_JitStressRegs' : '1'], 'jitstressregs2' : ['COMPlus_JitStressRegs' : '2'],
  46                'jitstressregs3' : ['COMPlus_JitStressRegs' : '3'], 'jitstressregs4' : ['COMPlus_JitStressRegs' : '4'],
  47                'jitstressregs8' : ['COMPlus_JitStressRegs' : '8'], 'jitstressregs0x10' : ['COMPlus_JitStressRegs' : '0x10'],
  48                'jitstressregs0x80' : ['COMPlus_JitStressRegs' : '0x80'],
  49                'jitstress2_jitstressregs1'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  50                'jitstress2_jitstressregs2'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  51                'jitstress2_jitstressregs3'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  52                'jitstress2_jitstressregs4'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  53                'jitstress2_jitstressregs8'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  54                'jitstress2_jitstressregs0x10' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  55                'jitstress2_jitstressregs0x80' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  56                'corefx_baseline' : [ : ], // corefx baseline
  57                'corefx_minopts' : ['COMPlus_JITMinOpts' : '1'],
  58                'corefx_jitstress1' : ['COMPlus_JitStress' : '1'],
  59                'corefx_jitstress2' : ['COMPlus_JitStress' : '2'],
  60                'corefx_jitstressregs1' : ['COMPlus_JitStressRegs' : '1'], 'corefx_jitstressregs2' : ['COMPlus_JitStressRegs' : '2'],
  61                'corefx_jitstressregs3' : ['COMPlus_JitStressRegs' : '3'], 'corefx_jitstressregs4' : ['COMPlus_JitStressRegs' : '4'],
  62                'corefx_jitstressregs8' : ['COMPlus_JitStressRegs' : '8'], 'corefx_jitstressregs0x10' : ['COMPlus_JitStressRegs' : '0x10'],
  63                'corefx_jitstressregs0x80' : ['COMPlus_JitStressRegs' : '0x80'],
  64                'gcstress0x3' : ['COMPlus_GCStress'  : '0x3'], 'gcstress0xc' : ['COMPlus_GCStress'  : '0xC'],
  65                'zapdisable' : ['COMPlus_ZapDisable'  : '0xC'],
  66                'heapverify1' : ['COMPlus_HeapVerify'  : '1'],
  67                'gcstress0xc_zapdisable' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_ZapDisable'  : '1'],
  68                'gcstress0xc_zapdisable_jitstress2' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_ZapDisable'  : '1', 'COMPlus_JitStress'  : '2'],
  69                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_ZapDisable'  : '1', 'COMPlus_HeapVerify'  : '1'],
  70                'gcstress0xc_jitstress1' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_JitStress'  : '1'],
  71                'gcstress0xc_jitstress2' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_JitStress'  : '2'],
  72                'gcstress0xc_minopts_heapverify1' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_JITMinOpts'  : '1', 'COMPlus_HeapVerify'  : '1']
  73                ]
  74     // This is a set of r2r jit stress scenarios
  75     def static r2rJitStressScenarios = ['r2r_jitstress1', 'r2r_jitstress2', 'r2r_jitstressregs1', 'r2r_jitstressregs2', 'r2r_jitstressregs3',
  76                                         'r2r_jitstressregs4', 'r2r_jitstressregs8', 'r2r_jitstressregsx10', 'r2r_jitstressregsx80',
  77                                         'r2r_jitminopts', 'r2r_jitforcerelocs']
  78     // This is the basic set of scenarios
  79     def static basicScenarios = ['default', 'pri1', 'ilrt', 'r2r', 'pri1r2r', 'gcstress15_pri1r2r', 'longgc', 'coverage', 'formatting', 'gcsimulator'] + r2rJitStressScenarios
  80     def static configurationList = ['Debug', 'Checked', 'Release']
  81     // This is the set of architectures
  82     def static architectureList = ['arm', 'arm64', 'x64', 'x86ryujit', 'x86lb']
  83 }
  84
  85 def static setMachineAffinity(def job, def os, def architecture) {
  86     if (architecture == 'arm64' && os == 'Windows_NT') {
  87         // For cross compilation
  88         job.with {
  89             label('arm64')
  90         }
  91     } else if ((architecture == 'arm' || architecture == 'arm64') && os == 'Ubuntu') {
  92         Utilities.setMachineAffinity(job, os, 'arm-cross-latest');
  93     } else {
  94         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
  95     }
  96 }
  97
  98 def static isJITStressJob(def scenario) {
  99     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 100            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 101 }
 102
 103 def static isGCStressRelatedTesting(def scenario) {
 104     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 105     // Detect it and make it a GCStress related.
 106     if (scenario == 'gcstress15_pri1r2r')
 107     {
 108         return true;
 109     }
 110
 111     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 112     def scenarioName = scenario.toLowerCase()
 113     def isGCStressTesting = false
 114     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 115         if (k in gcStressTestEnvVars) {
 116             isGCStressTesting = true;
 117         }
 118     }
 119     return isGCStressTesting
 120 }
 121
 122 def static isCorefxTesting(def scenario) {
 123     def corefx_prefix = 'corefx_'
 124     if (scenario.length() < corefx_prefix.length()) {
 125         return false
 126     }
 127     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 128 }
 129
 130 def static isR2R(def scenario) {
 131     return (scenario == 'r2r' || scenario == 'pri1r2r')
 132 }
 133
 134 def static isCoverage(def scenario) {
 135     return (scenario == 'coverage')
 136 }
 137
 138 def static isLongGc(def scenario) {
 139     return (scenario == 'longgc' || scenario == 'gcsimulator')
 140 }
 141
 142 def static setTestJobTimeOut(newJob, scenario) {
 143     if (isGCStressRelatedTesting(scenario)) {
 144         Utilities.setJobTimeout(newJob, 4320)
 145     }
 146     else if (isCorefxTesting(scenario)) {
 147         Utilities.setJobTimeout(newJob, 360)
 148     }
 149     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 150         Utilities.setJobTimeout(newJob, 240)
 151     }
 152     else if (isR2R(scenario)) {
 153         Utilities.setJobTimeout(newJob, 240)
 154     }
 155     else if (isCoverage(scenario)) {
 156         Utilities.setJobTimeout(newJob, 1440)
 157     }
 158     else if (isLongGc(scenario)) {
 159         Utilities.setJobTimeout(newJob, 1440)
 160     }
 161     // Non-test jobs use the default timeout value.
 162 }
 163
 164 def static getStressModeDisplayName(def scenario) {
 165     def displayStr = ''
 166     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 167         def prefixLength = 'COMPlus_'.length()
 168         if (k.length() >= prefixLength) {
 169             def modeName = k.substring(prefixLength, k.length())
 170             displayStr += ' ' + modeName + '=' + v
 171         }
 172     }
 173     return displayStr
 174 }
 175
 176 // Generates the string for creating a file that sets environment variables
 177 // that makes it possible to run stress modes.  Writes the script to a file called
 178 // SetStressModes.[sh/cmd]
 179 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 180     def stepScript = ''
 181     if (os == 'Windows_NT') {
 182         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 183         stepScript += "del ${stepScriptLocation}\r\n"
 184
 185         // Timeout in ms, default is 10 minutes. For stress
 186         // modes up this to 30 minutes
 187         def timeout = 1800000
 188
 189         // Set the Timeout
 190         stepScript += "set __TestTimeout=${timeout}\r\n"
 191         stressModeVars.each{ k, v ->
 192             // Write out what we are writing to the script file
 193             stepScript += "echo Setting ${k}=${v}\r\n"
 194             // Write out the set itself to the script file`
 195             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 196         }
 197     }
 198     else {
 199         stepScript += "echo Setting variables for ${stressModeName}\n"
 200         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 201         stressModeVars.each{ k, v ->
 202             // Write out what we are writing to the script file
 203             stepScript += "echo Setting ${k}=${v}\n"
 204             // Write out the set itself to the script file`
 205             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 206         }
 207         stepScript += "chmod +x ${stepScriptLocation}\n"
 208     }
 209     return stepScript
 210 }
 211
 212 // Corefx doesn't have a support to pass stress mode environment variables. This function
 213 // generates commands to set or export environment variables
 214 def static getStressModeEnvSetCmd(def os, def stressModeName) {
 215     def envVars = Constants.jitStressModeScenarios[stressModeName]
 216     def setEnvVars = ''
 217     if (os == 'Windows_NT') {
 218         envVars.each{ VarName, Value   ->
 219             if (VarName != '') {
 220                 setEnvVars += "set ${VarName}=${Value}\n"
 221             }
 222         }
 223     }
 224     else {
 225         envVars.each{ VarName, Value   ->
 226             if (VarName != '') {
 227                 setEnvVars += "export ${VarName}=${Value}\n"
 228             }
 229         }
 230     }
 231     return setEnvVars
 232 }
 233
 234 // Calculates the name of the build job based on some typical parameters.
 235 //
 236 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly, def isLinuxEmulatorBuild = false) {
 237     // If the architecture is x64, do not add that info into the build name.
 238     // Need to change around some systems and other builds to pick up the right builds
 239     // to do that.
 240
 241     def suffix = scenario != 'default' ? "_${scenario}" : '';
 242     if (isBuildOnly) {
 243         suffix += '_bld'
 244     }
 245     def baseName = ''
 246     switch (architecture) {
 247         case 'x64':
 248             if (scenario == 'default') {
 249                 // For now we leave x64 off of the name for compatibility with other jobs
 250                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 251             }
 252             else if (scenario == 'formatting') {
 253                 // we don't care about the configuration for the formatting job. It runs all configs
 254                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 255             }
 256             else {
 257                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 258             }
 259             break
 260         case 'arm64':
 261         case 'arm':
 262             // These are cross builds
 263             if (isLinuxEmulatorBuild == false) {
 264                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 265             }
 266             else {
 267                 baseName = architecture.toLowerCase() + '_emulator_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 268             }
 269             break
 270         case 'x86ryujit':
 271             baseName = 'x86_ryujit_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 272             break
 273         case 'x86lb':
 274             baseName = 'x86_lb_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 275             break
 276         default:
 277             println("Unknown architecture: ${architecture}");
 278             assert false
 279             break
 280     }
 281
 282     return baseName + suffix
 283 }
 284
 285 static void addEmailPublisher(def job, def recipient) {
 286     job.with {
 287         publishers {
 288             extendedEmail(recipient, '$DEFAULT_SUBJECT', '$DEFAULT_CONTENT') {
 289                 trigger('Aborted', '$PROJECT_DEFAULT_SUBJECT', '$PROJECT_DEFAULT_CONTENT', null, true, true, true, true)
 290                 trigger('Failure', '$PROJECT_DEFAULT_SUBJECT', '$PROJECT_DEFAULT_CONTENT', null, true, true, true, true)
 291             }
 292         }
 293     }
 294 }
 295
 296 // **************************
 297 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 298 // of coreclr builds over linux/osx/freebsd/windows and debug/release/checked.  In addition, the windows
 299 // builds will do a couple extra steps.
 300 // **************************
 301
 302 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 303 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 304 // it's just used for internal builds
 305 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 306 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def isLinuxEmulatorBuild) {
 307     if (isWindowsBuildOnlyJob) {
 308         return
 309     }
 310
 311     def bidailyCrossList = ['RHEL7.2', 'Debian8.4', 'OpenSUSE13.2']
 312     // Non pull request builds.
 313     if (!isPR) {
 314         // Check scenario.
 315         switch (scenario) {
 316             case 'default':
 317                 switch (architecture) {
 318                     case 'x64':
 319                     case 'x86ryujit':
 320                     case 'x86lb':
 321                         if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 322                             Utilities.addGithubPushTrigger(job)
 323                         }
 324                         break
 325                     case 'arm':
 326                         Utilities.addGithubPushTrigger(job)
 327                         break
 328                     case 'arm64':
 329                         if (os == 'Windows_NT') {
 330                             Utilities.addGithubPushTrigger(job)
 331                             addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 332                         }
 333                         break
 334                     default:
 335                         println("Unknown architecture: ${architecture}");
 336                         assert false
 337                         break
 338                 }
 339                 break
 340             case 'pri1':
 341                 // Pri one gets a push trigger, and only for release
 342                 if (architecture == 'x64') {
 343                     if (configuration == 'Release') {
 344                         // We expect release jobs to be Windows, or in the cross list
 345                         assert (os == 'Windows_NT') || (os in Constants.crossList)
 346                         if (!os in bidailyCrossList) {
 347                             if (isFlowJob || os == 'Windows_NT') {
 348                                 Utilities.addGithubPushTrigger(job)
 349                             }
 350                         }
 351                         else {
 352                             if (isFlowJob) {
 353                                 Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 354                             }
 355                         }
 356                     }
 357                 }
 358                 break
 359             case 'r2r':
 360                 //r2r jobs that aren't pri1 can only be triggered by phrase
 361                 break
 362             case 'pri1r2r':
 363                 assert !(os in bidailyCrossList)
 364                 //pri1 r2r gets a push trigger for checked/release
 365                 if (configuration == 'Checked' || configuration == 'Release') {
 366                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 367                     if (architecture == 'x64' && os != 'OSX') {
 368                         //Flow jobs should be Windows, Ubuntu, OSX, or CentOS
 369                         if (isFlowJob || os == 'Windows_NT') {
 370                             Utilities.addGithubPushTrigger(job)
 371                         }
 372                     // OSX pri1r2r jobs should only run every 12 hours, not daily.
 373                     } else if (architecture == 'x64' && os == 'OSX'){
 374                         if (isFlowJob) {
 375                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 376                         }
 377                     }
 378                     // For x86, only add per-commit jobs for Windows
 379                     else if (architecture == 'x86ryujit' || architecture == 'x86lb') {
 380                         if (os == 'Windows_NT') {
 381                             Utilities.addGithubPushTrigger(job)
 382                         }
 383                     }
 384                     // arm64 pri1r2r jobs should only run every 12 hours.
 385                     else if (architecture == 'arm64') {
 386                         if (os == 'Windows_NT') {
 387                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 388                             addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 389                         }
 390                     }
 391                 }
 392                 break
 393             case 'r2r_jitstress1':
 394             case 'r2r_jitstress2':
 395             case 'r2r_jitstressregs1':
 396             case 'r2r_jitstressregs2':
 397             case 'r2r_jitstressregs3':
 398             case 'r2r_jitstressregs4':
 399             case 'r2r_jitstressregs8':
 400             case 'r2r_jitstressregsx10':
 401             case 'r2r_jitstressregsx80':
 402             case 'r2r_jitminopts':
 403             case 'r2r_jitforcerelocs':
 404             case 'gcstress15_pri1r2r':
 405                 assert !(os in bidailyCrossList)
 406
 407                 // GCStress=C is currently not supported on OS X
 408                 if (os == 'OSX' && isGCStressRelatedTesting(scenario)) {
 409                     break
 410                 }
 411
 412                 //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 413                 if (configuration == 'Checked' || configuration == 'Release') {
 414                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 415                     if (architecture == 'x64') {
 416                         //Flow jobs should be Windows, Ubuntu, OSX, or CentOS
 417                         if (isFlowJob || os == 'Windows_NT') {
 418                             // Add a weekly periodic trigger
 419                             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 420                         }
 421                     }
 422                     // For x86, only add per-commit jobs for Windows
 423                     else if (architecture == 'x86ryujit' || architecture == 'x86lb') {
 424                         if (os == 'Windows_NT') {
 425                             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 426                         }
 427                     }
 428                 }
 429                 break
 430             case 'longgc':
 431                 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX')
 432                 assert configuration == 'Release'
 433                 assert architecture == 'x64'
 434                 Utilities.addPeriodicTrigger(job, '@daily')
 435                 addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 436                 break
 437             case 'gcsimulator':
 438                 assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX')
 439                 assert configuration == 'Release'
 440                 assert architecture == 'x64'
 441                 Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 442                 addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 443                 break
 444             case 'ilrt':
 445                 assert !(os in bidailyCrossList)
 446                 // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 447                 if (architecture == 'x64' && configuration == 'Release') {
 448                     // We don't expect to see a job generated except in these scenarios
 449                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 450                     if (isFlowJob || os == 'Windows_NT') {
 451                         Utilities.addPeriodicTrigger(job, '@daily')
 452                     }
 453                 }
 454                 break
 455             case 'coverage':
 456                 assert (os == 'Ubuntu' || os == 'Windows_NT')
 457                 assert configuration == 'Release'
 458                 assert architecture == 'x64'
 459                 Utilities.addPeriodicTrigger(job, '@weekly')
 460                 break
 461             case 'formatting':
 462                 assert (os == 'Windows_NT' || os == "Ubuntu")
 463                 assert architecture == 'x64'
 464                 Utilities.addGithubPushTrigger(job)
 465                 break
 466             case 'jitstressregs1':
 467             case 'jitstressregs2':
 468             case 'jitstressregs3':
 469             case 'jitstressregs4':
 470             case 'jitstressregs8':
 471             case 'jitstressregs0x10':
 472             case 'jitstressregs0x80':
 473             case 'minopts':
 474             case 'forcerelocs':
 475             case 'jitstress1':
 476             case 'jitstress2':
 477             case 'jitstress2_jitstressregs1':
 478             case 'jitstress2_jitstressregs2':
 479             case 'jitstress2_jitstressregs3':
 480             case 'jitstress2_jitstressregs4':
 481             case 'jitstress2_jitstressregs8':
 482             case 'jitstress2_jitstressregs0x10':
 483             case 'jitstress2_jitstressregs0x80':
 484             case 'corefx_baseline':
 485             case 'corefx_minopts':
 486             case 'corefx_jitstress1':
 487             case 'corefx_jitstress2':
 488             case 'corefx_jitstressregs1':
 489             case 'corefx_jitstressregs2':
 490             case 'corefx_jitstressregs3':
 491             case 'corefx_jitstressregs4':
 492             case 'corefx_jitstressregs8':
 493             case 'corefx_jitstressregs0x10':
 494             case 'corefx_jitstressregs0x80':
 495             case 'zapdisable':
 496                 if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 497                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 498                     Utilities.addPeriodicTrigger(job, '@daily')
 499                 }
 500                 break
 501             case 'heapverify1':
 502             case 'gcstress0x3':
 503                 if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 504                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 505                     if (architecture == 'arm64') {
 506                         assert (os == 'Windows_NT')
 507                         Utilities.addPeriodicTrigger(job, '@daily')
 508                         addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 509                     }
 510                     else {
 511                         Utilities.addPeriodicTrigger(job, '@weekly')
 512                     }
 513                 }
 514                 break
 515             case 'gcstress0xc':
 516             case 'gcstress0xc_zapdisable':
 517             case 'gcstress0xc_zapdisable_jitstress2':
 518             case 'gcstress0xc_zapdisable_heapverify1':
 519             case 'gcstress0xc_jitstress1':
 520             case 'gcstress0xc_jitstress2':
 521             case 'gcstress0xc_minopts_heapverify1':
 522                 // GCStress=C is currently not supported on OS X
 523                 if (os != 'CentOS7.1' && os != 'OSX' && !(os in bidailyCrossList)) {
 524                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 525                     if (architecture == 'arm64') {
 526                         assert (os == 'Windows_NT')
 527                         // TODO: Enable a periodic trigger after tests are updated.
 528                         // Utilities.addPeriodicTrigger(job, '@daily')
 529                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 530                     }
 531                     else {
 532                         Utilities.addPeriodicTrigger(job, '@weekly')
 533                     }
 534                 }
 535                 break
 536             default:
 537                 println("Unknown scenario: ${scenario}");
 538                 assert false
 539                 break
 540         }
 541         return
 542     }
 543     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 544     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 545     def osGroup = getOSGroup(os)
 546     switch (architecture) {
 547         case 'x64':
 548             if (scenario == 'coverage') {
 549                 assert configuration == 'Release'
 550                 if (os == 'Ubuntu') {
 551                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 552                 }
 553                 break
 554             }
 555
 556             if (scenario == 'formatting') {
 557                 assert configuration == 'Checked'
 558                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 559                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting", "(?i).*test\\W+${os}\\W+formatting.*")
 560                 }
 561                 break
 562             }
 563
 564             switch (os) {
 565                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 566                 case 'OpenSUSE13.2':
 567                 case 'Debian8.4':
 568                 case 'RHEL7.2':
 569                     if (scenario == 'default') {
 570                         assert !isFlowJob
 571                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 572                     }
 573                     else if (scenario == 'pri1' && isFlowJob) {
 574                         assert (configuration == 'Release')
 575                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 576                     }
 577                     break
 578                 case 'Fedora23':
 579                 case 'Ubuntu16.04':
 580                 case 'OpenSUSE42.1':
 581                     assert !isFlowJob
 582                     assert scenario == 'default'
 583                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 584                     break
 585                 case 'Ubuntu':
 586                 case 'OSX':
 587                     // Triggers on the non-flow jobs aren't necessary here
 588                     // Corefx testing uses non-flow jobs.
 589                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 590                         break
 591                     }
 592                     switch (scenario) {
 593                         case 'default':
 594                             // Ubuntu uses checked for default PR tests
 595                             if (configuration == 'Checked') {
 596                                 // Default trigger
 597                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 598                             }
 599                             break
 600                         case 'pri1':
 601                             if (configuration == 'Release') {
 602                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 603                             }
 604                             break
 605                         case 'ilrt':
 606                             if (configuration == 'Release') {
 607                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 608                             }
 609                             break
 610                         case 'r2r':
 611                             if (configuration == 'Release' || configuration == 'Checked') {
 612                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 613                             }
 614                             break
 615                         case 'pri1r2r':
 616                             if (configuration == 'Release' || configuration == 'Checked') {
 617                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 618                             }
 619                             break
 620                         case 'gcstress15_pri1r2r':
 621                             if (configuration == 'Release' || configuration == 'Checked') {
 622                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 623                             }
 624                             break
 625                         case 'r2r_jitstress1':
 626                             if (configuration == 'Release' || configuration == 'Checked') {
 627                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 628                             }
 629                             break
 630                         case 'r2r_jitstress2':
 631                             if (configuration == 'Release' || configuration == 'Checked') {
 632                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 633                             }
 634                             break
 635                         case 'r2r_jitstressregs1':
 636                             if (configuration == 'Release' || configuration == 'Checked') {
 637                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 638                             }
 639                             break
 640                         case 'r2r_jitstressregs2':
 641                             if (configuration == 'Release' || configuration == 'Checked') {
 642                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 643                             }
 644                             break
 645                         case 'r2r_jitstressregs3':
 646                             if (configuration == 'Release' || configuration == 'Checked') {
 647                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 648                             }
 649                             break
 650                         case 'r2r_jitstressregs4':
 651                             if (configuration == 'Release' || configuration == 'Checked') {
 652                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 653                             }
 654                             break
 655                         case 'r2r_jitstressregs8':
 656                             if (configuration == 'Release' || configuration == 'Checked') {
 657                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 658                             }
 659                             break
 660                         case 'r2r_jitstressregsx10':
 661                             if (configuration == 'Release' || configuration == 'Checked') {
 662                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 663                             }
 664                             break
 665                         case 'r2r_jitstressregsx80':
 666                             if (configuration == 'Release' || configuration == 'Checked') {
 667                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 668                             }
 669                             break
 670                         case 'r2r_jitminopts':
 671                             if (configuration == 'Release' || configuration == 'Checked') {
 672                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} JITMinOpts R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 673                             }
 674                             break
 675                         case 'r2r_jitforcerelocs':
 676                             if (configuration == 'Release' || configuration == 'Checked') {
 677                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ForceRelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 678                             }
 679                             break
 680                         case 'longgc':
 681                             if (configuration == 'Release') {
 682                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 683                             }
 684                             break
 685                         case 'gcsimulator':
 686                             if (configuration == 'Release') {
 687                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 688                             }
 689                             break
 690                         case 'minopts':
 691                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 692                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - MinOpts)",
 693                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 694                             break
 695                         case 'jitstress1':
 696                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 697                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=1)",
 698                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 699                             break
 700                         case 'jitstress2':
 701                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 702                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=2)",
 703                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 704                             break
 705                         case 'forcerelocs':
 706                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 707                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ForceRelocs)",
 708                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 709                         case 'jitstressregs1':
 710                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 711                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=1)",
 712                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 713                             break
 714                         case 'jitstressregs2':
 715                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 716                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=2)",
 717                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 718                             break
 719                         case 'jitstressregs3':
 720                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 721                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=3)",
 722                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 723                             break
 724                         case 'jitstressregs4':
 725                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 726                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=4)",
 727                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 728                             break
 729                         case 'jitstressregs8':
 730                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 731                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=8)",
 732                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 733                             break
 734                         case 'jitstressregs0x10':
 735                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 736                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x10)",
 737                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 738                             break
 739                         case 'jitstressregs0x80':
 740                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 741                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x80)",
 742                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 743                             break
 744                         case 'jitstress2_jitstressregs1':
 745                         case 'jitstress2_jitstressregs2':
 746                         case 'jitstress2_jitstressregs3':
 747                         case 'jitstress2_jitstressregs4':
 748                         case 'jitstress2_jitstressregs8':
 749                         case 'jitstress2_jitstressregs0x10':
 750                         case 'jitstress2_jitstressregs0x80':
 751                         case 'gcstress0x3':
 752                         case 'gcstress0xc':
 753                         case 'zapdisable':
 754                         case 'heapverify1':
 755                         case 'gcstress0xc_zapdisable':
 756                         case 'gcstress0xc_zapdisable_jitstress2':
 757                         case 'gcstress0xc_zapdisable_heapverify1':
 758                         case 'gcstress0xc_jitstress1':
 759                         case 'gcstress0xc_jitstress2':
 760                         case 'gcstress0xc_minopts_heapverify1':
 761                             def displayStr = getStressModeDisplayName(scenario)
 762                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 763                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 764                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 765                             break
 766                         case 'corefx_baseline':
 767                         case 'corefx_minopts':
 768                         case 'corefx_jitstress1':
 769                         case 'corefx_jitstress2':
 770                         case 'corefx_jitstressregs1':
 771                         case 'corefx_jitstressregs2':
 772                         case 'corefx_jitstressregs3':
 773                         case 'corefx_jitstressregs4':
 774                         case 'corefx_jitstressregs8':
 775                         case 'corefx_jitstressregs0x10':
 776                         case 'corefx_jitstressregs0x80':
 777                             def displayName = 'CoreFx' + getStressModeDisplayName(scenario)
 778                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 779                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 780                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 781                             break
 782                         default:
 783                             println("Unknown scenario: ${scenario}");
 784                             assert false
 785                             break
 786                     }
 787                     break
 788                 case 'CentOS7.1':
 789                     switch (scenario) {
 790                         case 'pri1':
 791                             if (configuration == 'Release') {
 792                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 793                             }
 794                             break
 795                         case 'r2r':
 796                             if (configuration == 'Checked' || configuration == 'Release') {
 797                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 798                             }
 799                             break
 800                         case 'pri1r2r':
 801                             if (configuration == 'Checked' || configuration == 'Release') {
 802                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 803                             }
 804                             break
 805                         case 'gcstress15_pri1r2r':
 806                             if (configuration == 'Release' || configuration == 'Checked') {
 807                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 808                             }
 809                             break
 810                         case 'r2r_jitstress1':
 811                             if (configuration == 'Release' || configuration == 'Checked') {
 812                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 813                             }
 814                             break
 815                         case 'r2r_jitstress2':
 816                             if (configuration == 'Release' || configuration == 'Checked') {
 817                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 818                             }
 819                             break
 820                         case 'r2r_jitstressregs1':
 821                             if (configuration == 'Release' || configuration == 'Checked') {
 822                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 823                             }
 824                             break
 825                         case 'r2r_jitstressregs2':
 826                             if (configuration == 'Release' || configuration == 'Checked') {
 827                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 828                             }
 829                             break
 830                         case 'r2r_jitstressregs3':
 831                             if (configuration == 'Release' || configuration == 'Checked') {
 832                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 833                             }
 834                             break
 835                         case 'r2r_jitstressregs4':
 836                             if (configuration == 'Release' || configuration == 'Checked') {
 837                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 838                             }
 839                             break
 840                         case 'r2r_jitstressregs8':
 841                             if (configuration == 'Release' || configuration == 'Checked') {
 842                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 843                             }
 844                             break
 845                         case 'r2r_jitstressregsx10':
 846                             if (configuration == 'Release' || configuration == 'Checked') {
 847                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 848                             }
 849                             break
 850                         case 'r2r_jitstressregsx80':
 851                             if (configuration == 'Release' || configuration == 'Checked') {
 852                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 853                             }
 854                             break
 855                         case 'r2r_jitminopts':
 856                             if (configuration == 'Release' || configuration == 'Checked') {
 857                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} JITMinOpts R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 858                             }
 859                             break
 860                         case 'r2r_jitforcerelocs':
 861                             if (configuration == 'Release' || configuration == 'Checked') {
 862                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ForceRelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 863                             }
 864                             break
 865                         default:
 866                             break
 867                     }
 868                 case 'Windows_NT':
 869                     switch (scenario) {
 870                         case 'default':
 871                             // Default trigger
 872                             if (configuration == 'Debug') {
 873                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 874                             }
 875                             break
 876                         case 'pri1':
 877                             // Default trigger
 878                             if (configuration == 'Release') {
 879                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 880                             }
 881                             break
 882                         case 'ilrt':
 883                             if (configuration == 'Release') {
 884                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 885                             }
 886                             break
 887                         case 'r2r':
 888                             if (configuration == 'Checked' || configuration == 'Release') {
 889                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 890                             }
 891                             break
 892                         case 'pri1r2r':
 893                             if (configuration == 'Checked' || configuration == 'Release') {
 894                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 895                             }
 896                             break
 897                         case 'gcstress15_pri1r2r':
 898                             if (configuration == 'Release' || configuration == 'Checked') {
 899                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 900                             }
 901                             break
 902                         case 'r2r_jitstress1':
 903                             if (configuration == 'Release' || configuration == 'Checked') {
 904                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 905                             }
 906                             break
 907                         case 'r2r_jitstress2':
 908                             if (configuration == 'Release' || configuration == 'Checked') {
 909                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 910                             }
 911                             break
 912                         case 'r2r_jitstressregs1':
 913                             if (configuration == 'Release' || configuration == 'Checked') {
 914                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 915                             }
 916                             break
 917                         case 'r2r_jitstressregs2':
 918                             if (configuration == 'Release' || configuration == 'Checked') {
 919                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 920                             }
 921                             break
 922                         case 'r2r_jitstressregs3':
 923                             if (configuration == 'Release' || configuration == 'Checked') {
 924                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 925                             }
 926                             break
 927                         case 'r2r_jitstressregs4':
 928                             if (configuration == 'Release' || configuration == 'Checked') {
 929                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 930                             }
 931                             break
 932                         case 'r2r_jitstressregs8':
 933                             if (configuration == 'Release' || configuration == 'Checked') {
 934                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 935                             }
 936                             break
 937                         case 'r2r_jitstressregsx10':
 938                             if (configuration == 'Release' || configuration == 'Checked') {
 939                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 940                             }
 941                             break
 942                         case 'r2r_jitstressregsx80':
 943                             if (configuration == 'Release' || configuration == 'Checked') {
 944                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 945                             }
 946                             break
 947                         case 'r2r_jitminopts':
 948                             if (configuration == 'Release' || configuration == 'Checked') {
 949                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} JITMinOpts R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 950                             }
 951                             break
 952                         case 'r2r_jitforcerelocs':
 953                             if (configuration == 'Release' || configuration == 'Checked') {
 954                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ForceRelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 955                             }
 956                             break
 957                         case 'longgc':
 958                             if (configuration == 'Release') {
 959                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 960                             }
 961                             break
 962                         case 'gcsimulator':
 963                             if (configuration == 'Release') {
 964                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 965                             }
 966                             break
 967                         case 'minopts':
 968                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 969                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - MinOpts)",
 970                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 971                             break
 972                         case 'forcerelocs':
 973                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 974                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ForceRelocs)",
 975                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 976                             break
 977                         case 'jitstress1':
 978                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 979                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=1)",
 980                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 981                             break
 982                         case 'jitstress2':
 983                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 984                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=2)",
 985                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 986                             break
 987                         case 'jitstressregs1':
 988                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 989                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=1)",
 990                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 991                             break
 992                         case 'jitstressregs2':
 993                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 994                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=2)",
 995                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 996                             break
 997                         case 'jitstressregs3':
 998                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 999                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=3)",
1000                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1001                             break
1002                         case 'jitstressregs4':
1003                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1004                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=4)",
1005                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1006                             break
1007                         case 'jitstressregs8':
1008                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1009                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=8)",
1010                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1011                             break
1012                         case 'jitstressregs0x10':
1013                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1014                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x10)",
1015                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1016                             break
1017                         case 'jitstressregs0x80':
1018                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1019                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x80)",
1020                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1021                             break
1022                         case 'jitstress2_jitstressregs1':
1023                         case 'jitstress2_jitstressregs2':
1024                         case 'jitstress2_jitstressregs3':
1025                         case 'jitstress2_jitstressregs4':
1026                         case 'jitstress2_jitstressregs8':
1027                         case 'jitstress2_jitstressregs0x10':
1028                         case 'jitstress2_jitstressregs0x80':
1029                         case 'gcstress0x3':
1030                         case 'gcstress0xc':
1031                         case 'zapdisable':
1032                         case 'heapverify1':
1033                         case 'gcstress0xc_zapdisable':
1034                         case 'gcstress0xc_zapdisable_jitstress2':
1035                         case 'gcstress0xc_zapdisable_heapverify1':
1036                         case 'gcstress0xc_jitstress1':
1037                         case 'gcstress0xc_jitstress2':
1038                         case 'gcstress0xc_minopts_heapverify1':
1039                             def displayStr = getStressModeDisplayName(scenario)
1040                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1041                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1042                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1043                             break
1044                         case 'corefx_baseline':
1045                         case 'corefx_minopts':
1046                         case 'corefx_jitstress1':
1047                         case 'corefx_jitstress2':
1048                         case 'corefx_jitstressregs1':
1049                         case 'corefx_jitstressregs2':
1050                         case 'corefx_jitstressregs3':
1051                         case 'corefx_jitstressregs4':
1052                         case 'corefx_jitstressregs8':
1053                         case 'corefx_jitstressregs0x10':
1054                         case 'corefx_jitstressregs0x80':
1055                             def displayName = 'CoreFx ' + getStressModeDisplayName(scenario)
1056                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1057                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1058                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1059                             break
1060                         default:
1061                             println("Unknown scenario: ${scenario}");
1062                             assert false
1063                             break
1064                     }
1065                     break
1066                 case 'FreeBSD':
1067                     assert scenario == 'default'
1068                     if (configuration == 'Checked') {
1069                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1070                     }
1071                     break
1072                 default:
1073                     println("Unknown os: ${os}");
1074                     assert false
1075                     break
1076             }
1077             break
1078         case 'arm':
1079             assert scenario == 'default'
1080             switch (os) {
1081                 case 'Ubuntu':
1082                     if (isLinuxEmulatorBuild == false) {
1083                         // Removing the regex will cause this to run on each PR.
1084                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+Linux\\W+arm\\W+cross\\W+${configuration}.*")
1085                     }
1086                     else {
1087                         Utilities.addGithubPRTriggerForBranch(job, branch, "Linux ARM Emulator Cross ${configuration} Build")
1088                     }
1089                     break
1090                 default:
1091                     println("NYI os: ${os}");
1092                     assert false
1093                     break
1094             }
1095             break
1096         case 'arm64':
1097             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1098
1099             // Set up a private trigger
1100             def contextString = "${os} ${architecture} Cross ${configuration}"
1101             if (scenario != 'default')
1102                 contextString += " ${scenario}"
1103             contextString += " Build"
1104             // Debug builds only.
1105             if (configuration != 'Debug') {
1106                contextString += " and Test"
1107             }
1108
1109             def arm64Users = ['erozenfeld', 'kyulee1', 'pgavlin', 'russellhadley', 'swaroop-sridhar', 'JosephTremoulet', 'jashook', 'RussKeldorph', 'gkhanna79', 'briansull', 'cmckinsey', 'jkotas', 'ramarag', 'markwilkie', 'rahku', 'tzwlai', 'weshaggard']
1110             switch (os) {
1111                 case 'Windows_NT':
1112                     switch (scenario) {
1113                         case 'default':
1114                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1115                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1116                             break
1117                         case 'pri1r2r':
1118                         case 'gcstress0x3':
1119                         case 'gcstress0xc':
1120                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1121                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1122                             break
1123                     }
1124                     break
1125                 default:
1126                     println("NYI os: ${os}");
1127                     assert false
1128                     break
1129             }
1130             break
1131         case 'x86ryujit':
1132             def arch = 'x86'
1133             def jit = 'ryujit'
1134             switch (scenario) {
1135                 case 'default':
1136                     // Default trigger
1137                     if (os == 'Windows_NT') {
1138                         if (configuration == 'Checked') {
1139                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test")
1140                         }
1141                     }
1142                     else {
1143                         // default trigger
1144                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build", "(?i).*test\\W+${arch}\\W+${osGroup}.\\W+${jit}.*")
1145                     }
1146                     break
1147                 case 'pri1':
1148                     // Default trigger
1149                     if (configuration == 'Release') {
1150                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Priority 1 Build and Test")
1151                     }
1152                     break
1153                 case 'ilrt':
1154                     if (configuration == 'Release') {
1155                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1156                     }
1157                     break
1158                 case 'r2r':
1159                     if (configuration == 'Checked' || configuration == 'Release') {
1160                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1161                     }
1162                     break
1163                 case 'pri1r2r':
1164                     if (configuration == 'Checked' || configuration == 'Release') {
1165                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1166                     }
1167                     break
1168                 case 'gcstress15_pri1r2r':
1169                     if (configuration == 'Release' || configuration == 'Checked') {
1170                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1171                     }
1172                     break
1173                 case 'r2r_jitstress1':
1174                     if (configuration == 'Release' || configuration == 'Checked') {
1175                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1176                     }
1177                     break
1178                 case 'r2r_jitstress2':
1179                     if (configuration == 'Release' || configuration == 'Checked') {
1180                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1181                     }
1182                     break
1183                 case 'r2r_jitstressregs1':
1184                     if (configuration == 'Release' || configuration == 'Checked') {
1185                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1186                     }
1187                     break
1188                 case 'r2r_jitstressregs2':
1189                     if (configuration == 'Release' || configuration == 'Checked') {
1190                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1191                     }
1192                     break
1193                 case 'r2r_jitstressregs3':
1194                     if (configuration == 'Release' || configuration == 'Checked') {
1195                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1196                     }
1197                     break
1198                 case 'r2r_jitstressregs4':
1199                     if (configuration == 'Release' || configuration == 'Checked') {
1200                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1201                     }
1202                     break
1203                 case 'r2r_jitstressregs8':
1204                     if (configuration == 'Release' || configuration == 'Checked') {
1205                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1206                     }
1207                     break
1208                 case 'r2r_jitstressregsx10':
1209                     if (configuration == 'Release' || configuration == 'Checked') {
1210                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1211                     }
1212                     break
1213                 case 'r2r_jitstressregsx80':
1214                     if (configuration == 'Release' || configuration == 'Checked') {
1215                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1216                     }
1217                     break
1218                 case 'r2r_jitminopts':
1219                     if (configuration == 'Release' || configuration == 'Checked') {
1220                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} JITMinOpts R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1221                     }
1222                     break
1223                 case 'r2r_jitforcerelocs':
1224                     if (configuration == 'Release' || configuration == 'Checked') {
1225                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} ForceRelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1226                     }
1227                     break
1228                 case 'longgc':
1229                     if (configuration == 'Release') {
1230                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1231                     }
1232                     break
1233                 case 'gcsimulator':
1234                     if (configuration == 'Release') {
1235                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1236                     }
1237                     break
1238                 case 'minopts':
1239                     assert (os == 'Windows_NT')
1240                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - MinOpts)",
1241                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1242                     break
1243                 case 'forcerelocs':
1244                     assert (os == 'Windows_NT')
1245                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - ForceRelocs)",
1246                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1247                     break
1248                 case 'jitstress1':
1249                     assert (os == 'Windows_NT')
1250                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStress=1)",
1251                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1252                     break
1253                 case 'jitstress2':
1254                     assert (os == 'Windows_NT')
1255                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStress=2)",
1256                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1257                     break
1258                 case 'jitstressregs1':
1259                     assert (os == 'Windows_NT')
1260                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=1)",
1261                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1262                     break
1263                 case 'jitstressregs2':
1264                     assert (os == 'Windows_NT')
1265                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=2)",
1266                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1267                     break
1268                 case 'jitstressregs3':
1269                     assert (os == 'Windows_NT')
1270                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=3)",
1271                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1272                     break
1273                 case 'jitstressregs4':
1274                     assert (os == 'Windows_NT')
1275                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=4)",
1276                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1277                     break
1278                 case 'jitstressregs8':
1279                     assert (os == 'Windows_NT')
1280                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=8)",
1281                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1282                     break
1283                 case 'jitstressregs0x10':
1284                     assert (os == 'Windows_NT')
1285                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=0x10)",
1286                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1287                     break
1288                 case 'jitstressregs0x80':
1289                     assert (os == 'Windows_NT')
1290                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - JitStressRegs=0x80)",
1291                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1292                     break
1293                 case 'jitstress2_jitstressregs1':
1294                 case 'jitstress2_jitstressregs2':
1295                 case 'jitstress2_jitstressregs3':
1296                 case 'jitstress2_jitstressregs4':
1297                 case 'jitstress2_jitstressregs8':
1298                 case 'jitstress2_jitstressregs0x10':
1299                 case 'jitstress2_jitstressregs0x80':
1300                 case 'gcstress0x3':
1301                 case 'gcstress0xc':
1302                 case 'zapdisable':
1303                 case 'heapverify1':
1304                 case 'gcstress0xc_zapdisable':
1305                 case 'gcstress0xc_zapdisable_jitstress2':
1306                 case 'gcstress0xc_zapdisable_heapverify1':
1307                 case 'gcstress0xc_jitstress1':
1308                 case 'gcstress0xc_jitstress2':
1309                 case 'gcstress0xc_minopts_heapverify1':
1310                     def displayStr = getStressModeDisplayName(scenario)
1311                     assert (os == 'Windows_NT')
1312                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - ${displayStr})",
1313                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1314                     break
1315                 default:
1316                     println("Unknown scenario: ${arch} ${jit} ${scenario}");
1317                     assert false
1318                     break
1319             }
1320             break
1321         case 'x86lb':
1322             assert (scenario == 'default' || scenario == 'r2r' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r' || scenario == 'longgc' || scenario == 'gcsimulator' ||
1323                     Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
1324             // For windows, x86 runs by default
1325             def arch = 'x86'
1326             def jit = 'ryujit'
1327             if (architecture == 'x86lb') {
1328                 jit = 'legacy_backend'
1329             }
1330
1331             if (scenario == 'default') {
1332                 if (os == 'Windows_NT') {
1333                     if (configuration == 'Checked') {
1334                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test")
1335                     }
1336                 }
1337                 else {
1338                     // default trigger
1339                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build", "(?i).*test\\W+${arch}\\W+${osGroup}.\\W+${jit}.*")
1340                 }
1341             }
1342             else if (scenario == 'r2r') {
1343                 if (os == 'Windows_NT') {
1344                     if (configuration == 'Release') {
1345                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1346                     }
1347                 }
1348             }
1349             else if (scenario == 'pri1r2r') {
1350                 if (os == 'Windows_NT') {
1351                     if (configuration == 'Release') {
1352                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1353                     }
1354                 }
1355             }
1356             else if (scenario == 'gcstress15_pri1r2r'){
1357                 if (os == 'Windows_NT'){
1358                     if (configuration == 'Release'){
1359                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1360                     }
1361                 }
1362             }
1363             else if (scenario == 'r2r_jitstress1'){
1364                 if (configuration == 'Release' || configuration == 'Checked') {
1365                     if (os == 'Windows_NT'){
1366                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstress1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1367                     }
1368                 }
1369             }
1370             else if (scenario == 'r2r_jitstress2'){
1371                 if (configuration == 'Release' || configuration == 'Checked') {
1372                     if (os == 'Windows_NT'){
1373                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstress2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1374                     }
1375                 }
1376             }
1377             else if (scenario == 'r2r_jitstressregs1'){
1378                 if (configuration == 'Release' || configuration == 'Checked') {
1379                     if (os == 'Windows_NT'){
1380                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs1 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1381                     }
1382                 }
1383             }
1384             else if (scenario == 'r2r_jitstressregs2'){
1385                 if (configuration == 'Release' || configuration == 'Checked') {
1386                     if (os == 'Windows_NT'){
1387                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs2 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1388                     }
1389                 }
1390             }
1391             else if (scenario == 'r2r_jitstressregs3'){
1392                 if (configuration == 'Release' || configuration == 'Checked') {
1393                     if (os == 'Windows_NT'){
1394                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs3 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1395                     }
1396                 }
1397             }
1398             else if (scenario == 'r2r_jitstressregs4'){
1399                 if (configuration == 'Release' || configuration == 'Checked') {
1400                     if (os == 'Windows_NT'){
1401                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs4 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1402                     }
1403                 }
1404             }
1405             else if (scenario == 'r2r_jitstressregs8'){
1406                 if (configuration == 'Release' || configuration == 'Checked') {
1407                     if (os == 'Windows_NT'){
1408                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregs8 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1409                     }
1410                 }
1411             }
1412             else if (scenario == 'r2r_jitstressregsx10'){
1413                 if (configuration == 'Release' || configuration == 'Checked') {
1414                     if (os == 'Windows_NT'){
1415                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregsx10 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1416                     }
1417                 }
1418             }
1419             else if (scenario == 'r2r_jitstressregsx80'){
1420                 if (configuration == 'Release' || configuration == 'Checked') {
1421                     if (os == 'Windows_NT'){
1422                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitstressregsx80 R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1423                     }
1424                 }
1425             }
1426             else if (scenario == 'r2r_jitminopts'){
1427                 if (configuration == 'Release' || configuration == 'Checked') {
1428                     if (os == 'Windows_NT'){
1429                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitminopts R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1430                     }
1431                 }
1432             }
1433             else if (scenario == 'r2r_jitforcerelocs'){
1434                 if (configuration == 'Release' || configuration == 'Checked') {
1435                     if (os == 'Windows_NT'){
1436                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} jitforcerelocs R2R Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1437                     }
1438                 }
1439             }
1440             else if (scenario == 'longgc') {
1441                 if (os == 'Windows_NT'){
1442                     if (configuration == 'Release'){
1443                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1444                     }
1445                 }
1446             }
1447             else if (scenario == 'gcsimulator') {
1448                 if (os == 'Windows_NT') {
1449                     if (configuration == 'Release') {
1450                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1451                     }
1452                 }
1453             }
1454             break
1455         default:
1456             println("Unknown architecture: ${architecture}");
1457             assert false
1458             break
1459     }
1460 }
1461
1462 // Additional scenario which can alter behavior
1463
1464 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1465 combinedScenarios.each { scenario ->
1466     [true, false].each { isPR ->
1467         Constants.architectureList.each { architecture ->
1468             Constants.configurationList.each { configuration ->
1469                 Constants.osList.each { os ->
1470                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1471                     // and reset the os to Windows_NT
1472                     def isBuildOnly = false
1473                     if (os == 'Windows_NT_BuildOnly') {
1474                         isBuildOnly = true
1475                         os = 'Windows_NT'
1476                     }
1477
1478                     // If the OS is LinuxARMEmulator and arch is arm, set the isLinuxEmulatorBuild
1479                     // flag to true and reset the os to Ubuntu
1480                     // The isLinuxEmulatorBuild flag will be used at a later time to execute the right
1481                     // set of build commands
1482                     // The tuples (LinuxARMEmulator, other architectures) are not handled and control returns
1483                     def isLinuxEmulatorBuild = false
1484                     if (os == 'LinuxARMEmulator' && architecture == 'arm') {
1485                         // Cross Builds only in Debug and Release modes allowed
1486                         if ( configuration == 'Checked' ) {
1487                             return
1488                         }
1489
1490                         isLinuxEmulatorBuild = true
1491                         os = 'Ubuntu'
1492                     } else if (os == 'LinuxARMEmulator') {
1493                         return
1494                     }
1495
1496                     // Skip totally unimplemented (in CI) configurations.
1497                     switch (architecture) {
1498                         case 'arm64':
1499                             // Windows only
1500                             if (os != 'Windows_NT' || isBuildOnly) {
1501                                 return
1502                             }
1503                             break
1504                         case 'arm':
1505                             // Only Ubuntu cross implemented
1506                             if (os != 'Ubuntu') {
1507                                 return
1508                             }
1509                             break
1510                         case 'x86ryujit':
1511                         case 'x86lb':
1512                             // Skip non-windows
1513                             if (os != 'Windows_NT') {
1514                                 return
1515                             }
1516                             break
1517                         case 'x64':
1518                             // Everything implemented
1519                             break
1520                         default:
1521                             println("Unknown architecture: ${architecture}")
1522                             assert false
1523                             break
1524                     }
1525                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1526                     // with checked builds
1527                     def enableCorefxTesting = false
1528                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1529                         if (configuration != 'Checked') {
1530                             return
1531                         }
1532
1533                         enableCorefxTesting = isCorefxTesting(scenario)
1534
1535                         // Since these are just execution time differences,
1536                         // skip platforms that don't execute the tests here (Windows_NT only)
1537                         def isEnabledOS = os == 'Windows_NT' || (os == 'Ubuntu' && enableCorefxTesting)
1538                         if (!isEnabledOS || isBuildOnly) {
1539                             return
1540                         }
1541
1542                         switch (architecture) {
1543                             case 'arm64':
1544                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1545                                     return
1546                                 }
1547                                 break
1548                             case 'x64':
1549                                 // Everything implemented
1550                                 break
1551                             case 'x86ryujit':
1552                                 if (enableCorefxTesting) {
1553                                     return
1554                                 }
1555                                 break
1556                             default:
1557                                 return
1558                         }
1559                     }
1560                     else {
1561                         // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
1562                         // and configuration is not Checked, bail out.
1563                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1564                             return;
1565                         }
1566
1567                         // Skip scenarios
1568                         switch (scenario) {
1569                             case 'pri1':
1570                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1571                                 // the default scenario build
1572                                 if (os != 'Windows_NT') {
1573                                     return
1574                                 }
1575                                 // Only x64 for now
1576                                 if (architecture != 'x64') {
1577                                     return
1578                                 }
1579                                 break
1580                             case 'ilrt':
1581                                 // The ilrt build isn't necessary except for Windows_NT.  Non-Windows NT uses
1582                                 // the default scenario build
1583                                 if (os != 'Windows_NT') {
1584                                     return
1585                                 }
1586                                 // Only x64 for now
1587                                 if (architecture != 'x64') {
1588                                     return
1589                                 }
1590                                 // Release only
1591                                 if (configuration != 'Release') {
1592                                     return
1593                                 }
1594                                 break
1595                             case 'r2r':
1596                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1597                                 // the default scenario build
1598                                 if (os != 'Windows_NT') {
1599                                     return
1600                                 }
1601                                 if (architecture != 'x64') {
1602                                     return
1603                                 }
1604                                 break
1605                             case 'pri1r2r':
1606                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1607                                 // the default scenario build
1608                                 if (os != 'Windows_NT') {
1609                                     return
1610                                 }
1611                                 if (architecture != 'x64') {
1612                                     if (architecture != 'arm64' || configuration == 'Debug') {
1613                                         return
1614                                     }
1615                                 }
1616                                 break
1617                             case 'gcstress15_pri1r2r':
1618                             case 'r2r_jitstress1':
1619                             case 'r2r_jitstress2':
1620                             case 'r2r_jitstressregs1':
1621                             case 'r2r_jitstressregs2':
1622                             case 'r2r_jitstressregs3':
1623                             case 'r2r_jitstressregs4':
1624                             case 'r2r_jitstressregs8':
1625                             case 'r2r_jitstressregsx10':
1626                             case 'r2r_jitstressregsx80':
1627                             case 'r2r_jitminopts':
1628                             case 'r2r_jitforcerelocs':
1629                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
1630                                 // the default scenario build
1631                                 if (os != 'Windows_NT') {
1632                                     return
1633                                 }
1634                                 if (architecture != 'x64') {
1635                                     return
1636                                 }
1637                                 break
1638                             case 'longgc':
1639                             case 'gcsimulator':
1640                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX') {
1641                                     return
1642                                 }
1643                                 if (architecture != 'x64') {
1644                                     return
1645                                 }
1646                                 if (configuration != 'Release') {
1647                                     return
1648                                 }
1649                                 break
1650                             // We need Windows x64 Release bits for the code coverage build
1651                             case 'coverage':
1652                                 if (os != 'Windows_NT') {
1653                                     return
1654                                 }
1655                                 if (architecture != 'x64') {
1656                                     return
1657                                 }
1658                                 if (configuration != 'Release') {
1659                                     return
1660                                 }
1661                                 break
1662                             // We only run Windows and Ubuntu x64 Checked for formatting right now
1663                             case 'formatting':
1664                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
1665                                     return
1666                                 }
1667                                 if (architecture != 'x64') {
1668                                     return
1669                                 }
1670                                 if (configuration != 'Checked') {
1671                                     return
1672                                 }
1673                                 if (isBuildOnly) {
1674                                     return
1675                                 }
1676                                 break
1677                             case 'default':
1678                                 // Nothing skipped
1679                                 break
1680                             default:
1681                                 println("Unknown scenario: ${scenario}")
1682                                 assert false
1683                                 break
1684                         }
1685                     }
1686
1687                     // Calculate names
1688                     def lowerConfiguration = configuration.toLowerCase()
1689                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly, isLinuxEmulatorBuild)
1690                     def folderName = isJITStressJob(scenario) ? 'jitstress' : '';
1691
1692                     // Create the new job
1693                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
1694
1695                     setMachineAffinity(newJob, os, architecture)
1696
1697                     // Add all the standard options
1698                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
1699                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly, isLinuxEmulatorBuild)
1700
1701                     def buildCommands = [];
1702                     def osGroup = getOSGroup(os)
1703
1704                     // Calculate the build steps, archival, and xunit results
1705                     switch (os) {
1706                         case 'Windows_NT':
1707                             switch (architecture) {
1708                                 case 'x64':
1709                                 case 'x86ryujit':
1710                                 case 'x86lb':
1711                                     def arch = architecture
1712                                     if (architecture == 'x86ryujit' || architecture == 'x86lb') {
1713                                         arch = 'x86'
1714                                     }
1715
1716                                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1717                                             scenario == 'default' ||
1718                                             scenario == 'r2r' ||
1719                                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1720                                         buildOpts = enableCorefxTesting ? 'skiptests' : ''
1721                                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1722                                     }
1723
1724                                     // For Pri 1 tests, we must shorten the output test binary path names.
1725                                     // if __TestIntermediateDir is already set, build-test.cmd will
1726                                     // output test binaries to that directory. If it is not set, the
1727                                     // binaries are sent to a default directory whose name is about
1728                                     // 35 characters long.
1729
1730                                     else if (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage') {
1731                                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} -priority=1"
1732                                     }
1733                                     else if (scenario == 'ilrt') {
1734                                         // First do the build with skiptests and then build the tests with ilasm roundtrip
1735                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} skiptests"
1736                                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch} -ilasmroundtrip"
1737                                     }
1738                                     else if (isLongGc(scenario)) {
1739                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} skiptests"
1740                                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1741                                     }
1742                                     else if (scenario == 'formatting') {
1743                                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1744                                         break
1745                                     }
1746                                     else {
1747                                         println("Unknown scenario: ${scenario}")
1748                                         assert false
1749                                     }
1750
1751                                     // If we are running a stress mode, we should write out the set of key
1752                                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1753
1754                                     if (!isBuildOnly) {
1755                                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1756                                         def crossgenStr = ''
1757                                         def runcrossgentestsStr = ''
1758                                         def runjitstressStr = ''
1759                                         def runjitstressregsStr = ''
1760                                         def runjitmioptsStr = ''
1761                                         def runjitforcerelocsStr = ''
1762                                         def gcstressStr = ''
1763                                         def runtestArguments = ''
1764                                         def gcTestArguments = ''
1765
1766                                         if (scenario == 'r2r' ||
1767                                             scenario == 'pri1r2r' ||
1768                                             scenario == 'gcstress15_pri1r2r' ||
1769                                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1770                                                 crossgenStr = 'crossgen'
1771                                                 runcrossgentestsStr = 'runcrossgentests'
1772
1773                                                 if (scenario == 'r2r_jitstress1'){
1774                                                     runjitstressStr = 'jitstress 1'
1775                                                 }
1776                                                 else if (scenario == 'r2r_jitstress2') {
1777                                                     runjitstressStr = 'jitstress 2'
1778                                                 }
1779                                                 else if (scenario == 'r2r_jitstressregs1'){
1780                                                     runjitstressregsStr = 'jitstressregs 1'
1781                                                 }
1782                                                 else if (scenario == 'r2r_jitstressregs2') {
1783                                                     runjitstressregsStr = 'jitstressregs 2'
1784                                                 }
1785                                                 else if (scenario == 'r2r_jitstressregs3') {
1786                                                     runjitstressregsStr = 'jitstressregs 3'
1787                                                 }
1788                                                 else if (scenario == 'r2r_jitstressregs4') {
1789                                                     runjitstressregsStr = 'jitstressregs 4'
1790                                                 }
1791                                                 else if (scenario == 'r2r_jitstressregs8') {
1792                                                     runjitstressregsStr = 'jitstressregs 8'
1793                                                 }
1794                                                 else if (scenario == 'r2r_jitstressregsx10') {
1795                                                     runjitstressregsStr = 'jitstressregs x10'
1796                                                 }
1797                                                 else if (scenario == 'r2r_jitstressregsx80') {
1798                                                     runjitstressregsStr = 'jitstressregs x80'
1799                                                 }
1800                                                 else if (scenario == 'r2r_jitminopts') {
1801                                                     runjitmioptsStr = 'jitminopts'
1802                                                 }
1803                                                 else if (scenario == 'r2r_jitforcerelocs') {
1804                                                     runjitforcerelocsStr = 'jitforcerelocs'
1805                                                 }
1806                                         }
1807                                         if (scenario == 'gcstress15_pri1r2r')
1808                                         {
1809                                            gcstressStr = 'gcstresslevel 0xF'
1810                                         }
1811
1812                                         if (isLongGc(scenario)) {
1813                                             gcTestArguments = "${scenario} sequential"
1814                                         }
1815
1816                                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${gcTestArguments}"
1817                                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1818                                             if (enableCorefxTesting) {
1819                                                 // Sync to corefx repo
1820                                                 // Move coreclr files to a subdirectory, %workspace%/clr. Otherwise, corefx build
1821                                                 // thinks that %workspace% is the project base directory.
1822                                                 buildCommands += "powershell new-item clr -type directory -force"
1823                                                 buildCommands += 'powershell foreach ($x in get-childitem -force) { if (\$x.name -ne \'clr\') { move-item $x clr }}'
1824                                                 buildCommands += "git clone https://github.com/dotnet/corefx fx"
1825
1826                                                 buildCommands += getStressModeEnvSetCmd(os, scenario);
1827
1828                                                 // Run corefx build and testing
1829                                                 buildCommands += "cd fx && call \"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat\" x86 && Build.cmd -Release -- /p:BUILDTOOLS_OVERRIDE_RUNTIME=%WORKSPACE%\\clr\\bin\\Product\\Windows_NT.x64.Checked "
1830                                             }
1831                                             else {
1832                                                 def stepScriptLocation = "%WORKSPACE%\\bin\\tests\\SetStressModes.bat"
1833
1834                                                 if (architecture == 'x86ryujit'){
1835                                                     def x86Vars = ['COMPLUS_AltJit' : '*', 'COMPLUS_AltJitName' : 'protojit.dll', 'COMPLUS_NoGuiOnAssert' : '1']
1836                                                     buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario] + x86Vars, stepScriptLocation)
1837                                                 }
1838                                                 else {
1839                                                     buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1840                                                 }
1841
1842                                                 // Run tests with the
1843                                                 buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1844                                             }
1845                                         }
1846                                         else if (architecture == 'x64') {
1847                                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1848                                         }
1849                                         else if (architecture == 'x86ryujit') {
1850                                             def testEnvLocation = "%WORKSPACE%\\tests\\x86\\ryujit_x86_testenv.cmd"
1851
1852                                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv ${testEnvLocation}"
1853                                         }
1854                                         else if (architecture == 'x86lb') {
1855                                             buildCommands += "tests\\runtest.cmd ${runtestArguments} Exclude0 x86_legacy_backend_issues.targets"
1856                                         }
1857                                     }
1858
1859                                     if (!enableCorefxTesting) {
1860                                         // Run the rest of the build
1861                                         // Build the mscorlib for the other OS's
1862                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1863                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1864                                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1865
1866                                         // Zip up the tests directory so that we don't use so much space/time copying
1867                                         // 10s of thousands of files around.
1868                                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1869
1870                                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1871                                             // For windows, pull full test results and test drops for x86/x64.
1872                                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1873                                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip")
1874                                         }
1875
1876                                         if (!isBuildOnly) {
1877                                             if (architecture == 'x64' || !isPR) {
1878                                                 Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml')
1879                                             }
1880                                             setTestJobTimeOut(newJob, scenario)
1881                                         }
1882                                     }
1883                                     else {
1884                                         // Archive only result xml files since corefx/bin/tests is very large around 10 GB.
1885                                         // For windows, pull full test results and test drops for x86/x64
1886                                         Utilities.addArchival(newJob, "fx/bin/tests/**/testResults.xml")
1887
1888                                         // Set timeout
1889                                         setTestJobTimeOut(newJob, scenario)
1890
1891                                         if (architecture == 'x64' || !isPR) {
1892                                             Utilities.addXUnitDotNETResults(newJob, 'fx/bin/tests/**/testResults.xml')
1893                                         }
1894                                     }
1895
1896                                     break
1897                                 case 'arm64':
1898                                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1899                                     // Set time out
1900                                     setTestJobTimeOut(newJob, scenario)
1901
1902                                     // Debug runs take too long to run. So build job only.
1903                                     if (lowerConfiguration == "debug") {
1904                                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1905                                     }
1906                                     else {
1907                                        buildCommands += "set __TestIntermediateDir=int&&build.cmd skiptests ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1908                                        // Test build and run are launched together.
1909                                        buildCommands += "Z:\\arm64\\common\\scripts\\arm64PostLauncher.cmd %WORKSPACE% ${architecture} ${lowerConfiguration} ${scenario}"
1910                                        Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1911                                     }
1912
1913                                     // Add archival.
1914                                     Utilities.addArchival(newJob, "bin/Product/**")
1915                                     break
1916                                 default:
1917                                     println("Unknown architecture: ${architecture}");
1918                                     assert false
1919                                     break
1920                             }
1921                             break
1922                         case 'Ubuntu':
1923                         case 'Ubuntu16.04':
1924                         case 'Debian8.4':
1925                         case 'OSX':
1926                         case 'FreeBSD':
1927                         case 'CentOS7.1':
1928                         case 'RHEL7.2':
1929                         case 'OpenSUSE13.2':
1930                         case 'OpenSUSE42.1':
1931                         case 'Fedora23':
1932                             switch (architecture) {
1933                                 case 'x64':
1934                                 case 'x86ryujit':
1935                                 case 'x86lb':
1936                                     def arch = architecture
1937                                     if (architecture == 'x86ryujit' || architecture == 'x86lb') {
1938                                         arch = 'x86'
1939                                     }
1940
1941                                     if (scenario == 'formatting') {
1942                                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${arch}"
1943                                         break
1944                                     }
1945
1946                                     if (!enableCorefxTesting) {
1947                                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1948                                         // only on supported OS platforms.
1949                                         if ((os == 'FreeBSD') || (os == 'OpenSUSE13.2'))
1950                                         {
1951                                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${arch}"
1952                                         }
1953                                         else
1954                                         {
1955                                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1956                                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1957                                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${arch}"
1958                                         }
1959                                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${arch}.${configuration} \${WORKSPACE}/bin/paltestout"
1960
1961                                         // Set time out
1962                                         setTestJobTimeOut(newJob, scenario)
1963                                         // Basic archiving of the build
1964                                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so")
1965                                         // And pal tests
1966                                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1967                                     }
1968                                     else {
1969                                         // Corefx stress testing
1970                                         assert os == 'Ubuntu'
1971                                         assert architecture == 'x64'
1972                                         assert lowerConfiguration == 'checked'
1973
1974                                         // Build coreclr and move it to clr directory
1975                                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1976                                         buildCommands += "rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr"
1977
1978                                         // Get corefx
1979                                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1980
1981                                         // Set environment variable
1982                                         def setEnvVar = getStressModeEnvSetCmd(os, scenario)
1983
1984                                         // Build and text corefx
1985                                         buildCommands += "rm -rf \$WORKSPACE/fx_home; mkdir \$WORKSPACE/fx_home"
1986                                         buildCommands += setEnvVar
1987                                         buildCommands += "cd fx; export HOME=\$WORKSPACE/fx_home; ./build.sh -Release -Outerloop -TestWithLocalLibraries -- /p:BUILDTOOLS_OVERRIDE_RUNTIME=\$WORKSPACE/clr/bin/Product/Linux.x64.Checked"
1988
1989                                         // Archive and process test result
1990                                         Utilities.addArchival(newJob, "fx/bin/tests/**/testResults.xml")
1991                                         setTestJobTimeOut(newJob, scenario)
1992                                         Utilities.addXUnitDotNETResults(newJob, 'fx/bin/tests/**/testResults.xml')
1993                                     }
1994                                     break
1995                                 case 'arm64':
1996                                     // We don't run the cross build except on Ubuntu
1997                                     assert os == 'Ubuntu'
1998
1999                                     buildCommands += """echo \"Using rootfs in /opt/aarch64-linux-gnu-root\"
2000                                         ROOTFS_DIR=/opt/aarch64-linux-gnu-root ./build.sh skipmscorlib arm64 cross verbose ${lowerConfiguration}"""
2001
2002                                     // Basic archiving of the build, no pal tests
2003                                     Utilities.addArchival(newJob, "bin/Product/**")
2004                                     break
2005                                 case 'arm':
2006                                     // All builds for ARM architecture are run on Ubuntu currently
2007                                     assert os == 'Ubuntu'
2008                                     if (isLinuxEmulatorBuild == false) {
2009                                         buildCommands += """echo \"Using rootfs in /opt/arm-liux-genueabihf-root\"
2010                                             ROOTFS_DIR=/opt/arm-linux-genueabihf-root ./build.sh skipmscorlib arm cross verbose ${lowerConfiguration}"""
2011
2012                                         // Basic archiving of the build, no pal tests
2013                                         Utilities.addArchival(newJob, "bin/Product/**")
2014                                         break
2015                                     }
2016                                     else {
2017                                         // Make sure the build configuration is either of debug or release
2018                                         assert ( lowerConfiguration == 'debug' ) || ( lowerConfiguration == 'release' )
2019
2020                                         // Setup variables to hold emulator folder path and the rootfs mount path
2021                                         def armemul_path = '/opt/linux-arm-emulator'
2022                                         def armrootfs_mountpath = '/opt/linux-arm-emulator-root'
2023
2024                                         // Unzip the Windows test binaries first. Exit with 0
2025                                         buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
2026
2027                                         // Unpack the corefx binaries
2028                                         buildCommands += "tar -xf ./bin/build.tar.gz"
2029
2030                                         // Call the ARM emulator build script to cross build and test using the ARM emulator rootfs
2031                                         buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2032                                         --emulatorPath=${armemul_path} \\
2033                                         --mountPath=${armrootfs_mountpath} \\
2034                                         --buildConfig=${lowerConfiguration} \\
2035                                         --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2036                                         --coreFxNativeBinDir=./bin/Linux.arm-softfp.${configuration} \\
2037                                         --coreFxBinDir=\"./bin/Linux.AnyCPU.${configuration};./bin/Unix.AnyCPU.${configuration};./bin/AnyOS.AnyCPU.${configuration}\" \\
2038                                         --testDirFile=./tests/testsRunningInsideARM.txt"""
2039
2040
2041                                         // Basic archiving of the build
2042                                         Utilities.addArchival(newJob, "bin/Product/**")
2043                                         break
2044                                     }
2045                                 default:
2046                                     println("Unknown architecture: ${architecture}");
2047                                     assert false
2048                                     break
2049                             }
2050                             break
2051                         default:
2052                             println("Unknown os: ${os}");
2053                             assert false
2054                             break
2055                     }
2056
2057                     newJob.with {
2058                         steps {
2059                             if (os == 'Windows_NT') {
2060                                 buildCommands.each { buildCommand ->
2061                                     batchFile(buildCommand)
2062                                 }
2063                             }
2064                             else {
2065                                 // Setup corefx and Windows test binaries for Linux ARM Emulator Build
2066                                 if (isLinuxEmulatorBuild) {
2067                                     // Define the Windows Tests and Corefx build job names
2068                                     def WindowTestsName = projectFolder + '/' +
2069                                                           Utilities.getFullJobName(project,
2070                                                                                    getJobName(lowerConfiguration,
2071                                                                                               'x64' ,
2072                                                                                               'windows_nt',
2073                                                                                               'default',
2074                                                                                               true),
2075                                                                                    false)
2076                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2077                                                        Utilities.getFolderName(branch)
2078
2079                                     // Copy the Windows test binaries and the Corefx build binaries
2080                                     copyArtifacts(WindowTestsName) {
2081                                         excludePatterns('**/testResults.xml', '**/*.ni.dll')
2082                                         buildSelector {
2083                                             latestSuccessful(true)
2084                                         }
2085                                     }
2086                                     copyArtifacts("${corefxFolder}/linuxarmemulator_cross_${lowerConfiguration}") {
2087                                         includePatterns('bin/build.tar.gz')
2088                                         buildSelector {
2089                                             latestSuccessful(true)
2090                                         }
2091                                     }
2092                                 }
2093
2094                                 buildCommands.each { buildCommand ->
2095                                     shell(buildCommand)
2096                                 }
2097                             }
2098                         }
2099                     }
2100
2101                 } // os
2102             } // configuration
2103         } // architecture
2104     } // isPR
2105 } // scenario
2106
2107
2108 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2109 combinedScenarios.each { scenario ->
2110     [true, false].each { isPR ->
2111         // Architectures.  x64 only at this point
2112         ['x64'].each { architecture ->
2113             // Put the OS's supported for coreclr cross testing here
2114             Constants.crossList.each { os ->
2115                 Constants.configurationList.each { configuration ->
2116
2117                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2118                         if (configuration != 'Checked') {
2119                             return
2120                         }
2121                         if (isCorefxTesting(scenario)) {
2122                             return
2123                         }
2124                         //Skip stress modes for these scenarios
2125                         if (os == 'RHEL7.2' || os == 'Debian8.4' || os == 'OpenSUSE13.2') {
2126                             return
2127                         }
2128                     }
2129                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2130                     // and configuration is not Checked, bail out.
2131                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2132                         return;
2133                     }
2134                     // For CentOS, we only want Checked/Release pri1 builds.
2135                     else if (os == 'CentOS7.1') {
2136                         if (scenario != 'pri1' &&
2137                             scenario != 'r2r' &&
2138                             scenario != 'pri1r2r' &&
2139                             scenario != 'gcstress15_pri1r2r' &&
2140                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2141                             return
2142                         }
2143                         if (configuration != 'Checked' && configuration != 'Release') {
2144                             return
2145                         }
2146                     }
2147                     // For RedHat, Debian, and OpenSUSE, we only do Release pri1 builds.
2148                     else if (os == 'RHEL7.2' || os == 'Debian8.4' || os == 'OpenSUSE13.2') {
2149                         if (scenario != 'pri1') {
2150                             return
2151                         }
2152                         if (configuration != 'Release') {
2153                             return
2154                         }
2155                     }
2156                     else {
2157                         // Skip scenarios
2158                         switch (scenario) {
2159                             case 'pri1':
2160                                 // Nothing skipped
2161                                 break
2162                             case 'ilrt':
2163                                 // Release only
2164                                 if (configuration != 'Release') {
2165                                     return
2166                                 }
2167                                 break
2168                             case 'r2r':
2169                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2170                                 if (configuration != 'Checked' && configuration != 'Release') {
2171                                     return
2172                                 }
2173                                 break
2174                             case 'pri1r2r':
2175                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2176                                 if (configuration != 'Checked' && configuration != 'Release') {
2177                                     return
2178                                 }
2179                                 break
2180                             case 'gcstress15_pri1r2r':
2181                             case 'r2r_jitstress1':
2182                             case 'r2r_jitstress2':
2183                             case 'r2r_jitstressregs1':
2184                             case 'r2r_jitstressregs2':
2185                             case 'r2r_jitstressregs3':
2186                             case 'r2r_jitstressregs4':
2187                             case 'r2r_jitstressregs8':
2188                             case 'r2r_jitstressregsx10':
2189                             case 'r2r_jitstressregsx80':
2190                             case 'r2r_jitminopts':
2191                             case 'r2r_jitforcerelocs':
2192                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2193                                 if (configuration != 'Checked' && configuration != 'Release') {
2194                                     return
2195                                 }
2196                                 break
2197                             case 'longgc':
2198                             case 'gcsimulator':
2199                                 // Long GC tests take a long time on non-Release builds
2200                                 if (configuration != 'Release') {
2201                                     return
2202                                 }
2203                                 break
2204                             case 'coverage':
2205                                 //We only want Ubuntu Release for coverage
2206                                 if (os != 'Ubuntu') {
2207                                     return
2208                                 }
2209                                 if (configuration != 'Release') {
2210                                     return
2211                                 }
2212                             case 'formatting':
2213                                 return
2214                             case 'default':
2215                                 // Nothing skipped
2216                                 break
2217                             default:
2218                                 println("Unknown scenario: ${scenario}")
2219                                 assert false
2220                                 break
2221                         }
2222                     }
2223
2224                     def lowerConfiguration = configuration.toLowerCase()
2225                     def osGroup = getOSGroup(os)
2226                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2227
2228                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2229                     def inputScenario = 'default'
2230                     if (scenario == 'coverage') {
2231                         inputScenario = 'coverage'
2232                     }
2233                     def inputCoreCLRBuildName = projectFolder + '/' +
2234                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2235                     // If this is a stress scenario, there isn't any difference in the build job
2236                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2237                     // from the default scenario
2238                     def testBuildScenario = scenario
2239                     if (testBuildScenario == 'coverage' || testBuildScenario == 'pri1r2r'|| testBuildScenario == 'gcstress15_pri1r2r') {
2240                         testBuildScenario = 'pri1'
2241                     }
2242                     else if ( testBuildScenario == 'r2r' || isLongGc(testBuildScenario)) {
2243                         testBuildScenario = 'default'
2244                     }
2245                     def inputWindowTestsBuildName = ''
2246                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2247                         inputWindowTestsBuildName = projectFolder + '/' +
2248                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', 'default', true), isPR)
2249                     }
2250                     else {
2251                         inputWindowTestsBuildName = projectFolder + '/' +
2252                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', testBuildScenario, true), isPR)
2253                     }
2254                     // Enable Server GC for Ubuntu PR builds
2255                     def serverGCString = ''
2256
2257                     // Whether or not this test run should be run sequentially instead
2258                     // of in parallel. Only used for long GC tests.
2259                     def sequentialString = ''
2260
2261                     // Whether or not this test run should run a specific playlist.
2262                     // Only used for long GC tests.
2263
2264                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2265                     // for running long GC and GCSimulator tests, respectively. We don't use them
2266                     // here because using a playlist file produces much more readable output on the CI machines
2267                     // and reduces running time.
2268                     def playlistString = ''
2269
2270                     if (os == 'Ubuntu' && isPR){
2271                         serverGCString = '--useServerGC'
2272                     }
2273
2274                     // pass --crossgen to runtest.sh for crossgen builds
2275                     def crossgenStr = ''
2276                     def runcrossgentestsStr = ''
2277                     def runjitstressStr = ''
2278                     def runjitstressregsStr = ''
2279                     def runjitmioptsStr = ''
2280                     def runjitforcerelocsStr = ''
2281                     def gcstressStr = ''
2282
2283                     if (scenario == 'r2r' ||
2284                         scenario == 'pri1r2r' ||
2285                         scenario == 'gcstress15_pri1r2r' ||
2286                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2287                             crossgenStr = '--crossgen'
2288                             runcrossgentestsStr = '--runcrossgentests'
2289
2290                             if (scenario == 'r2r_jitstress1'){
2291                                 runjitstressStr = '--jitstress=1'
2292                             }
2293                             else if (scenario == 'r2r_jitstress2') {
2294                                 runjitstressStr = '--jitstress=2'
2295                             }
2296                             else if (scenario == 'r2r_jitstressregs1'){
2297                                 runjitstressregsStr = '--jitstressregs=1'
2298                             }
2299                             else if (scenario == 'r2r_jitstressregs2') {
2300                                 runjitstressregsStr = '--jitstressregs=2'
2301                             }
2302                             else if (scenario == 'r2r_jitstressregs3') {
2303                                 runjitstressregsStr = '--jitstressregs=3'
2304                             }
2305                             else if (scenario == 'r2r_jitstressregs4') {
2306                                 runjitstressregsStr = '--jitstressregs=4'
2307                             }
2308                             else if (scenario == 'r2r_jitstressregs8') {
2309                                 runjitstressregsStr = '--jitstressregs=8'
2310                             }
2311                             else if (scenario == 'r2r_jitstressregsx10') {
2312                                 runjitstressregsStr = '--jitstressregs=x10'
2313                             }
2314                             else if (scenario == 'r2r_jitstressregsx80') {
2315                                 runjitstressregsStr = '--jitstressregs=x80'
2316                             }
2317                             else if (scenario == 'r2r_jitminopts') {
2318                                 runjitmioptsStr = '--jitminopts'
2319                             }
2320                             else if (scenario == 'r2r_jitforcerelocs') {
2321                                 runjitforcerelocsStr = '--jitforcerelocs'
2322                             }
2323                     }
2324                     if  (scenario == 'gcstress15_pri1r2r')
2325                     {
2326                         gcstressStr = '--gcstresslevel=0xF'
2327                     }
2328
2329                     if (isLongGc(scenario)) {
2330                         // Long GC tests behave very poorly when they are not
2331                         // the only test running (many of them allocate until OOM).
2332                         sequentialString = '--sequential'
2333
2334                         // The Long GC playlist contains all of the tests that are
2335                         // going to be run. The GCSimulator playlist contains all of
2336                         // the GC simulator tests.
2337                         if (scenario == 'longgc') {
2338                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2339                         }
2340                         else if (scenario == 'gcsimulator') {
2341                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2342                         }
2343                     }
2344
2345                     def folder = isJITStressJob(scenario) ? 'jitstress' : ''
2346                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2347                         // Add parameters for the inputs
2348
2349                         parameters {
2350                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2351                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2352                         }
2353
2354                         steps {
2355                             // Set up the copies
2356
2357                             // Coreclr build containing the tests and mscorlib
2358
2359                             copyArtifacts(inputWindowTestsBuildName) {
2360                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2361                                 buildSelector {
2362                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2363                                 }
2364                             }
2365
2366                             if (scenario == 'coverage') {
2367
2368                                 // Move coreclr to clr directory
2369                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2370
2371                                 // Build coreclr
2372                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2373
2374                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2375                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2376                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2377                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2378                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2379                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2380                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2381                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2382
2383                                 // Run PAL tests
2384                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2385
2386                                 // Remove obj files for PAL tests so they're not included in coverage results
2387                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2388
2389                                 // Unzip the tests first.  Exit with 0
2390                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2391
2392                                 // Get corefx
2393                                 shell("git clone https://github.com/dotnet/corefx fx")
2394
2395                                 // Build Linux corefx
2396                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2397                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2398
2399                                 def testEnvOpt = ""
2400                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2401                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2402                                 shell("${createScriptCmds}")
2403                                 testEnvOpt = "--test-env=" + scriptFileName
2404
2405                                 // Run corefx tests
2406                                 shell("""./fx/run-test.sh \\
2407                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2408                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2409                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2410                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2411                 --configurationGroup Release""")
2412
2413
2414                                 // Run coreclr tests w/ workstation GC
2415                                 shell("""./clr/tests/runtest.sh \\
2416                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2417                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2418                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2419                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2420                 --coreFxBinDir=\"\$(pwd)/fx/bin/${osGroup}.AnyCPU.Release;\$(pwd)/fx/bin/Unix.AnyCPU.Release;\$(pwd)/fx/bin/AnyOS.AnyCPU.Release\" \\
2421                 --coreFxNativeBinDir=\"\$(pwd)/fx/bin/${osGroup}.${architecture}.Release\" \\
2422                 --crossgen --runcrossgentests""")
2423
2424                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2425                                 shell("""./clr/tests/runtest.sh \\
2426                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2427                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2428                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2429                 --useServerGC ${testEnvOpt}""")
2430
2431                                  // Run long-running coreclr GC tests & produce coverage reports
2432                                 shell("""./clr/tests/runtest.sh \\
2433                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2434                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2435                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2436                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2437                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2438                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2439                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2440
2441                             }
2442                             else {
2443
2444                                 // Coreclr build we are trying to test
2445
2446                                 copyArtifacts(inputCoreCLRBuildName) {
2447                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2448                                     buildSelector {
2449                                         buildNumber('${CORECLR_BUILD}')
2450                                     }
2451                                 }
2452
2453                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2454
2455                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2456                                 def osJobName = (os == 'Ubuntu') ? 'ubuntu14.04' : os.toLowerCase()
2457                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2458                                     includePatterns('bin/build.tar.gz')
2459                                     buildSelector {
2460                                         latestSuccessful(true)
2461                                     }
2462                                 }
2463
2464                                 // Unpack the corefx binaries
2465                                 shell("tar -xf ./bin/build.tar.gz")
2466
2467                                 // Unzip the tests first.  Exit with 0
2468                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2469
2470                                 // Execute the tests
2471                                 // If we are running a stress mode, we'll set those variables first
2472                                 def testEnvOpt = ""
2473                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2474                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2475                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2476                                     if (createScriptCmds != "") {
2477                                         shell("${createScriptCmds}")
2478                                         testEnvOpt = "--test-env=" + scriptFileName
2479                                     }
2480                                 }
2481
2482                                 if (isGCStressRelatedTesting(scenario)) {
2483                                     shell('./init-tools.sh')
2484                                 }
2485
2486                                 shell("""./tests/runtest.sh \\
2487                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2488                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2489                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2490                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2491                 --coreFxBinDir=\"\${WORKSPACE}/bin/${osGroup}.AnyCPU.Release;\${WORKSPACE}/bin/Unix.AnyCPU.Release;\${WORKSPACE}/bin/AnyOS.AnyCPU.Release\" \\
2492                 --coreFxNativeBinDir=\"\${WORKSPACE}/bin/${osGroup}.${architecture}.Release\" \\
2493                 --limitedDumpGeneration \\
2494                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${sequentialString} ${playlistString}""")
2495                             }
2496                         }
2497                     }
2498
2499                     if (scenario == 'coverage') {
2500                         // Publish coverage reports
2501                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2502                         addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2503                     }
2504
2505                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2506                     if (os in ['Ubuntu']) {
2507                         SummaryBuilder summaries = new SummaryBuilder()
2508                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2509                         summaries.emit(newJob)
2510                     }
2511
2512                     setMachineAffinity(newJob, os, architecture)
2513                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2514                     // Set timeouts to 240.
2515                     setTestJobTimeOut(newJob, scenario)
2516                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2517
2518                     // Create a build flow to join together the build and tests required to run this
2519                     // test.
2520                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2521                     // Linux CoreCLR test
2522                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2523                     def fullTestJobName = projectFolder + '/' + newJob.name
2524                     // Add a reference to the input jobs for report purposes
2525                     JobReport.Report.addReference(inputCoreCLRBuildName)
2526                     JobReport.Report.addReference(inputWindowTestsBuildName)
2527                     JobReport.Report.addReference(fullTestJobName)
2528                     def newFlowJob;
2529
2530                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2531                     // so that coverage data can be preserved.
2532                     if (scenario == 'coverage') {
2533                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2534                         buildFlow("""
2535 // Build the input Windows job
2536 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2537
2538 // And then build the test build
2539 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2540 """)
2541                         }
2542                     // Normal jobs copy a Windows build & a non-Windows build
2543                     } else {
2544                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2545                         buildFlow("""
2546 // Build the input jobs in parallel
2547 parallel (
2548     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2549     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2550 )
2551
2552 // And then build the test build
2553 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2554                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2555 """)
2556                         }
2557                     }
2558
2559                     setMachineAffinity(newFlowJob, os, architecture)
2560                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2561                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false, false)
2562                 } // configuration
2563             } // os
2564         } // architecture
2565     } // isPR
2566 } // scenario
2567
2568 JobReport.Report.generateJobReport(out)
2569
2570 // Make the call to generate the help job
2571 Utilities.createHelperJob(this, project, branch,
2572     "Welcome to the ${project} Repository",  // This is prepended to the help message
2573     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.