netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.Utilities;
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 def static getOSGroup(def os) {
  12     def osGroupMap = ['Ubuntu':'Linux',
  13         'RHEL7.2': 'Linux',
  14         'Ubuntu15.10': 'Linux',
  15         'Debian8.2':'Linux',
  16         'OSX':'OSX',
  17         'Windows_NT':'Windows_NT',
  18         'FreeBSD':'FreeBSD',
  19         'CentOS7.1': 'Linux',
  20         'OpenSUSE13.2': 'Linux']
  21     def osGroup = osGroupMap.get(os, null)
  22     assert osGroup != null : "Could not find os group for ${os}"
  23     return osGroupMap[os]
  24 }
  25
  26 // We use this class (vs variables) so that the static functions can access data here.
  27 class Constants {
  28     // Innerloop build OS's
  29     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  30     // test execution in the build flow runs.  It generates the exact same build
  31     // as Windows_NT but without the tests.
  32     def static osList = ['Ubuntu', 'Debian8.2', 'OSX', 'Windows_NT', 'Windows_NT_BuildOnly', 'FreeBSD', 'CentOS7.1', 'OpenSUSE13.2', 'Ubuntu15.10', 'RHEL7.2']
  33     def static crossList = ['Ubuntu', 'OSX', 'CentOS7.1']
  34     // This is a set of JIT stress modes combined with the set of variables that
  35     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  36     // the values are the environment variables
  37     def static jitStressModeScenarios = ['minopts' : ['COMPlus_JITMinOpts' : '1'], 'forcerelocs' : ['COMPlus_ForceRelocs' : '1'],
  38                'jitstress1' : ['COMPlus_JitStress' : '1'], 'jitstress2' : ['COMPlus_JitStress' : '2'],
  39                'jitstressregs1' : ['COMPlus_JitStressRegs' : '1'], 'jitstressregs2' : ['COMPlus_JitStressRegs' : '2'],
  40                'jitstressregs3' : ['COMPlus_JitStressRegs' : '3'], 'jitstressregs4' : ['COMPlus_JitStressRegs' : '4'],
  41                'jitstressregs8' : ['COMPlus_JitStressRegs' : '8'], 'jitstressregs0x10' : ['COMPlus_JitStressRegs' : '0x10'],
  42                'jitstressregs0x80' : ['COMPlus_JitStressRegs' : '0x80'],
  43                'jitstress2_jitstressregs1'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  44                'jitstress2_jitstressregs2'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  45                'jitstress2_jitstressregs3'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  46                'jitstress2_jitstressregs4'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  47                'jitstress2_jitstressregs8'    : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  48                'jitstress2_jitstressregs0x10' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  49                'jitstress2_jitstressregs0x80' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  50                'corefx_baseline' : [ : ], // corefx baseline
  51                'corefx_minopts' : ['COMPlus_JITMinOpts' : '1'],
  52                'corefx_jitstress1' : ['COMPlus_JitStress' : '1'],
  53                'corefx_jitstress2' : ['COMPlus_JitStress' : '2'],
  54                'corefx_jitstressregs1' : ['COMPlus_JitStressRegs' : '1'], 'corefx_jitstressregs2' : ['COMPlus_JitStressRegs' : '2'],
  55                'corefx_jitstressregs3' : ['COMPlus_JitStressRegs' : '3'], 'corefx_jitstressregs4' : ['COMPlus_JitStressRegs' : '4'],
  56                'corefx_jitstressregs8' : ['COMPlus_JitStressRegs' : '8'], 'corefx_jitstressregs0x10' : ['COMPlus_JitStressRegs' : '0x10'],
  57                'corefx_jitstressregs0x80' : ['COMPlus_JitStressRegs' : '0x80'],
  58                'gcstress0x3' : ['COMPlus_GCStress'  : '0x3'], 'gcstress0xc' : ['COMPlus_GCStress'  : '0xC'],
  59                'zapdisable' : ['COMPlus_ZapDisable'  : '0xC'],
  60                'heapverify1' : ['COMPlus_HeapVerify'  : '1'],
  61                'gcstress0xc_zapdisable' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_ZapDisable'  : '1'],
  62                'gcstress0xc_zapdisable_jitstress2' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_ZapDisable'  : '1', 'COMPlus_JitStress'  : '2'],
  63                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_ZapDisable'  : '1', 'COMPlus_HeapVerify'  : '1'],
  64                'gcstress0xc_jitstress1' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_JitStress'  : '1'],
  65                'gcstress0xc_jitstress2' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_JitStress'  : '2'],
  66                'gcstress0xc_minopts_heapverify1' : ['COMPlus_GCStress'  : '0xC', 'COMPlus_JITMinOpts'  : '1', 'COMPlus_HeapVerify'  : '1']
  67                ]
  68     // This is the basic set of scenarios
  69     def static basicScenarios = ['default', 'pri1', 'ilrt', 'r2r', 'pri1r2r', 'gcstress15_pri1r2r']
  70     // This is the set of configurations
  71     def static configurationList = ['Debug', 'Checked', 'Release']
  72     // This is the set of architectures
  73     def static architectureList = ['arm', 'arm64', 'x64', 'x86']
  74 }
  75
  76 def static setMachineAffinity(def job, def os, def architecture) {
  77     if (architecture == 'arm64' && os == 'Windows_NT') {
  78         // For cross compilation
  79         job.with {
  80             label('arm64')
  81         }
  82     } else if ((architecture == 'arm' || architecture == 'arm64') && os == 'Ubuntu') {
  83         Utilities.setMachineAffinity(job, os, 'arm-cross-latest');
  84     } else {
  85         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
  86     }
  87 }
  88
  89 def static isGCStressRelatedTesting(def scenario) {
  90     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
  91     def scenarioName = scenario.toLowerCase()
  92     def isGCStressTesting = false
  93     Constants.jitStressModeScenarios[scenario].each{ k, v ->
  94         if (k in gcStressTestEnvVars) {
  95             isGCStressTesting = true;
  96         }
  97     }
  98     return isGCStressTesting
  99 }
 100
 101 def static isCorefxTesting(def scenario) {
 102     def corefx_prefix = 'corefx_'
 103     if (scenario.length() < corefx_prefix.length()) {
 104         return false
 105     }
 106     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 107 }
 108
 109 def static isR2R(def scenario) {
 110     return (scenario == 'r2r' || scenario == 'pri1r2r')
 111 }
 112
 113 def static setTestJobTimeOut(newJob, scenario) {
 114     if (isGCStressRelatedTesting(scenario)) {
 115         Utilities.setJobTimeout(newJob, 1440)
 116     }
 117     else if (isCorefxTesting(scenario)) {
 118         Utilities.setJobTimeout(newJob, 360)
 119     }
 120     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 121         Utilities.setJobTimeout(newJob, 240)
 122     }
 123     else if (isR2R(scenario)) {
 124         Utilities.setJobTimeout(newJob, 240)
 125     }
 126     // Non-test jobs use the default timeout value.
 127 }
 128
 129 def static getStressModeDisplayName(def scenario) {
 130     def displayStr = ''
 131     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 132         def prefixLength = 'COMPlus_'.length()
 133         if (k.length() >= prefixLength) {
 134             def modeName = k.substring(prefixLength, k.length())
 135             displayStr += ' ' + modeName + '=' + v
 136         }
 137     }
 138     return displayStr
 139 }
 140
 141 // Generates the string for creating a file that sets environment variables
 142 // that makes it possible to run stress modes.  Writes the script to a file called
 143 // SetStressModes.[sh/cmd]
 144 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 145     def stepScript = ''
 146     if (os == 'Windows_NT') {
 147         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 148         stepScript += "del ${stepScriptLocation}\r\n"
 149         stressModeVars.each{ k, v ->
 150             // Write out what we are writing to the script file
 151             stepScript += "echo Setting ${k}=${v}\r\n"
 152             // Write out the set itself to the script file`
 153             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 154         }
 155     }
 156     else {
 157         // For these we don't use a script, we use directly
 158         stepScript += "echo Setting variables for ${stressModeName}\n"
 159         stepScript += "rm -f ${stepScriptLocation}\n"
 160         stressModeVars.each{ k, v ->
 161             // Write out what we are writing to the script file
 162             stepScript += "echo Setting ${k}=${v}\n"
 163             // Write out the set itself to the script file`
 164             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 165         }
 166     }
 167     return stepScript
 168 }
 169
 170 // Corefx doesn't have a support to pass stress mode environment variables. This function
 171 // generates commands to set or export environment variables
 172 def static getStressModeEnvSetCmd(def os, def stressModeName) {
 173     def envVars = Constants.jitStressModeScenarios[stressModeName]
 174     def setEnvVars = ''
 175     if (os == 'Windows_NT') {
 176         envVars.each{ VarName, Value   ->
 177             if (VarName != '') {
 178                 setEnvVars += "set ${VarName}=${Value}\n"
 179             }
 180         }
 181     }
 182     else {
 183         envVars.each{ VarName, Value   ->
 184             if (VarName != '') {
 185                 setEnvVars += "export ${VarName}=${Value}\n"
 186             }
 187         }
 188     }
 189     return setEnvVars
 190 }
 191
 192 // Calculates the name of the build job based on some typical parameters.
 193 //
 194 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 195     // If the architecture is x64, do not add that info into the build name.
 196     // Need to change around some systems and other builds to pick up the right builds
 197     // to do that.
 198
 199     def suffix = scenario != 'default' ? "_${scenario}" : '';
 200     if (isBuildOnly) {
 201         suffix += '_bld'
 202     }
 203     def baseName = ''
 204     switch (architecture) {
 205         case 'x64':
 206             if (scenario == 'default') {
 207                 // For now we leave x64 off of the name for compatibility with other jobs
 208                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 209             }
 210             else {
 211                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 212             }
 213             break
 214         case 'arm64':
 215         case 'arm':
 216             // These are cross builds
 217             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 218             break
 219         case 'x86':
 220             baseName = architecture.toLowerCase() + '_lb_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 221             break
 222         default:
 223             println("Unknown architecture: ${architecture}");
 224             assert false
 225             break
 226     }
 227
 228     return baseName + suffix
 229 }
 230
 231 // **************************
 232 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 233 // of coreclr builds over linux/osx/freebsd/windows and debug/release/checked.  In addition, the windows
 234 // builds will do a couple extra steps.
 235 // **************************
 236
 237 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 238 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 239 // it's just used for internal builds
 240 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 241     if (isWindowsBuildOnlyJob) {
 242         return
 243     }
 244
 245     // Non pull request builds.
 246     if (!isPR) {
 247         // Check scenario.
 248         switch (scenario) {
 249             case 'default':
 250                 switch (architecture) {
 251                     case 'x64':
 252                     case 'x86':
 253                         if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 254                             Utilities.addGithubPushTrigger(job)
 255                         }
 256                         break
 257                     case 'arm':
 258                     case 'arm64':
 259                         Utilities.addGithubPushTrigger(job)
 260                         break
 261                     default:
 262                         println("Unknown architecture: ${architecture}");
 263                         assert false
 264                         break
 265                 }
 266                 break
 267             case 'pri1':
 268                 // Pri one gets a push trigger, and only for release
 269                 if (architecture == 'x64') {
 270                     if (configuration == 'Release') {
 271                         // We expect release jobs to be Windows, Ubuntu, CentOS or OSX
 272                         assert (os == 'Windows_NT') || (os in Constants.crossList)
 273                         if (isFlowJob || os == 'Windows_NT') {
 274                             Utilities.addGithubPushTrigger(job)
 275                         }
 276                     }
 277                 }
 278                 break
 279             case 'r2r':
 280                 //r2r jobs that aren't pri1 can only be triggered by phrase
 281                 break
 282             case 'pri1r2r':
 283                 //pri1 r2r gets a push trigger for checked/release
 284                 if (configuration == 'Checked' || configuration == 'Release') {
 285                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 286                     if (architecture == 'x64') {
 287                         //Flow jobs should be Windows, Ubuntu, OSX, or CentOS
 288                         if (isFlowJob || os == 'Windows_NT') {
 289                             Utilities.addGithubPushTrigger(job)
 290                         }
 291                     }
 292                     // For x86, only add per-commit jobs for Windows
 293                     else if (architecture == 'x86') {
 294                         if (os == 'Windows_NT') {
 295                             Utilities.addGithubPushTrigger(job)
 296                         }
 297                     }
 298                 }
 299                 break
 300             case 'gcstress15_pri1r2r':
 301                 //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 302                 if (configuration == 'Checked' || configuration == 'Release') {
 303                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 304                     if (architecture == 'x64') {
 305                         //Flow jobs should be Windows, Ubuntu, OSX, or CentOS
 306                         if (isFlowJob || os == 'Windows_NT') {
 307                             Utilities.addGithubPushTrigger(job)
 308                         }
 309                     }
 310                     // For x86, only add per-commit jobs for Windows
 311                     else if (architecture == 'x86') {
 312                         if (os == 'Windows_NT') {
 313                             Utilities.addGithubPushTrigger(job)
 314                         }
 315                     }
 316                 }
 317                 break
 318             case 'ilrt':
 319                 // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 320                 if (architecture == 'x64' && configuration == 'Release') {
 321                     // We don't expect to see a job generated except in these scenarios
 322                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 323                     if (isFlowJob || os == 'Windows_NT') {
 324                         Utilities.addPeriodicTrigger(job, '@daily')
 325                     }
 326                 }
 327                 break
 328             case 'jitstressregs1':
 329             case 'jitstressregs2':
 330             case 'jitstressregs3':
 331             case 'jitstressregs4':
 332             case 'jitstressregs8':
 333             case 'jitstressregs0x10':
 334             case 'jitstressregs0x80':
 335             case 'minopts':
 336             case 'forcerelocs':
 337             case 'jitstress1':
 338             case 'jitstress2':
 339             case 'jitstress2_jitstressregs1':
 340             case 'jitstress2_jitstressregs2':
 341             case 'jitstress2_jitstressregs3':
 342             case 'jitstress2_jitstressregs4':
 343             case 'jitstress2_jitstressregs8':
 344             case 'jitstress2_jitstressregs0x10':
 345             case 'jitstress2_jitstressregs0x80':
 346             case 'corefx_baseline':
 347             case 'corefx_minopts':
 348             case 'corefx_jitstress1':
 349             case 'corefx_jitstress2':
 350             case 'corefx_jitstressregs1':
 351             case 'corefx_jitstressregs2':
 352             case 'corefx_jitstressregs3':
 353             case 'corefx_jitstressregs4':
 354             case 'corefx_jitstressregs8':
 355             case 'corefx_jitstressregs0x10':
 356             case 'corefx_jitstressregs0x80':
 357             case 'zapdisable':
 358                 if (os != 'CentOS7.1') {
 359                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 360                     Utilities.addPeriodicTrigger(job, '@daily')
 361                 }
 362                 break
 363             case 'gcstress0x3':
 364             case 'gcstress0xc':
 365             case 'heapverify1':
 366             case 'gcstress0xc_zapdisable':
 367             case 'gcstress0xc_zapdisable_jitstress2':
 368             case 'gcstress0xc_zapdisable_heapverify1':
 369             case 'gcstress0xc_jitstress1':
 370             case 'gcstress0xc_jitstress2':
 371             case 'gcstress0xc_minopts_heapverify1':
 372                 if (os != 'CentOS7.1') {
 373                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 374                     Utilities.addPeriodicTrigger(job, '@weekly')
 375                 }
 376                 break
 377             default:
 378                 println("Unknown scenario: ${scenario}");
 379                 assert false
 380                 break
 381         }
 382         return
 383     }
 384     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 385     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 386     def osGroup = getOSGroup(os)
 387     switch (architecture) {
 388         case 'x64':
 389             switch (os) {
 390                 case 'OpenSUSE13.2':
 391                     assert !isFlowJob
 392                     assert scenario == 'default'
 393                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", '(?i).*test\\W+suse.*')
 394                     break
 395                 case 'Debian8.2':
 396                     assert !isFlowJob
 397                     assert scenario == 'default'
 398                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", '(?i).*test\\W+debian.*')
 399                     break
 400                 case 'Ubuntu15.10':
 401                     assert !isFlowJob
 402                     assert scenario == 'default'
 403                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", '(?i).*test\\W+Ubuntu15\\.10.*')
 404                     break
 405                 case 'RHEL7.2':
 406                     assert !isFlowJob
 407                     assert scenario == 'default'
 408                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", '(?i).*test\\W+RHEL7\\.2.*')
 409                     break
 410                 case 'Ubuntu':
 411                 case 'OSX':
 412                     // Triggers on the non-flow jobs aren't necessary here
 413                     // Corefx testing uses non-flow jobs.
 414                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 415                         break
 416                     }
 417                     switch (scenario) {
 418                         case 'default':
 419                             // Ubuntu uses checked for default PR tests
 420                             if (configuration == 'Checked') {
 421                                 // Default trigger
 422                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 423                             }
 424                             break
 425                         case 'pri1':
 426                             if (configuration == 'Release') {
 427                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 428                             }
 429                             break
 430                         case 'ilrt':
 431                             if (configuration == 'Release') {
 432                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 433                             }
 434                             break
 435                         case 'r2r':
 436                             if (configuration == 'Release' || configuration == 'Checked') {
 437                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 438                             }
 439                             break
 440                         case 'pri1r2r':
 441                             if (configuration == 'Release' || configuration == 'Checked') {
 442                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 443                             }
 444                             break
 445                         case 'gcstress15_pri1r2r':
 446                             if (configuration == 'Release' || configuration == 'Checked') {
 447                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 448                             }
 449                             break
 450                         case 'minopts':
 451                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 452                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - MinOpts)",
 453                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 454                             break
 455                         case 'jitstress1':
 456                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 457                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=1)",
 458                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 459                             break
 460                         case 'jitstress2':
 461                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 462                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=2)",
 463                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 464                             break
 465                         case 'forcerelocs':
 466                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 467                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ForceRelocs)",
 468                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 469                         case 'jitstressregs1':
 470                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 471                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=1)",
 472                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 473                             break
 474                         case 'jitstressregs2':
 475                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 476                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=2)",
 477                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 478                             break
 479                         case 'jitstressregs3':
 480                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 481                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=3)",
 482                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 483                             break
 484                         case 'jitstressregs4':
 485                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 486                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=4)",
 487                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 488                             break
 489                         case 'jitstressregs8':
 490                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 491                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=8)",
 492                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 493                             break
 494                         case 'jitstressregs0x10':
 495                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 496                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x10)",
 497                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 498                             break
 499                         case 'jitstressregs0x80':
 500                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 501                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x80)",
 502                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 503                             break
 504                         case 'jitstress2_jitstressregs1':
 505                         case 'jitstress2_jitstressregs2':
 506                         case 'jitstress2_jitstressregs3':
 507                         case 'jitstress2_jitstressregs4':
 508                         case 'jitstress2_jitstressregs8':
 509                         case 'jitstress2_jitstressregs0x10':
 510                         case 'jitstress2_jitstressregs0x80':
 511                         case 'gcstress0x3':
 512                         case 'gcstress0xc':
 513                         case 'zapdisable':
 514                         case 'heapverify1':
 515                         case 'gcstress0xc_zapdisable':
 516                         case 'gcstress0xc_zapdisable_jitstress2':
 517                         case 'gcstress0xc_zapdisable_heapverify1':
 518                         case 'gcstress0xc_jitstress1':
 519                         case 'gcstress0xc_jitstress2':
 520                         case 'gcstress0xc_minopts_heapverify1':
 521                             def displayStr = getStressModeDisplayName(scenario)
 522                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 523                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 524                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 525                             break
 526                         case 'corefx_baseline':
 527                         case 'corefx_minopts':
 528                         case 'corefx_jitstress1':
 529                         case 'corefx_jitstress2':
 530                         case 'corefx_jitstressregs1':
 531                         case 'corefx_jitstressregs2':
 532                         case 'corefx_jitstressregs3':
 533                         case 'corefx_jitstressregs4':
 534                         case 'corefx_jitstressregs8':
 535                         case 'corefx_jitstressregs0x10':
 536                         case 'corefx_jitstressregs0x80':
 537                             def displayName = 'CoreFx' + getStressModeDisplayName(scenario)
 538                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 539                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 540                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 541                             break
 542                         default:
 543                             println("Unknown scenario: ${scenario}");
 544                             assert false
 545                             break
 546                     }
 547                     break
 548                 case 'CentOS7.1':
 549                     switch (scenario) {
 550                         case 'pri1':
 551                             if (configuration == 'Release') {
 552                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 553                             }
 554                             break
 555                         case 'r2r':
 556                             if (configuration == 'Checked' || configuration == 'Release') {
 557                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 558                             }
 559                             break
 560                         case 'pri1r2r':
 561                             if (configuration == 'Checked' || configuration == 'Release') {
 562                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 563                             }
 564                             break
 565                         case 'gcstress15_pri1r2r':
 566                             if (configuration == 'Release' || configuration == 'Checked') {
 567                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 568                             }
 569                             break
 570                         default:
 571                             break
 572                     }
 573                 case 'OpenSUSE13.2':
 574                     if (configuration == 'Checked' && !isFlowJob && scenario == 'default') {
 575                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
 576                     }
 577                     break
 578                 case 'Windows_NT':
 579                     switch (scenario) {
 580                         case 'default':
 581                             // Default trigger
 582                             if (configuration == 'Debug') {
 583                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 584                             }
 585                             break
 586                         case 'pri1':
 587                             // Default trigger
 588                             if (configuration == 'Release') {
 589                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 590                             }
 591                             break
 592                         case 'ilrt':
 593                             if (configuration == 'Release') {
 594                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 595                             }
 596                             break
 597                         case 'r2r':
 598                             if (configuration == 'Checked' || configuration == 'Release') {
 599                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 600                             }
 601                             break
 602                         case 'pri1r2r':
 603                             if (configuration == 'Checked' || configuration == 'Release') {
 604                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 605                             }
 606                             break
 607                         case 'gcstress15_pri1r2r':
 608                             if (configuration == 'Release' || configuration == 'Checked') {
 609                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 610                             }
 611                             break
 612                         case 'minopts':
 613                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 614                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - MinOpts)",
 615                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 616                             break
 617                         case 'forcerelocs':
 618                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 619                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ForceRelocs)",
 620                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 621                             break
 622                         case 'jitstress1':
 623                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 624                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=1)",
 625                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 626                             break
 627                         case 'jitstress2':
 628                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 629                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStress=2)",
 630                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 631                             break
 632                         case 'jitstressregs1':
 633                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 634                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=1)",
 635                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 636                             break
 637                         case 'jitstressregs2':
 638                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 639                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=2)",
 640                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 641                             break
 642                         case 'jitstressregs3':
 643                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 644                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=3)",
 645                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 646                             break
 647                         case 'jitstressregs4':
 648                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 649                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=4)",
 650                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 651                             break
 652                         case 'jitstressregs8':
 653                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 654                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=8)",
 655                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 656                             break
 657                         case 'jitstressregs0x10':
 658                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 659                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x10)",
 660                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 661                             break
 662                         case 'jitstressregs0x80':
 663                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 664                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - JitStressRegs=0x80)",
 665                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 666                             break
 667                         case 'jitstress2_jitstressregs1':
 668                         case 'jitstress2_jitstressregs2':
 669                         case 'jitstress2_jitstressregs3':
 670                         case 'jitstress2_jitstressregs4':
 671                         case 'jitstress2_jitstressregs8':
 672                         case 'jitstress2_jitstressregs0x10':
 673                         case 'jitstress2_jitstressregs0x80':
 674                         case 'gcstress0x3':
 675                         case 'gcstress0xc':
 676                         case 'zapdisable':
 677                         case 'heapverify1':
 678                         case 'gcstress0xc_zapdisable':
 679                         case 'gcstress0xc_zapdisable_jitstress2':
 680                         case 'gcstress0xc_zapdisable_heapverify1':
 681                         case 'gcstress0xc_jitstress1':
 682                         case 'gcstress0xc_jitstress2':
 683                         case 'gcstress0xc_minopts_heapverify1':
 684                             def displayStr = getStressModeDisplayName(scenario)
 685                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 686                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 687                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 688                             break
 689                         case 'corefx_baseline':
 690                         case 'corefx_minopts':
 691                         case 'corefx_jitstress1':
 692                         case 'corefx_jitstress2':
 693                         case 'corefx_jitstressregs1':
 694                         case 'corefx_jitstressregs2':
 695                         case 'corefx_jitstressregs3':
 696                         case 'corefx_jitstressregs4':
 697                         case 'corefx_jitstressregs8':
 698                         case 'corefx_jitstressregs0x10':
 699                         case 'corefx_jitstressregs0x80':
 700                             def displayName = 'CoreFx ' + getStressModeDisplayName(scenario)
 701                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 702                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 703                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 704                             break
 705                         default:
 706                             println("Unknown scenario: ${scenario}");
 707                             assert false
 708                             break
 709                     }
 710                     break
 711                 case 'FreeBSD':
 712                     assert scenario == 'default'
 713                     if (configuration == 'Checked') {
 714                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
 715                     }
 716                     break
 717                 default:
 718                     println("Unknown os: ${os}");
 719                     assert false
 720                     break
 721             }
 722             break
 723         case 'arm64':
 724         case 'arm':
 725             assert scenario == 'default'
 726             switch (os) {
 727                 case 'Ubuntu':
 728                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 729                     break
 730                 case 'Windows_NT':
 731                     // Set up a private trigger
 732                     Utilities.addPrivateGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
 733                         "(?i).*test\\W+${os}\\W+${architecture}.*", null, ['erozenfeld', 'kyulee1', 'pgavlin', 'russellhadley', 'swaroop-sridhar', 'JosephTremoulet', 'jashook', 'RussKeldorph', 'gkhanna79', 'briansull', 'cmckinsey', 'jkotas', 'ramarag', 'markwilkie', 'rahku', 'tzwlai', 'weshaggard', 'LLITCHEV'])
 734                     break
 735             }
 736             break
 737         case 'x86':
 738             assert (scenario == 'default' || scenario == 'r2r' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r')
 739             // For windows, x86 runs by default
 740             if (scenario == 'default') {
 741                 if (os == 'Windows_NT') {
 742                     if (configuration != 'Checked') {
 743                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Legacy Backend Build and Test")
 744                     }
 745                 }
 746                 else {
 747                     // default trigger
 748                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${architecture}\\W+${osGroup}.*")
 749                 }
 750             }
 751             else if (scenario == 'r2r') {
 752                 if (os == 'Windows_NT') {
 753                     if (configuration == 'Release') {
 754                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Legacy Backend Build & Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 755                     }
 756                 }
 757             }
 758             else if (scenario == 'pri1r2r') {
 759                 if (os == 'Windows_NT') {
 760                     if (configuration == 'Release') {
 761                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Legacy Backend Build & Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 762                     }
 763                 }
 764             }
 765             else if (scenario == 'gcstress15_pri1r2r'){
 766                 if (os == 'Windows_NT'){
 767                     if (configuration == 'Release'){
 768                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 769                     }
 770                 }
 771             }
 772             break
 773         default:
 774             println("Unknown architecture: ${architecture}");
 775             assert false
 776             break
 777     }
 778 }
 779
 780 // Additional scenario which can alter behavior
 781
 782 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
 783 combinedScenarios.each { scenario ->
 784     [true, false].each { isPR ->
 785         Constants.architectureList.each { architecture ->
 786             Constants.configurationList.each { configuration ->
 787                 Constants.osList.each { os ->
 788                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
 789                     // and reset the os to Windows_NT
 790                     def isBuildOnly = false
 791                     if (os == 'Windows_NT_BuildOnly') {
 792                         isBuildOnly = true
 793                         os = 'Windows_NT'
 794                     }
 795
 796                                         // Skip totally unimplemented (in CI) configurations.
 797                     switch (architecture) {
 798                         case 'arm64':
 799                             // Windows or cross compiled Ubuntu
 800                             if (os != 'Windows_NT' && os != 'Ubuntu') {
 801                                 return
 802                             }
 803                             break
 804                         case 'arm':
 805                             // Only Ubuntu cross implemented
 806                             if (os != 'Ubuntu') {
 807                                 return
 808                             }
 809                             break
 810                         case 'x86':
 811                             // Skip non-windows
 812                             if (os != 'Windows_NT') {
 813                                 return
 814                             }
 815                             break
 816                         case 'x64':
 817                             // Everything implemented
 818                             break
 819                         default:
 820                             println("Unknown architecture: ${architecture}")
 821                             assert false
 822                             break
 823                     }
 824                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
 825                     // with checked builds
 826                     def enableCorefxTesting = false
 827                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 828                         if (configuration != 'Checked') {
 829                             return
 830                         }
 831
 832                         // Since these are just execution time differences,
 833                         // skip platforms that don't execute the tests here (Windows_NT only)
 834                         def isEnabledOS = os == 'Windows_NT' || os == 'Ubuntu'
 835                         if (!isEnabledOS || isBuildOnly) {
 836                             return
 837                         }
 838
 839                         // No stress modes except on x64 right now (mainly because of bad test state on x86)
 840                         if (architecture != 'x64') {
 841                             return
 842                         }
 843
 844                         enableCorefxTesting = isCorefxTesting(scenario)
 845                     }
 846                     else {
 847                         // Skip scenarios
 848                         switch (scenario) {
 849                             case 'pri1':
 850                                 // The pri1 build isn't necessary except for os's in the cross list or Windows_NT (native OS runs)
 851                                 if (os != 'Windows_NT' && !(os in Constants.crossList)) {
 852                                     return
 853                                 }
 854                                 // Only x64 for now
 855                                 if (architecture != 'x64') {
 856                                     return
 857                                 }
 858                                 break
 859                             case 'ilrt':
 860                                 // The ilrt build isn't necessary except for os's in the cross list or Windows_NT (native OS runs)
 861                                 if (os != 'Windows_NT' && !(os in Constants.crossList)) {
 862                                     return
 863                                 }
 864                                 // Only x64 for now
 865                                 if (architecture != 'x64') {
 866                                     return
 867                                 }
 868                                 break
 869                             case 'r2r':
 870                                 // The r2r build isn't necessary except for os's in the cross list or Windows_NT (native OS runs)
 871                                 if (os != 'Windows_NT' && !(os in Constants.crossList)) {
 872                                     return
 873                                 }
 874                                 // only x64 or x86 for now
 875                                 if (architecture != 'x64' && architecture != 'x86') {
 876                                     return
 877                                 }
 878                                 break
 879                             case 'pri1r2r':
 880                                 // The pri1 r2r build isn't necessary except for os's in the cross list or Windows_NT (native OS runs)
 881                                 if (os != 'Windows_NT' && !(os in Constants.crossList)) {
 882                                     return
 883                                 }
 884                                 // only x64 or x86 for now
 885                                 if (architecture != 'x64' && architecture != 'x86') {
 886                                     return
 887                                 }
 888                                 break
 889                             case 'gcstress15_pri1r2r':
 890                                 // The GC stress r2r build isn't necessary except for os's in the cross list or Windows_NT (native OS runs)
 891                                 if (os != 'Windows_NT' && !(os in Constants.crossList)) {
 892                                     return
 893                                 }
 894                                 // only x64 or x86 for now
 895                                 if (architecture != 'x64' && architecture != 'x86') {
 896                                     return
 897                                 }
 898                                 break
 899                             case 'default':
 900                                 // Nothing skipped
 901                                 break
 902                             default:
 903                                 println("Unknown scenario: ${scenario}")
 904                                 assert false
 905                                 break
 906                         }
 907                     }
 908
 909                     // Calculate names
 910                     def lowerConfiguration = configuration.toLowerCase()
 911                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
 912
 913                     // Create the new job
 914                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {}
 915
 916                     setMachineAffinity(newJob, os, architecture)
 917
 918                     // Add all the standard options
 919                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 920                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
 921
 922                     def buildCommands = [];
 923                     def osGroup = getOSGroup(os)
 924
 925                     // Calculate the build steps, archival, and xunit results
 926                     switch (os) {
 927                         case 'Windows_NT':
 928                             switch (architecture) {
 929                                 case 'x64':
 930                                 case 'x86':
 931
 932                                     if (Constants.jitStressModeScenarios.containsKey(scenario) || scenario == 'default') {
 933                                         buildOpts = enableCorefxTesting ? 'skiptests' : ''
 934                                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} ${buildOpts}"
 935                                     }
 936
 937                                     // For Pri 1 tests, we must shorten the output test binary path names.
 938                                     // if __TestIntermediateDir is already set, buildtest.cmd will
 939                                     // output test binaries to that directory. If it is not set, the
 940                                     // binaries are sent to a default directory whose name is about
 941                                     // 35 characters long.
 942
 943                                     else if (scenario == 'pri1') {
 944                                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} Priority 1"
 945                                     }
 946                                     else if (scenario == 'ilrt') {
 947                                         // First do the build with skiptests and then build the tests with ilasm roundtrip
 948                                         buildCommands += "build.cmd ${lowerConfiguration} ${architecture} skiptests"
 949                                         buildCommands += "set __TestIntermediateDir=int&&tests\\buildtest.cmd ${lowerConfiguration} ${architecture} ilasmroundtrip"
 950                                     }
 951                                     else if (scenario == 'r2r') {
 952                                         buildCommands += "build.cmd ${lowerConfiguration} ${architecture} docrossgen skiptests"
 953                                         buildCommands += "set __TestIntermediateDir=int&&tests\\buildtest.cmd ${lowerConfiguration} ${architecture} crossgen"
 954                                     }
 955                                     else if (scenario == 'pri1r2r') {
 956                                         buildCommands += "build.cmd ${lowerConfiguration} ${architecture} docrossgen skiptests"
 957                                         buildCommands += "set __TestIntermediateDir=int&&tests\\buildtest.cmd ${lowerConfiguration} ${architecture} crossgen Priority 1"
 958                                     }
 959                                     else if (scenario == 'gcstress15_pri1r2r') {
 960                                         //Build pri1 R2R tests with GC stress level 15
 961                                         buildCommands += "build.cmd ${lowerConfiguration} ${architecture} docrossgen skiptests"
 962                                         buildCommands += "set __TestIntermediateDir=int&&tests\\buildtest.cmd ${lowerConfiguration} ${architecture} crossgen Priority 1 gcstresslevel 15"
 963                                     }
 964                                     else {
 965                                         println("Unknown scenario: ${scenario}")
 966                                         assert false
 967                                     }
 968
 969                                     // If we are running a stress mode, we should write out the set of key
 970                                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
 971
 972                                     if (!isBuildOnly) {
 973                                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 974                                             if (enableCorefxTesting) {
 975                                                 // Sync to corefx repo
 976                                                 // Move coreclr files to a subdirectory, %workspace%/clr. Otherwise, corefx build
 977                                                 // thinks that %workspace% is the project base directory.
 978                                                 buildCommands += "powershell new-item clr -type directory -force"
 979                                                 buildCommands += 'powershell foreach ($x in get-childitem -force) { if (\$x.name -ne \'clr\') { move-item $x clr }}'
 980                                                 buildCommands += "git clone https://github.com/dotnet/corefx fx"
 981
 982                                                 buildCommands += getStressModeEnvSetCmd(os, scenario);
 983
 984                                                 // Run corefx build and testing
 985                                                 buildCommands += "cd fx && call \"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat\" x86 && Build.cmd /p:ConfigurationGroup=Release  /p:BUILDTOOLS_OVERRIDE_RUNTIME=%WORKSPACE%\\clr\\bin\\Product\\Windows_NT.x64.Checked "
 986                                             }
 987                                             else {
 988                                                 def stepScriptLocation = "%WORKSPACE%\\bin\\tests\\SetStressModes.bat"
 989                                                 buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
 990
 991                                                 // Run tests with the
 992
 993                                                 buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} TestEnv ${stepScriptLocation}"
 994                                             }
 995                                         }
 996                                         else if (architecture == 'x64') {
 997                                             buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture}"
 998                                         }
 999                                         else if (architecture == 'x86') {
1000                                             buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} Exclude0 x86_legacy_backend_issues.targets"
1001                                         }
1002                                     }
1003
1004                                     if (!enableCorefxTesting) {
1005                                         // Run the rest of the build
1006                                         // Build the mscorlib for the other OS's
1007                                         buildCommands += "build.cmd ${lowerConfiguration} ${architecture} linuxmscorlib"
1008                                         buildCommands += "build.cmd ${lowerConfiguration} ${architecture} freebsdmscorlib"
1009                                         buildCommands += "build.cmd ${lowerConfiguration} ${architecture} osxmscorlib"
1010
1011                                         // Zip up the tests directory so that we don't use so much space/time copying
1012                                         // 10s of thousands of files around.
1013                                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1014
1015                                         // For windows, pull full test results and test drops for x86/x64
1016                                         Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip")
1017
1018                                         if (!isBuildOnly) {
1019                                             if (architecture == 'x64' || !isPR) {
1020                                                 Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml')
1021                                             }
1022                                             setTestJobTimeOut(newJob, scenario)
1023                                         }
1024                                     }
1025                                     else {
1026                                         // Archive only result xml files since corefx/bin/tests is very large around 10 GB.
1027                                         // For windows, pull full test results and test drops for x86/x64
1028                                         Utilities.addArchival(newJob, "fx/bin/tests/**/testResults.xml")
1029
1030                                         // Set timeout
1031                                         setTestJobTimeOut(newJob, scenario)
1032
1033                                         if (architecture == 'x64' || !isPR) {
1034                                             Utilities.addXUnitDotNETResults(newJob, 'fx/bin/tests/**/testResults.xml')
1035                                         }
1036                                     }
1037
1038                                     break
1039                                 case 'arm64':
1040                                     assert scenario == 'default'
1041
1042                                     // Up the timeout for arm64 jobs.
1043                                     Utilities.setJobTimeout(newJob, 360);
1044                                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} /toolset_dir C:\\ats"
1045
1046                                     // Debug runs take too long to run.
1047                                     if (lowerConfiguration != "debug") {
1048                                        buildCommands += "C:\\arm64PostBuild.cmd %WORKSPACE% ${architecture} ${lowerConfiguration}"
1049                                     }
1050
1051                                     // Add archival.  No xunit results for arm64 windows
1052                                     Utilities.addArchival(newJob, "bin/Product/**")
1053                                     break
1054                                 default:
1055                                     println("Unknown architecture: ${architecture}");
1056                                     assert false
1057                                     break
1058                             }
1059                             break
1060                         case 'Ubuntu':
1061                         case 'Ubuntu15.10':
1062                         case 'Debian8.2':
1063                         case 'OSX':
1064                         case 'FreeBSD':
1065                         case 'CentOS7.1':
1066                         case 'RHEL7.2':
1067                         case 'OpenSUSE13.2':
1068                             switch (architecture) {
1069                                 case 'x64':
1070                                 case 'x86':
1071                                     if (!enableCorefxTesting) {
1072                                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1073                                         // only on supported OS platforms.
1074                                         if ((os == 'FreeBSD') || (os == 'OpenSUSE13.2'))
1075                                         {
1076                                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture}"
1077                                         }
1078                                         else
1079                                         {
1080                                             buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1081                                         }
1082                                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1083
1084                                         // Set time out
1085                                         setTestJobTimeOut(newJob, scenario)
1086                                         // Basic archiving of the build
1087                                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**")
1088                                         // And pal tests
1089                                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1090                                     }
1091                                     else {
1092                                         // Corefx stress testing
1093                                         assert os == 'Ubuntu'
1094                                         assert architecture == 'x64'
1095                                         assert lowerConfiguration == 'checked'
1096
1097                                         // Build coreclr and move it to clr directory
1098                                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1099                                         buildCommands += "rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr"
1100
1101                                         // Get corefx
1102                                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1103
1104                                         // Set environment variable
1105                                         def setEnvVar = getStressModeEnvSetCmd(os, scenario)
1106
1107                                         // Build and text corefx
1108                                         buildCommands += "rm -rf \$WORKSPACE/fx_home; mkdir \$WORKSPACE/fx_home"
1109                                         buildCommands += setEnvVar
1110                                         buildCommands += "cd fx; export HOME=\$WORKSPACE/fx_home; ./build.sh /p:ConfigurationGroup=Release /p:BUILDTOOLS_OVERRIDE_RUNTIME=\$WORKSPACE/clr/bin/Product/Linux.x64.Checked"
1111
1112                                         // Archive and process test result
1113                                         Utilities.addArchival(newJob, "fx/bin/tests/**/testResults.xml")
1114                                         setTestJobTimeOut(newJob, scenario)
1115                                         Utilities.addXUnitDotNETResults(newJob, 'fx/bin/tests/**/testResults.xml')
1116                                     }
1117                                     break
1118                                 case 'arm64':
1119                                     // We don't run the cross build except on Ubuntu
1120                                     assert os == 'Ubuntu'
1121
1122                                     buildCommands += """echo \"Using rootfs in /opt/aarch64-linux-gnu-root\"
1123                                         ROOTFS_DIR=/opt/aarch64-linux-gnu-root ./build.sh skipmscorlib arm64 cross verbose ${lowerConfiguration}"""
1124
1125                                     // Basic archiving of the build, no pal tests
1126                                     Utilities.addArchival(newJob, "bin/Product/**")
1127                                     break
1128                                 case 'arm':
1129                                     // We don't run the cross build except on Ubuntu
1130                                     assert os == 'Ubuntu'
1131                                     buildCommands += """echo \"Using rootfs in /opt/arm-liux-genueabihf-root\"
1132                                         ROOTFS_DIR=/opt/arm-linux-genueabihf-root ./build.sh skipmscorlib arm cross verbose ${lowerConfiguration}"""
1133
1134                                     // Basic archiving of the build, no pal tests
1135                                     Utilities.addArchival(newJob, "bin/Product/**")
1136                                     break
1137                                 default:
1138                                     println("Unknown architecture: ${architecture}");
1139                                     assert false
1140                                     break
1141                             }
1142                             break
1143                         default:
1144                             println("Unknown os: ${os}");
1145                             assert false
1146                             break
1147                     }
1148
1149                     newJob.with {
1150                         steps {
1151                             if (os == 'Windows_NT') {
1152                                 buildCommands.each { buildCommand ->
1153                                     batchFile(buildCommand)
1154                                 }
1155                             }
1156                             else {
1157                                 buildCommands.each { buildCommand ->
1158                                     shell(buildCommand)
1159                                 }
1160                             }
1161                         }
1162                     }
1163                 } // os
1164             } // configuration
1165         } // architecture
1166     } // isPR
1167 } // scenario
1168
1169 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
1170 combinedScenarios.each { scenario ->
1171     [true, false].each { isPR ->
1172         // Architectures.  x64 only at this point
1173         ['x64'].each { architecture ->
1174             // Put the OS's supported for coreclr cross testing here
1175             Constants.crossList.each { os ->
1176                 Constants.configurationList.each { configuration ->
1177
1178                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1179                         if (configuration != 'Checked') {
1180                             return
1181                         }
1182                         if (isCorefxTesting(scenario)) {
1183                             return
1184                         }
1185                     }
1186                     // For CentOS, we only want Checked/Release pri1 builds.
1187                     else if (os == 'CentOS7.1') {
1188                         if (scenario != 'pri1' && scenario != 'r2r' && scenario != 'pri1r2r' && scenario != 'gcstress15_pri1r2r') {
1189                             return
1190                         }
1191                         if (configuration != 'Checked' && configuration != 'Release') {
1192                             return
1193                         }
1194                     }
1195                     else {
1196                         // Skip scenarios
1197                         switch (scenario) {
1198                             case 'pri1':
1199                                 // Nothing skipped
1200                                 break
1201                             case 'ilrt':
1202                                 break
1203                             case 'r2r':
1204                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
1205                                 if (configuration != 'Checked' && configuration != 'Release') {
1206                                     return
1207                                 }
1208                                 break
1209                             case 'pri1r2r':
1210                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
1211                                 if (configuration != 'Checked' && configuration != 'Release') {
1212                                     return
1213                                 }
1214                                 break
1215                             case 'gcstress15_pri1r2r':
1216                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
1217                                 if (configuration != 'Checked' && configuration != 'Release') {
1218                                     return
1219                                 }
1220                                 break
1221                             case 'default':
1222                                 // Nothing skipped
1223                                 break
1224                             default:
1225                                 println("Unknown scenario: ${scenario}")
1226                                 assert false
1227                                 break
1228                         }
1229                     }
1230
1231                     def lowerConfiguration = configuration.toLowerCase()
1232                     def osGroup = getOSGroup(os)
1233                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
1234                     def inputCoreCLRBuildName = projectFolder + '/' +
1235                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
1236                     // If this is a stress scenario, there isn't any difference in the build job
1237                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
1238                     // from the default scenario
1239                     def inputWindowTestsBuildName = ''
1240                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1241                         inputWindowTestsBuildName = projectFolder + '/' +
1242                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', 'default', true), isPR)
1243                     }
1244                     else {
1245                         inputWindowTestsBuildName = projectFolder + '/' +
1246                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', scenario, true), isPR)
1247                     }
1248                     // Enable Server GC for Ubuntu PR builds
1249                     def serverGCString = ''
1250
1251                     if (os == 'Ubuntu' && isPR){
1252                         serverGCString = '--useServerGC'
1253                     }
1254
1255
1256                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
1257                         // Add parameters for the inputs
1258
1259                         parameters {
1260                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
1261                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
1262                         }
1263
1264                         steps {
1265                             // Set up the copies
1266
1267                             // Coreclr build we are trying to test
1268
1269                             copyArtifacts(inputCoreCLRBuildName) {
1270                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
1271                                 buildSelector {
1272                                     buildNumber('${CORECLR_BUILD}')
1273                                 }
1274                             }
1275
1276                             // Coreclr build containing the tests and mscorlib
1277
1278                             copyArtifacts(inputWindowTestsBuildName) {
1279                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
1280                                 buildSelector {
1281                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
1282                                 }
1283                             }
1284
1285                             // Corefx native component.
1286                             // Pull from main folder in corefx for now, once the corefx branchify PR gets merged this will chnage
1287                             def corefxFolder = Utilities.getFolderName('dotnet/corefx')
1288                             copyArtifacts("${corefxFolder}/nativecomp_${os.toLowerCase()}_release") {
1289                                 includePatterns('bin/**')
1290                                 buildSelector {
1291                                     latestSuccessful(true)
1292                                 }
1293                             }
1294
1295                             // CoreFX Linux binaries
1296                             copyArtifacts("${corefxFolder}/${os.toLowerCase()}_release_bld") {
1297                                 includePatterns('bin/build.pack')
1298                                 buildSelector {
1299                                     latestSuccessful(true)
1300                                 }
1301                             }
1302
1303                             // Unpack the corefx binaries
1304                             shell("unpacker ./bin/build.pack ./bin")
1305
1306                             // Unzip the tests first.  Exit with 0
1307                             shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
1308
1309                             // Execute the tests
1310                             // If we are running a stress mode, we'll set those variables first
1311                             def testEnvOpt = ""
1312                             if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1313                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1314                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1315                                 if (createScriptCmds != "") {
1316                                     shell("${createScriptCmds}")
1317                                     testEnvOpt = "--test-env=" + scriptFileName
1318                                 }
1319                             }
1320
1321                             if (isGCStressRelatedTesting(scenario)) {
1322                                 shell('./init-tools.sh')
1323                             }
1324
1325                             shell("""./tests/runtest.sh \\
1326             --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
1327             --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
1328             --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
1329             --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
1330             --coreFxBinDir=\"\${WORKSPACE}/bin/${osGroup}.AnyCPU.Release\" \\
1331             --coreFxNativeBinDir=\"\${WORKSPACE}/bin/${osGroup}.${architecture}.Release\" \\
1332             ${testEnvOpt} ${serverGCString}""")
1333                         }
1334                     }
1335
1336                     setMachineAffinity(newJob, os, architecture)
1337                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
1338                     // Set timeouts to 240.
1339                     setTestJobTimeOut(newJob, scenario)
1340                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
1341
1342                     // Create a build flow to join together the build and tests required to run this
1343                     // test.
1344                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
1345                     // Linux CoreCLR test
1346                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
1347                     def fullTestJobName = projectFolder + '/' + newJob.name
1348                     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR)) {
1349                         buildFlow("""
1350 // Build the input jobs in parallel
1351 parallel (
1352     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
1353     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
1354 )
1355
1356 // And then build the test build
1357 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
1358                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
1359 """)
1360                     }
1361
1362                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
1363                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
1364                 } // configuration
1365             } // os
1366         } // architecture
1367     } // isPR
1368 } // scenario