netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'CentOS7.1',
  49                'RHEL7.2',
  50                'Ubuntu16.04',
  51                'Ubuntu16.10',
  52                'Fedora24',
  53                'Tizen']
  54
  55     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  56
  57     // This is a set of JIT stress modes combined with the set of variables that
  58     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  59     // the values are the environment variables
  60     def static jitStressModeScenarios = [
  61                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  62                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  63                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  64                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  65                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  66                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  67                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  68                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  69                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  70                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  71                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  72                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  73                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  74                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  75                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  76                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  77                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  78                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  79                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  80                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  81                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  82                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  83                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  84                'corefx_baseline'                : [ : ], // corefx baseline
  85                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  86                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  87                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  88                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  89                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  90                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  91                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  92                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  93                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  94                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  95                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  96                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  97                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  98                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
  99                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 100                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 101                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 103                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 104                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 105                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 106                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 107                ]
 108
 109     // This is a set of r2r jit stress scenarios
 110     def static r2rJitStressScenarios = [
 111                'r2r_jitstress1',
 112                'r2r_jitstress2',
 113                'r2r_jitstressregs1',
 114                'r2r_jitstressregs2',
 115                'r2r_jitstressregs3',
 116                'r2r_jitstressregs4',
 117                'r2r_jitstressregs8',
 118                'r2r_jitstressregs0x10',
 119                'r2r_jitstressregs0x80',
 120                'r2r_jitstressregs0x1000',
 121                'r2r_jitminopts',
 122                'r2r_jitforcerelocs']
 123
 124     // This is the basic set of scenarios
 125     def static basicScenarios = [
 126                'default',
 127                'pri1',
 128                'ilrt',
 129                'r2r',
 130                'pri1r2r',
 131                'gcstress15_pri1r2r',
 132                'longgc',
 133                'coverage',
 134                'formatting',
 135                'gcsimulator',
 136                'jitdiff',
 137                'standalone_gc',
 138                'gc_reliability_framework',
 139                'illink'] + r2rJitStressScenarios
 140
 141     def static configurationList = ['Debug', 'Checked', 'Release']
 142
 143     // This is the set of architectures
 144     def static architectureList = ['arm', 'arm64', 'x64', 'x86', 'x86lb']
 145 }
 146
 147 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 148     if (architecture == 'arm64' && os == 'Windows_NT') {
 149         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 150     } else if (architecture == 'arm64' && os != 'Windows_NT' && options == null) {
 151         Utilities.setMachineAffinity(job, os, 'arm64-small-page-size');
 152     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['large_pages'] == true) {
 153         Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size');
 154     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['is_build_only'] == true) {
 155         Utilities.setMachineAffinity(job, os, 'arm64-cross-latest');
 156     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 157         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 158     } else if ((architecture == 'arm') && (os == 'Windows_NT') && options['use_arm64_build_machine'] == true) {
 159         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 160     }else {
 161         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 162     }
 163 }
 164
 165 def static isJITStressJob(def scenario) {
 166     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 167            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 168 }
 169
 170 def static isGCStressRelatedTesting(def scenario) {
 171     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 172     // Detect it and make it a GCStress related.
 173     if (scenario == 'gcstress15_pri1r2r')
 174     {
 175         return true;
 176     }
 177
 178     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 179     def scenarioName = scenario.toLowerCase()
 180     def isGCStressTesting = false
 181     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 182         if (k in gcStressTestEnvVars) {
 183             isGCStressTesting = true;
 184         }
 185     }
 186     return isGCStressTesting
 187 }
 188
 189 def static isCorefxTesting(def scenario) {
 190     def corefx_prefix = 'corefx_'
 191     if (scenario.length() < corefx_prefix.length()) {
 192         return false
 193     }
 194     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 195 }
 196
 197 def static isR2R(def scenario) {
 198     return (scenario == 'r2r' || scenario == 'pri1r2r')
 199 }
 200
 201 def static isCoverage(def scenario) {
 202     return (scenario == 'coverage')
 203 }
 204
 205 def static isLongGc(def scenario) {
 206     return (scenario == 'longgc' || scenario == 'gcsimulator')
 207 }
 208
 209 def static isJitDiff(def scenario) {
 210     return (scenario == 'jitdiff')
 211 }
 212
 213 def static isGcReliabilityFramework(def scenario) {
 214     return (scenario == 'gc_reliability_framework')
 215 }
 216
 217 def static scenarioNeedsPri1Build(def scenario) {
 218     return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario) || isLongGc(scenario))
 219 }
 220
 221 def static setTestJobTimeOut(newJob, scenario) {
 222     if (isGCStressRelatedTesting(scenario)) {
 223         Utilities.setJobTimeout(newJob, 4320)
 224     }
 225     else if (isCorefxTesting(scenario)) {
 226         Utilities.setJobTimeout(newJob, 360)
 227     }
 228     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 229         Utilities.setJobTimeout(newJob, 240)
 230     }
 231     else if (isR2R(scenario)) {
 232         Utilities.setJobTimeout(newJob, 240)
 233     }
 234     else if (isCoverage(scenario)) {
 235         Utilities.setJobTimeout(newJob, 1440)
 236     }
 237     else if (isLongGc(scenario)) {
 238         Utilities.setJobTimeout(newJob, 1440)
 239     }
 240     else if (isJitDiff(scenario)) {
 241         Utilities.setJobTimeout(newJob, 240)
 242     }
 243     else if (isGcReliabilityFramework(scenario)) {
 244         Utilities.setJobTimeout(newJob, 1440)
 245     }
 246     // Non-test jobs use the default timeout value.
 247 }
 248
 249 def static getJobFolder(def scenario) {
 250     if (isJITStressJob(scenario)) {
 251         return 'jitstress'
 252     }
 253     if (scenario == 'illink') {
 254         return 'illink'
 255     }
 256     return ''
 257 }
 258
 259 def static getStressModeDisplayName(def scenario) {
 260     def displayStr = ''
 261     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 262         def prefixLength = 'COMPlus_'.length()
 263         if (k.length() >= prefixLength) {
 264             def modeName = k.substring(prefixLength, k.length())
 265             displayStr += ' ' + modeName + '=' + v
 266         }
 267     }
 268     return displayStr
 269 }
 270
 271 def static getR2RStressModeDisplayName(def scenario) {
 272     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 273     // "r2r_" prefix.
 274     def displayStr = scenario
 275     def prefixLength = 'r2r_'.length()
 276     if (displayStr.length() >= prefixLength) {
 277         displayStr = displayStr.substring(prefixLength, displayStr.length())
 278     }
 279     return displayStr
 280 }
 281
 282 // Generates the string for creating a file that sets environment variables
 283 // that makes it possible to run stress modes.  Writes the script to the file
 284 // specified by the stepScriptLocation parameter.
 285 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 286     def stepScript = ''
 287     if (os == 'Windows_NT') {
 288         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 289         stepScript += "del ${stepScriptLocation}\r\n"
 290
 291         // Timeout in ms, default is 10 minutes. For stress
 292         // modes up this to 30 minutes
 293         def timeout = 1800000
 294
 295         // Set the Timeout
 296         stepScript += "set __TestTimeout=${timeout}\r\n"
 297         stepScript += "echo. > ${stepScriptLocation}\r\n"
 298         stressModeVars.each{ k, v ->
 299             // Write out what we are writing to the script file
 300             stepScript += "echo Setting ${k}=${v}\r\n"
 301             // Write out the set itself to the script file`
 302             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 303         }
 304     }
 305     else {
 306         stepScript += "echo Setting variables for ${stressModeName}\n"
 307         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 308         stressModeVars.each{ k, v ->
 309             // Write out what we are writing to the script file
 310             stepScript += "echo Setting ${k}=${v}\n"
 311             // Write out the set itself to the script file`
 312             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 313         }
 314         stepScript += "chmod +x ${stepScriptLocation}\n"
 315     }
 316     return stepScript
 317 }
 318
 319 // Calculates the name of the build job based on some typical parameters.
 320 //
 321 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 322     // If the architecture is x64, do not add that info into the build name.
 323     // Need to change around some systems and other builds to pick up the right builds
 324     // to do that.
 325
 326     def suffix = scenario != 'default' ? "_${scenario}" : '';
 327     if (isBuildOnly) {
 328         suffix += '_bld'
 329     }
 330     def baseName = ''
 331     switch (architecture) {
 332         case 'x64':
 333             if (scenario == 'default') {
 334                 // For now we leave x64 off of the name for compatibility with other jobs
 335                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 336             }
 337             else if (scenario == 'formatting') {
 338                 // we don't care about the configuration for the formatting job. It runs all configs
 339                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 340             }
 341             else {
 342                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 343             }
 344             break
 345         case 'arm64':
 346             if (os.toLowerCase() == "windows_nt") {
 347                 // These are cross builds
 348                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 349             }
 350             else {
 351                 // Defaults to a small page size set of machines.
 352                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 353             }
 354             break
 355         case 'arm':
 356             // These are cross builds
 357             if (os == 'Tizen') {
 358                 // ABI: softfp
 359                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 360             }
 361             else {
 362                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 363             }
 364             break
 365         case 'x86':
 366         case 'x86lb':
 367             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 368             break
 369         default:
 370             println("Unknown architecture: ${architecture}");
 371             assert false
 372             break
 373     }
 374
 375     return baseName + suffix
 376 }
 377
 378 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 379     // Check scenario.
 380     switch (scenario) {
 381         case 'default':
 382             switch (architecture) {
 383                 case 'x64':
 384                 case 'x86':
 385                 case 'x86lb':
 386                     if (architecture == 'x86' && os == 'Ubuntu') {
 387                         Utilities.addPeriodicTrigger(job, '@daily')
 388                     }
 389                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 390                         Utilities.addGithubPushTrigger(job)
 391                     }
 392                     break
 393                 case 'arm':
 394                     Utilities.addGithubPushTrigger(job)
 395                     break
 396                 case 'arm64':
 397                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 398                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 399                     break
 400                 default:
 401                     println("Unknown architecture: ${architecture}");
 402                     assert false
 403                     break
 404             }
 405             break
 406         case 'pri1':
 407             // Pri one gets a push trigger, and only for release
 408             if (architecture == 'x64') {
 409                 if (configuration == 'Release') {
 410                     // We expect release jobs to be Windows, or in the cross list
 411                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 412                     if (!os in bidailyCrossList) {
 413                         if (isFlowJob || os == 'Windows_NT') {
 414                             Utilities.addGithubPushTrigger(job)
 415                         }
 416                     }
 417                     else {
 418                         if (isFlowJob) {
 419                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 420                         }
 421                     }
 422                 }
 423             }
 424             break
 425         case 'r2r':
 426             //r2r jobs that aren't pri1 can only be triggered by phrase
 427             break
 428         case 'pri1r2r':
 429             assert !(os in bidailyCrossList)
 430             //pri1 r2r gets a push trigger for checked/release
 431             if (configuration == 'Checked' || configuration == 'Release') {
 432                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 433                 if (architecture == 'x64' && os != 'OSX10.12') {
 434                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 435                     if (isFlowJob || os == 'Windows_NT') {
 436                         Utilities.addGithubPushTrigger(job)
 437                     }
 438                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 439                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 440                     if (isFlowJob) {
 441                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 442                     }
 443                 }
 444                 // For x86, only add per-commit jobs for Windows
 445                 else if (architecture == 'x86' || architecture == 'x86lb') {
 446                     if (os == 'Windows_NT') {
 447                         Utilities.addGithubPushTrigger(job)
 448                     }
 449                 }
 450                 // arm64 pri1r2r jobs should only run every 12 hours.
 451                 else if (architecture == 'arm64') {
 452                     if (os == 'Windows_NT') {
 453                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 454                         // TODO: Add once external email sending is available again
 455                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 456                     }
 457                 }
 458             }
 459             break
 460         case 'r2r_jitstress1':
 461         case 'r2r_jitstress2':
 462         case 'r2r_jitstressregs1':
 463         case 'r2r_jitstressregs2':
 464         case 'r2r_jitstressregs3':
 465         case 'r2r_jitstressregs4':
 466         case 'r2r_jitstressregs8':
 467         case 'r2r_jitstressregs0x10':
 468         case 'r2r_jitstressregs0x80':
 469         case 'r2r_jitstressregs0x1000':
 470         case 'r2r_jitminopts':
 471         case 'r2r_jitforcerelocs':
 472         case 'gcstress15_pri1r2r':
 473             assert !(os in bidailyCrossList)
 474
 475             // GCStress=C is currently not supported on OS X
 476             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 477                 break
 478             }
 479
 480             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 481             if (configuration == 'Checked' || configuration == 'Release') {
 482                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 483                 if (architecture == 'x64') {
 484                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 485                     if (isFlowJob || os == 'Windows_NT') {
 486                         // Add a weekly periodic trigger
 487                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 488                     }
 489                 }
 490                 // For x86, only add per-commit jobs for Windows
 491                 else if (architecture == 'x86') {
 492                     if (os == 'Windows_NT') {
 493                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 494                     }
 495                 }
 496             }
 497             break
 498         case 'longgc':
 499             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 500             assert configuration == 'Release'
 501             assert architecture == 'x64'
 502             Utilities.addPeriodicTrigger(job, '@daily')
 503             // TODO: Add once external email sending is available again
 504             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 505             break
 506         case 'gcsimulator':
 507             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 508             assert configuration == 'Release'
 509             assert architecture == 'x64'
 510             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 511             // TODO: Add once external email sending is available again
 512             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 513             break
 514         case 'standalone_gc':
 515             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 516             assert (configuration == 'Release' || configuration == 'Checked')
 517             // TODO: Add once external email sending is available again
 518             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 519             Utilities.addPeriodicTrigger(job, '@weekly')
 520             break
 521         case 'gc_reliability_framework':
 522             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 523             assert (configuration == 'Release' || configuration == 'Checked')
 524             // Only triggered by phrase.
 525             break
 526         case 'ilrt':
 527             assert !(os in bidailyCrossList)
 528             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 529             if (architecture == 'x64' && configuration == 'Release') {
 530                 // We don't expect to see a job generated except in these scenarios
 531                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 532                 if (isFlowJob || os == 'Windows_NT') {
 533                     Utilities.addPeriodicTrigger(job, '@daily')
 534                 }
 535             }
 536             break
 537         case 'jitdiff':
 538             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 539             assert configuration == 'Checked'
 540             assert (architecture == 'x64' || architecture == 'x86')
 541             Utilities.addGithubPushTrigger(job)
 542             break
 543         case 'coverage':
 544             assert (os == 'Ubuntu' || os == 'Windows_NT')
 545             assert configuration == 'Release'
 546             assert architecture == 'x64'
 547             Utilities.addPeriodicTrigger(job, '@weekly')
 548             break
 549         case 'formatting':
 550             assert (os == 'Windows_NT' || os == "Ubuntu")
 551             assert architecture == 'x64'
 552             Utilities.addGithubPushTrigger(job)
 553             break
 554         case 'jitstressregs1':
 555         case 'jitstressregs2':
 556         case 'jitstressregs3':
 557         case 'jitstressregs4':
 558         case 'jitstressregs8':
 559         case 'jitstressregs0x10':
 560         case 'jitstressregs0x80':
 561         case 'jitstressregs0x1000':
 562         case 'minopts':
 563         case 'forcerelocs':
 564         case 'jitstress1':
 565         case 'jitstress2':
 566         case 'jitstress2_jitstressregs1':
 567         case 'jitstress2_jitstressregs2':
 568         case 'jitstress2_jitstressregs3':
 569         case 'jitstress2_jitstressregs4':
 570         case 'jitstress2_jitstressregs8':
 571         case 'jitstress2_jitstressregs0x10':
 572         case 'jitstress2_jitstressregs0x80':
 573         case 'jitstress2_jitstressregs0x1000':
 574         case 'tailcallstress':
 575         case 'jitsse2only':
 576         case 'corefx_baseline':
 577         case 'corefx_minopts':
 578         case 'corefx_jitstress1':
 579         case 'corefx_jitstress2':
 580         case 'corefx_jitstressregs1':
 581         case 'corefx_jitstressregs2':
 582         case 'corefx_jitstressregs3':
 583         case 'corefx_jitstressregs4':
 584         case 'corefx_jitstressregs8':
 585         case 'corefx_jitstressregs0x10':
 586         case 'corefx_jitstressregs0x80':
 587         case 'corefx_jitstressregs0x1000':
 588         case 'zapdisable':
 589             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 590             assert (os == 'Windows_NT') || (os in Constants.crossList)
 591             Utilities.addPeriodicTrigger(job, '@daily')
 592         }
 593         break
 594         case 'heapverify1':
 595         case 'gcstress0x3':
 596             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 597                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 598                 if (architecture == 'arm64') {
 599                     if (os == 'Windows_NT') {
 600                         Utilities.addPeriodicTrigger(job, '@daily')
 601                     }
 602                     // TODO: Add once external email sending is available again
 603                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 604                 }
 605                 else {
 606                     Utilities.addPeriodicTrigger(job, '@weekly')
 607                 }
 608             }
 609             break
 610         case 'gcstress0xc':
 611         case 'gcstress0xc_zapdisable':
 612         case 'gcstress0xc_zapdisable_jitstress2':
 613         case 'gcstress0xc_zapdisable_heapverify1':
 614         case 'gcstress0xc_jitstress1':
 615         case 'gcstress0xc_jitstress2':
 616         case 'gcstress0xc_minopts_heapverify1':
 617             // GCStress=C is currently not supported on OS X
 618             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 619                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 620                 if (architecture == 'arm64') {
 621                     // TODO: Enable a periodic trigger after tests are updated.
 622                     // Utilities.addPeriodicTrigger(job, '@daily')
 623                     // TODO: Add once external email sending is available again
 624                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 625                 }
 626                 else {
 627                     Utilities.addPeriodicTrigger(job, '@weekly')
 628                 }
 629             }
 630             break
 631
 632         case 'illink':
 633             // Testing on other operating systems TBD
 634             assert (os == 'Windows_NT' || os == 'Ubuntu')
 635             if (architecture == 'x64' || architecture == 'x86') {
 636                 if (configuration == 'Checked') {
 637                     Utilities.addPeriodicTrigger(job, '@daily')
 638                 }
 639             }
 640             break
 641
 642         case 'tieredcompilation':
 643         case 'corefx_tieredcompilation':
 644             // No periodic jobs just yet, still testing
 645             break
 646
 647         default:
 648             println("Unknown scenario: ${scenario}");
 649             assert false
 650             break
 651     }
 652     return
 653 }
 654
 655 // **************************
 656 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 657 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 658 // builds will do a couple extra steps.
 659 // **************************
 660
 661 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 662 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 663 // it's just used for internal builds
 664 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 665 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 666     if (isWindowsBuildOnlyJob) {
 667         return
 668     }
 669
 670     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 671     // Non pull request builds.
 672     if (!isPR) {
 673         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 674         return
 675     }
 676
 677      def arm64Users = [
 678         'adiaaida',
 679         'AndyAyersMS',
 680         'briansull',
 681         'BruceForstall',
 682         'CarolEidt',
 683         'cmckinsey',
 684         'erozenfeld',
 685         'jashook',
 686         'JosephTremoulet',
 687         'pgavlin',
 688         'russellhadley',
 689         'RussKeldorph',
 690         'sandreenko',
 691         'sdmaclea',
 692         'sivarv',
 693         'swaroop-sridhar',
 694         'gkhanna79',
 695         'jkotas',
 696         'markwilkie',
 697         'rahku',
 698         'ramarag',
 699         'tzwlai',
 700         'weshaggard'
 701     ]
 702
 703     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 704     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 705     def osGroup = getOSGroup(os)
 706     switch (architecture) {
 707         case 'x64': // editor brace matching: {
 708             if (scenario == 'coverage') {
 709                 assert configuration == 'Release'
 710                 if (os == 'Ubuntu') {
 711                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 712                 }
 713                 break
 714             }
 715
 716             if (scenario == 'formatting') {
 717                 assert configuration == 'Checked'
 718                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 719                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 720                 }
 721                 break
 722             }
 723
 724             switch (os) {
 725                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 726                 case 'Debian8.4':
 727                 case 'RHEL7.2':
 728                     if (scenario == 'default') {
 729                         assert !isFlowJob
 730                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 731                     }
 732                     else if (scenario == 'pri1' && isFlowJob) {
 733                         assert (configuration == 'Release')
 734                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 735                     }
 736                     break
 737                 case 'Ubuntu16.04':
 738                     assert !isFlowJob
 739                     assert scenario == 'default'
 740                     // Distinguish with the other architectures (arm and x86)
 741                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 742                     break
 743                 case 'Fedora24':
 744                 case 'Ubuntu16.10':
 745                     assert !isFlowJob
 746                     assert scenario == 'default'
 747                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 748                     break
 749                 case 'Ubuntu':
 750                     if (scenario == 'illink') {
 751                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 752                         break
 753                     }
 754                     // fall through
 755                 case 'OSX10.12':
 756                     // Triggers on the non-flow jobs aren't necessary here
 757                     // Corefx testing uses non-flow jobs.
 758                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 759                         break
 760                     }
 761                     switch (scenario) {
 762                         case 'default':
 763                             // Ubuntu uses checked for default PR tests
 764                             if (configuration == 'Checked') {
 765                                 // Default trigger
 766                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 767                             }
 768                             break
 769                         case 'pri1':
 770                             if (configuration == 'Release') {
 771                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 772                             }
 773                             break
 774                         case 'jitdiff':
 775                             if (configuration == 'Checked') {
 776                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 777                             }
 778                             break
 779                         case 'ilrt':
 780                             if (configuration == 'Release') {
 781                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 782                             }
 783                             break
 784                         case 'r2r':
 785                             if (configuration == 'Release' || configuration == 'Checked') {
 786                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 787                             }
 788                             break
 789                         case 'pri1r2r':
 790                             if (configuration == 'Release' || configuration == 'Checked') {
 791                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 792                             }
 793                             break
 794                         case 'gcstress15_pri1r2r':
 795                             if (configuration == 'Release' || configuration == 'Checked') {
 796                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 797                             }
 798                             break
 799                         case 'r2r_jitstress1':
 800                         case 'r2r_jitstress2':
 801                         case 'r2r_jitstressregs1':
 802                         case 'r2r_jitstressregs2':
 803                         case 'r2r_jitstressregs3':
 804                         case 'r2r_jitstressregs4':
 805                         case 'r2r_jitstressregs8':
 806                         case 'r2r_jitstressregs0x10':
 807                         case 'r2r_jitstressregs0x80':
 808                         case 'r2r_jitstressregs0x1000':
 809                         case 'r2r_jitminopts':
 810                         case 'r2r_jitforcerelocs':
 811                             if (configuration == 'Release' || configuration == 'Checked') {
 812                                 def displayStr = getR2RStressModeDisplayName(scenario)
 813                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 814                             }
 815                             break
 816                         case 'longgc':
 817                             if (configuration == 'Release') {
 818                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 819                             }
 820                             break
 821                         case 'gcsimulator':
 822                             if (configuration == 'Release') {
 823                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 824                             }
 825                             break
 826                         case 'standalone_gc':
 827                             if (configuration == 'Release' || configuration == 'Checked') {
 828                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 829                             }
 830                             break
 831                         case 'gc_reliability_framework':
 832                             if (configuration == 'Release' || configuration == 'Checked') {
 833                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 834                             }
 835                             break
 836                         case 'minopts':
 837                         case 'tieredcompilation':
 838                         case 'forcerelocs':
 839                         case 'jitstress1':
 840                         case 'jitstress2':
 841                         case 'jitstressregs1':
 842                         case 'jitstressregs2':
 843                         case 'jitstressregs3':
 844                         case 'jitstressregs4':
 845                         case 'jitstressregs8':
 846                         case 'jitstressregs0x10':
 847                         case 'jitstressregs0x80':
 848                         case 'jitstressregs0x1000':
 849                         case 'jitstress2_jitstressregs1':
 850                         case 'jitstress2_jitstressregs2':
 851                         case 'jitstress2_jitstressregs3':
 852                         case 'jitstress2_jitstressregs4':
 853                         case 'jitstress2_jitstressregs8':
 854                         case 'jitstress2_jitstressregs0x10':
 855                         case 'jitstress2_jitstressregs0x80':
 856                         case 'jitstress2_jitstressregs0x1000':
 857                         case 'tailcallstress':
 858                         case 'jitsse2only':
 859                         case 'gcstress0x3':
 860                         case 'gcstress0xc':
 861                         case 'zapdisable':
 862                         case 'heapverify1':
 863                         case 'gcstress0xc_zapdisable':
 864                         case 'gcstress0xc_zapdisable_jitstress2':
 865                         case 'gcstress0xc_zapdisable_heapverify1':
 866                         case 'gcstress0xc_jitstress1':
 867                         case 'gcstress0xc_jitstress2':
 868                         case 'gcstress0xc_minopts_heapverify1':
 869                             def displayStr = getStressModeDisplayName(scenario)
 870                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 871                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 872                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 873                             break
 874                         case 'corefx_baseline':
 875                         case 'corefx_minopts':
 876                         case 'corefx_tieredcompilation':
 877                         case 'corefx_jitstress1':
 878                         case 'corefx_jitstress2':
 879                         case 'corefx_jitstressregs1':
 880                         case 'corefx_jitstressregs2':
 881                         case 'corefx_jitstressregs3':
 882                         case 'corefx_jitstressregs4':
 883                         case 'corefx_jitstressregs8':
 884                         case 'corefx_jitstressregs0x10':
 885                         case 'corefx_jitstressregs0x80':
 886                         case 'corefx_jitstressregs0x1000':
 887                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 888                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 889                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 890                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 891                             break
 892                         default:
 893                             println("Unknown scenario: ${scenario}");
 894                             assert false
 895                             break
 896                     }
 897                     break
 898                 case 'CentOS7.1':
 899                     switch (scenario) {
 900                         case 'pri1':
 901                             if (configuration == 'Release') {
 902                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 903                             }
 904                             break
 905                         case 'r2r':
 906                             if (configuration == 'Checked' || configuration == 'Release') {
 907                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 908                             }
 909                             break
 910                         case 'pri1r2r':
 911                             if (configuration == 'Checked' || configuration == 'Release') {
 912                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 913                             }
 914                             break
 915                         case 'gcstress15_pri1r2r':
 916                             if (configuration == 'Release' || configuration == 'Checked') {
 917                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 918                             }
 919                             break
 920                         case 'r2r_jitstress1':
 921                         case 'r2r_jitstress2':
 922                         case 'r2r_jitstressregs1':
 923                         case 'r2r_jitstressregs2':
 924                         case 'r2r_jitstressregs3':
 925                         case 'r2r_jitstressregs4':
 926                         case 'r2r_jitstressregs8':
 927                         case 'r2r_jitstressregs0x10':
 928                         case 'r2r_jitstressregs0x80':
 929                         case 'r2r_jitstressregs0x1000':
 930                         case 'r2r_jitminopts':
 931                         case 'r2r_jitforcerelocs':
 932                             if (configuration == 'Release' || configuration == 'Checked') {
 933                                 def displayStr = getR2RStressModeDisplayName(scenario)
 934                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 935                             }
 936                             break
 937                         default:
 938                             break
 939                     }
 940                 case 'Windows_NT':
 941                     switch (scenario) {
 942                         case 'default':
 943                             // Default trigger
 944                             if (configuration == 'Debug') {
 945                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 946                             }
 947                             break
 948                         case 'pri1':
 949                             // Default trigger
 950                             if (configuration == 'Release') {
 951                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 952                             }
 953                             break
 954                         case 'jitdiff':
 955                             if (configuration == 'Checked') {
 956                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 957                             }
 958                             break
 959                         case 'ilrt':
 960                             if (configuration == 'Release') {
 961                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 962                             }
 963                             break
 964                         case 'r2r':
 965                             if (configuration == 'Checked' || configuration == 'Release') {
 966                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 967                             }
 968                             break
 969                         case 'pri1r2r':
 970                             if (configuration == 'Checked' || configuration == 'Release') {
 971                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 972                             }
 973                             break
 974                         case 'gcstress15_pri1r2r':
 975                             if (configuration == 'Release' || configuration == 'Checked') {
 976                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 977                             }
 978                             break
 979                         case 'r2r_jitstress1':
 980                         case 'r2r_jitstress2':
 981                         case 'r2r_jitstressregs1':
 982                         case 'r2r_jitstressregs2':
 983                         case 'r2r_jitstressregs3':
 984                         case 'r2r_jitstressregs4':
 985                         case 'r2r_jitstressregs8':
 986                         case 'r2r_jitstressregs0x10':
 987                         case 'r2r_jitstressregs0x80':
 988                         case 'r2r_jitstressregs0x1000':
 989                         case 'r2r_jitminopts':
 990                         case 'r2r_jitforcerelocs':
 991                             if (configuration == 'Release' || configuration == 'Checked') {
 992                                 def displayStr = getR2RStressModeDisplayName(scenario)
 993                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 994                             }
 995                             break
 996                         case 'longgc':
 997                             if (configuration == 'Release') {
 998                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 999                             }
1000                             break
1001                         case 'gcsimulator':
1002                             if (configuration == 'Release') {
1003                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1004                             }
1005                             break
1006                         case 'standalone_gc':
1007                             if (configuration == 'Release' || configuration == 'Checked') {
1008                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1009                             }
1010                             break
1011                         case 'gc_reliability_framework':
1012                             if (configuration == 'Release' || configuration == 'Checked') {
1013                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1014                             }
1015                             break
1016                         case 'minopts':
1017                         case 'tieredcompilation':
1018                         case 'forcerelocs':
1019                         case 'jitstress1':
1020                         case 'jitstress2':
1021                         case 'jitstressregs1':
1022                         case 'jitstressregs2':
1023                         case 'jitstressregs3':
1024                         case 'jitstressregs4':
1025                         case 'jitstressregs8':
1026                         case 'jitstressregs0x10':
1027                         case 'jitstressregs0x80':
1028                         case 'jitstressregs0x1000':
1029                         case 'jitstress2_jitstressregs1':
1030                         case 'jitstress2_jitstressregs2':
1031                         case 'jitstress2_jitstressregs3':
1032                         case 'jitstress2_jitstressregs4':
1033                         case 'jitstress2_jitstressregs8':
1034                         case 'jitstress2_jitstressregs0x10':
1035                         case 'jitstress2_jitstressregs0x80':
1036                         case 'jitstress2_jitstressregs0x1000':
1037                         case 'tailcallstress':
1038                         case 'jitsse2only':
1039                         case 'gcstress0x3':
1040                         case 'gcstress0xc':
1041                         case 'zapdisable':
1042                         case 'heapverify1':
1043                         case 'gcstress0xc_zapdisable':
1044                         case 'gcstress0xc_zapdisable_jitstress2':
1045                         case 'gcstress0xc_zapdisable_heapverify1':
1046                         case 'gcstress0xc_jitstress1':
1047                         case 'gcstress0xc_jitstress2':
1048                         case 'gcstress0xc_minopts_heapverify1':
1049                             def displayStr = getStressModeDisplayName(scenario)
1050                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1051                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1052                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1053                             break
1054                         case 'corefx_baseline':
1055                         case 'corefx_minopts':
1056                         case 'corefx_tieredcompilation':
1057                         case 'corefx_jitstress1':
1058                         case 'corefx_jitstress2':
1059                         case 'corefx_jitstressregs1':
1060                         case 'corefx_jitstressregs2':
1061                         case 'corefx_jitstressregs3':
1062                         case 'corefx_jitstressregs4':
1063                         case 'corefx_jitstressregs8':
1064                         case 'corefx_jitstressregs0x10':
1065                         case 'corefx_jitstressregs0x80':
1066                         case 'corefx_jitstressregs0x1000':
1067                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1068                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1069                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1070                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1071                             break
1072                         case 'illink':
1073                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1074                                     break
1075                         default:
1076                             println("Unknown scenario: ${scenario}");
1077                             assert false
1078                             break
1079                     }
1080                     break
1081                 default:
1082                     println("Unknown os: ${os}");
1083                     assert false
1084                     break
1085             }
1086             break
1087         // editor brace matching: }
1088         case 'arm': // editor brace matching: {
1089             switch (os) {
1090                 case 'Ubuntu':
1091                 case 'Ubuntu16.04':
1092                     assert scenario == 'default'
1093                     job.with {
1094                         publishers {
1095                             azureVMAgentPostBuildAction {
1096                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1097                             }
1098                         }
1099                     }
1100                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1101                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1102                     }
1103                     else {
1104                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1105                     }
1106                     break;
1107                 case 'Tizen':
1108                     architecture='armel'
1109                     job.with {
1110                         publishers {
1111                             azureVMAgentPostBuildAction {
1112                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1113                             }
1114                         }
1115                     }
1116                     // Removing the regex will cause this to run on each PR.
1117                     if (configuration == 'Release' || configuration == 'Debug') {
1118                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1119                     }
1120                     else {
1121                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1122                     }
1123                     break;
1124                 case 'Windows_NT':
1125                     // Set up a private trigger
1126                     def contextString = "${os} ${architecture} Cross ${configuration}"
1127                     if (scenario != 'default')
1128                         contextString += " ${scenario}"
1129                     contextString += " Build"
1130                     // Debug builds only.
1131                     if (configuration != 'Debug') {
1132                         contextString += " and Test"
1133                     }
1134                     switch (scenario) {
1135                         case 'default':
1136                             // For now only run Debug and Release build jobs on PR Trigger. Note this is not a private trigger.
1137                             if (configuration == 'Debug' || configuration == 'Release')
1138                             {
1139                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1140                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1141                             }
1142                             else
1143                             {
1144                                 // Checked jobs will run on private trigger and run tests.
1145                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1146                             }
1147                             break
1148                         case 'pri1r2r':
1149                         case 'gcstress0x3':
1150                         case 'gcstress0xc':
1151                             // Stress jobs will will run on private trigger and run tests.
1152                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1153                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1154                             break
1155                     }
1156                     break
1157                 default:
1158                     println("NYI os: ${os}");
1159                     assert false
1160                     break
1161             }
1162             break
1163         // editor brace matching: }
1164         case 'arm64': // editor brace matching: {
1165             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1166
1167             // Set up a private trigger
1168             def contextString = "${os} ${architecture} Cross ${configuration}"
1169             if (scenario != 'default')
1170                 contextString += " ${scenario}"
1171             contextString += " Build"
1172             // Debug builds only.
1173             if (configuration != 'Debug') {
1174                contextString += " and Test"
1175             }
1176
1177             switch (os) {
1178                 case 'Ubuntu':
1179                 case 'Ubuntu16.04':
1180                     switch (scenario) {
1181                         case 'pri1':
1182                             if (configuration == 'Release') {
1183                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1184                             }
1185                             break
1186                         case 'r2r':
1187                             if (configuration == 'Checked' || configuration == 'Release') {
1188                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1189                             }
1190                             break
1191                         case 'pri1r2r':
1192                             if (configuration == 'Checked' || configuration == 'Release') {
1193                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1194                             }
1195                             break
1196                         case 'gcstress15_pri1r2r':
1197                             if (configuration == 'Release' || configuration == 'Checked') {
1198                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1199                             }
1200                             break
1201                         case 'r2r_jitstress1':
1202                         case 'r2r_jitstress2':
1203                         case 'r2r_jitstressregs1':
1204                         case 'r2r_jitstressregs2':
1205                         case 'r2r_jitstressregs3':
1206                         case 'r2r_jitstressregs4':
1207                         case 'r2r_jitstressregs8':
1208                         case 'r2r_jitstressregs0x10':
1209                         case 'r2r_jitstressregs0x80':
1210                         case 'r2r_jitstressregs0x1000':
1211                         case 'r2r_jitminopts':
1212                         case 'r2r_jitforcerelocs':
1213                             if (configuration == 'Release' || configuration == 'Checked') {
1214                                 def displayStr = getR2RStressModeDisplayName(scenario)
1215                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1216                             }
1217                             break
1218                         default:
1219                             break
1220                     }
1221                 case 'Windows_NT':
1222                     switch (scenario) {
1223                         case 'default':
1224                             if (isFlowJob == true) {
1225                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration}", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1226                             }
1227
1228                             // For now only run Debug jobs on PR Trigger.
1229                             else if (configuration != 'Debug') {
1230                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1231                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1232                             }
1233                             else {
1234                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1235                                 // are at higher risk of ARM64-breaking changes.
1236                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1237                             }
1238                             break
1239                         case 'pri1r2r':
1240                         case 'gcstress0x3':
1241                         case 'gcstress0xc':
1242                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1243                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1244                             break
1245                     }
1246                     break
1247                 default:
1248                     println("NYI os: ${os}");
1249                     assert false
1250                     break
1251             }
1252             break
1253         // editor brace matching: }
1254         case 'x86': // editor brace matching: {
1255             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1256             if (os == 'Ubuntu') {
1257                 // on-demand only for ubuntu x86
1258                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1259                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1260                 break
1261             }
1262             switch (scenario) {
1263                 case 'default':
1264                     if (configuration == 'Checked') {
1265                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1266                     }
1267                     else if (configuration == 'Release') {
1268                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1269                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1270                     }
1271                     break
1272                 case 'pri1':
1273                     if (configuration == 'Release') {
1274                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1275                     }
1276                     break
1277                 case 'ilrt':
1278                     if (configuration == 'Release') {
1279                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1280                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1281                     }
1282                     break
1283                 case 'r2r':
1284                     if (configuration == 'Checked' || configuration == 'Release') {
1285                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1286                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1287                     }
1288                     break
1289                 case 'pri1r2r':
1290                     if (configuration == 'Checked' || configuration == 'Release') {
1291                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1292                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1293                     }
1294                     break
1295                 case 'gcstress15_pri1r2r':
1296                     if (configuration == 'Release' || configuration == 'Checked') {
1297                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1298                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1299                     }
1300                     break
1301                 case 'r2r_jitstress1':
1302                 case 'r2r_jitstress2':
1303                 case 'r2r_jitstressregs1':
1304                 case 'r2r_jitstressregs2':
1305                 case 'r2r_jitstressregs3':
1306                 case 'r2r_jitstressregs4':
1307                 case 'r2r_jitstressregs8':
1308                 case 'r2r_jitstressregs0x10':
1309                 case 'r2r_jitstressregs0x80':
1310                 case 'r2r_jitstressregs0x1000':
1311                 case 'r2r_jitminopts':
1312                 case 'r2r_jitforcerelocs':
1313                     if (configuration == 'Release' || configuration == 'Checked') {
1314                         def displayStr = getR2RStressModeDisplayName(scenario)
1315                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1316                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1317                     }
1318                     break
1319                 case 'longgc':
1320                     if (configuration == 'Release') {
1321                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1322                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1323                     }
1324                     break
1325                 case 'gcsimulator':
1326                     if (configuration == 'Release') {
1327                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1328                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1329                     }
1330                     break
1331                 case 'standalone_gc':
1332                     if (configuration == 'Release' || configuration == 'Checked') {
1333                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1334                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1335                     }
1336                     break
1337                 case 'minopts':
1338                 case 'tieredcompilation':
1339                 case 'forcerelocs':
1340                 case 'jitstress1':
1341                 case 'jitstress2':
1342                 case 'jitstressregs1':
1343                 case 'jitstressregs2':
1344                 case 'jitstressregs3':
1345                 case 'jitstressregs4':
1346                 case 'jitstressregs8':
1347                 case 'jitstressregs0x10':
1348                 case 'jitstressregs0x80':
1349                 case 'jitstressregs0x1000':
1350                 case 'jitstress2_jitstressregs1':
1351                 case 'jitstress2_jitstressregs2':
1352                 case 'jitstress2_jitstressregs3':
1353                 case 'jitstress2_jitstressregs4':
1354                 case 'jitstress2_jitstressregs8':
1355                 case 'jitstress2_jitstressregs0x10':
1356                 case 'jitstress2_jitstressregs0x80':
1357                 case 'jitstress2_jitstressregs0x1000':
1358                 case 'tailcallstress':
1359                 case 'jitsse2only':
1360                 case 'gcstress0x3':
1361                 case 'gcstress0xc':
1362                 case 'zapdisable':
1363                 case 'heapverify1':
1364                 case 'gcstress0xc_zapdisable':
1365                 case 'gcstress0xc_zapdisable_jitstress2':
1366                 case 'gcstress0xc_zapdisable_heapverify1':
1367                 case 'gcstress0xc_jitstress1':
1368                 case 'gcstress0xc_jitstress2':
1369                 case 'gcstress0xc_minopts_heapverify1':
1370                     def displayStr = getStressModeDisplayName(scenario)
1371                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1372                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1373                     break
1374                 case 'corefx_baseline':
1375                 case 'corefx_minopts':
1376                 case 'corefx_tieredcompilation':
1377                 case 'corefx_jitstress1':
1378                 case 'corefx_jitstress2':
1379                 case 'corefx_jitstressregs1':
1380                 case 'corefx_jitstressregs2':
1381                 case 'corefx_jitstressregs3':
1382                 case 'corefx_jitstressregs4':
1383                 case 'corefx_jitstressregs8':
1384                 case 'corefx_jitstressregs0x10':
1385                 case 'corefx_jitstressregs0x80':
1386                 case 'corefx_jitstressregs0x1000':
1387                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1388                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1389                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1390                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1391                     break
1392                 case 'illink':
1393                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1394                     break
1395                 default:
1396                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1397                     assert false
1398                     break
1399             }
1400             break
1401          // editor brace matching: }
1402         case 'x86lb': // editor brace matching: {
1403             assert (os == 'Windows_NT')
1404             assert (scenario == 'default' || Constants.r2rJitStressScenarios.indexOf(scenario) !=1)
1405
1406             def arch = 'x86'
1407             def jit = 'legacy_backend'
1408             switch (scenario) {
1409                 case 'default':
1410                     if (configuration == 'Checked') {
1411                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1412                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1413                     }
1414                     break
1415                 default:
1416                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1417                     assert false
1418                     break
1419             }
1420             break
1421         // editor brace matching: }
1422         default:
1423             println("Unknown architecture: ${architecture}");
1424             assert false
1425             break
1426     }
1427 }
1428
1429 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1430     def buildCommands = [];
1431     def osGroup = getOSGroup(os)
1432     def lowerConfiguration = configuration.toLowerCase()
1433
1434     // Calculate the build steps, archival, and xunit results
1435     switch (os) {
1436         case 'Windows_NT': // editor brace matching: {
1437             switch (architecture) {
1438                 case 'x64':
1439                 case 'x86':
1440                 case 'x86lb':
1441                     def arch = architecture
1442                     def buildOpts = ''
1443                     if (architecture == 'x86lb') {
1444                         arch = 'x86'
1445                     }
1446
1447                     if (scenario == 'illink') {
1448                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1449                     }
1450
1451                     // If it is a release build for windows, ensure PGO is used, else fail the build
1452                     if ((lowerConfiguration == 'release') && (scenario in Constants.basicScenarios) && (architecture != 'x86lb')) {
1453                         buildOpts += ' enforcepgo'
1454                     }
1455
1456                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1457                             scenario == 'default' ||
1458                             scenario == 'r2r' ||
1459                             scenario == 'jitdiff' ||
1460                             scenario == 'ilrt' ||
1461                             scenario == 'illink' ||
1462                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1463                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1464                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1465                     }
1466
1467                     // For Pri 1 tests, we must shorten the output test binary path names.
1468                     // if __TestIntermediateDir is already set, build-test.cmd will
1469                     // output test binaries to that directory. If it is not set, the
1470                     // binaries are sent to a default directory whose name is about
1471                     // 35 characters long.
1472
1473                     else if (scenarioNeedsPri1Build(scenario)) {
1474                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1475                     }
1476                     else if (isLongGc(scenario)) {
1477                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1478                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1479                     }
1480                     else if (scenario == 'standalone_gc') {
1481                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1482                     }
1483                     else if (scenario == 'formatting') {
1484                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1485                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1486                         break
1487                     }
1488                     else {
1489                         println("Unknown scenario: ${scenario}")
1490                         assert false
1491                     }
1492
1493                     // If we are running a stress mode, we should write out the set of key
1494                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1495
1496                     if (!isBuildOnly) {
1497                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1498                         def crossgenStr = ''
1499                         def runcrossgentestsStr = ''
1500                         def runjitstressStr = ''
1501                         def runjitstressregsStr = ''
1502                         def runjitmioptsStr = ''
1503                         def runjitforcerelocsStr = ''
1504                         def runjitdisasmStr = ''
1505                         def runilasmroundtripStr = ''
1506                         def gcstressStr = ''
1507                         def runtestArguments = ''
1508                         def gcTestArguments = ''
1509                         def illinkArguments = ''
1510
1511                         if (scenario == 'r2r' ||
1512                             scenario == 'pri1r2r' ||
1513                             scenario == 'gcstress15_pri1r2r' ||
1514                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1515                                 crossgenStr = 'crossgen'
1516                                 runcrossgentestsStr = 'runcrossgentests'
1517
1518                                 if (scenario == 'r2r_jitstress1'){
1519                                     runjitstressStr = 'jitstress 1'
1520                                 }
1521                                 else if (scenario == 'r2r_jitstress2') {
1522                                     runjitstressStr = 'jitstress 2'
1523                                 }
1524                                 else if (scenario == 'r2r_jitstressregs1'){
1525                                     runjitstressregsStr = 'jitstressregs 1'
1526                                 }
1527                                 else if (scenario == 'r2r_jitstressregs2') {
1528                                     runjitstressregsStr = 'jitstressregs 2'
1529                                 }
1530                                 else if (scenario == 'r2r_jitstressregs3') {
1531                                     runjitstressregsStr = 'jitstressregs 3'
1532                                 }
1533                                 else if (scenario == 'r2r_jitstressregs4') {
1534                                     runjitstressregsStr = 'jitstressregs 4'
1535                                 }
1536                                 else if (scenario == 'r2r_jitstressregs8') {
1537                                     runjitstressregsStr = 'jitstressregs 8'
1538                                 }
1539                                 else if (scenario == 'r2r_jitstressregs0x10') {
1540                                     runjitstressregsStr = 'jitstressregs 0x10'
1541                                 }
1542                                 else if (scenario == 'r2r_jitstressregs0x80') {
1543                                     runjitstressregsStr = 'jitstressregs 0x80'
1544                                 }
1545                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1546                                     runjitstressregsStr = 'jitstressregs 0x1000'
1547                                 }
1548                                 else if (scenario == 'r2r_jitminopts') {
1549                                     runjitmioptsStr = 'jitminopts'
1550                                 }
1551                                 else if (scenario == 'r2r_jitforcerelocs') {
1552                                     runjitforcerelocsStr = 'jitforcerelocs'
1553                                 }
1554                         }
1555                         if (scenario == 'gcstress15_pri1r2r')
1556                         {
1557                             gcstressStr = 'gcstresslevel 0xF'
1558                         }
1559
1560                         if (scenario == 'jitdiff')
1561                         {
1562                             runjitdisasmStr = 'jitdisasm crossgen'
1563                         }
1564
1565                         if (scenario == 'ilrt')
1566                         {
1567                             runilasmroundtripStr = 'ilasmroundtrip'
1568                         }
1569
1570                         if (isLongGc(scenario)) {
1571                             gcTestArguments = "${scenario} sequential"
1572                         }
1573
1574                         if (scenario == 'illink')
1575                         {
1576                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1577                         }
1578
1579                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1580
1581                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1582                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1583                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1584
1585                             if (enableCorefxTesting) {
1586                                 def workspaceRelativeFxRoot = "_/fx"
1587                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1588
1589                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1590
1591                                 setTestJobTimeOut(newJob, scenario)
1592
1593                                 // Archive and process (only) the test results
1594                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1595                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1596
1597                                 //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1598                                 Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1599                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1600                             }
1601                             else {
1602                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1603                             }
1604                         }
1605                         else if (isGcReliabilityFramework(scenario)) {
1606                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1607                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1608                         }
1609                         else if (architecture == 'x64' || architecture == 'x86') {
1610                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1611                         }
1612                         else if (architecture == 'x86lb') {
1613                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd"
1614                         }
1615                     }
1616
1617                     if (!enableCorefxTesting) {
1618                         // Run the rest of the build
1619                         // Build the mscorlib for the other OS's
1620                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1621                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1622
1623                         if (arch == "x64") {
1624                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1625                         }
1626
1627                         // Zip up the tests directory so that we don't use so much space/time copying
1628                         // 10s of thousands of files around.
1629                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1630
1631                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1632                             // For windows, pull full test results and test drops for x86/x64.
1633                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1634                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1635                         }
1636
1637                         if (scenario == 'jitdiff') {
1638                             // retrive jit-dasm output for base commit, and run jit-diff
1639                             if (!isBuildOnly) {
1640                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1641                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1642                             }
1643                         }
1644
1645                         if (!isBuildOnly) {
1646                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1647                             setTestJobTimeOut(newJob, scenario)
1648                         }
1649                     }
1650                     break
1651                 case 'arm':
1652                     def validArmWindowsScenarios = [ "default",
1653                                                      "pri1r2r",
1654                                                      "zapdisable",
1655                                                      "minopts",
1656                                                      "tieredcompilation",
1657                                                      "tailcallstress",
1658                                                      "jitstress1",
1659                                                      "jitstress2",
1660                                                      "gcstress0x3",
1661                                                      "gcstress0xc",
1662                                                      "jitstressregs1",
1663                                                      "jitstressregs2",
1664                                                      "gcstress0xc_jitstress1",
1665                                                      "gcstress0xc_jitstress2"]
1666
1667                     assert validArmWindowsScenarios.contains(scenario)
1668
1669                     // Set time out
1670                     setTestJobTimeOut(newJob, scenario)
1671
1672                     if ( lowerConfiguration == "debug" ) {
1673                         // For Debug builds, we will do a P1 test build
1674                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1675                     }
1676                     else if (lowerConfiguration == "checked") {
1677
1678                         if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1679                         {
1680                            // Up the timeout for arm checked testing only.
1681                            // Keep the longer timeout for gcstress.
1682                            Utilities.setJobTimeout(newJob, 240)
1683                         }
1684
1685                         def machineAffinityOptions = ['use_arm64_build_machine' : true]
1686                         setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1687                         // For checked runs we will also run testing.
1688                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1689                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1690                     }
1691                     else if (lowerConfiguration == "release") {
1692                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1693                     }
1694                     // Add archival.
1695                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1696                     break
1697                 case 'arm64':
1698                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1699
1700                     // Set time out
1701                     setTestJobTimeOut(newJob, scenario)
1702
1703                     // Debug runs take too long to run. So build job only.
1704                     if (lowerConfiguration == "debug") {
1705                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1706                     }
1707                     else {
1708                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1709                        {
1710                            // Up the timeout for arm64 checked testing only.
1711                            // Keep the longer timeout for gcstress.
1712                            Utilities.setJobTimeout(newJob, 240)
1713                        }
1714
1715                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=1"
1716                        // Test build and run are launched together.
1717                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1718                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1719                     }
1720
1721                     // Add archival.
1722                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1723                     break
1724                 default:
1725                     println("Unknown architecture: ${architecture}");
1726                     assert false
1727                     break
1728             }
1729             break
1730         // editor brace matching: }
1731         case 'Ubuntu':
1732         case 'Ubuntu16.04':
1733         case 'Ubuntu16.10':
1734         case 'Debian8.4':
1735         case 'OSX10.12':
1736         case 'CentOS7.1':
1737         case 'RHEL7.2':
1738         case 'Tizen':
1739         case 'Fedora24': // editor brace matching: {
1740             switch (architecture) {
1741                 case 'x64':
1742                 case 'x86':
1743                     if (architecture == 'x86' && os == 'Ubuntu') {
1744                         // build and PAL test
1745                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1746                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1747                         break;
1748                     }
1749
1750                     if (scenario == 'formatting') {
1751                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1752                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1753                         break
1754                     }
1755
1756                     if (scenario == 'illink') {
1757                         assert(os == 'Ubuntu')
1758                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1759                     }
1760
1761                     def standaloneGc = ''
1762                     if (scenario == 'standalone_gc') {
1763                         standaloneGc = 'buildstandalonegc'
1764                     }
1765
1766                     if (!enableCorefxTesting) {
1767                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1768                         // only on supported OS platforms.
1769                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1770                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1771                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1772                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1773
1774                         // Set time out
1775                         setTestJobTimeOut(newJob, scenario)
1776                         // Basic archiving of the build
1777                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1778                         // And pal tests
1779                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1780                     }
1781                     else {
1782                         // Corefx stress testing
1783                         assert os == 'Ubuntu'
1784                         assert architecture == 'x64'
1785                         assert lowerConfiguration == 'checked'
1786                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1787
1788                         // Build coreclr
1789                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1790
1791                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1792                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1793
1794                         // Build and text corefx
1795                         def workspaceRelativeFxRoot = "_/fx"
1796                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1797
1798                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1799
1800                         setTestJobTimeOut(newJob, scenario)
1801
1802                         // Archive and process (only) the test results
1803                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1804                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1805                     }
1806                     break
1807                 case 'arm64':
1808                     def standaloneGc = ''
1809                     if (scenario == 'standalone_gc') {
1810                         standaloneGc = 'buildstandalonegc'
1811                     }
1812
1813                     if (!enableCorefxTesting) {
1814                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8 ${standaloneGc}"
1815
1816                         // HACK -- Arm64 does not have corefx jobs yet.
1817                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1818                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1819                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1820                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1821
1822                         // Set time out
1823                         setTestJobTimeOut(newJob, scenario)
1824                         // Basic archiving of the build
1825                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1826                     }
1827                     break
1828                 case 'arm':
1829                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1830                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1831
1832                     // default values for Ubuntu
1833                     def arm_abi="arm"
1834                     def linuxCodeName="trusty"
1835                     if (os == 'Ubuntu16.04') {
1836                         linuxCodeName="xenial"
1837                     }
1838                     else if (os == 'Tizen') {
1839                         arm_abi="armel"
1840                         linuxCodeName="tizen"
1841                     }
1842
1843                     // Unzip the Windows test binaries first. Exit with 0
1844                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1845
1846                     // Unpack the corefx binaries
1847                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1848                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1849                     if (os != 'Tizen') {
1850                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1851                     }
1852                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1853                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1854                     if (os == 'Ubuntu') {
1855                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1856                     }
1857
1858                     // Call the ARM CI script to cross build and test using docker
1859                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1860                     --mode=docker \\
1861                     --${arm_abi} \\
1862                     --linuxCodeName=${linuxCodeName} \\
1863                     --buildConfig=${lowerConfiguration} \\
1864                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1865                     --coreFxBinDir=./bin/CoreFxBinDir \\
1866                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1867
1868                     // Basic archiving of the build, no pal tests
1869                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1870                     break
1871                 default:
1872                     println("Unknown architecture: ${architecture}");
1873                     assert false
1874                     break
1875             }
1876             break
1877         // editor brace matching: }
1878         default:
1879             println("Unknown os: ${os}");
1880             assert false
1881             break
1882     } // os
1883
1884     return buildCommands
1885 }
1886
1887 // Additional scenario which can alter behavior
1888
1889 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1890 combinedScenarios.each { scenario ->
1891     [true, false].each { isPR ->
1892         Constants.architectureList.each { architecture ->
1893             Constants.configurationList.each { configuration ->
1894                 Constants.osList.each { os ->
1895                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1896                     // and reset the os to Windows_NT
1897                     def isBuildOnly = false
1898                     if (os == 'Windows_NT_BuildOnly') {
1899                         isBuildOnly = true
1900                         os = 'Windows_NT'
1901                     }
1902
1903                     // Tizen is only supported for arm architecture
1904                     if (os == 'Tizen' && architecture != 'arm') {
1905                         return
1906                     }
1907
1908                     // Skip totally unimplemented (in CI) configurations.
1909                     switch (architecture) {
1910                         case 'arm64':
1911                             if (os == 'Ubuntu16.04') {
1912                                 os = 'Ubuntu'
1913                             }
1914
1915                             // Windows and Ubuntu only
1916                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1917                                 return
1918                             }
1919                             break
1920                         case 'arm':
1921                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1922                                 return
1923                             }
1924                             break
1925                         case 'x86':
1926                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1927                                 return
1928                             }
1929                             break
1930                         case 'x86lb':
1931                             if (os != 'Windows_NT') {
1932                                 return
1933                             }
1934                             break
1935                         case 'x64':
1936                             // Everything implemented
1937                             break
1938                         default:
1939                             println("Unknown architecture: ${architecture}")
1940                             assert false
1941                             break
1942                     }
1943
1944                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1945                     // with checked builds
1946                     def enableCorefxTesting = false
1947                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1948                         if (configuration != 'Checked') {
1949                             return
1950                         }
1951
1952                         enableCorefxTesting = isCorefxTesting(scenario)
1953
1954                         // Since these are just execution time differences,
1955                         // skip platforms that don't execute the tests here (Windows_NT only)
1956                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1957                         if (!isEnabledOS || isBuildOnly) {
1958                             return
1959                         }
1960
1961                         switch (architecture) {
1962                             case 'arm':
1963                                 if ((scenario != 'gcstress0x3') &&
1964                                     (scenario != 'gcstress0xc') &&
1965                                     (scenario != 'jitstress1') &&
1966                                     (scenario != 'jitstress2') &&
1967                                     (scenario != 'jitstressregs1') &&
1968                                     (scenario != 'jitstressregs2') &&
1969                                     (scenario != 'gcstress0xc_jitstress1') &&
1970                                     (scenario != 'gcstress0xc_jitstress2') &&
1971                                     (scenario != 'minopts') &&
1972                                     (scenario != 'tieredcompilation') &&
1973                                     (scenario != 'tailcallstress') &&
1974                                     (scenario != 'zapdisable')) {
1975                                         return
1976                                     }
1977                                     break
1978                             case 'arm64':
1979                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1980                                     return
1981                                 }
1982                                 break
1983                             case 'x64':
1984                             case 'x86':
1985                                 // x86 ubuntu: default only
1986                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1987                                     return
1988                                 }
1989                                 // Windows: Everything implemented
1990                                 break
1991                             case 'x86lb':
1992                                 // No stress modes for legacy jit.
1993                                 // (There's no technical reason we couldn't allow these.)
1994                                 return
1995                             default:
1996                                 return
1997                         }
1998                     }
1999                     else {
2000                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
2001                         // and configuration is not Checked, bail out.
2002                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2003                             return;
2004                         }
2005
2006                         // Skip scenarios
2007                         switch (scenario) {
2008                             case 'pri1':
2009                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
2010                                 // the default scenario build
2011                                 if (os != 'Windows_NT') {
2012                                     return
2013                                 }
2014                                 // Only x64 for now
2015                                 if (architecture != 'x64') {
2016                                     return
2017                                 }
2018                                 break
2019                             case 'ilrt':
2020                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2021                                 // the default scenario build
2022                                 if (os != 'Windows_NT') {
2023                                     return
2024                                 }
2025                                 // Only x64 for now
2026                                 if (architecture != 'x64') {
2027                                     return
2028                                 }
2029                                 // Release only
2030                                 if (configuration != 'Release') {
2031                                     return
2032                                 }
2033                                 break
2034                             case 'jitdiff':
2035                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2036                                     return
2037                                 }
2038                                 if (architecture != 'x64') {
2039                                     return
2040                                 }
2041                                 if (configuration != 'Checked') {
2042                                     return
2043                                 }
2044                                 break
2045                             case 'r2r':
2046                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2047                                 // the default scenario build
2048                                 if (os != 'Windows_NT') {
2049                                     return
2050                                 }
2051                                 if (architecture != 'x64') {
2052                                     return
2053                                 }
2054                                 break
2055                             case 'pri1r2r':
2056                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2057                                 // the default scenario build
2058                                 if (os != 'Windows_NT') {
2059                                     return
2060                                 }
2061                                 if (architecture != 'x64') {
2062                                     if ((architecture != 'arm64' && architecture != 'arm') || (configuration == 'Debug')) {
2063                                         return
2064                                     }
2065                                 }
2066                                 break
2067                             case 'gcstress15_pri1r2r':
2068                             case 'r2r_jitstress1':
2069                             case 'r2r_jitstress2':
2070                             case 'r2r_jitstressregs1':
2071                             case 'r2r_jitstressregs2':
2072                             case 'r2r_jitstressregs3':
2073                             case 'r2r_jitstressregs4':
2074                             case 'r2r_jitstressregs8':
2075                             case 'r2r_jitstressregs0x10':
2076                             case 'r2r_jitstressregs0x80':
2077                             case 'r2r_jitstressregs0x1000':
2078                             case 'r2r_jitminopts':
2079                             case 'r2r_jitforcerelocs':
2080                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
2081                                 // the default scenario build
2082                                 if (os != 'Windows_NT') {
2083                                     return
2084                                 }
2085                                 if (architecture != 'x64' && architecture != 'x86') {
2086                                     return
2087                                 }
2088                                 break
2089                             case 'longgc':
2090                             case 'gcsimulator':
2091                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2092                                     return
2093                                 }
2094                                 if (architecture != 'x64') {
2095                                     return
2096                                 }
2097                                 if (configuration != 'Release') {
2098                                     return
2099                                 }
2100                                 break
2101                             case 'gc_reliability_framework':
2102                             case 'standalone_gc':
2103                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2104                                     return
2105                                 }
2106
2107                                 if (architecture != 'x64') {
2108                                     return
2109                                 }
2110
2111                                 if (configuration != 'Release' && configuration != 'Checked') {
2112                                     return
2113                                 }
2114                                 break
2115                             // We need Windows x64 Release bits for the code coverage build
2116                             case 'coverage':
2117                                 if (os != 'Windows_NT') {
2118                                     return
2119                                 }
2120                                 if (architecture != 'x64') {
2121                                     return
2122                                 }
2123                                 if (configuration != 'Release') {
2124                                     return
2125                                 }
2126                                 break
2127                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2128                             case 'formatting':
2129                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2130                                     return
2131                                 }
2132                                 if (architecture != 'x64') {
2133                                     return
2134                                 }
2135                                 if (configuration != 'Checked') {
2136                                     return
2137                                 }
2138                                 if (isBuildOnly) {
2139                                     return
2140                                 }
2141                                 break
2142                             case 'illink':
2143                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2144                                     return
2145                                 }
2146                                 if (architecture != 'x64' && architecture != 'x86') {
2147                                     return
2148                                 }
2149                                 if (isBuildOnly) {
2150                                     return
2151                                 }
2152                                 break
2153                             case 'default':
2154                                 // Nothing skipped
2155                                 break
2156                             default:
2157                                 println("Unknown scenario: ${scenario}")
2158                                 assert false
2159                                 break
2160                         }
2161                     }
2162
2163                     // Calculate names
2164                     def lowerConfiguration = configuration.toLowerCase()
2165                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2166                     def folderName = getJobFolder(scenario)
2167
2168                     // Create the new job
2169                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2170
2171                     def machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2172                     machineAffinityOptions = architecture == 'arm' ? ['use_arm64_build_machine': false] : machineAffinityOptions
2173                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2174
2175                     // Add all the standard options
2176                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2177                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2178
2179                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
2180                     def osGroup = getOSGroup(os)
2181
2182                     newJob.with {
2183                         steps {
2184                             if (os == 'Windows_NT') {
2185                                 buildCommands.each { buildCommand ->
2186                                     batchFile(buildCommand)
2187                                 }
2188                             }
2189                             else {
2190                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2191                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2192                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2193                                     // Define the Windows Tests and Corefx build job names
2194                                     def WindowTestsName = projectFolder + '/' +
2195                                                           Utilities.getFullJobName(project,
2196                                                                                    getJobName(lowerConfiguration,
2197                                                                                               'x64' ,
2198                                                                                               'windows_nt',
2199                                                                                               'default',
2200                                                                                               true),
2201                                                                                    false)
2202                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2203                                                        Utilities.getFolderName(branch)
2204
2205                                     // Copy the Windows test binaries and the Corefx build binaries
2206                                     copyArtifacts(WindowTestsName) {
2207                                         includePatterns('bin/tests/tests.zip')
2208                                         buildSelector {
2209                                             latestSuccessful(true)
2210                                         }
2211                                     }
2212
2213                                     def arm_abi = 'arm'
2214                                     def corefx_os = 'linux'
2215                                     if (os == 'Tizen') {
2216                                         arm_abi = 'armel'
2217                                         corefx_os = 'tizen'
2218                                     }
2219
2220                                     // Let's use release CoreFX to test checked CoreCLR,
2221                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2222                                     def corefx_lowerConfiguration = lowerConfiguration
2223                                     if ( lowerConfiguration == 'checked' ) {
2224                                         corefx_lowerConfiguration='release'
2225                                     }
2226
2227                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2228                                         includePatterns('bin/build.tar.gz')
2229                                         buildSelector {
2230                                             latestSuccessful(true)
2231                                         }
2232                                     }
2233                                 }
2234
2235                                 buildCommands.each { buildCommand ->
2236                                     shell(buildCommand)
2237                                 }
2238                             }
2239                         }
2240                     } // newJob.with
2241
2242                 } // os
2243             } // configuration
2244         } // architecture
2245     } // isPR
2246 } // scenario
2247
2248
2249 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2250 combinedScenarios.each { scenario ->
2251     [true, false].each { isPR ->
2252         // Architectures.  x64 only at this point
2253         ['x64', 'arm64'].each { architecture ->
2254             // Put the OS's supported for coreclr cross testing here
2255             Constants.crossList.each { os ->
2256                 if (architecture == 'arm64') {
2257                     if (os != "Ubuntu") {
2258                         return
2259                     }
2260                 }
2261
2262                 Constants.configurationList.each { configuration ->
2263
2264                     if (architecture == 'arm64') {
2265                         if (scenario != 'default' && scenario != 'pri1r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2266                             return
2267                         }
2268                     }
2269
2270                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2271                         if (configuration != 'Checked') {
2272                             return
2273                         }
2274                         if (isCorefxTesting(scenario)) {
2275                             return
2276                         }
2277                         //Skip stress modes for these scenarios
2278                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2279                             return
2280                         }
2281                     }
2282                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2283                     // and configuration is not Checked, bail out.
2284                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2285                         return;
2286                     }
2287                     // For CentOS, we only want Checked/Release pri1 builds.
2288                     else if (os == 'CentOS7.1') {
2289                         if (scenario != 'pri1' &&
2290                             scenario != 'r2r' &&
2291                             scenario != 'pri1r2r' &&
2292                             scenario != 'gcstress15_pri1r2r' &&
2293                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2294                             return
2295                         }
2296                         if (configuration != 'Checked' && configuration != 'Release') {
2297                             return
2298                         }
2299                     }
2300                     // For RedHat and Debian, we only do Release pri1 builds.
2301                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2302                         if (scenario != 'pri1') {
2303                             return
2304                         }
2305                         if (configuration != 'Release') {
2306                             return
2307                         }
2308                     }
2309                     else {
2310                         // Skip scenarios
2311                         switch (scenario) {
2312                             case 'pri1':
2313                                 // Nothing skipped
2314                                 break
2315                             case 'ilrt':
2316                                 // Release only
2317                                 if (configuration != 'Release') {
2318                                     return
2319                                 }
2320                                 break
2321                             case 'jitdiff':
2322                                 if (configuration != 'Checked') {
2323                                     return;
2324                                 }
2325                             case 'r2r':
2326                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2327                                 if (configuration != 'Checked' && configuration != 'Release') {
2328                                     return
2329                                 }
2330                                 break
2331                             case 'pri1r2r':
2332                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2333                                 if (configuration != 'Checked' && configuration != 'Release') {
2334                                     return
2335                                 }
2336                                 break
2337                             case 'gcstress15_pri1r2r':
2338                             case 'r2r_jitstress1':
2339                             case 'r2r_jitstress2':
2340                             case 'r2r_jitstressregs1':
2341                             case 'r2r_jitstressregs2':
2342                             case 'r2r_jitstressregs3':
2343                             case 'r2r_jitstressregs4':
2344                             case 'r2r_jitstressregs8':
2345                             case 'r2r_jitstressregs0x10':
2346                             case 'r2r_jitstressregs0x80':
2347                             case 'r2r_jitstressregs0x1000':
2348                             case 'r2r_jitminopts':
2349                             case 'r2r_jitforcerelocs':
2350                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2351                                 if (configuration != 'Checked' && configuration != 'Release') {
2352                                     return
2353                                 }
2354                                 break
2355                             case 'longgc':
2356                             case 'gcsimulator':
2357                                 // Long GC tests take a long time on non-Release builds
2358                                 if (configuration != 'Release') {
2359                                     return
2360                                 }
2361                                 break
2362                             case 'gc_reliability_framework':
2363                             case 'standalone_gc':
2364                                 if (configuration != 'Release' && configuration != 'Checked') {
2365                                     return
2366                                 }
2367                                 break
2368                             case 'coverage':
2369                                 //We only want Ubuntu Release for coverage
2370                                 if (os != 'Ubuntu') {
2371                                     return
2372                                 }
2373                                 if (configuration != 'Release') {
2374                                     return
2375                                 }
2376                             case 'formatting':
2377                                 return
2378                             case 'illink':
2379                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2380                                     return
2381                                 }
2382                                 break
2383                             case 'default':
2384                                 // Nothing skipped
2385                                 break
2386                             default:
2387                                 println("Unknown scenario: ${scenario}")
2388                                 assert false
2389                                 break
2390                         }
2391                     }
2392
2393                     def lowerConfiguration = configuration.toLowerCase()
2394                     def osGroup = getOSGroup(os)
2395                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2396
2397                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2398                     def inputScenario = 'default'
2399                     if (scenario == 'coverage') {
2400                         inputScenario = 'coverage'
2401                     }
2402                     def inputCoreCLRBuildName = projectFolder + '/' +
2403                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2404                     // If this is a stress scenario, there isn't any difference in the build job
2405                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2406                     // from the default scenario
2407                     def testBuildScenario = scenario
2408                     if (scenarioNeedsPri1Build(scenario)) {
2409                         testBuildScenario = 'pri1'
2410                     }
2411                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2412                         testBuildScenario = 'default'
2413                     }
2414                     def inputWindowTestsBuildName = ''
2415                     def inputWindowsTestBuildArch = architecture
2416                     if (architecture == "arm64") {
2417                         // Use the x64 test build for arm64 unix
2418                         inputWindowsTestBuildArch = "x64"
2419                     }
2420                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2421                         inputWindowTestsBuildName = projectFolder + '/' +
2422                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', 'default', true), isPR)
2423                     }
2424                     else {
2425                         inputWindowTestsBuildName = projectFolder + '/' +
2426                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2427                     }
2428                     // Enable Server GC for Ubuntu PR builds
2429                     def serverGCString = ''
2430
2431                     // Whether or not this test run should be run sequentially instead
2432                     // of in parallel. Only used for long GC tests.
2433                     def sequentialString = ''
2434
2435                     // Whether or not this test run should run a specific playlist.
2436                     // Only used for long GC tests.
2437
2438                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2439                     // for running long GC and GCSimulator tests, respectively. We don't use them
2440                     // here because using a playlist file produces much more readable output on the CI machines
2441                     // and reduces running time.
2442                     def playlistString = ''
2443
2444                     if (os == 'Ubuntu' && isPR){
2445                         serverGCString = '--useServerGC'
2446                     }
2447
2448                     // pass --crossgen to runtest.sh for crossgen builds
2449                     def crossgenStr = ''
2450                     def runcrossgentestsStr = ''
2451                     def runjitstressStr = ''
2452                     def runjitstressregsStr = ''
2453                     def runjitmioptsStr = ''
2454                     def runjitforcerelocsStr = ''
2455                     def runjitdisasmStr = ''
2456                     def runilasmroundtripStr = ''
2457                     def gcstressStr = ''
2458                     def illinkStr = ''
2459                     def layoutOnlyStr =''
2460
2461                     if (scenario == 'r2r' ||
2462                         scenario == 'pri1r2r' ||
2463                         scenario == 'gcstress15_pri1r2r' ||
2464                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2465                             crossgenStr = '--crossgen'
2466                             runcrossgentestsStr = '--runcrossgentests'
2467
2468                             if (scenario == 'r2r_jitstress1'){
2469                                 runjitstressStr = '--jitstress=1'
2470                             }
2471                             else if (scenario == 'r2r_jitstress2') {
2472                                 runjitstressStr = '--jitstress=2'
2473                             }
2474                             else if (scenario == 'r2r_jitstressregs1'){
2475                                 runjitstressregsStr = '--jitstressregs=1'
2476                             }
2477                             else if (scenario == 'r2r_jitstressregs2') {
2478                                 runjitstressregsStr = '--jitstressregs=2'
2479                             }
2480                             else if (scenario == 'r2r_jitstressregs3') {
2481                                 runjitstressregsStr = '--jitstressregs=3'
2482                             }
2483                             else if (scenario == 'r2r_jitstressregs4') {
2484                                 runjitstressregsStr = '--jitstressregs=4'
2485                             }
2486                             else if (scenario == 'r2r_jitstressregs8') {
2487                                 runjitstressregsStr = '--jitstressregs=8'
2488                             }
2489                             else if (scenario == 'r2r_jitstressregs0x10') {
2490                                 runjitstressregsStr = '--jitstressregs=0x10'
2491                             }
2492                             else if (scenario == 'r2r_jitstressregs0x80') {
2493                                 runjitstressregsStr = '--jitstressregs=0x80'
2494                             }
2495                             else if (scenario == 'r2r_jitstressregs0x1000') {
2496                                 runjitstressregsStr = '--jitstressregs=0x1000'
2497                             }
2498                             else if (scenario == 'r2r_jitminopts') {
2499                                 runjitmioptsStr = '--jitminopts'
2500                             }
2501                             else if (scenario == 'r2r_jitforcerelocs') {
2502                                 runjitforcerelocsStr = '--jitforcerelocs'
2503                             }
2504                     }
2505                     if  (scenario == 'gcstress15_pri1r2r')
2506                     {
2507                         gcstressStr = '--gcstresslevel=0xF'
2508                     }
2509
2510                     if (scenario == 'jitdiff')
2511                     {
2512                         runjitdisasmStr = '--jitdisasm --crossgen'
2513                     }
2514
2515                     if (scenario == 'illink')
2516                     {
2517                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2518                     }
2519
2520                     if (isLongGc(scenario)) {
2521                         // Long GC tests behave very poorly when they are not
2522                         // the only test running (many of them allocate until OOM).
2523                         sequentialString = '--sequential'
2524
2525                         // The Long GC playlist contains all of the tests that are
2526                         // going to be run. The GCSimulator playlist contains all of
2527                         // the GC simulator tests.
2528                         if (scenario == 'longgc') {
2529                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2530                         }
2531                         else if (scenario == 'gcsimulator') {
2532                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2533                         }
2534                     }
2535
2536                     if (isGcReliabilityFramework(scenario)) {
2537                         layoutOnlyStr = '--build-overlay-only'
2538                     }
2539
2540                     def folder = getJobFolder(scenario)
2541                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2542                         // Add parameters for the inputs
2543
2544                         parameters {
2545                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2546                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2547                         }
2548
2549                         steps {
2550                             // Set up the copies
2551
2552                             // Coreclr build containing the tests and mscorlib
2553
2554                             copyArtifacts(inputWindowTestsBuildName) {
2555                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2556                                 buildSelector {
2557                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2558                                 }
2559                             }
2560
2561                             if (scenario == 'coverage') {
2562
2563                                 // Move coreclr to clr directory
2564                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2565
2566                                 // Build coreclr
2567                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2568
2569                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2570                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2571                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2572                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2573                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2574                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2575                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2576                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2577
2578                                 // Run PAL tests
2579                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2580
2581                                 // Remove obj files for PAL tests so they're not included in coverage results
2582                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2583
2584                                 // Unzip the tests first.  Exit with 0
2585                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2586
2587                                 // Get corefx
2588                                 shell("git clone https://github.com/dotnet/corefx fx")
2589
2590                                 // Build Linux corefx
2591                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2592                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2593
2594                                 def testEnvOpt = ""
2595                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2596                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2597                                 shell("${createScriptCmds}")
2598                                 testEnvOpt = "--test-env=" + scriptFileName
2599
2600                                 // Run corefx tests
2601                                 shell("""./fx/run-test.sh \\
2602                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2603                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2604                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2605                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2606                 --configurationGroup Release""")
2607
2608
2609                                 // Run coreclr tests w/ workstation GC
2610                                 shell("""./clr/tests/runtest.sh \\
2611                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2612                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2613                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2614                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2615                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2616                 --crossgen --runcrossgentests""")
2617
2618                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2619                                 shell("""./clr/tests/runtest.sh \\
2620                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2621                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2622                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2623                 --useServerGC ${testEnvOpt}""")
2624
2625                                  // Run long-running coreclr GC tests & produce coverage reports
2626                                 shell("""./clr/tests/runtest.sh \\
2627                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2628                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2629                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2630                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2631                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2632                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2633                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2634
2635                             }
2636                             else {
2637
2638                                 // Coreclr build we are trying to test
2639
2640                                 copyArtifacts(inputCoreCLRBuildName) {
2641                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2642                                     buildSelector {
2643                                         buildNumber('${CORECLR_BUILD}')
2644                                     }
2645                                 }
2646
2647                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2648
2649                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2650                                 def osJobName
2651                                 if (os == 'Ubuntu') {
2652                                     osJobName = 'ubuntu14.04'
2653                                 }
2654                                 else {
2655                                     osJobName = os.toLowerCase()
2656                                 }
2657                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2658                                     includePatterns('bin/build.tar.gz')
2659                                     buildSelector {
2660                                         latestSuccessful(true)
2661                                     }
2662                                 }
2663
2664                                 shell ("mkdir ./bin/CoreFxBinDir")
2665                                 // Unpack the corefx binaries
2666                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2667
2668                                 // HACK -- Arm64 does not have corefx jobs yet.
2669                                 // Clone corefx and build the native packages overwriting the x64 packages.
2670                                 if (architecture == 'arm64') {
2671                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2672                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2673                                 }
2674
2675                                 // Unzip the tests first.  Exit with 0
2676                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2677
2678                                 // Execute the tests
2679                                 // If we are running a stress mode, we'll set those variables first
2680                                 def testEnvOpt = ""
2681                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2682                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2683                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2684                                     shell("${createScriptCmds}")
2685                                     testEnvOpt = "--test-env=" + scriptFileName
2686                                 }
2687
2688                                 if (isGCStressRelatedTesting(scenario)) {
2689                                     shell('./init-tools.sh')
2690                                 }
2691
2692                                 shell("""./tests/runtest.sh \\
2693                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2694                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2695                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2696                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2697                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2698                 --limitedDumpGeneration \\
2699                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
2700                 ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
2701                 ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
2702
2703                                 if (isGcReliabilityFramework(scenario)) {
2704                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2705                                     if (serverGCString != '') {
2706                                         shell("export COMPlus_gcServer=1")
2707                                     }
2708
2709                                     shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2710                                 }
2711                             }
2712                         }
2713                     }
2714
2715                     if (scenario == 'coverage') {
2716                         // Publish coverage reports
2717                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2718                         // TODO: Add once external email sending is available again
2719                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2720                     }
2721
2722                     if (scenario == 'jitdiff') {
2723                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2724                     }
2725
2726                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2727                     if (os in ['Ubuntu']) {
2728                         SummaryBuilder summaries = new SummaryBuilder()
2729                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2730                         summaries.emit(newJob)
2731                     }
2732
2733                     setMachineAffinity(newJob, os, architecture)
2734                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2735                     // Set timeouts to 240.
2736                     setTestJobTimeOut(newJob, scenario)
2737
2738                     if (architecture == 'arm64') {
2739                         Utilities.setJobTimeout(newJob, 240)
2740                     }
2741
2742                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2743
2744                     // Create a build flow to join together the build and tests required to run this
2745                     // test.
2746                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2747                     // Linux CoreCLR test
2748                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2749                     def fullTestJobName = projectFolder + '/' + newJob.name
2750                     // Add a reference to the input jobs for report purposes
2751                     JobReport.Report.addReference(inputCoreCLRBuildName)
2752                     JobReport.Report.addReference(inputWindowTestsBuildName)
2753                     JobReport.Report.addReference(fullTestJobName)
2754                     def newFlowJob;
2755
2756                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2757                     // so that coverage data can be preserved.
2758                     if (scenario == 'coverage') {
2759                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2760                         buildFlow("""
2761 // Build the input Windows job
2762 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2763
2764 // And then build the test build
2765 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2766 """)
2767                         }
2768                     // Normal jobs copy a Windows build & a non-Windows build
2769                     } else {
2770                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2771                         buildFlow("""
2772 // Build the input jobs in parallel
2773 parallel (
2774     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2775     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2776 )
2777
2778 // And then build the test build
2779 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2780                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2781 """)
2782                         }
2783                     }
2784
2785                     setMachineAffinity(newFlowJob, os, architecture)
2786                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2787                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2788                 } // configuration
2789             } // os
2790         } // architecture
2791     } // isPR
2792 } // scenario
2793
2794 JobReport.Report.generateJobReport(out)
2795
2796 // Make the call to generate the help job
2797 Utilities.createHelperJob(this, project, branch,
2798     "Welcome to the ${project} Repository",  // This is prepended to the help message
2799     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2800
2801 Utilities.addCROSSCheck(this, project, branch)