netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs
  12 folder('jitstress')
  13
  14 // Create a folder for testing via illink
  15 folder('illink')
  16
  17 def static getOSGroup(def os) {
  18     def osGroupMap = ['Ubuntu':'Linux',
  19         'RHEL7.2': 'Linux',
  20         'Ubuntu16.04': 'Linux',
  21         'Ubuntu16.10': 'Linux',
  22         'Debian8.4':'Linux',
  23         'Fedora23':'Linux',
  24         'OSX10.12':'OSX',
  25         'Windows_NT':'Windows_NT',
  26         'FreeBSD':'FreeBSD',
  27         'CentOS7.1': 'Linux',
  28         'OpenSUSE42.1': 'Linux',
  29         'Tizen': 'Linux',
  30         'LinuxARMEmulator': 'Linux']
  31     def osGroup = osGroupMap.get(os, null)
  32     assert osGroup != null : "Could not find os group for ${os}"
  33     return osGroupMap[os]
  34 }
  35
  36 // We use this class (vs variables) so that the static functions can access data here.
  37 class Constants {
  38
  39     // Innerloop build OS's
  40     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  41     // test execution in the build flow runs.  It generates the exact same build
  42     // as Windows_NT but without the tests.
  43     def static osList = [
  44                'Ubuntu',
  45                'Debian8.4',
  46                'OSX10.12',
  47                'Windows_NT',
  48                'Windows_NT_BuildOnly',
  49                'FreeBSD',
  50                'CentOS7.1',
  51                'OpenSUSE42.1',
  52                'RHEL7.2',
  53                'LinuxARMEmulator',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Tizen',
  57                'Fedora23']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  67                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  68                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  69                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  70                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  71                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  72                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  73                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  74                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  75                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  76                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  77                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  78                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  79                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  80                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  81                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  82                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  83                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  84                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  85                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  86                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  87                'corefx_baseline'                : [ : ], // corefx baseline
  88                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  89                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  90                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  91                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  92                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  93                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  94                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  95                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  96                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  97                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  98                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  99                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 100                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 101                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 103                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 104                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 105                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 107                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 109                ]
 110
 111     // This is a set of r2r jit stress scenarios
 112     def static r2rJitStressScenarios = [
 113                'r2r_jitstress1',
 114                'r2r_jitstress2',
 115                'r2r_jitstressregs1',
 116                'r2r_jitstressregs2',
 117                'r2r_jitstressregs3',
 118                'r2r_jitstressregs4',
 119                'r2r_jitstressregs8',
 120                'r2r_jitstressregs0x10',
 121                'r2r_jitstressregs0x80',
 122                'r2r_jitstressregs0x1000',
 123                'r2r_jitminopts',
 124                'r2r_jitforcerelocs']
 125
 126     // This is the basic set of scenarios
 127     def static basicScenarios = [
 128                'default',
 129                'pri1',
 130                'ilrt',
 131                'r2r',
 132                'pri1r2r',
 133                'gcstress15_pri1r2r',
 134                'longgc',
 135                'coverage',
 136                'formatting',
 137                'gcsimulator',
 138                'jitdiff',
 139                'standalone_gc',
 140                'illink'] + r2rJitStressScenarios
 141
 142     def static configurationList = ['Debug', 'Checked', 'Release']
 143
 144     // This is the set of architectures
 145     def static architectureList = ['arm', 'arm64', 'x64', 'x86', 'x86compatjit']
 146 }
 147
 148 def static setMachineAffinity(def job, def os, def architecture) {
 149     if (architecture == 'arm64' && os == 'Windows_NT') {
 150         // For cross compilation
 151         job.with {
 152             label('arm64')
 153         }
 154     } else if (architecture == 'arm64' && os == 'Ubuntu') {
 155         Utilities.setMachineAffinity(job, os, 'arm-cross-latest');
 156     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 157         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 158     } else {
 159         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 160     }
 161 }
 162
 163 def static isJITStressJob(def scenario) {
 164     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 165            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 166 }
 167
 168 def static isGCStressRelatedTesting(def scenario) {
 169     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 170     // Detect it and make it a GCStress related.
 171     if (scenario == 'gcstress15_pri1r2r')
 172     {
 173         return true;
 174     }
 175
 176     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 177     def scenarioName = scenario.toLowerCase()
 178     def isGCStressTesting = false
 179     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 180         if (k in gcStressTestEnvVars) {
 181             isGCStressTesting = true;
 182         }
 183     }
 184     return isGCStressTesting
 185 }
 186
 187 def static isCorefxTesting(def scenario) {
 188     def corefx_prefix = 'corefx_'
 189     if (scenario.length() < corefx_prefix.length()) {
 190         return false
 191     }
 192     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 193 }
 194
 195 def static isR2R(def scenario) {
 196     return (scenario == 'r2r' || scenario == 'pri1r2r')
 197 }
 198
 199 def static isCoverage(def scenario) {
 200     return (scenario == 'coverage')
 201 }
 202
 203 def static isLongGc(def scenario) {
 204     return (scenario == 'longgc' || scenario == 'gcsimulator')
 205 }
 206
 207 def static isJitDiff(def scenario) {
 208     return (scenario == 'jitdiff')
 209 }
 210
 211 def static setTestJobTimeOut(newJob, scenario) {
 212     if (isGCStressRelatedTesting(scenario)) {
 213         Utilities.setJobTimeout(newJob, 4320)
 214     }
 215     else if (isCorefxTesting(scenario)) {
 216         Utilities.setJobTimeout(newJob, 360)
 217     }
 218     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 219         Utilities.setJobTimeout(newJob, 240)
 220     }
 221     else if (isR2R(scenario)) {
 222         Utilities.setJobTimeout(newJob, 240)
 223     }
 224     else if (isCoverage(scenario)) {
 225         Utilities.setJobTimeout(newJob, 1440)
 226     }
 227     else if (isLongGc(scenario)) {
 228         Utilities.setJobTimeout(newJob, 1440)
 229     }
 230     else if (isJitDiff(scenario)) {
 231         Utilities.setJobTimeout(newJob, 240)
 232     }
 233     // Non-test jobs use the default timeout value.
 234 }
 235
 236 def static getJobFolder(def scenario) {
 237     if (isJITStressJob(scenario)) {
 238         return 'jitstress'
 239     }
 240     if (scenario == 'illink') {
 241         return 'illink'
 242     }
 243     return ''
 244 }
 245
 246 def static getStressModeDisplayName(def scenario) {
 247     def displayStr = ''
 248     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 249         def prefixLength = 'COMPlus_'.length()
 250         if (k.length() >= prefixLength) {
 251             def modeName = k.substring(prefixLength, k.length())
 252             displayStr += ' ' + modeName + '=' + v
 253         }
 254     }
 255     return displayStr
 256 }
 257
 258 def static getR2RStressModeDisplayName(def scenario) {
 259     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 260     // "r2r_" prefix.
 261     def displayStr = scenario
 262     def prefixLength = 'r2r_'.length()
 263     if (displayStr.length() >= prefixLength) {
 264         displayStr = displayStr.substring(prefixLength, displayStr.length())
 265     }
 266     return displayStr
 267 }
 268
 269 // Generates the string for creating a file that sets environment variables
 270 // that makes it possible to run stress modes.  Writes the script to the file
 271 // specified by the stepScriptLocation parameter.
 272 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 273     def stepScript = ''
 274     if (os == 'Windows_NT') {
 275         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 276         stepScript += "del ${stepScriptLocation}\r\n"
 277
 278         // Timeout in ms, default is 10 minutes. For stress
 279         // modes up this to 30 minutes
 280         def timeout = 1800000
 281
 282         // Set the Timeout
 283         stepScript += "set __TestTimeout=${timeout}\r\n"
 284         stepScript += "echo. > ${stepScriptLocation}\r\n"
 285         stressModeVars.each{ k, v ->
 286             // Write out what we are writing to the script file
 287             stepScript += "echo Setting ${k}=${v}\r\n"
 288             // Write out the set itself to the script file`
 289             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 290         }
 291     }
 292     else {
 293         stepScript += "echo Setting variables for ${stressModeName}\n"
 294         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 295         stressModeVars.each{ k, v ->
 296             // Write out what we are writing to the script file
 297             stepScript += "echo Setting ${k}=${v}\n"
 298             // Write out the set itself to the script file`
 299             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 300         }
 301         stepScript += "chmod +x ${stepScriptLocation}\n"
 302     }
 303     return stepScript
 304 }
 305
 306 // Calculates the name of the build job based on some typical parameters.
 307 //
 308 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly, def isLinuxEmulatorBuild = false) {
 309     // If the architecture is x64, do not add that info into the build name.
 310     // Need to change around some systems and other builds to pick up the right builds
 311     // to do that.
 312
 313     def suffix = scenario != 'default' ? "_${scenario}" : '';
 314     if (isBuildOnly) {
 315         suffix += '_bld'
 316     }
 317     def baseName = ''
 318     switch (architecture) {
 319         case 'x64':
 320             if (scenario == 'default') {
 321                 // For now we leave x64 off of the name for compatibility with other jobs
 322                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 323             }
 324             else if (scenario == 'formatting') {
 325                 // we don't care about the configuration for the formatting job. It runs all configs
 326                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 327             }
 328             else if (scenario == 'illink') {
 329                 // Identify that the job ran post-illink
 330                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase() + '_illink'
 331             }
 332             else {
 333                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 334             }
 335             break
 336         case 'arm64':
 337             // These are cross builds
 338             if (isLinuxEmulatorBuild == false) {
 339                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 340             }
 341             else {
 342                 baseName = architecture.toLowerCase() + '_emulator_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 343             }
 344             break
 345         case 'arm':
 346             // These are cross builds
 347             if (isLinuxEmulatorBuild == false) {
 348                 if (os == 'Tizen') {
 349                     // ABI: softfp
 350                     baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 351                 }
 352                 else {
 353                     baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 354                 }
 355             }
 356             else {
 357                 baseName = architecture.toLowerCase() + '_emulator_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 358             }
 359             break
 360         case 'x86':
 361             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 362             break
 363         case 'x86compatjit':
 364             baseName = 'x86_compatjit_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 365             break
 366         default:
 367             println("Unknown architecture: ${architecture}");
 368             assert false
 369             break
 370     }
 371
 372     return baseName + suffix
 373 }
 374
 375 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def isLinuxEmulatorBuild, def bidailyCrossList) {
 376     // Check scenario.
 377     switch (scenario) {
 378         case 'default':
 379             switch (architecture) {
 380                 case 'x64':
 381                 case 'x86':
 382                 case 'x86compatjit':
 383                     if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 384                         Utilities.addGithubPushTrigger(job)
 385                     }
 386                     break
 387                 case 'arm':
 388                     Utilities.addGithubPushTrigger(job)
 389                     break
 390                 case 'arm64':
 391                     if (os == 'Windows_NT') {
 392                         Utilities.addGithubPushTrigger(job)
 393                         // TODO: Add once external email sending is available again
 394                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 395                     }
 396                     break
 397                 default:
 398                     println("Unknown architecture: ${architecture}");
 399                     assert false
 400                     break
 401             }
 402             break
 403         case 'pri1':
 404             // Pri one gets a push trigger, and only for release
 405             if (architecture == 'x64') {
 406                 if (configuration == 'Release') {
 407                     // We expect release jobs to be Windows, or in the cross list
 408                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 409                     if (!os in bidailyCrossList) {
 410                         if (isFlowJob || os == 'Windows_NT') {
 411                             Utilities.addGithubPushTrigger(job)
 412                         }
 413                     }
 414                     else {
 415                         if (isFlowJob) {
 416                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 417                         }
 418                     }
 419                 }
 420             }
 421             break
 422         case 'r2r':
 423             //r2r jobs that aren't pri1 can only be triggered by phrase
 424             break
 425         case 'pri1r2r':
 426             assert !(os in bidailyCrossList)
 427             //pri1 r2r gets a push trigger for checked/release
 428             if (configuration == 'Checked' || configuration == 'Release') {
 429                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 430                 if (architecture == 'x64' && os != 'OSX10.12') {
 431                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 432                     if (isFlowJob || os == 'Windows_NT') {
 433                         Utilities.addGithubPushTrigger(job)
 434                     }
 435                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 436                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 437                     if (isFlowJob) {
 438                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 439                     }
 440                 }
 441                 // For x86, only add per-commit jobs for Windows
 442                 else if (architecture == 'x86' || architecture == 'x86compatjit') {
 443                     if (os == 'Windows_NT') {
 444                         Utilities.addGithubPushTrigger(job)
 445                     }
 446                 }
 447                 // arm64 pri1r2r jobs should only run every 12 hours.
 448                 else if (architecture == 'arm64') {
 449                     if (os == 'Windows_NT') {
 450                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 451                         // TODO: Add once external email sending is available again
 452                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 453                     }
 454                 }
 455             }
 456             break
 457         case 'r2r_jitstress1':
 458         case 'r2r_jitstress2':
 459         case 'r2r_jitstressregs1':
 460         case 'r2r_jitstressregs2':
 461         case 'r2r_jitstressregs3':
 462         case 'r2r_jitstressregs4':
 463         case 'r2r_jitstressregs8':
 464         case 'r2r_jitstressregs0x10':
 465         case 'r2r_jitstressregs0x80':
 466         case 'r2r_jitstressregs0x1000':
 467         case 'r2r_jitminopts':
 468         case 'r2r_jitforcerelocs':
 469         case 'gcstress15_pri1r2r':
 470             assert !(os in bidailyCrossList)
 471
 472             // GCStress=C is currently not supported on OS X
 473             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 474                 break
 475             }
 476
 477             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 478             if (configuration == 'Checked' || configuration == 'Release') {
 479                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 480                 if (architecture == 'x64') {
 481                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 482                     if (isFlowJob || os == 'Windows_NT') {
 483                         // Add a weekly periodic trigger
 484                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 485                     }
 486                 }
 487                 // For x86, only add per-commit jobs for Windows
 488                 else if (architecture == 'x86' || architecture == 'x86compatjit') {
 489                     if (os == 'Windows_NT') {
 490                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 491                     }
 492                 }
 493             }
 494             break
 495         case 'longgc':
 496             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 497             assert configuration == 'Release'
 498             assert architecture == 'x64'
 499             Utilities.addPeriodicTrigger(job, '@daily')
 500             // TODO: Add once external email sending is available again
 501             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 502             break
 503         case 'gcsimulator':
 504             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 505             assert configuration == 'Release'
 506             assert architecture == 'x64'
 507             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 508             // TODO: Add once external email sending is available again
 509             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 510             break
 511         case 'standalone_gc':
 512             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 513             assert (configuration == 'Release' || configuration == 'Checked')
 514             // TODO: Add once external email sending is available again
 515             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 516             Utilities.addPeriodicTrigger(job, '@weekly')
 517             break
 518         case 'ilrt':
 519             assert !(os in bidailyCrossList)
 520             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 521             if (architecture == 'x64' && configuration == 'Release') {
 522                 // We don't expect to see a job generated except in these scenarios
 523                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 524                 if (isFlowJob || os == 'Windows_NT') {
 525                     Utilities.addPeriodicTrigger(job, '@daily')
 526                 }
 527             }
 528             break
 529         case 'jitdiff':
 530             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 531             assert configuration == 'Checked'
 532             assert (architecture == 'x64' || architecture == 'x86')
 533             Utilities.addGithubPushTrigger(job)
 534             break
 535         case 'coverage':
 536             assert (os == 'Ubuntu' || os == 'Windows_NT')
 537             assert configuration == 'Release'
 538             assert architecture == 'x64'
 539             Utilities.addPeriodicTrigger(job, '@weekly')
 540             break
 541         case 'formatting':
 542             assert (os == 'Windows_NT' || os == "Ubuntu")
 543             assert architecture == 'x64'
 544             Utilities.addGithubPushTrigger(job)
 545             break
 546         case 'jitstressregs1':
 547         case 'jitstressregs2':
 548         case 'jitstressregs3':
 549         case 'jitstressregs4':
 550         case 'jitstressregs8':
 551         case 'jitstressregs0x10':
 552         case 'jitstressregs0x80':
 553         case 'jitstressregs0x1000':
 554         case 'minopts':
 555         case 'forcerelocs':
 556         case 'jitstress1':
 557         case 'jitstress2':
 558         case 'jitstress2_jitstressregs1':
 559         case 'jitstress2_jitstressregs2':
 560         case 'jitstress2_jitstressregs3':
 561         case 'jitstress2_jitstressregs4':
 562         case 'jitstress2_jitstressregs8':
 563         case 'jitstress2_jitstressregs0x10':
 564         case 'jitstress2_jitstressregs0x80':
 565         case 'jitstress2_jitstressregs0x1000':
 566         case 'tailcallstress':
 567         case 'jitsse2only':
 568         case 'corefx_baseline':
 569         case 'corefx_minopts':
 570         case 'corefx_jitstress1':
 571         case 'corefx_jitstress2':
 572         case 'corefx_jitstressregs1':
 573         case 'corefx_jitstressregs2':
 574         case 'corefx_jitstressregs3':
 575         case 'corefx_jitstressregs4':
 576         case 'corefx_jitstressregs8':
 577         case 'corefx_jitstressregs0x10':
 578         case 'corefx_jitstressregs0x80':
 579         case 'corefx_jitstressregs0x1000':
 580         case 'zapdisable':
 581             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 582             assert (os == 'Windows_NT') || (os in Constants.crossList)
 583             Utilities.addPeriodicTrigger(job, '@daily')
 584         }
 585         break
 586         case 'heapverify1':
 587         case 'gcstress0x3':
 588             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 589                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 590                 if (architecture == 'arm64') {
 591                     assert (os == 'Windows_NT')
 592                     Utilities.addPeriodicTrigger(job, '@daily')
 593                     // TODO: Add once external email sending is available again
 594                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 595                 }
 596                 else {
 597                     Utilities.addPeriodicTrigger(job, '@weekly')
 598                 }
 599             }
 600             break
 601         case 'gcstress0xc':
 602         case 'gcstress0xc_zapdisable':
 603         case 'gcstress0xc_zapdisable_jitstress2':
 604         case 'gcstress0xc_zapdisable_heapverify1':
 605         case 'gcstress0xc_jitstress1':
 606         case 'gcstress0xc_jitstress2':
 607         case 'gcstress0xc_minopts_heapverify1':
 608             // GCStress=C is currently not supported on OS X
 609             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 610                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 611                 if (architecture == 'arm64') {
 612                     assert (os == 'Windows_NT')
 613                     // TODO: Enable a periodic trigger after tests are updated.
 614                     // Utilities.addPeriodicTrigger(job, '@daily')
 615                     // TODO: Add once external email sending is available again
 616                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 617                 }
 618                 else {
 619                     Utilities.addPeriodicTrigger(job, '@weekly')
 620                 }
 621             }
 622             break
 623
 624         case 'illink':
 625             // Testing on other operating systems TBD
 626             assert (os == 'Windows_NT')
 627             assert (architecture == 'x64' || architecture == 'x86' || architecture == 'x86compatjit')
 628             Utilities.addPeriodicTrigger(job, '@daily')
 629             break
 630
 631         default:
 632             println("Unknown scenario: ${scenario}");
 633             assert false
 634             break
 635     }
 636     return
 637 }
 638
 639 // **************************
 640 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 641 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 642 // builds will do a couple extra steps.
 643 // **************************
 644
 645 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 646 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 647 // it's just used for internal builds
 648 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 649 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def isLinuxEmulatorBuild) {
 650     if (isWindowsBuildOnlyJob) {
 651         return
 652     }
 653
 654     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 655     // Non pull request builds.
 656     if (!isPR) {
 657         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, isLinuxEmulatorBuild, bidailyCrossList)
 658         return
 659     }
 660     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 661     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 662     def osGroup = getOSGroup(os)
 663     switch (architecture) {
 664         case 'x64': // editor brace matching: {
 665             if (scenario == 'coverage') {
 666                 assert configuration == 'Release'
 667                 if (os == 'Ubuntu') {
 668                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 669                 }
 670                 break
 671             }
 672
 673             if (scenario == 'formatting') {
 674                 assert configuration == 'Checked'
 675                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 676                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 677                 }
 678                 break
 679             }
 680
 681             switch (os) {
 682                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 683                 case 'Debian8.4':
 684                 case 'RHEL7.2':
 685                     if (scenario == 'default') {
 686                         assert !isFlowJob
 687                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 688                     }
 689                     else if (scenario == 'pri1' && isFlowJob) {
 690                         assert (configuration == 'Release')
 691                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 692                     }
 693                     break
 694                 case 'Fedora23':
 695                 case 'Ubuntu16.04':
 696                 case 'Ubuntu16.10':
 697                 case 'OpenSUSE42.1':
 698                     assert !isFlowJob
 699                     assert scenario == 'default'
 700                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 701                     break
 702                 case 'Ubuntu':
 703                 case 'OSX10.12':
 704                     // Triggers on the non-flow jobs aren't necessary here
 705                     // Corefx testing uses non-flow jobs.
 706                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 707                         break
 708                     }
 709                     switch (scenario) {
 710                         case 'default':
 711                             // Ubuntu uses checked for default PR tests
 712                             if (configuration == 'Checked') {
 713                                 // Default trigger
 714                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 715                             }
 716                             break
 717                         case 'pri1':
 718                             if (configuration == 'Release') {
 719                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 720                             }
 721                             break
 722                         case 'jitdiff':
 723                             if (configuration == 'Checked') {
 724                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 725                             }
 726                             break
 727                         case 'ilrt':
 728                             if (configuration == 'Release') {
 729                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 730                             }
 731                             break
 732                         case 'r2r':
 733                             if (configuration == 'Release' || configuration == 'Checked') {
 734                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 735                             }
 736                             break
 737                         case 'pri1r2r':
 738                             if (configuration == 'Release' || configuration == 'Checked') {
 739                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 740                             }
 741                             break
 742                         case 'gcstress15_pri1r2r':
 743                             if (configuration == 'Release' || configuration == 'Checked') {
 744                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 745                             }
 746                             break
 747                         case 'r2r_jitstress1':
 748                         case 'r2r_jitstress2':
 749                         case 'r2r_jitstressregs1':
 750                         case 'r2r_jitstressregs2':
 751                         case 'r2r_jitstressregs3':
 752                         case 'r2r_jitstressregs4':
 753                         case 'r2r_jitstressregs8':
 754                         case 'r2r_jitstressregs0x10':
 755                         case 'r2r_jitstressregs0x80':
 756                         case 'r2r_jitstressregs0x1000':
 757                         case 'r2r_jitminopts':
 758                         case 'r2r_jitforcerelocs':
 759                             if (configuration == 'Release' || configuration == 'Checked') {
 760                                 def displayStr = getR2RStressModeDisplayName(scenario)
 761                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 762                             }
 763                             break
 764                         case 'longgc':
 765                             if (configuration == 'Release') {
 766                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 767                             }
 768                             break
 769                         case 'gcsimulator':
 770                             if (configuration == 'Release') {
 771                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 772                             }
 773                             break
 774                         case 'minopts':
 775                         case 'forcerelocs':
 776                         case 'jitstress1':
 777                         case 'jitstress2':
 778                         case 'jitstressregs1':
 779                         case 'jitstressregs2':
 780                         case 'jitstressregs3':
 781                         case 'jitstressregs4':
 782                         case 'jitstressregs8':
 783                         case 'jitstressregs0x10':
 784                         case 'jitstressregs0x80':
 785                         case 'jitstressregs0x1000':
 786                         case 'jitstress2_jitstressregs1':
 787                         case 'jitstress2_jitstressregs2':
 788                         case 'jitstress2_jitstressregs3':
 789                         case 'jitstress2_jitstressregs4':
 790                         case 'jitstress2_jitstressregs8':
 791                         case 'jitstress2_jitstressregs0x10':
 792                         case 'jitstress2_jitstressregs0x80':
 793                         case 'jitstress2_jitstressregs0x1000':
 794                         case 'tailcallstress':
 795                         case 'jitsse2only':
 796                         case 'gcstress0x3':
 797                         case 'gcstress0xc':
 798                         case 'zapdisable':
 799                         case 'heapverify1':
 800                         case 'gcstress0xc_zapdisable':
 801                         case 'gcstress0xc_zapdisable_jitstress2':
 802                         case 'gcstress0xc_zapdisable_heapverify1':
 803                         case 'gcstress0xc_jitstress1':
 804                         case 'gcstress0xc_jitstress2':
 805                         case 'gcstress0xc_minopts_heapverify1':
 806                             def displayStr = getStressModeDisplayName(scenario)
 807                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 808                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 809                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 810                             break
 811                         case 'corefx_baseline':
 812                         case 'corefx_minopts':
 813                         case 'corefx_jitstress1':
 814                         case 'corefx_jitstress2':
 815                         case 'corefx_jitstressregs1':
 816                         case 'corefx_jitstressregs2':
 817                         case 'corefx_jitstressregs3':
 818                         case 'corefx_jitstressregs4':
 819                         case 'corefx_jitstressregs8':
 820                         case 'corefx_jitstressregs0x10':
 821                         case 'corefx_jitstressregs0x80':
 822                         case 'corefx_jitstressregs0x1000':
 823                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 824                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 825                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 826                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 827                             break
 828                         default:
 829                             println("Unknown scenario: ${scenario}");
 830                             assert false
 831                             break
 832                     }
 833                     break
 834                 case 'CentOS7.1':
 835                     switch (scenario) {
 836                         case 'pri1':
 837                             if (configuration == 'Release') {
 838                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 839                             }
 840                             break
 841                         case 'r2r':
 842                             if (configuration == 'Checked' || configuration == 'Release') {
 843                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 844                             }
 845                             break
 846                         case 'pri1r2r':
 847                             if (configuration == 'Checked' || configuration == 'Release') {
 848                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 849                             }
 850                             break
 851                         case 'gcstress15_pri1r2r':
 852                             if (configuration == 'Release' || configuration == 'Checked') {
 853                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 854                             }
 855                             break
 856                         case 'r2r_jitstress1':
 857                         case 'r2r_jitstress2':
 858                         case 'r2r_jitstressregs1':
 859                         case 'r2r_jitstressregs2':
 860                         case 'r2r_jitstressregs3':
 861                         case 'r2r_jitstressregs4':
 862                         case 'r2r_jitstressregs8':
 863                         case 'r2r_jitstressregs0x10':
 864                         case 'r2r_jitstressregs0x80':
 865                         case 'r2r_jitstressregs0x1000':
 866                         case 'r2r_jitminopts':
 867                         case 'r2r_jitforcerelocs':
 868                             if (configuration == 'Release' || configuration == 'Checked') {
 869                                 def displayStr = getR2RStressModeDisplayName(scenario)
 870                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 871                             }
 872                             break
 873                         default:
 874                             break
 875                     }
 876                 case 'Windows_NT':
 877                     switch (scenario) {
 878                         case 'default':
 879                             // Default trigger
 880                             if (configuration == 'Debug') {
 881                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 882                             }
 883                             break
 884                         case 'pri1':
 885                             // Default trigger
 886                             if (configuration == 'Release') {
 887                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 888                             }
 889                             break
 890                         case 'jitdiff':
 891                             if (configuration == 'Checked') {
 892                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 893                             }
 894                             break
 895                         case 'ilrt':
 896                             if (configuration == 'Release') {
 897                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 898                             }
 899                             break
 900                         case 'r2r':
 901                             if (configuration == 'Checked' || configuration == 'Release') {
 902                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 903                             }
 904                             break
 905                         case 'pri1r2r':
 906                             if (configuration == 'Checked' || configuration == 'Release') {
 907                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 908                             }
 909                             break
 910                         case 'gcstress15_pri1r2r':
 911                             if (configuration == 'Release' || configuration == 'Checked') {
 912                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 913                             }
 914                             break
 915                         case 'r2r_jitstress1':
 916                         case 'r2r_jitstress2':
 917                         case 'r2r_jitstressregs1':
 918                         case 'r2r_jitstressregs2':
 919                         case 'r2r_jitstressregs3':
 920                         case 'r2r_jitstressregs4':
 921                         case 'r2r_jitstressregs8':
 922                         case 'r2r_jitstressregs0x10':
 923                         case 'r2r_jitstressregs0x80':
 924                         case 'r2r_jitstressregs0x1000':
 925                         case 'r2r_jitminopts':
 926                         case 'r2r_jitforcerelocs':
 927                             if (configuration == 'Release' || configuration == 'Checked') {
 928                                 def displayStr = getR2RStressModeDisplayName(scenario)
 929                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 930                             }
 931                             break
 932                         case 'longgc':
 933                             if (configuration == 'Release') {
 934                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 935                             }
 936                             break
 937                         case 'gcsimulator':
 938                             if (configuration == 'Release') {
 939                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 940                             }
 941                             break
 942                         case 'standalone_gc':
 943                             if (configuration == 'Release' || configuration == 'Checked') {
 944                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 945                             }
 946                             break
 947                         case 'minopts':
 948                         case 'forcerelocs':
 949                         case 'jitstress1':
 950                         case 'jitstress2':
 951                         case 'jitstressregs1':
 952                         case 'jitstressregs2':
 953                         case 'jitstressregs3':
 954                         case 'jitstressregs4':
 955                         case 'jitstressregs8':
 956                         case 'jitstressregs0x10':
 957                         case 'jitstressregs0x80':
 958                         case 'jitstressregs0x1000':
 959                         case 'jitstress2_jitstressregs1':
 960                         case 'jitstress2_jitstressregs2':
 961                         case 'jitstress2_jitstressregs3':
 962                         case 'jitstress2_jitstressregs4':
 963                         case 'jitstress2_jitstressregs8':
 964                         case 'jitstress2_jitstressregs0x10':
 965                         case 'jitstress2_jitstressregs0x80':
 966                         case 'jitstress2_jitstressregs0x1000':
 967                         case 'tailcallstress':
 968                         case 'jitsse2only':
 969                         case 'gcstress0x3':
 970                         case 'gcstress0xc':
 971                         case 'zapdisable':
 972                         case 'heapverify1':
 973                         case 'gcstress0xc_zapdisable':
 974                         case 'gcstress0xc_zapdisable_jitstress2':
 975                         case 'gcstress0xc_zapdisable_heapverify1':
 976                         case 'gcstress0xc_jitstress1':
 977                         case 'gcstress0xc_jitstress2':
 978                         case 'gcstress0xc_minopts_heapverify1':
 979                             def displayStr = getStressModeDisplayName(scenario)
 980                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 981                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 982                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 983                             break
 984                         case 'corefx_baseline':
 985                         case 'corefx_minopts':
 986                         case 'corefx_jitstress1':
 987                         case 'corefx_jitstress2':
 988                         case 'corefx_jitstressregs1':
 989                         case 'corefx_jitstressregs2':
 990                         case 'corefx_jitstressregs3':
 991                         case 'corefx_jitstressregs4':
 992                         case 'corefx_jitstressregs8':
 993                         case 'corefx_jitstressregs0x10':
 994                         case 'corefx_jitstressregs0x80':
 995                         case 'corefx_jitstressregs0x1000':
 996                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 997                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 998                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 999                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1000                             break
1001                         case 'illink':
1002                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+illink\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1003                                     break
1004                         default:
1005                             println("Unknown scenario: ${scenario}");
1006                             assert false
1007                             break
1008                     }
1009                     break
1010                 case 'FreeBSD':
1011                     assert scenario == 'default'
1012                     if (configuration == 'Checked') {
1013                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1014                     }
1015                     break
1016                 default:
1017                     println("Unknown os: ${os}");
1018                     assert false
1019                     break
1020             }
1021             break
1022         // editor brace matching: }
1023         case 'arm': // editor brace matching: {
1024             assert scenario == 'default'
1025             switch (os) {
1026                 case 'Ubuntu':
1027                     if (isLinuxEmulatorBuild == false) {
1028                         // Removing the regex will cause this to run on each PR.
1029                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+Ubuntu\\W+arm\\W+cross\\W+${configuration}.*")
1030                     }
1031                     else {
1032                         Utilities.addGithubPRTriggerForBranch(job, branch, "Linux ARM Emulator Cross ${configuration} Build")
1033                     }
1034                     break
1035                 case 'Ubuntu16.04':
1036                     // Removing the regex will cause this to run on each PR.
1037                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+Ubuntu16.04\\W+arm\\W+cross\\W+${configuration}.*")
1038                     break;
1039                 case 'Tizen':
1040                     architecture='armel'
1041                     // Removing the regex will cause this to run on each PR.
1042                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+Tizen\\W+armel\\W+cross\\W+${configuration}.*")
1043                     break;
1044                 case 'Windows_NT':
1045                     if (configuration == 'Debug' || configuration == 'Release')
1046                     {
1047                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1048                     }
1049                     break
1050                 default:
1051                     println("NYI os: ${os}");
1052                     assert false
1053                     break
1054             }
1055             break
1056         // editor brace matching: }
1057         case 'arm64': // editor brace matching: {
1058             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1059
1060             // Set up a private trigger
1061             def contextString = "${os} ${architecture} Cross ${configuration}"
1062             if (scenario != 'default')
1063                 contextString += " ${scenario}"
1064             contextString += " Build"
1065             // Debug builds only.
1066             if (configuration != 'Debug') {
1067                contextString += " and Test"
1068             }
1069
1070             def arm64Users = [
1071                 'adiaaida',
1072                 'AndyAyersMS',
1073                 'briansull',
1074                 'BruceForstall',
1075                 'CarolEidt',
1076                 'cmckinsey',
1077                 'erozenfeld',
1078                 'jashook',
1079                 'JosephTremoulet',
1080                 'pgavlin',
1081                 'pkukol',
1082                 'russellhadley',
1083                 'RussKeldorph',
1084                 'sandreenko',
1085                 'sivarv',
1086                 'swaroop-sridhar',
1087                 'gkhanna79',
1088                 'jkotas',
1089                 'markwilkie',
1090                 'rahku',
1091                 'ramarag',
1092                 'tzwlai',
1093                 'weshaggard'
1094             ]
1095
1096             switch (os) {
1097                 case 'Windows_NT':
1098                     switch (scenario) {
1099                         case 'default':
1100                             // For now only run Debug jobs on PR Trigger.
1101                             if (configuration != 'Debug') {
1102                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1103                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1104                             }
1105                             else {
1106                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1107                                 // are at higher risk of ARM64-breaking changes.
1108                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1109                             }
1110                             break
1111                         case 'pri1r2r':
1112                         case 'gcstress0x3':
1113                         case 'gcstress0xc':
1114                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1115                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1116                             break
1117                     }
1118                     break
1119                 default:
1120                     println("NYI os: ${os}");
1121                     assert false
1122                     break
1123             }
1124             break
1125         // editor brace matching: }
1126         case 'x86': // editor brace matching: {
1127             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1128             if (os == 'Ubuntu') {
1129                 // on-demand only for ubuntu x86
1130                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1131                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1132                 break
1133             }
1134             switch (scenario) {
1135                 case 'default':
1136                     if (configuration == 'Checked') {
1137                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1138                     }
1139                     else if (configuration == 'Release') {
1140                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1141                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1142                     }
1143                     break
1144                 case 'pri1':
1145                     if (configuration == 'Release') {
1146                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1147                     }
1148                     break
1149                 case 'ilrt':
1150                     if (configuration == 'Release') {
1151                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1152                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1153                     }
1154                     break
1155                 case 'r2r':
1156                     if (configuration == 'Checked' || configuration == 'Release') {
1157                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1158                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1159                     }
1160                     break
1161                 case 'pri1r2r':
1162                     if (configuration == 'Checked' || configuration == 'Release') {
1163                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1164                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1165                     }
1166                     break
1167                 case 'gcstress15_pri1r2r':
1168                     if (configuration == 'Release' || configuration == 'Checked') {
1169                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1170                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1171                     }
1172                     break
1173                 case 'r2r_jitstress1':
1174                 case 'r2r_jitstress2':
1175                 case 'r2r_jitstressregs1':
1176                 case 'r2r_jitstressregs2':
1177                 case 'r2r_jitstressregs3':
1178                 case 'r2r_jitstressregs4':
1179                 case 'r2r_jitstressregs8':
1180                 case 'r2r_jitstressregs0x10':
1181                 case 'r2r_jitstressregs0x80':
1182                 case 'r2r_jitstressregs0x1000':
1183                 case 'r2r_jitminopts':
1184                 case 'r2r_jitforcerelocs':
1185                     if (configuration == 'Release' || configuration == 'Checked') {
1186                         def displayStr = getR2RStressModeDisplayName(scenario)
1187                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1188                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1189                     }
1190                     break
1191                 case 'longgc':
1192                     if (configuration == 'Release') {
1193                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1194                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1195                     }
1196                     break
1197                 case 'gcsimulator':
1198                     if (configuration == 'Release') {
1199                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1200                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1201                     }
1202                     break
1203                 case 'standalone_gc':
1204                     if (configuration == 'Release' || configuration == 'Checked') {
1205                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1206                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1207                     }
1208                     break
1209                 case 'minopts':
1210                 case 'forcerelocs':
1211                 case 'jitstress1':
1212                 case 'jitstress2':
1213                 case 'jitstressregs1':
1214                 case 'jitstressregs2':
1215                 case 'jitstressregs3':
1216                 case 'jitstressregs4':
1217                 case 'jitstressregs8':
1218                 case 'jitstressregs0x10':
1219                 case 'jitstressregs0x80':
1220                 case 'jitstressregs0x1000':
1221                 case 'jitstress2_jitstressregs1':
1222                 case 'jitstress2_jitstressregs2':
1223                 case 'jitstress2_jitstressregs3':
1224                 case 'jitstress2_jitstressregs4':
1225                 case 'jitstress2_jitstressregs8':
1226                 case 'jitstress2_jitstressregs0x10':
1227                 case 'jitstress2_jitstressregs0x80':
1228                 case 'jitstress2_jitstressregs0x1000':
1229                 case 'tailcallstress':
1230                 case 'jitsse2only':
1231                 case 'gcstress0x3':
1232                 case 'gcstress0xc':
1233                 case 'zapdisable':
1234                 case 'heapverify1':
1235                 case 'gcstress0xc_zapdisable':
1236                 case 'gcstress0xc_zapdisable_jitstress2':
1237                 case 'gcstress0xc_zapdisable_heapverify1':
1238                 case 'gcstress0xc_jitstress1':
1239                 case 'gcstress0xc_jitstress2':
1240                 case 'gcstress0xc_minopts_heapverify1':
1241                     def displayStr = getStressModeDisplayName(scenario)
1242                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1243                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1244                     break
1245                 case 'corefx_baseline':
1246                 case 'corefx_minopts':
1247                 case 'corefx_jitstress1':
1248                 case 'corefx_jitstress2':
1249                 case 'corefx_jitstressregs1':
1250                 case 'corefx_jitstressregs2':
1251                 case 'corefx_jitstressregs3':
1252                 case 'corefx_jitstressregs4':
1253                 case 'corefx_jitstressregs8':
1254                 case 'corefx_jitstressregs0x10':
1255                 case 'corefx_jitstressregs0x80':
1256                 case 'corefx_jitstressregs0x1000':
1257                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1258                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1259                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1260                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1261                     break
1262                 case 'illink':
1263                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+illink\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1264                     break
1265                 default:
1266                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1267                     assert false
1268                     break
1269             }
1270             break
1271         // editor brace matching: }
1272         case 'x86compatjit': // editor brace matching: {
1273             assert (os == 'Windows_NT')
1274             def arch = 'x86'
1275             def jit = 'compatjit'
1276             switch (scenario) {
1277                 case 'default':
1278                     if (configuration == 'Checked') {
1279                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1280                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1281                     }
1282                     break
1283                 case 'pri1':
1284                     if (configuration == 'Release') {
1285                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Priority 1 Build and Test",
1286                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+Priority 1 Build and Test.*")
1287                     }
1288                     break
1289                 case 'ilrt':
1290                     if (configuration == 'Release') {
1291                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} IL RoundTrip Build and Test",
1292                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1293                     }
1294                     break
1295                 case 'r2r':
1296                     if (configuration == 'Checked' || configuration == 'Release') {
1297                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri0 Build & Test",
1298                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1299                     }
1300                     break
1301                 case 'pri1r2r':
1302                     if (configuration == 'Checked' || configuration == 'Release') {
1303                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri1 Build & Test",
1304                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1305                     }
1306                     break
1307                 case 'gcstress15_pri1r2r':
1308                     if (configuration == 'Release' || configuration == 'Checked') {
1309                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GCStress 15 R2R pri1 Build & Test",
1310                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1311                     }
1312                     break
1313                 case 'r2r_jitstress1':
1314                 case 'r2r_jitstress2':
1315                 case 'r2r_jitstressregs1':
1316                 case 'r2r_jitstressregs2':
1317                 case 'r2r_jitstressregs3':
1318                 case 'r2r_jitstressregs4':
1319                 case 'r2r_jitstressregs8':
1320                 case 'r2r_jitstressregs0x10':
1321                 case 'r2r_jitstressregs0x80':
1322                 case 'r2r_jitstressregs0x1000':
1323                 case 'r2r_jitminopts':
1324                 case 'r2r_jitforcerelocs':
1325                     if (configuration == 'Release' || configuration == 'Checked') {
1326                         def displayStr = getR2RStressModeDisplayName(scenario)
1327                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} ${displayStr} R2R Build & Test",
1328                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1329                     }
1330                     break
1331                 case 'longgc':
1332                     if (configuration == 'Release') {
1333                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Long-Running GC Build & Test",
1334                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1335                     }
1336                     break
1337                 case 'gcsimulator':
1338                     if (configuration == 'Release') {
1339                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GC Simulator",
1340                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1341                     }
1342                     break
1343                 case 'standalone_gc':
1344                     if (configuration == 'Release' || configuration == 'Checked') {
1345                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1346                     }
1347                     break
1348                 case 'minopts':
1349                 case 'forcerelocs':
1350                 case 'jitstress1':
1351                 case 'jitstress2':
1352                 case 'jitstressregs1':
1353                 case 'jitstressregs2':
1354                 case 'jitstressregs3':
1355                 case 'jitstressregs4':
1356                 case 'jitstressregs8':
1357                 case 'jitstressregs0x10':
1358                 case 'jitstressregs0x80':
1359                 case 'jitstressregs0x1000':
1360                 case 'jitstress2_jitstressregs1':
1361                 case 'jitstress2_jitstressregs2':
1362                 case 'jitstress2_jitstressregs3':
1363                 case 'jitstress2_jitstressregs4':
1364                 case 'jitstress2_jitstressregs8':
1365                 case 'jitstress2_jitstressregs0x10':
1366                 case 'jitstress2_jitstressregs0x80':
1367                 case 'jitstress2_jitstressregs0x1000':
1368                 case 'tailcallstress':
1369                 case 'jitsse2only':
1370                 case 'gcstress0x3':
1371                 case 'gcstress0xc':
1372                 case 'zapdisable':
1373                 case 'heapverify1':
1374                 case 'gcstress0xc_zapdisable':
1375                 case 'gcstress0xc_zapdisable_jitstress2':
1376                 case 'gcstress0xc_zapdisable_heapverify1':
1377                 case 'gcstress0xc_jitstress1':
1378                 case 'gcstress0xc_jitstress2':
1379                 case 'gcstress0xc_minopts_heapverify1':
1380                     def displayStr = getStressModeDisplayName(scenario)
1381                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - ${displayStr})",
1382                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1383                     break
1384                 case 'corefx_baseline':
1385                 case 'corefx_minopts':
1386                 case 'corefx_jitstress1':
1387                 case 'corefx_jitstress2':
1388                 case 'corefx_jitstressregs1':
1389                 case 'corefx_jitstressregs2':
1390                 case 'corefx_jitstressregs3':
1391                 case 'corefx_jitstressregs4':
1392                 case 'corefx_jitstressregs8':
1393                 case 'corefx_jitstressregs0x10':
1394                 case 'corefx_jitstressregs0x80':
1395                 case 'corefx_jitstressregs0x1000':
1396                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1397                     assert (os == 'Windows_NT')
1398                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - ${displayName})",
1399                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1400                     break
1401                 case 'illink':
1402                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+illink\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1403                     break
1404                 default:
1405                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1406                     assert false
1407                     break
1408             }
1409             break
1410         // editor brace matching: }
1411         default:
1412             println("Unknown architecture: ${architecture}");
1413             assert false
1414             break
1415     }
1416 }
1417
1418 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly, def isLinuxEmulatorBuild) {
1419     def buildCommands = [];
1420     def osGroup = getOSGroup(os)
1421     def lowerConfiguration = configuration.toLowerCase()
1422
1423     // Calculate the build steps, archival, and xunit results
1424     switch (os) {
1425         case 'Windows_NT': // editor brace matching: {
1426             switch (architecture) {
1427                 case 'x64':
1428                 case 'x86':
1429                 case 'x86compatjit':
1430                     def arch = architecture
1431                     def buildOpts = ''
1432
1433                     // We need to explicitly run build-test.cmd with Exclude for x86compatjit, so skip tests.
1434                     if (architecture == 'x86compatjit') {
1435                         arch = 'x86'
1436                         buildOpts = 'compatjitcrossgen skiptests'
1437                     }
1438
1439                     // build_illink.cmd performs the following tasks:
1440                     // - Clone and build illink for netcoreapp2.0
1441                     // - sets ILLINK=<path to illink.exe> in the environment.
1442                     if (scenario == 'illink') {
1443                         def illinkArgs = (architecture == 'x86compatjit') ? 'x86' : architecture
1444                         buildCommands += "tests\\build_illink.cmd clone setenv ${illinkArgs}"
1445                     }
1446
1447                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1448                             scenario == 'default' ||
1449                             scenario == 'r2r' ||
1450                             scenario == 'jitdiff' ||
1451                             scenario == 'illink' ||
1452                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1453                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1454                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1455                     }
1456
1457                     // For Pri 1 tests, we must shorten the output test binary path names.
1458                     // if __TestIntermediateDir is already set, build-test.cmd will
1459                     // output test binaries to that directory. If it is not set, the
1460                     // binaries are sent to a default directory whose name is about
1461                     // 35 characters long.
1462
1463                     else if (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage') {
1464                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1465                     }
1466                     else if (scenario == 'ilrt') {
1467                         // First do the build with skiptests and then build the tests with ilasm roundtrip
1468                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1469                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch} -ilasmroundtrip"
1470                     }
1471                     else if (isLongGc(scenario)) {
1472                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1473                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1474                     }
1475                     else if (scenario == 'standalone_gc') {
1476                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1477                     }
1478                     else if (scenario == 'formatting') {
1479                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1480                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1481                         break
1482                     }
1483                     else {
1484                         println("Unknown scenario: ${scenario}")
1485                         assert false
1486                     }
1487
1488                     // If we are running a stress mode, we should write out the set of key
1489                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1490
1491                     if (!isBuildOnly) {
1492                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1493                         def crossgenStr = ''
1494                         def runcrossgentestsStr = ''
1495                         def runjitstressStr = ''
1496                         def runjitstressregsStr = ''
1497                         def runjitmioptsStr = ''
1498                         def runjitforcerelocsStr = ''
1499                         def runjitdisasmStr = ''
1500                         def gcstressStr = ''
1501                         def runtestArguments = ''
1502                         def gcTestArguments = ''
1503                         def illinkArguments = ''
1504
1505                         if (scenario == 'r2r' ||
1506                             scenario == 'pri1r2r' ||
1507                             scenario == 'gcstress15_pri1r2r' ||
1508                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1509                                 crossgenStr = 'crossgen'
1510                                 runcrossgentestsStr = 'runcrossgentests'
1511
1512                                 if (scenario == 'r2r_jitstress1'){
1513                                     runjitstressStr = 'jitstress 1'
1514                                 }
1515                                 else if (scenario == 'r2r_jitstress2') {
1516                                     runjitstressStr = 'jitstress 2'
1517                                 }
1518                                 else if (scenario == 'r2r_jitstressregs1'){
1519                                     runjitstressregsStr = 'jitstressregs 1'
1520                                 }
1521                                 else if (scenario == 'r2r_jitstressregs2') {
1522                                     runjitstressregsStr = 'jitstressregs 2'
1523                                 }
1524                                 else if (scenario == 'r2r_jitstressregs3') {
1525                                     runjitstressregsStr = 'jitstressregs 3'
1526                                 }
1527                                 else if (scenario == 'r2r_jitstressregs4') {
1528                                     runjitstressregsStr = 'jitstressregs 4'
1529                                 }
1530                                 else if (scenario == 'r2r_jitstressregs8') {
1531                                     runjitstressregsStr = 'jitstressregs 8'
1532                                 }
1533                                 else if (scenario == 'r2r_jitstressregs0x10') {
1534                                     runjitstressregsStr = 'jitstressregs 0x10'
1535                                 }
1536                                 else if (scenario == 'r2r_jitstressregs0x80') {
1537                                     runjitstressregsStr = 'jitstressregs 0x80'
1538                                 }
1539                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1540                                     runjitstressregsStr = 'jitstressregs 0x1000'
1541                                 }
1542                                 else if (scenario == 'r2r_jitminopts') {
1543                                     runjitmioptsStr = 'jitminopts'
1544                                 }
1545                                 else if (scenario == 'r2r_jitforcerelocs') {
1546                                     runjitforcerelocsStr = 'jitforcerelocs'
1547                                 }
1548                         }
1549                         if (scenario == 'gcstress15_pri1r2r')
1550                         {
1551                             gcstressStr = 'gcstresslevel 0xF'
1552                         }
1553
1554                         if (scenario == 'jitdiff')
1555                         {
1556                             runjitdisasmStr = 'jitdisasm crossgen'
1557                         }
1558
1559                         if (isLongGc(scenario)) {
1560                             gcTestArguments = "${scenario} sequential"
1561                         }
1562
1563                         if (scenario == 'illink')
1564                         {
1565                             illinkArguments = 'link %ILLINK%'
1566                         }
1567
1568                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1569
1570                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1571                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1572                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1573
1574                             if (enableCorefxTesting) {
1575                                 def workspaceRelativeFxRoot = "_/fx"
1576                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1577
1578                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1579
1580                                 setTestJobTimeOut(newJob, scenario)
1581
1582                                 // Archive and process (only) the test results
1583                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1584                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1585                             }
1586                             else {
1587                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1588                             }
1589                         }
1590                         else if (architecture == 'x64' || architecture == 'x86') {
1591                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1592                         }
1593                         else if (architecture == 'x86compatjit') {
1594                             def testEnvLocation = "%WORKSPACE%\\tests\\x86\\compatjit_x86_testenv.cmd"
1595                             buildCommands += "build-test.cmd ${runtestArguments}"
1596                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv ${testEnvLocation}"
1597                         }
1598                     }
1599
1600                     if (!enableCorefxTesting) {
1601                         // Run the rest of the build
1602                         // Build the mscorlib for the other OS's
1603                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1604                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1605                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1606
1607                         // Zip up the tests directory so that we don't use so much space/time copying
1608                         // 10s of thousands of files around.
1609                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1610
1611                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1612                             // For windows, pull full test results and test drops for x86/x64.
1613                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1614                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1615                         }
1616
1617                         if (scenario == 'jitdiff') {
1618                             // retrive jit-dasm output for base commit, and run jit-diff
1619                             if (!isBuildOnly) {
1620                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1621                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1622                             }
1623                         }
1624
1625                         if (!isBuildOnly) {
1626                             if (architecture == 'x64' || !isPR) {
1627                                 Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml')
1628                             }
1629                             setTestJobTimeOut(newJob, scenario)
1630                         }
1631                     }
1632                     break
1633                 case 'arm':
1634                     assert (scenario == 'default')
1635
1636                     // Set time out
1637                     setTestJobTimeOut(newJob, scenario)
1638
1639                     if ( lowerConfiguration == "debug" ) {
1640                         // For Debug builds, we will do a P1 test build
1641                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1642                     }
1643                     else {
1644                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1645                     }
1646                     // Add archival.
1647                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1648                     break
1649                 case 'arm64':
1650                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1651
1652                     // Set time out
1653                     setTestJobTimeOut(newJob, scenario)
1654
1655                     // Debug runs take too long to run. So build job only.
1656                     if (lowerConfiguration == "debug") {
1657                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1658                     }
1659                     else {
1660                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1661                        {
1662                            // Up the timeout for arm64 checked testing only.
1663                            // Keep the longer timeout for gcstress.
1664                            Utilities.setJobTimeout(newJob, 240)
1665                        }
1666
1667                        buildCommands += "set __TestIntermediateDir=int&&build.cmd skiptests ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1668                        // Test build and run are launched together.
1669                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1670                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1671                     }
1672
1673                     // Add archival.
1674                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1675                     break
1676                 default:
1677                     println("Unknown architecture: ${architecture}");
1678                     assert false
1679                     break
1680             }
1681             break
1682         // editor brace matching: }
1683         case 'Ubuntu':
1684         case 'Ubuntu16.04':
1685         case 'Ubuntu16.10':
1686         case 'Debian8.4':
1687         case 'OSX10.12':
1688         case 'FreeBSD':
1689         case 'CentOS7.1':
1690         case 'RHEL7.2':
1691         case 'OpenSUSE42.1':
1692         case 'Tizen':
1693         case 'Fedora23': // editor brace matching: {
1694             switch (architecture) {
1695                 case 'x64':
1696                 case 'x86':
1697                 case 'x86compatjit':
1698                     def arch = architecture
1699                     if (architecture == 'x86compatjit') {
1700                         arch = 'x86'
1701                     }
1702
1703                     if (architecture == 'x86' && os == 'Ubuntu') {
1704                         // build only, not test yet
1705                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1706                         break;
1707                     }
1708
1709                     if (scenario == 'formatting') {
1710                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${arch}"
1711                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1712                         break
1713                     }
1714
1715                     def standaloneGc = ''
1716                     if (scenario == 'standalone_gc') {
1717                         standaloneGc = 'buildstandalonegc'
1718                     }
1719
1720                     if (!enableCorefxTesting) {
1721                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1722                         // only on supported OS platforms.
1723                         if (os == 'FreeBSD')
1724                         {
1725                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${arch} ${standaloneGc}"
1726                         }
1727                         else
1728                         {
1729                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1730                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1731                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${arch} ${standaloneGc}"
1732                         }
1733                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${arch}.${configuration} \${WORKSPACE}/bin/paltestout"
1734
1735                         // Set time out
1736                         setTestJobTimeOut(newJob, scenario)
1737                         // Basic archiving of the build
1738                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1739                         // And pal tests
1740                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1741                     }
1742                     else {
1743                         // Corefx stress testing
1744                         assert os == 'Ubuntu'
1745                         assert architecture == 'x64'
1746                         assert lowerConfiguration == 'checked'
1747                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1748
1749                         // Build coreclr
1750                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1751
1752                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1753                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1754
1755                         // Build and text corefx
1756                         def workspaceRelativeFxRoot = "_/fx"
1757                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1758
1759                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1760
1761                         setTestJobTimeOut(newJob, scenario)
1762
1763                         // Archive and process (only) the test results
1764                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1765                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1766                     }
1767                     break
1768                 case 'arm64':
1769                     // We don't run the cross build except on Ubuntu
1770                     assert os == 'Ubuntu'
1771
1772                     buildCommands += """echo \"Using rootfs in /opt/aarch64-linux-gnu-root\"
1773                         ROOTFS_DIR=/opt/aarch64-linux-gnu-root ./build.sh skipmscorlib arm64 cross verbose ${lowerConfiguration}"""
1774
1775                     // Basic archiving of the build, no pal tests
1776                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1777                     break
1778                 case 'arm':
1779                     if (isLinuxEmulatorBuild == false) {
1780                         // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1781                         assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1782
1783                         // default values for Ubuntu
1784                         def arm_abi="arm"
1785                         def linuxCodeName="trusty"
1786                         if (os == 'Ubuntu16.04') {
1787                             linuxCodeName="xenial"
1788                         }
1789                         else if (os == 'Tizen') {
1790                             arm_abi="armel"
1791                             linuxCodeName="tizen"
1792                         }
1793
1794                         // Unzip the Windows test binaries first. Exit with 0
1795                         buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1796
1797                         // Unpack the corefx binaries
1798                         buildCommands += "mkdir ./bin/CoreFxBinDir"
1799                         buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1800
1801                         // Call the ARM CI script to cross build and test using docker
1802                         buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1803                         --mode=docker \\
1804                         --${arm_abi} \\
1805                         --linuxCodeName=${linuxCodeName} \\
1806                         --buildConfig=${lowerConfiguration} \\
1807                         --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1808                         --coreFxBinDir=./bin/CoreFxBinDir \\
1809                         --testDirFile=./tests/testsRunningInsideARM.txt"""
1810
1811                         // Basic archiving of the build, no pal tests
1812                         Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1813                         break
1814                     }
1815                     else {
1816                         assert os == 'Ubuntu'
1817                         // Make sure the build configuration is either of debug or release
1818                         assert ( lowerConfiguration == 'debug' ) || ( lowerConfiguration == 'release' )
1819
1820                         // Setup variables to hold emulator folder path and the rootfs mount path
1821                         def armemul_path = '/opt/linux-arm-emulator'
1822                         def armrootfs_mountpath = '/opt/linux-arm-emulator-root'
1823
1824                         // Unzip the Windows test binaries first. Exit with 0
1825                         buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1826
1827                         // Unpack the corefx binaries
1828                         buildCommands += "mkdir ./bin/CoreFxBinDir"
1829                         buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1830
1831                         // Call the ARM emulator build script to cross build and test using the ARM emulator rootfs
1832                         buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1833                         --emulatorPath=${armemul_path} \\
1834                         --mountPath=${armrootfs_mountpath} \\
1835                         --buildConfig=${lowerConfiguration} \\
1836                         --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1837                         --coreFxNativeBinDir=./bin/Linux.armel.${configuration} \\
1838                         --coreFxBinDir=\"./bin/Linux.AnyCPU.${configuration};./bin/Unix.AnyCPU.${configuration};./bin/AnyOS.AnyCPU.${configuration}\" \\
1839                         --testDirFile=./tests/testsRunningInsideARM.txt"""
1840
1841
1842                         // Basic archiving of the build
1843                         Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1844                         break
1845                     }
1846                 default:
1847                     println("Unknown architecture: ${architecture}");
1848                     assert false
1849                     break
1850             }
1851             break
1852         // editor brace matching: }
1853         default:
1854             println("Unknown os: ${os}");
1855             assert false
1856             break
1857     } // os
1858
1859     return buildCommands
1860 }
1861
1862 // Additional scenario which can alter behavior
1863
1864 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1865 combinedScenarios.each { scenario ->
1866     [true, false].each { isPR ->
1867         Constants.architectureList.each { architecture ->
1868             Constants.configurationList.each { configuration ->
1869                 Constants.osList.each { os ->
1870                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1871                     // and reset the os to Windows_NT
1872                     def isBuildOnly = false
1873                     if (os == 'Windows_NT_BuildOnly') {
1874                         isBuildOnly = true
1875                         os = 'Windows_NT'
1876                     }
1877
1878                     // WinArm32 is only built for Debug and Release
1879                     if (os == 'Windows_NT' && architecture == 'arm')
1880                     {
1881                         if (configuration == 'Checked')
1882                         {
1883                             return
1884                         }
1885                     }
1886                     // If the OS is LinuxARMEmulator and arch is arm, set the isLinuxEmulatorBuild
1887                     // flag to true and reset the os to Ubuntu
1888                     // The isLinuxEmulatorBuild flag will be used at a later time to execute the right
1889                     // set of build commands
1890                     // The tuples (LinuxARMEmulator, other architectures) are not handled and control returns
1891                     def isLinuxEmulatorBuild = false
1892                     if (os == 'LinuxARMEmulator' && architecture == 'arm') {
1893                         // Cross Builds only in Debug and Release modes allowed
1894                         if ( configuration == 'Checked' ) {
1895                             return
1896                         }
1897
1898                         isLinuxEmulatorBuild = true
1899                         os = 'Ubuntu'
1900                     } else if (os == 'LinuxARMEmulator') {
1901                         return
1902                     }
1903                     // Tizen is only supported for arm architecture
1904                     if (os == 'Tizen' && architecture != 'arm') {
1905                         return
1906                     }
1907
1908                     // Skip totally unimplemented (in CI) configurations.
1909                     switch (architecture) {
1910                         case 'arm64':
1911                             // Windows only
1912                             if (os != 'Windows_NT' || isBuildOnly) {
1913                                 return
1914                             }
1915                             break
1916                         case 'arm':
1917                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1918                                 return
1919                             }
1920                             break
1921                         case 'x86':
1922                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1923                                 return
1924                             }
1925                             break
1926                         case 'x86compatjit':
1927                             // Skip non-windows
1928                             if (os != 'Windows_NT') {
1929                                 return
1930                             }
1931                             break
1932                         case 'x64':
1933                             // Everything implemented
1934                             break
1935                         default:
1936                             println("Unknown architecture: ${architecture}")
1937                             assert false
1938                             break
1939                     }
1940
1941                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1942                     // with checked builds
1943                     def enableCorefxTesting = false
1944                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1945                         if (configuration != 'Checked') {
1946                             return
1947                         }
1948
1949                         enableCorefxTesting = isCorefxTesting(scenario)
1950
1951                         // Since these are just execution time differences,
1952                         // skip platforms that don't execute the tests here (Windows_NT only)
1953                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1954                         if (!isEnabledOS || isBuildOnly) {
1955                             return
1956                         }
1957
1958                         switch (architecture) {
1959                             case 'arm64':
1960                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1961                                     return
1962                                 }
1963                                 break
1964                             case 'x64':
1965                             case 'x86':
1966                                 // x86 ubuntu: default only
1967                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1968                                     return
1969                                 }
1970                                 // Windows: Everything implemented
1971                                 break
1972                             case 'x86compatjit':
1973                                 // No stress modes for compatjit.dll.
1974                                 // (There's no technical reason we couldn't allow these.)
1975                                 return
1976                             default:
1977                                 return
1978                         }
1979                     }
1980                     else {
1981                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
1982                         // and configuration is not Checked, bail out.
1983                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1984                             return;
1985                         }
1986
1987                         // Skip scenarios
1988                         switch (scenario) {
1989                             case 'pri1':
1990                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1991                                 // the default scenario build
1992                                 if (os != 'Windows_NT') {
1993                                     return
1994                                 }
1995                                 // Only x64 for now
1996                                 if (architecture != 'x64') {
1997                                     return
1998                                 }
1999                                 break
2000                             case 'ilrt':
2001                                 // The ilrt build isn't necessary except for Windows_NT.  Non-Windows NT uses
2002                                 // the default scenario build
2003                                 if (os != 'Windows_NT') {
2004                                     return
2005                                 }
2006                                 // Only x64 for now
2007                                 if (architecture != 'x64') {
2008                                     return
2009                                 }
2010                                 // Release only
2011                                 if (configuration != 'Release') {
2012                                     return
2013                                 }
2014                                 break
2015                             case 'jitdiff':
2016                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2017                                     return
2018                                 }
2019                                 if (architecture != 'x64') {
2020                                     return
2021                                 }
2022                                 if (configuration != 'Checked') {
2023                                     return
2024                                 }
2025                                 break
2026                             case 'r2r':
2027                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2028                                 // the default scenario build
2029                                 if (os != 'Windows_NT') {
2030                                     return
2031                                 }
2032                                 if (architecture != 'x64') {
2033                                     return
2034                                 }
2035                                 break
2036                             case 'pri1r2r':
2037                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2038                                 // the default scenario build
2039                                 if (os != 'Windows_NT') {
2040                                     return
2041                                 }
2042                                 if (architecture != 'x64') {
2043                                     if (architecture != 'arm64' || configuration == 'Debug') {
2044                                         return
2045                                     }
2046                                 }
2047                                 break
2048                             case 'gcstress15_pri1r2r':
2049                             case 'r2r_jitstress1':
2050                             case 'r2r_jitstress2':
2051                             case 'r2r_jitstressregs1':
2052                             case 'r2r_jitstressregs2':
2053                             case 'r2r_jitstressregs3':
2054                             case 'r2r_jitstressregs4':
2055                             case 'r2r_jitstressregs8':
2056                             case 'r2r_jitstressregs0x10':
2057                             case 'r2r_jitstressregs0x80':
2058                             case 'r2r_jitstressregs0x1000':
2059                             case 'r2r_jitminopts':
2060                             case 'r2r_jitforcerelocs':
2061                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
2062                                 // the default scenario build
2063                                 if (os != 'Windows_NT') {
2064                                     return
2065                                 }
2066                                 if (architecture != 'x64') {
2067                                     return
2068                                 }
2069                                 break
2070                             case 'longgc':
2071                             case 'gcsimulator':
2072                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2073                                     return
2074                                 }
2075                                 if (architecture != 'x64') {
2076                                     return
2077                                 }
2078                                 if (configuration != 'Release') {
2079                                     return
2080                                 }
2081                                 break
2082                             case 'standalone_gc':
2083                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2084                                     return
2085                                 }
2086
2087                                 if (architecture != 'x64') {
2088                                     return
2089                                 }
2090
2091                                 if (configuration != 'Release' && configuration != 'Checked') {
2092                                     return
2093                                 }
2094                                 break
2095                             // We need Windows x64 Release bits for the code coverage build
2096                             case 'coverage':
2097                                 if (os != 'Windows_NT') {
2098                                     return
2099                                 }
2100                                 if (architecture != 'x64') {
2101                                     return
2102                                 }
2103                                 if (configuration != 'Release') {
2104                                     return
2105                                 }
2106                                 break
2107                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2108                             case 'formatting':
2109                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2110                                     return
2111                                 }
2112                                 if (architecture != 'x64') {
2113                                     return
2114                                 }
2115                                 if (configuration != 'Checked') {
2116                                     return
2117                                 }
2118                                 if (isBuildOnly) {
2119                                     return
2120                                 }
2121                                 break
2122                             case 'illink':
2123                                 if (os != 'Windows_NT') {
2124                                     return
2125                                 }
2126                                 if (architecture != 'x64' && architecture != 'x86' && architecture != 'x86compatjit') {
2127                                     return
2128                                 }
2129                                 if (isBuildOnly) {
2130                                     return
2131                                 }
2132                                 break
2133                             case 'default':
2134                                 // Nothing skipped
2135                                 break
2136                             default:
2137                                 println("Unknown scenario: ${scenario}")
2138                                 assert false
2139                                 break
2140                         }
2141                     }
2142
2143                     // Calculate names
2144                     def lowerConfiguration = configuration.toLowerCase()
2145                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly, isLinuxEmulatorBuild)
2146                     def folderName = getJobFolder(scenario)
2147
2148                     // Create the new job
2149                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2150
2151                     setMachineAffinity(newJob, os, architecture)
2152
2153                     // Add all the standard options
2154                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2155                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly, isLinuxEmulatorBuild)
2156
2157                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly, isLinuxEmulatorBuild)
2158                     def osGroup = getOSGroup(os)
2159
2160                     newJob.with {
2161                         steps {
2162                             if (os == 'Windows_NT') {
2163                                 buildCommands.each { buildCommand ->
2164                                     batchFile(buildCommand)
2165                                 }
2166                             }
2167                             else {
2168                                 // Setup corefx and Windows test binaries for Linux ARM Emulator Build
2169                                 // and cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2170                                 if (isLinuxEmulatorBuild) {
2171                                     // Define the Windows Tests and Corefx build job names
2172                                     def WindowTestsName = projectFolder + '/' +
2173                                                           Utilities.getFullJobName(project,
2174                                                                                    getJobName(lowerConfiguration,
2175                                                                                               'x64' ,
2176                                                                                               'windows_nt',
2177                                                                                               'default',
2178                                                                                               true),
2179                                                                                    false)
2180                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2181                                                        Utilities.getFolderName(branch)
2182
2183                                     // Copy the Windows test binaries and the Corefx build binaries
2184                                     copyArtifacts(WindowTestsName) {
2185                                         excludePatterns('**/testResults.xml', '**/*.ni.dll')
2186                                         buildSelector {
2187                                             latestSuccessful(true)
2188                                         }
2189                                     }
2190                                     copyArtifacts("${corefxFolder}/tizen_armel_cross_${lowerConfiguration}") {
2191                                         includePatterns('bin/build.tar.gz')
2192                                         buildSelector {
2193                                             latestSuccessful(true)
2194                                         }
2195                                     }
2196                                 }
2197                                 else if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2198                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2199                                     // Define the Windows Tests and Corefx build job names
2200                                     def WindowTestsName = projectFolder + '/' +
2201                                                           Utilities.getFullJobName(project,
2202                                                                                    getJobName(lowerConfiguration,
2203                                                                                               'x64' ,
2204                                                                                               'windows_nt',
2205                                                                                               'default',
2206                                                                                               true),
2207                                                                                    false)
2208                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2209                                                        Utilities.getFolderName(branch)
2210
2211                                     // Copy the Windows test binaries and the Corefx build binaries
2212                                     copyArtifacts(WindowTestsName) {
2213                                         excludePatterns('**/testResults.xml', '**/*.ni.dll')
2214                                         buildSelector {
2215                                             latestSuccessful(true)
2216                                         }
2217                                     }
2218
2219                                     // Defaults for Ubuntu
2220                                     def arm_abi = 'arm'
2221                                     def corefx_os = 'ubuntu14.04'
2222                                     if (os == 'Ubuntu16.04') {
2223                                         arm_abi = 'arm'
2224                                         corefx_os = 'ubuntu16.04'
2225                                     }
2226                                     else if (os == 'Tizen') {
2227                                         arm_abi = 'armel'
2228                                         corefx_os = 'tizen'
2229                                     }
2230
2231                                     // Let's use release CoreFX to test checked CoreCLR,
2232                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2233                                     def corefx_lowerConfiguration = lowerConfiguration
2234                                     if ( lowerConfiguration == 'checked' ) {
2235                                         corefx_lowerConfiguration='release'
2236                                     }
2237                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2238                                         includePatterns('bin/build.tar.gz')
2239                                         buildSelector {
2240                                             latestSuccessful(true)
2241                                         }
2242                                     }
2243                                 }
2244
2245                                 buildCommands.each { buildCommand ->
2246                                     shell(buildCommand)
2247                                 }
2248                             }
2249                         }
2250                     } // newJob.with
2251
2252                 } // os
2253             } // configuration
2254         } // architecture
2255     } // isPR
2256 } // scenario
2257
2258
2259 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2260 combinedScenarios.each { scenario ->
2261     [true, false].each { isPR ->
2262         // Architectures.  x64 only at this point
2263         ['x64'].each { architecture ->
2264             // Put the OS's supported for coreclr cross testing here
2265             Constants.crossList.each { os ->
2266                 Constants.configurationList.each { configuration ->
2267
2268                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2269                         if (configuration != 'Checked') {
2270                             return
2271                         }
2272                         if (isCorefxTesting(scenario)) {
2273                             return
2274                         }
2275                         //Skip stress modes for these scenarios
2276                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2277                             return
2278                         }
2279                     }
2280                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2281                     // and configuration is not Checked, bail out.
2282                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2283                         return;
2284                     }
2285                     // For CentOS, we only want Checked/Release pri1 builds.
2286                     else if (os == 'CentOS7.1') {
2287                         if (scenario != 'pri1' &&
2288                             scenario != 'r2r' &&
2289                             scenario != 'pri1r2r' &&
2290                             scenario != 'gcstress15_pri1r2r' &&
2291                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2292                             return
2293                         }
2294                         if (configuration != 'Checked' && configuration != 'Release') {
2295                             return
2296                         }
2297                     }
2298                     // For RedHat and Debian, we only do Release pri1 builds.
2299                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2300                         if (scenario != 'pri1') {
2301                             return
2302                         }
2303                         if (configuration != 'Release') {
2304                             return
2305                         }
2306                     }
2307                     else {
2308                         // Skip scenarios
2309                         switch (scenario) {
2310                             case 'pri1':
2311                                 // Nothing skipped
2312                                 break
2313                             case 'ilrt':
2314                                 // Release only
2315                                 if (configuration != 'Release') {
2316                                     return
2317                                 }
2318                                 break
2319                             case 'jitdiff':
2320                                 if (configuration != 'Checked') {
2321                                     return;
2322                                 }
2323                             case 'r2r':
2324                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2325                                 if (configuration != 'Checked' && configuration != 'Release') {
2326                                     return
2327                                 }
2328                                 break
2329                             case 'pri1r2r':
2330                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2331                                 if (configuration != 'Checked' && configuration != 'Release') {
2332                                     return
2333                                 }
2334                                 break
2335                             case 'gcstress15_pri1r2r':
2336                             case 'r2r_jitstress1':
2337                             case 'r2r_jitstress2':
2338                             case 'r2r_jitstressregs1':
2339                             case 'r2r_jitstressregs2':
2340                             case 'r2r_jitstressregs3':
2341                             case 'r2r_jitstressregs4':
2342                             case 'r2r_jitstressregs8':
2343                             case 'r2r_jitstressregs0x10':
2344                             case 'r2r_jitstressregs0x80':
2345                             case 'r2r_jitstressregs0x1000':
2346                             case 'r2r_jitminopts':
2347                             case 'r2r_jitforcerelocs':
2348                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2349                                 if (configuration != 'Checked' && configuration != 'Release') {
2350                                     return
2351                                 }
2352                                 break
2353                             case 'longgc':
2354                             case 'gcsimulator':
2355                                 // Long GC tests take a long time on non-Release builds
2356                                 if (configuration != 'Release') {
2357                                     return
2358                                 }
2359                                 break
2360                             case 'standalone_gc':
2361                                 if (configuration != 'Release' && configuration != 'Checked') {
2362                                     return
2363                                 }
2364                             case 'coverage':
2365                                 //We only want Ubuntu Release for coverage
2366                                 if (os != 'Ubuntu') {
2367                                     return
2368                                 }
2369                                 if (configuration != 'Release') {
2370                                     return
2371                                 }
2372                             case 'formatting':
2373                                 return
2374                             case 'illink':
2375                                 if (os != 'Windows_NT') {
2376                                     return
2377                                 }
2378                                 break
2379                             case 'default':
2380                                 // Nothing skipped
2381                                 break
2382                             default:
2383                                 println("Unknown scenario: ${scenario}")
2384                                 assert false
2385                                 break
2386                         }
2387                     }
2388
2389                     def lowerConfiguration = configuration.toLowerCase()
2390                     def osGroup = getOSGroup(os)
2391                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2392
2393                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2394                     def inputScenario = 'default'
2395                     if (scenario == 'coverage') {
2396                         inputScenario = 'coverage'
2397                     }
2398                     def inputCoreCLRBuildName = projectFolder + '/' +
2399                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2400                     // If this is a stress scenario, there isn't any difference in the build job
2401                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2402                     // from the default scenario
2403                     def testBuildScenario = scenario
2404                     if (testBuildScenario == 'coverage' || testBuildScenario == 'pri1r2r'|| testBuildScenario == 'gcstress15_pri1r2r') {
2405                         testBuildScenario = 'pri1'
2406                     }
2407                     else if ( testBuildScenario == 'r2r' || isLongGc(testBuildScenario)) {
2408                         testBuildScenario = 'default'
2409                     }
2410                     def inputWindowTestsBuildName = ''
2411                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2412                         inputWindowTestsBuildName = projectFolder + '/' +
2413                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', 'default', true), isPR)
2414                     }
2415                     else {
2416                         inputWindowTestsBuildName = projectFolder + '/' +
2417                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', testBuildScenario, true), isPR)
2418                     }
2419                     // Enable Server GC for Ubuntu PR builds
2420                     def serverGCString = ''
2421
2422                     // Whether or not this test run should be run sequentially instead
2423                     // of in parallel. Only used for long GC tests.
2424                     def sequentialString = ''
2425
2426                     // Whether or not this test run should run a specific playlist.
2427                     // Only used for long GC tests.
2428
2429                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2430                     // for running long GC and GCSimulator tests, respectively. We don't use them
2431                     // here because using a playlist file produces much more readable output on the CI machines
2432                     // and reduces running time.
2433                     def playlistString = ''
2434
2435                     if (os == 'Ubuntu' && isPR){
2436                         serverGCString = '--useServerGC'
2437                     }
2438
2439                     // pass --crossgen to runtest.sh for crossgen builds
2440                     def crossgenStr = ''
2441                     def runcrossgentestsStr = ''
2442                     def runjitstressStr = ''
2443                     def runjitstressregsStr = ''
2444                     def runjitmioptsStr = ''
2445                     def runjitforcerelocsStr = ''
2446                     def runjitdisasmStr = ''
2447                     def gcstressStr = ''
2448
2449                     if (scenario == 'r2r' ||
2450                         scenario == 'pri1r2r' ||
2451                         scenario == 'gcstress15_pri1r2r' ||
2452                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2453                             crossgenStr = '--crossgen'
2454                             runcrossgentestsStr = '--runcrossgentests'
2455
2456                             if (scenario == 'r2r_jitstress1'){
2457                                 runjitstressStr = '--jitstress=1'
2458                             }
2459                             else if (scenario == 'r2r_jitstress2') {
2460                                 runjitstressStr = '--jitstress=2'
2461                             }
2462                             else if (scenario == 'r2r_jitstressregs1'){
2463                                 runjitstressregsStr = '--jitstressregs=1'
2464                             }
2465                             else if (scenario == 'r2r_jitstressregs2') {
2466                                 runjitstressregsStr = '--jitstressregs=2'
2467                             }
2468                             else if (scenario == 'r2r_jitstressregs3') {
2469                                 runjitstressregsStr = '--jitstressregs=3'
2470                             }
2471                             else if (scenario == 'r2r_jitstressregs4') {
2472                                 runjitstressregsStr = '--jitstressregs=4'
2473                             }
2474                             else if (scenario == 'r2r_jitstressregs8') {
2475                                 runjitstressregsStr = '--jitstressregs=8'
2476                             }
2477                             else if (scenario == 'r2r_jitstressregs0x10') {
2478                                 runjitstressregsStr = '--jitstressregs=0x10'
2479                             }
2480                             else if (scenario == 'r2r_jitstressregs0x80') {
2481                                 runjitstressregsStr = '--jitstressregs=0x80'
2482                             }
2483                             else if (scenario == 'r2r_jitstressregs0x1000') {
2484                                 runjitstressregsStr = '--jitstressregs=0x1000'
2485                             }
2486                             else if (scenario == 'r2r_jitminopts') {
2487                                 runjitmioptsStr = '--jitminopts'
2488                             }
2489                             else if (scenario == 'r2r_jitforcerelocs') {
2490                                 runjitforcerelocsStr = '--jitforcerelocs'
2491                             }
2492                     }
2493                     if  (scenario == 'gcstress15_pri1r2r')
2494                     {
2495                         gcstressStr = '--gcstresslevel=0xF'
2496                     }
2497
2498                     if (scenario == 'jitdiff')
2499                     {
2500                         runjitdisasmStr = '--jitdisasm --crossgen'
2501                     }
2502
2503                     if (isLongGc(scenario)) {
2504                         // Long GC tests behave very poorly when they are not
2505                         // the only test running (many of them allocate until OOM).
2506                         sequentialString = '--sequential'
2507
2508                         // The Long GC playlist contains all of the tests that are
2509                         // going to be run. The GCSimulator playlist contains all of
2510                         // the GC simulator tests.
2511                         if (scenario == 'longgc') {
2512                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2513                         }
2514                         else if (scenario == 'gcsimulator') {
2515                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2516                         }
2517                     }
2518
2519                     def folder = getJobFolder(scenario)
2520                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2521                         // Add parameters for the inputs
2522
2523                         parameters {
2524                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2525                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2526                         }
2527
2528                         steps {
2529                             // Set up the copies
2530
2531                             // Coreclr build containing the tests and mscorlib
2532
2533                             copyArtifacts(inputWindowTestsBuildName) {
2534                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2535                                 buildSelector {
2536                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2537                                 }
2538                             }
2539
2540                             if (scenario == 'coverage') {
2541
2542                                 // Move coreclr to clr directory
2543                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2544
2545                                 // Build coreclr
2546                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2547
2548                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2549                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2550                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2551                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2552                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2553                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2554                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2555                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2556
2557                                 // Run PAL tests
2558                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2559
2560                                 // Remove obj files for PAL tests so they're not included in coverage results
2561                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2562
2563                                 // Unzip the tests first.  Exit with 0
2564                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2565
2566                                 // Get corefx
2567                                 shell("git clone https://github.com/dotnet/corefx fx")
2568
2569                                 // Build Linux corefx
2570                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2571                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2572
2573                                 def testEnvOpt = ""
2574                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2575                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2576                                 shell("${createScriptCmds}")
2577                                 testEnvOpt = "--test-env=" + scriptFileName
2578
2579                                 // Run corefx tests
2580                                 shell("""./fx/run-test.sh \\
2581                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2582                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2583                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2584                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2585                 --configurationGroup Release""")
2586
2587
2588                                 // Run coreclr tests w/ workstation GC
2589                                 shell("""./clr/tests/runtest.sh \\
2590                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2591                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2592                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2593                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2594                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2595                 --crossgen --runcrossgentests""")
2596
2597                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2598                                 shell("""./clr/tests/runtest.sh \\
2599                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2600                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2601                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2602                 --useServerGC ${testEnvOpt}""")
2603
2604                                  // Run long-running coreclr GC tests & produce coverage reports
2605                                 shell("""./clr/tests/runtest.sh \\
2606                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2607                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2608                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2609                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2610                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2611                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2612                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2613
2614                             }
2615                             else {
2616
2617                                 // Coreclr build we are trying to test
2618
2619                                 copyArtifacts(inputCoreCLRBuildName) {
2620                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2621                                     buildSelector {
2622                                         buildNumber('${CORECLR_BUILD}')
2623                                     }
2624                                 }
2625
2626                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2627
2628                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2629                                 def osJobName
2630                                 if (os == 'Ubuntu') {
2631                                     osJobName = 'ubuntu14.04'
2632                                 }
2633                                 else {
2634                                     osJobName = os.toLowerCase()
2635                                 }
2636                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2637                                     includePatterns('bin/build.tar.gz')
2638                                     buildSelector {
2639                                         latestSuccessful(true)
2640                                     }
2641                                 }
2642
2643                                 shell ("mkdir ./bin/CoreFxBinDir")
2644                                 // Unpack the corefx binaries
2645                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2646
2647                                 // Unzip the tests first.  Exit with 0
2648                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2649
2650                                 // Execute the tests
2651                                 // If we are running a stress mode, we'll set those variables first
2652                                 def testEnvOpt = ""
2653                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2654                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2655                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2656                                     shell("${createScriptCmds}")
2657                                     testEnvOpt = "--test-env=" + scriptFileName
2658                                 }
2659
2660                                 if (isGCStressRelatedTesting(scenario)) {
2661                                     shell('./init-tools.sh')
2662                                 }
2663
2664                                 shell("""./tests/runtest.sh \\
2665                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2666                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2667                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2668                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2669                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2670                 --limitedDumpGeneration \\
2671                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${sequentialString} ${playlistString}""")
2672                             }
2673                         }
2674                     }
2675
2676                     if (scenario == 'coverage') {
2677                         // Publish coverage reports
2678                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2679                         // TODO: Add once external email sending is available again
2680                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2681                     }
2682
2683                     if (scenario == 'jitdiff') {
2684                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2685                     }
2686
2687                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2688                     if (os in ['Ubuntu']) {
2689                         SummaryBuilder summaries = new SummaryBuilder()
2690                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2691                         summaries.emit(newJob)
2692                     }
2693
2694                     setMachineAffinity(newJob, os, architecture)
2695                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2696                     // Set timeouts to 240.
2697                     setTestJobTimeOut(newJob, scenario)
2698                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2699
2700                     // Create a build flow to join together the build and tests required to run this
2701                     // test.
2702                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2703                     // Linux CoreCLR test
2704                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2705                     def fullTestJobName = projectFolder + '/' + newJob.name
2706                     // Add a reference to the input jobs for report purposes
2707                     JobReport.Report.addReference(inputCoreCLRBuildName)
2708                     JobReport.Report.addReference(inputWindowTestsBuildName)
2709                     JobReport.Report.addReference(fullTestJobName)
2710                     def newFlowJob;
2711
2712                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2713                     // so that coverage data can be preserved.
2714                     if (scenario == 'coverage') {
2715                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2716                         buildFlow("""
2717 // Build the input Windows job
2718 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2719
2720 // And then build the test build
2721 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2722 """)
2723                         }
2724                     // Normal jobs copy a Windows build & a non-Windows build
2725                     } else {
2726                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2727                         buildFlow("""
2728 // Build the input jobs in parallel
2729 parallel (
2730     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2731     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2732 )
2733
2734 // And then build the test build
2735 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2736                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2737 """)
2738                         }
2739                     }
2740
2741                     setMachineAffinity(newFlowJob, os, architecture)
2742                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2743                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false, false)
2744                 } // configuration
2745             } // os
2746         } // architecture
2747     } // isPR
2748 } // scenario
2749
2750 JobReport.Report.generateJobReport(out)
2751
2752 // Make the call to generate the help job
2753 Utilities.createHelperJob(this, project, branch,
2754     "Welcome to the ${project} Repository",  // This is prepended to the help message
2755     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.