netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'FreeBSD':'FreeBSD',
  29         'CentOS7.1': 'Linux',
  30         'OpenSUSE42.1': 'Linux',
  31         'Tizen': 'Linux']
  32     def osGroup = osGroupMap.get(os, null)
  33     assert osGroup != null : "Could not find os group for ${os}"
  34     return osGroupMap[os]
  35 }
  36
  37 // We use this class (vs variables) so that the static functions can access data here.
  38 class Constants {
  39
  40     // Innerloop build OS's
  41     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  42     // test execution in the build flow runs.  It generates the exact same build
  43     // as Windows_NT but without the tests.
  44     def static osList = [
  45                'Ubuntu',
  46                'Debian8.4',
  47                'OSX10.12',
  48                'Windows_NT',
  49                'Windows_NT_BuildOnly',
  50                'FreeBSD',
  51                'CentOS7.1',
  52                'OpenSUSE42.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'corefx_baseline'                : [ : ], // corefx baseline
  89                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  90                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  91                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  92                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  93                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  94                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  95                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  96                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  97                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  98                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  99                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 100                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 101                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 102                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 103                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 104                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 105                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 106                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 107                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 108                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 109                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 110                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 111                ]
 112
 113     // This is a set of r2r jit stress scenarios
 114     def static r2rJitStressScenarios = [
 115                'r2r_jitstress1',
 116                'r2r_jitstress2',
 117                'r2r_jitstressregs1',
 118                'r2r_jitstressregs2',
 119                'r2r_jitstressregs3',
 120                'r2r_jitstressregs4',
 121                'r2r_jitstressregs8',
 122                'r2r_jitstressregs0x10',
 123                'r2r_jitstressregs0x80',
 124                'r2r_jitstressregs0x1000',
 125                'r2r_jitminopts',
 126                'r2r_jitforcerelocs']
 127
 128     // This is the basic set of scenarios
 129     def static basicScenarios = [
 130                'default',
 131                'pri1',
 132                'ilrt',
 133                'r2r',
 134                'pri1r2r',
 135                'gcstress15_pri1r2r',
 136                'longgc',
 137                'coverage',
 138                'formatting',
 139                'gcsimulator',
 140                'jitdiff',
 141                'standalone_gc',
 142                'gc_reliability_framework',
 143                'illink'] + r2rJitStressScenarios
 144
 145     def static configurationList = ['Debug', 'Checked', 'Release']
 146
 147     // This is the set of architectures
 148     def static architectureList = ['arm', 'arm64', 'x64', 'x86', 'x86lb']
 149 }
 150
 151 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 152     if (architecture == 'arm64' && os == 'Windows_NT') {
 153         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 154     } else if (architecture == 'arm64' && os != 'Windows_NT' && options == null) {
 155         Utilities.setMachineAffinity(job, os, 'arm64-small-page-size');
 156     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['large_pages'] == true) {
 157         Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size');
 158     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['is_build_only'] == true) {
 159         Utilities.setMachineAffinity(job, os, 'arm64-cross-latest');
 160     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 161         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 162     } else if ((architecture == 'arm') && (os == 'Windows_NT') && options['use_arm64_build_machine'] == true) {
 163         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 164     }else {
 165         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 166     }
 167 }
 168
 169 def static isJITStressJob(def scenario) {
 170     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 171            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 172 }
 173
 174 def static isGCStressRelatedTesting(def scenario) {
 175     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 176     // Detect it and make it a GCStress related.
 177     if (scenario == 'gcstress15_pri1r2r')
 178     {
 179         return true;
 180     }
 181
 182     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 183     def scenarioName = scenario.toLowerCase()
 184     def isGCStressTesting = false
 185     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 186         if (k in gcStressTestEnvVars) {
 187             isGCStressTesting = true;
 188         }
 189     }
 190     return isGCStressTesting
 191 }
 192
 193 def static isCorefxTesting(def scenario) {
 194     def corefx_prefix = 'corefx_'
 195     if (scenario.length() < corefx_prefix.length()) {
 196         return false
 197     }
 198     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 199 }
 200
 201 def static isR2R(def scenario) {
 202     return (scenario == 'r2r' || scenario == 'pri1r2r')
 203 }
 204
 205 def static isCoverage(def scenario) {
 206     return (scenario == 'coverage')
 207 }
 208
 209 def static isLongGc(def scenario) {
 210     return (scenario == 'longgc' || scenario == 'gcsimulator')
 211 }
 212
 213 def static isJitDiff(def scenario) {
 214     return (scenario == 'jitdiff')
 215 }
 216
 217 def static isGcReliabilityFramework(def scenario) {
 218     return (scenario == 'gc_reliability_framework')
 219 }
 220
 221 def static scenarioNeedsPri1Build(def scenario) {
 222     return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario))
 223 }
 224
 225 def static setTestJobTimeOut(newJob, scenario) {
 226     if (isGCStressRelatedTesting(scenario)) {
 227         Utilities.setJobTimeout(newJob, 4320)
 228     }
 229     else if (isCorefxTesting(scenario)) {
 230         Utilities.setJobTimeout(newJob, 360)
 231     }
 232     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 233         Utilities.setJobTimeout(newJob, 240)
 234     }
 235     else if (isR2R(scenario)) {
 236         Utilities.setJobTimeout(newJob, 240)
 237     }
 238     else if (isCoverage(scenario)) {
 239         Utilities.setJobTimeout(newJob, 1440)
 240     }
 241     else if (isLongGc(scenario)) {
 242         Utilities.setJobTimeout(newJob, 1440)
 243     }
 244     else if (isJitDiff(scenario)) {
 245         Utilities.setJobTimeout(newJob, 240)
 246     }
 247     else if (isGcReliabilityFramework(scenario)) {
 248         Utilities.setJobTimeout(newJob, 1440)
 249     }
 250     // Non-test jobs use the default timeout value.
 251 }
 252
 253 def static getJobFolder(def scenario) {
 254     if (isJITStressJob(scenario)) {
 255         return 'jitstress'
 256     }
 257     if (scenario == 'illink') {
 258         return 'illink'
 259     }
 260     return ''
 261 }
 262
 263 def static getStressModeDisplayName(def scenario) {
 264     def displayStr = ''
 265     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 266         def prefixLength = 'COMPlus_'.length()
 267         if (k.length() >= prefixLength) {
 268             def modeName = k.substring(prefixLength, k.length())
 269             displayStr += ' ' + modeName + '=' + v
 270         }
 271     }
 272     return displayStr
 273 }
 274
 275 def static getR2RStressModeDisplayName(def scenario) {
 276     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 277     // "r2r_" prefix.
 278     def displayStr = scenario
 279     def prefixLength = 'r2r_'.length()
 280     if (displayStr.length() >= prefixLength) {
 281         displayStr = displayStr.substring(prefixLength, displayStr.length())
 282     }
 283     return displayStr
 284 }
 285
 286 // Generates the string for creating a file that sets environment variables
 287 // that makes it possible to run stress modes.  Writes the script to the file
 288 // specified by the stepScriptLocation parameter.
 289 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 290     def stepScript = ''
 291     if (os == 'Windows_NT') {
 292         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 293         stepScript += "del ${stepScriptLocation}\r\n"
 294
 295         // Timeout in ms, default is 10 minutes. For stress
 296         // modes up this to 30 minutes
 297         def timeout = 1800000
 298
 299         // Set the Timeout
 300         stepScript += "set __TestTimeout=${timeout}\r\n"
 301         stepScript += "echo. > ${stepScriptLocation}\r\n"
 302         stressModeVars.each{ k, v ->
 303             // Write out what we are writing to the script file
 304             stepScript += "echo Setting ${k}=${v}\r\n"
 305             // Write out the set itself to the script file`
 306             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 307         }
 308     }
 309     else {
 310         stepScript += "echo Setting variables for ${stressModeName}\n"
 311         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 312         stressModeVars.each{ k, v ->
 313             // Write out what we are writing to the script file
 314             stepScript += "echo Setting ${k}=${v}\n"
 315             // Write out the set itself to the script file`
 316             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 317         }
 318         stepScript += "chmod +x ${stepScriptLocation}\n"
 319     }
 320     return stepScript
 321 }
 322
 323 // Calculates the name of the build job based on some typical parameters.
 324 //
 325 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 326     // If the architecture is x64, do not add that info into the build name.
 327     // Need to change around some systems and other builds to pick up the right builds
 328     // to do that.
 329
 330     def suffix = scenario != 'default' ? "_${scenario}" : '';
 331     if (isBuildOnly) {
 332         suffix += '_bld'
 333     }
 334     def baseName = ''
 335     switch (architecture) {
 336         case 'x64':
 337             if (scenario == 'default') {
 338                 // For now we leave x64 off of the name for compatibility with other jobs
 339                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 340             }
 341             else if (scenario == 'formatting') {
 342                 // we don't care about the configuration for the formatting job. It runs all configs
 343                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 344             }
 345             else {
 346                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 347             }
 348             break
 349         case 'arm64':
 350             if (os.toLowerCase() == "windows_nt") {
 351                 // These are cross builds
 352                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 353             }
 354             else {
 355                 // Defaults to a small page size set of machines.
 356                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 357             }
 358             break
 359         case 'arm':
 360             // These are cross builds
 361             if (os == 'Tizen') {
 362                 // ABI: softfp
 363                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 364             }
 365             else {
 366                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 367             }
 368             break
 369         case 'x86':
 370         case 'x86lb':
 371             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 372             break
 373         default:
 374             println("Unknown architecture: ${architecture}");
 375             assert false
 376             break
 377     }
 378
 379     return baseName + suffix
 380 }
 381
 382 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 383     // Check scenario.
 384     switch (scenario) {
 385         case 'default':
 386             switch (architecture) {
 387                 case 'x64':
 388                 case 'x86':
 389                 case 'x86lb':
 390                     if (architecture == 'x86' && os == 'Ubuntu') {
 391                         Utilities.addPeriodicTrigger(job, '@daily')
 392                     }
 393                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 394                         Utilities.addGithubPushTrigger(job)
 395                     }
 396                     break
 397                 case 'arm':
 398                     Utilities.addGithubPushTrigger(job)
 399                     break
 400                 case 'arm64':
 401                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 402                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 403                     break
 404                 default:
 405                     println("Unknown architecture: ${architecture}");
 406                     assert false
 407                     break
 408             }
 409             break
 410         case 'pri1':
 411             // Pri one gets a push trigger, and only for release
 412             if (architecture == 'x64') {
 413                 if (configuration == 'Release') {
 414                     // We expect release jobs to be Windows, or in the cross list
 415                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 416                     if (!os in bidailyCrossList) {
 417                         if (isFlowJob || os == 'Windows_NT') {
 418                             Utilities.addGithubPushTrigger(job)
 419                         }
 420                     }
 421                     else {
 422                         if (isFlowJob) {
 423                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 424                         }
 425                     }
 426                 }
 427             }
 428             break
 429         case 'r2r':
 430             //r2r jobs that aren't pri1 can only be triggered by phrase
 431             break
 432         case 'pri1r2r':
 433             assert !(os in bidailyCrossList)
 434             //pri1 r2r gets a push trigger for checked/release
 435             if (configuration == 'Checked' || configuration == 'Release') {
 436                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 437                 if (architecture == 'x64' && os != 'OSX10.12') {
 438                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 439                     if (isFlowJob || os == 'Windows_NT') {
 440                         Utilities.addGithubPushTrigger(job)
 441                     }
 442                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 443                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 444                     if (isFlowJob) {
 445                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 446                     }
 447                 }
 448                 // For x86, only add per-commit jobs for Windows
 449                 else if (architecture == 'x86' || architecture == 'x86lb') {
 450                     if (os == 'Windows_NT') {
 451                         Utilities.addGithubPushTrigger(job)
 452                     }
 453                 }
 454                 // arm64 pri1r2r jobs should only run every 12 hours.
 455                 else if (architecture == 'arm64') {
 456                     if (os == 'Windows_NT') {
 457                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 458                         // TODO: Add once external email sending is available again
 459                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 460                     }
 461                 }
 462             }
 463             break
 464         case 'r2r_jitstress1':
 465         case 'r2r_jitstress2':
 466         case 'r2r_jitstressregs1':
 467         case 'r2r_jitstressregs2':
 468         case 'r2r_jitstressregs3':
 469         case 'r2r_jitstressregs4':
 470         case 'r2r_jitstressregs8':
 471         case 'r2r_jitstressregs0x10':
 472         case 'r2r_jitstressregs0x80':
 473         case 'r2r_jitstressregs0x1000':
 474         case 'r2r_jitminopts':
 475         case 'r2r_jitforcerelocs':
 476         case 'gcstress15_pri1r2r':
 477             assert !(os in bidailyCrossList)
 478
 479             // GCStress=C is currently not supported on OS X
 480             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 481                 break
 482             }
 483
 484             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 485             if (configuration == 'Checked' || configuration == 'Release') {
 486                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 487                 if (architecture == 'x64') {
 488                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 489                     if (isFlowJob || os == 'Windows_NT') {
 490                         // Add a weekly periodic trigger
 491                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 492                     }
 493                 }
 494                 // For x86, only add per-commit jobs for Windows
 495                 else if (architecture == 'x86') {
 496                     if (os == 'Windows_NT') {
 497                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 498                     }
 499                 }
 500             }
 501             break
 502         case 'longgc':
 503             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 504             assert configuration == 'Release'
 505             assert architecture == 'x64'
 506             Utilities.addPeriodicTrigger(job, '@daily')
 507             // TODO: Add once external email sending is available again
 508             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 509             break
 510         case 'gcsimulator':
 511             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 512             assert configuration == 'Release'
 513             assert architecture == 'x64'
 514             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 515             // TODO: Add once external email sending is available again
 516             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 517             break
 518         case 'standalone_gc':
 519             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 520             assert (configuration == 'Release' || configuration == 'Checked')
 521             // TODO: Add once external email sending is available again
 522             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 523             Utilities.addPeriodicTrigger(job, '@weekly')
 524             break
 525         case 'gc_reliability_framework':
 526             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 527             assert (configuration == 'Release' || configuration == 'Checked')
 528             // Only triggered by phrase.
 529             break
 530         case 'ilrt':
 531             assert !(os in bidailyCrossList)
 532             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 533             if (architecture == 'x64' && configuration == 'Release') {
 534                 // We don't expect to see a job generated except in these scenarios
 535                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 536                 if (isFlowJob || os == 'Windows_NT') {
 537                     Utilities.addPeriodicTrigger(job, '@daily')
 538                 }
 539             }
 540             break
 541         case 'jitdiff':
 542             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 543             assert configuration == 'Checked'
 544             assert (architecture == 'x64' || architecture == 'x86')
 545             Utilities.addGithubPushTrigger(job)
 546             break
 547         case 'coverage':
 548             assert (os == 'Ubuntu' || os == 'Windows_NT')
 549             assert configuration == 'Release'
 550             assert architecture == 'x64'
 551             Utilities.addPeriodicTrigger(job, '@weekly')
 552             break
 553         case 'formatting':
 554             assert (os == 'Windows_NT' || os == "Ubuntu")
 555             assert architecture == 'x64'
 556             Utilities.addGithubPushTrigger(job)
 557             break
 558         case 'jitstressregs1':
 559         case 'jitstressregs2':
 560         case 'jitstressregs3':
 561         case 'jitstressregs4':
 562         case 'jitstressregs8':
 563         case 'jitstressregs0x10':
 564         case 'jitstressregs0x80':
 565         case 'jitstressregs0x1000':
 566         case 'minopts':
 567         case 'forcerelocs':
 568         case 'jitstress1':
 569         case 'jitstress2':
 570         case 'jitstress2_jitstressregs1':
 571         case 'jitstress2_jitstressregs2':
 572         case 'jitstress2_jitstressregs3':
 573         case 'jitstress2_jitstressregs4':
 574         case 'jitstress2_jitstressregs8':
 575         case 'jitstress2_jitstressregs0x10':
 576         case 'jitstress2_jitstressregs0x80':
 577         case 'jitstress2_jitstressregs0x1000':
 578         case 'tailcallstress':
 579         case 'jitsse2only':
 580         case 'corefx_baseline':
 581         case 'corefx_minopts':
 582         case 'corefx_jitstress1':
 583         case 'corefx_jitstress2':
 584         case 'corefx_jitstressregs1':
 585         case 'corefx_jitstressregs2':
 586         case 'corefx_jitstressregs3':
 587         case 'corefx_jitstressregs4':
 588         case 'corefx_jitstressregs8':
 589         case 'corefx_jitstressregs0x10':
 590         case 'corefx_jitstressregs0x80':
 591         case 'corefx_jitstressregs0x1000':
 592         case 'zapdisable':
 593             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 594             assert (os == 'Windows_NT') || (os in Constants.crossList)
 595             Utilities.addPeriodicTrigger(job, '@daily')
 596         }
 597         break
 598         case 'heapverify1':
 599         case 'gcstress0x3':
 600             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 601                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 602                 if (architecture == 'arm64') {
 603                     if (os == 'Windows_NT') {
 604                         Utilities.addPeriodicTrigger(job, '@daily')
 605                     }
 606                     // TODO: Add once external email sending is available again
 607                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 608                 }
 609                 else {
 610                     Utilities.addPeriodicTrigger(job, '@weekly')
 611                 }
 612             }
 613             break
 614         case 'gcstress0xc':
 615         case 'gcstress0xc_zapdisable':
 616         case 'gcstress0xc_zapdisable_jitstress2':
 617         case 'gcstress0xc_zapdisable_heapverify1':
 618         case 'gcstress0xc_jitstress1':
 619         case 'gcstress0xc_jitstress2':
 620         case 'gcstress0xc_minopts_heapverify1':
 621             // GCStress=C is currently not supported on OS X
 622             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 623                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 624                 if (architecture == 'arm64') {
 625                     // TODO: Enable a periodic trigger after tests are updated.
 626                     // Utilities.addPeriodicTrigger(job, '@daily')
 627                     // TODO: Add once external email sending is available again
 628                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 629                 }
 630                 else {
 631                     Utilities.addPeriodicTrigger(job, '@weekly')
 632                 }
 633             }
 634             break
 635
 636         case 'illink':
 637             // Testing on other operating systems TBD
 638             assert (os == 'Windows_NT' || os == 'Ubuntu')
 639             if (architecture == 'x64' || architecture == 'x86') {
 640                 if (configuration == 'Checked') {
 641                     Utilities.addPeriodicTrigger(job, '@daily')
 642                 }
 643             }
 644             break
 645
 646         default:
 647             println("Unknown scenario: ${scenario}");
 648             assert false
 649             break
 650     }
 651     return
 652 }
 653
 654 // **************************
 655 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 656 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 657 // builds will do a couple extra steps.
 658 // **************************
 659
 660 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 661 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 662 // it's just used for internal builds
 663 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 664 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 665     if (isWindowsBuildOnlyJob) {
 666         return
 667     }
 668
 669     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 670     // Non pull request builds.
 671     if (!isPR) {
 672         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 673         return
 674     }
 675
 676      def arm64Users = [
 677         'adiaaida',
 678         'AndyAyersMS',
 679         'briansull',
 680         'BruceForstall',
 681         'CarolEidt',
 682         'cmckinsey',
 683         'erozenfeld',
 684         'jashook',
 685         'JosephTremoulet',
 686         'pgavlin',
 687         'russellhadley',
 688         'RussKeldorph',
 689         'sandreenko',
 690         'sdmaclea',
 691         'sivarv',
 692         'swaroop-sridhar',
 693         'gkhanna79',
 694         'jkotas',
 695         'markwilkie',
 696         'rahku',
 697         'ramarag',
 698         'tzwlai',
 699         'weshaggard'
 700     ]
 701
 702     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 703     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 704     def osGroup = getOSGroup(os)
 705     switch (architecture) {
 706         case 'x64': // editor brace matching: {
 707             if (scenario == 'coverage') {
 708                 assert configuration == 'Release'
 709                 if (os == 'Ubuntu') {
 710                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 711                 }
 712                 break
 713             }
 714
 715             if (scenario == 'formatting') {
 716                 assert configuration == 'Checked'
 717                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 718                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 719                 }
 720                 break
 721             }
 722
 723             switch (os) {
 724                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 725                 case 'Debian8.4':
 726                 case 'RHEL7.2':
 727                     if (scenario == 'default') {
 728                         assert !isFlowJob
 729                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 730                     }
 731                     else if (scenario == 'pri1' && isFlowJob) {
 732                         assert (configuration == 'Release')
 733                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 734                     }
 735                     break
 736                 case 'Ubuntu16.04':
 737                     assert !isFlowJob
 738                     assert scenario == 'default'
 739                     // Distinguish with the other architectures (arm and x86)
 740                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 741                     break
 742                 case 'Fedora24':
 743                 case 'Ubuntu16.10':
 744                 case 'OpenSUSE42.1':
 745                     assert !isFlowJob
 746                     assert scenario == 'default'
 747                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 748                     break
 749                 case 'Ubuntu':
 750                     if (scenario == 'illink') {
 751                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 752                         break
 753                     }
 754                     // fall through
 755                 case 'OSX10.12':
 756                     // Triggers on the non-flow jobs aren't necessary here
 757                     // Corefx testing uses non-flow jobs.
 758                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 759                         break
 760                     }
 761                     switch (scenario) {
 762                         case 'default':
 763                             // Ubuntu uses checked for default PR tests
 764                             if (configuration == 'Checked') {
 765                                 // Default trigger
 766                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 767                             }
 768                             break
 769                         case 'pri1':
 770                             if (configuration == 'Release') {
 771                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 772                             }
 773                             break
 774                         case 'jitdiff':
 775                             if (configuration == 'Checked') {
 776                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 777                             }
 778                             break
 779                         case 'ilrt':
 780                             if (configuration == 'Release') {
 781                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 782                             }
 783                             break
 784                         case 'r2r':
 785                             if (configuration == 'Release' || configuration == 'Checked') {
 786                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 787                             }
 788                             break
 789                         case 'pri1r2r':
 790                             if (configuration == 'Release' || configuration == 'Checked') {
 791                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 792                             }
 793                             break
 794                         case 'gcstress15_pri1r2r':
 795                             if (configuration == 'Release' || configuration == 'Checked') {
 796                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 797                             }
 798                             break
 799                         case 'r2r_jitstress1':
 800                         case 'r2r_jitstress2':
 801                         case 'r2r_jitstressregs1':
 802                         case 'r2r_jitstressregs2':
 803                         case 'r2r_jitstressregs3':
 804                         case 'r2r_jitstressregs4':
 805                         case 'r2r_jitstressregs8':
 806                         case 'r2r_jitstressregs0x10':
 807                         case 'r2r_jitstressregs0x80':
 808                         case 'r2r_jitstressregs0x1000':
 809                         case 'r2r_jitminopts':
 810                         case 'r2r_jitforcerelocs':
 811                             if (configuration == 'Release' || configuration == 'Checked') {
 812                                 def displayStr = getR2RStressModeDisplayName(scenario)
 813                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 814                             }
 815                             break
 816                         case 'longgc':
 817                             if (configuration == 'Release') {
 818                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 819                             }
 820                             break
 821                         case 'gcsimulator':
 822                             if (configuration == 'Release') {
 823                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 824                             }
 825                             break
 826                         case 'standalone_gc':
 827                             if (configuration == 'Release' || configuration == 'Checked') {
 828                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 829                             }
 830                             break
 831                         case 'gc_reliability_framework':
 832                             if (configuration == 'Release' || configuration == 'Checked') {
 833                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 834                             }
 835                             break
 836                         case 'minopts':
 837                         case 'tieredcompilation':
 838                         case 'forcerelocs':
 839                         case 'jitstress1':
 840                         case 'jitstress2':
 841                         case 'jitstressregs1':
 842                         case 'jitstressregs2':
 843                         case 'jitstressregs3':
 844                         case 'jitstressregs4':
 845                         case 'jitstressregs8':
 846                         case 'jitstressregs0x10':
 847                         case 'jitstressregs0x80':
 848                         case 'jitstressregs0x1000':
 849                         case 'jitstress2_jitstressregs1':
 850                         case 'jitstress2_jitstressregs2':
 851                         case 'jitstress2_jitstressregs3':
 852                         case 'jitstress2_jitstressregs4':
 853                         case 'jitstress2_jitstressregs8':
 854                         case 'jitstress2_jitstressregs0x10':
 855                         case 'jitstress2_jitstressregs0x80':
 856                         case 'jitstress2_jitstressregs0x1000':
 857                         case 'tailcallstress':
 858                         case 'jitsse2only':
 859                         case 'gcstress0x3':
 860                         case 'gcstress0xc':
 861                         case 'zapdisable':
 862                         case 'heapverify1':
 863                         case 'gcstress0xc_zapdisable':
 864                         case 'gcstress0xc_zapdisable_jitstress2':
 865                         case 'gcstress0xc_zapdisable_heapverify1':
 866                         case 'gcstress0xc_jitstress1':
 867                         case 'gcstress0xc_jitstress2':
 868                         case 'gcstress0xc_minopts_heapverify1':
 869                             def displayStr = getStressModeDisplayName(scenario)
 870                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 871                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 872                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 873                             break
 874                         case 'corefx_baseline':
 875                         case 'corefx_minopts':
 876                         case 'corefx_tieredcompilation':
 877                         case 'corefx_jitstress1':
 878                         case 'corefx_jitstress2':
 879                         case 'corefx_jitstressregs1':
 880                         case 'corefx_jitstressregs2':
 881                         case 'corefx_jitstressregs3':
 882                         case 'corefx_jitstressregs4':
 883                         case 'corefx_jitstressregs8':
 884                         case 'corefx_jitstressregs0x10':
 885                         case 'corefx_jitstressregs0x80':
 886                         case 'corefx_jitstressregs0x1000':
 887                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 888                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 889                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 890                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 891                             break
 892                         default:
 893                             println("Unknown scenario: ${scenario}");
 894                             assert false
 895                             break
 896                     }
 897                     break
 898                 case 'CentOS7.1':
 899                     switch (scenario) {
 900                         case 'pri1':
 901                             if (configuration == 'Release') {
 902                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 903                             }
 904                             break
 905                         case 'r2r':
 906                             if (configuration == 'Checked' || configuration == 'Release') {
 907                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 908                             }
 909                             break
 910                         case 'pri1r2r':
 911                             if (configuration == 'Checked' || configuration == 'Release') {
 912                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 913                             }
 914                             break
 915                         case 'gcstress15_pri1r2r':
 916                             if (configuration == 'Release' || configuration == 'Checked') {
 917                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 918                             }
 919                             break
 920                         case 'r2r_jitstress1':
 921                         case 'r2r_jitstress2':
 922                         case 'r2r_jitstressregs1':
 923                         case 'r2r_jitstressregs2':
 924                         case 'r2r_jitstressregs3':
 925                         case 'r2r_jitstressregs4':
 926                         case 'r2r_jitstressregs8':
 927                         case 'r2r_jitstressregs0x10':
 928                         case 'r2r_jitstressregs0x80':
 929                         case 'r2r_jitstressregs0x1000':
 930                         case 'r2r_jitminopts':
 931                         case 'r2r_jitforcerelocs':
 932                             if (configuration == 'Release' || configuration == 'Checked') {
 933                                 def displayStr = getR2RStressModeDisplayName(scenario)
 934                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 935                             }
 936                             break
 937                         default:
 938                             break
 939                     }
 940                 case 'Windows_NT':
 941                     switch (scenario) {
 942                         case 'default':
 943                             // Default trigger
 944                             if (configuration == 'Debug') {
 945                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 946                             }
 947                             break
 948                         case 'pri1':
 949                             // Default trigger
 950                             if (configuration == 'Release') {
 951                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 952                             }
 953                             break
 954                         case 'jitdiff':
 955                             if (configuration == 'Checked') {
 956                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 957                             }
 958                             break
 959                         case 'ilrt':
 960                             if (configuration == 'Release') {
 961                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 962                             }
 963                             break
 964                         case 'r2r':
 965                             if (configuration == 'Checked' || configuration == 'Release') {
 966                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 967                             }
 968                             break
 969                         case 'pri1r2r':
 970                             if (configuration == 'Checked' || configuration == 'Release') {
 971                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 972                             }
 973                             break
 974                         case 'gcstress15_pri1r2r':
 975                             if (configuration == 'Release' || configuration == 'Checked') {
 976                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 977                             }
 978                             break
 979                         case 'r2r_jitstress1':
 980                         case 'r2r_jitstress2':
 981                         case 'r2r_jitstressregs1':
 982                         case 'r2r_jitstressregs2':
 983                         case 'r2r_jitstressregs3':
 984                         case 'r2r_jitstressregs4':
 985                         case 'r2r_jitstressregs8':
 986                         case 'r2r_jitstressregs0x10':
 987                         case 'r2r_jitstressregs0x80':
 988                         case 'r2r_jitstressregs0x1000':
 989                         case 'r2r_jitminopts':
 990                         case 'r2r_jitforcerelocs':
 991                             if (configuration == 'Release' || configuration == 'Checked') {
 992                                 def displayStr = getR2RStressModeDisplayName(scenario)
 993                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 994                             }
 995                             break
 996                         case 'longgc':
 997                             if (configuration == 'Release') {
 998                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 999                             }
1000                             break
1001                         case 'gcsimulator':
1002                             if (configuration == 'Release') {
1003                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1004                             }
1005                             break
1006                         case 'standalone_gc':
1007                             if (configuration == 'Release' || configuration == 'Checked') {
1008                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1009                             }
1010                             break
1011                         case 'gc_reliability_framework':
1012                             if (configuration == 'Release' || configuration == 'Checked') {
1013                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1014                             }
1015                             break
1016                         case 'minopts':
1017                         case 'tieredcompilation':
1018                         case 'forcerelocs':
1019                         case 'jitstress1':
1020                         case 'jitstress2':
1021                         case 'jitstressregs1':
1022                         case 'jitstressregs2':
1023                         case 'jitstressregs3':
1024                         case 'jitstressregs4':
1025                         case 'jitstressregs8':
1026                         case 'jitstressregs0x10':
1027                         case 'jitstressregs0x80':
1028                         case 'jitstressregs0x1000':
1029                         case 'jitstress2_jitstressregs1':
1030                         case 'jitstress2_jitstressregs2':
1031                         case 'jitstress2_jitstressregs3':
1032                         case 'jitstress2_jitstressregs4':
1033                         case 'jitstress2_jitstressregs8':
1034                         case 'jitstress2_jitstressregs0x10':
1035                         case 'jitstress2_jitstressregs0x80':
1036                         case 'jitstress2_jitstressregs0x1000':
1037                         case 'tailcallstress':
1038                         case 'jitsse2only':
1039                         case 'gcstress0x3':
1040                         case 'gcstress0xc':
1041                         case 'zapdisable':
1042                         case 'heapverify1':
1043                         case 'gcstress0xc_zapdisable':
1044                         case 'gcstress0xc_zapdisable_jitstress2':
1045                         case 'gcstress0xc_zapdisable_heapverify1':
1046                         case 'gcstress0xc_jitstress1':
1047                         case 'gcstress0xc_jitstress2':
1048                         case 'gcstress0xc_minopts_heapverify1':
1049                             def displayStr = getStressModeDisplayName(scenario)
1050                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1051                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1052                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1053                             break
1054                         case 'corefx_baseline':
1055                         case 'corefx_minopts':
1056                         case 'corefx_tieredcompilation':
1057                         case 'corefx_jitstress1':
1058                         case 'corefx_jitstress2':
1059                         case 'corefx_jitstressregs1':
1060                         case 'corefx_jitstressregs2':
1061                         case 'corefx_jitstressregs3':
1062                         case 'corefx_jitstressregs4':
1063                         case 'corefx_jitstressregs8':
1064                         case 'corefx_jitstressregs0x10':
1065                         case 'corefx_jitstressregs0x80':
1066                         case 'corefx_jitstressregs0x1000':
1067                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1068                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1069                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1070                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1071                             break
1072                         case 'illink':
1073                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1074                                     break
1075                         default:
1076                             println("Unknown scenario: ${scenario}");
1077                             assert false
1078                             break
1079                     }
1080                     break
1081                 case 'FreeBSD':
1082                     assert scenario == 'default'
1083                     if (configuration == 'Checked') {
1084                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1085                     }
1086                     break
1087                 default:
1088                     println("Unknown os: ${os}");
1089                     assert false
1090                     break
1091             }
1092             break
1093         // editor brace matching: }
1094         case 'arm': // editor brace matching: {
1095             switch (os) {
1096                 case 'Ubuntu':
1097                 case 'Ubuntu16.04':
1098                     assert scenario == 'default'
1099                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1100                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1101                     }
1102                     else {
1103                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1104                     }
1105                     break;
1106                 case 'Tizen':
1107                     architecture='armel'
1108                     // Removing the regex will cause this to run on each PR.
1109                     if (configuration == 'Release' || configuration == 'Debug') {
1110                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1111                     }
1112                     else {
1113                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1114                     }
1115                     break;
1116                 case 'Windows_NT':
1117                     // Set up a private trigger
1118                     def contextString = "${os} ${architecture} Cross ${configuration}"
1119                     if (scenario != 'default')
1120                         contextString += " ${scenario}"
1121                     contextString += " Build"
1122                     // Debug builds only.
1123                     if (configuration != 'Debug') {
1124                         contextString += " and Test"
1125                     }
1126                     switch (scenario) {
1127                         case 'default':
1128                             // For now only run Debug and Release build jobs on PR Trigger. Note this is not a private trigger.
1129                             if (configuration == 'Debug' || configuration == 'Release')
1130                             {
1131                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1132                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1133                             }
1134                             else
1135                             {
1136                                 // Checked jobs will run on private trigger and run tests.
1137                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1138                             }
1139                             break
1140                         case 'pri1r2r':
1141                         case 'gcstress0x3':
1142                         case 'gcstress0xc':
1143                             // Stress jobs will will run on private trigger and run tests.
1144                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1145                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1146                             break
1147                     }
1148                     break
1149                 default:
1150                     println("NYI os: ${os}");
1151                     assert false
1152                     break
1153             }
1154             break
1155         // editor brace matching: }
1156         case 'arm64': // editor brace matching: {
1157             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1158
1159             // Set up a private trigger
1160             def contextString = "${os} ${architecture} Cross ${configuration}"
1161             if (scenario != 'default')
1162                 contextString += " ${scenario}"
1163             contextString += " Build"
1164             // Debug builds only.
1165             if (configuration != 'Debug') {
1166                contextString += " and Test"
1167             }
1168
1169             switch (os) {
1170                 case 'Ubuntu':
1171                 case 'Ubuntu16.04':
1172                     switch (scenario) {
1173                         case 'pri1':
1174                             if (configuration == 'Release') {
1175                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1176                             }
1177                             break
1178                         case 'r2r':
1179                             if (configuration == 'Checked' || configuration == 'Release') {
1180                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1181                             }
1182                             break
1183                         case 'pri1r2r':
1184                             if (configuration == 'Checked' || configuration == 'Release') {
1185                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1186                             }
1187                             break
1188                         case 'gcstress15_pri1r2r':
1189                             if (configuration == 'Release' || configuration == 'Checked') {
1190                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1191                             }
1192                             break
1193                         case 'r2r_jitstress1':
1194                         case 'r2r_jitstress2':
1195                         case 'r2r_jitstressregs1':
1196                         case 'r2r_jitstressregs2':
1197                         case 'r2r_jitstressregs3':
1198                         case 'r2r_jitstressregs4':
1199                         case 'r2r_jitstressregs8':
1200                         case 'r2r_jitstressregs0x10':
1201                         case 'r2r_jitstressregs0x80':
1202                         case 'r2r_jitstressregs0x1000':
1203                         case 'r2r_jitminopts':
1204                         case 'r2r_jitforcerelocs':
1205                             if (configuration == 'Release' || configuration == 'Checked') {
1206                                 def displayStr = getR2RStressModeDisplayName(scenario)
1207                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1208                             }
1209                             break
1210                         default:
1211                             break
1212                     }
1213                 case 'Windows_NT':
1214                     switch (scenario) {
1215                         case 'default':
1216                             if (isFlowJob == true) {
1217                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration}", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1218                             }
1219
1220                             // For now only run Debug jobs on PR Trigger.
1221                             else if (configuration != 'Debug') {
1222                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1223                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1224                             }
1225                             else {
1226                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1227                                 // are at higher risk of ARM64-breaking changes.
1228                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1229                             }
1230                             break
1231                         case 'pri1r2r':
1232                         case 'gcstress0x3':
1233                         case 'gcstress0xc':
1234                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1235                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1236                             break
1237                     }
1238                     break
1239                 default:
1240                     println("NYI os: ${os}");
1241                     assert false
1242                     break
1243             }
1244             break
1245         // editor brace matching: }
1246         case 'x86': // editor brace matching: {
1247             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1248             if (os == 'Ubuntu') {
1249                 // on-demand only for ubuntu x86
1250                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1251                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1252                 break
1253             }
1254             switch (scenario) {
1255                 case 'default':
1256                     if (configuration == 'Checked') {
1257                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1258                     }
1259                     else if (configuration == 'Release') {
1260                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1261                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1262                     }
1263                     break
1264                 case 'pri1':
1265                     if (configuration == 'Release') {
1266                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1267                     }
1268                     break
1269                 case 'ilrt':
1270                     if (configuration == 'Release') {
1271                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1272                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1273                     }
1274                     break
1275                 case 'r2r':
1276                     if (configuration == 'Checked' || configuration == 'Release') {
1277                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1278                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1279                     }
1280                     break
1281                 case 'pri1r2r':
1282                     if (configuration == 'Checked' || configuration == 'Release') {
1283                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1284                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1285                     }
1286                     break
1287                 case 'gcstress15_pri1r2r':
1288                     if (configuration == 'Release' || configuration == 'Checked') {
1289                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1290                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1291                     }
1292                     break
1293                 case 'r2r_jitstress1':
1294                 case 'r2r_jitstress2':
1295                 case 'r2r_jitstressregs1':
1296                 case 'r2r_jitstressregs2':
1297                 case 'r2r_jitstressregs3':
1298                 case 'r2r_jitstressregs4':
1299                 case 'r2r_jitstressregs8':
1300                 case 'r2r_jitstressregs0x10':
1301                 case 'r2r_jitstressregs0x80':
1302                 case 'r2r_jitstressregs0x1000':
1303                 case 'r2r_jitminopts':
1304                 case 'r2r_jitforcerelocs':
1305                     if (configuration == 'Release' || configuration == 'Checked') {
1306                         def displayStr = getR2RStressModeDisplayName(scenario)
1307                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1308                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1309                     }
1310                     break
1311                 case 'longgc':
1312                     if (configuration == 'Release') {
1313                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1314                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1315                     }
1316                     break
1317                 case 'gcsimulator':
1318                     if (configuration == 'Release') {
1319                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1320                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1321                     }
1322                     break
1323                 case 'standalone_gc':
1324                     if (configuration == 'Release' || configuration == 'Checked') {
1325                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1326                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1327                     }
1328                     break
1329                 case 'minopts':
1330                 case 'tieredcompilation':
1331                 case 'forcerelocs':
1332                 case 'jitstress1':
1333                 case 'jitstress2':
1334                 case 'jitstressregs1':
1335                 case 'jitstressregs2':
1336                 case 'jitstressregs3':
1337                 case 'jitstressregs4':
1338                 case 'jitstressregs8':
1339                 case 'jitstressregs0x10':
1340                 case 'jitstressregs0x80':
1341                 case 'jitstressregs0x1000':
1342                 case 'jitstress2_jitstressregs1':
1343                 case 'jitstress2_jitstressregs2':
1344                 case 'jitstress2_jitstressregs3':
1345                 case 'jitstress2_jitstressregs4':
1346                 case 'jitstress2_jitstressregs8':
1347                 case 'jitstress2_jitstressregs0x10':
1348                 case 'jitstress2_jitstressregs0x80':
1349                 case 'jitstress2_jitstressregs0x1000':
1350                 case 'tailcallstress':
1351                 case 'jitsse2only':
1352                 case 'gcstress0x3':
1353                 case 'gcstress0xc':
1354                 case 'zapdisable':
1355                 case 'heapverify1':
1356                 case 'gcstress0xc_zapdisable':
1357                 case 'gcstress0xc_zapdisable_jitstress2':
1358                 case 'gcstress0xc_zapdisable_heapverify1':
1359                 case 'gcstress0xc_jitstress1':
1360                 case 'gcstress0xc_jitstress2':
1361                 case 'gcstress0xc_minopts_heapverify1':
1362                     def displayStr = getStressModeDisplayName(scenario)
1363                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1364                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1365                     break
1366                 case 'corefx_baseline':
1367                 case 'corefx_minopts':
1368                 case 'corefx_tieredcompilation':
1369                 case 'corefx_jitstress1':
1370                 case 'corefx_jitstress2':
1371                 case 'corefx_jitstressregs1':
1372                 case 'corefx_jitstressregs2':
1373                 case 'corefx_jitstressregs3':
1374                 case 'corefx_jitstressregs4':
1375                 case 'corefx_jitstressregs8':
1376                 case 'corefx_jitstressregs0x10':
1377                 case 'corefx_jitstressregs0x80':
1378                 case 'corefx_jitstressregs0x1000':
1379                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1380                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1381                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1382                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1383                     break
1384                 case 'illink':
1385                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1386                     break
1387                 default:
1388                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1389                     assert false
1390                     break
1391             }
1392             break
1393          // editor brace matching: }
1394         case 'x86lb': // editor brace matching: {
1395             assert (os == 'Windows_NT')
1396             assert (scenario == 'default' || Constants.r2rJitStressScenarios.indexOf(scenario) !=1)
1397
1398             def arch = 'x86'
1399             def jit = 'legacy_backend'
1400             switch (scenario) {
1401                 case 'default':
1402                     if (configuration == 'Checked') {
1403                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1404                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1405                     }
1406                     break
1407                 default:
1408                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1409                     assert false
1410                     break
1411             }
1412             break
1413         // editor brace matching: }
1414         default:
1415             println("Unknown architecture: ${architecture}");
1416             assert false
1417             break
1418     }
1419 }
1420
1421 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1422     def buildCommands = [];
1423     def osGroup = getOSGroup(os)
1424     def lowerConfiguration = configuration.toLowerCase()
1425
1426     // Calculate the build steps, archival, and xunit results
1427     switch (os) {
1428         case 'Windows_NT': // editor brace matching: {
1429             switch (architecture) {
1430                 case 'x64':
1431                 case 'x86':
1432                 case 'x86lb':
1433                     def arch = architecture
1434                     def buildOpts = ''
1435                     if (architecture == 'x86lb') {
1436                         arch = 'x86'
1437                     }
1438
1439                     if (scenario == 'illink') {
1440                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1441                     }
1442
1443                     // If it is a release build for windows, ensure PGO is used, else fail the build
1444                     if ((lowerConfiguration == 'release') && (scenario in Constants.basicScenarios) && (architecture != 'x86lb')) {
1445                         buildOpts += ' enforcepgo'
1446                     }
1447
1448                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1449                             scenario == 'default' ||
1450                             scenario == 'r2r' ||
1451                             scenario == 'jitdiff' ||
1452                             scenario == 'ilrt' ||
1453                             scenario == 'illink' ||
1454                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1455                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1456                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1457                     }
1458
1459                     // For Pri 1 tests, we must shorten the output test binary path names.
1460                     // if __TestIntermediateDir is already set, build-test.cmd will
1461                     // output test binaries to that directory. If it is not set, the
1462                     // binaries are sent to a default directory whose name is about
1463                     // 35 characters long.
1464
1465                     else if (scenarioNeedsPri1Build(scenario)) {
1466                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1467                     }
1468                     else if (isLongGc(scenario)) {
1469                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1470                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1471                     }
1472                     else if (scenario == 'standalone_gc') {
1473                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1474                     }
1475                     else if (scenario == 'formatting') {
1476                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1477                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1478                         break
1479                     }
1480                     else {
1481                         println("Unknown scenario: ${scenario}")
1482                         assert false
1483                     }
1484
1485                     // If we are running a stress mode, we should write out the set of key
1486                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1487
1488                     if (!isBuildOnly) {
1489                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1490                         def crossgenStr = ''
1491                         def runcrossgentestsStr = ''
1492                         def runjitstressStr = ''
1493                         def runjitstressregsStr = ''
1494                         def runjitmioptsStr = ''
1495                         def runjitforcerelocsStr = ''
1496                         def runjitdisasmStr = ''
1497                         def runilasmroundtripStr = ''
1498                         def gcstressStr = ''
1499                         def runtestArguments = ''
1500                         def gcTestArguments = ''
1501                         def illinkArguments = ''
1502
1503                         if (scenario == 'r2r' ||
1504                             scenario == 'pri1r2r' ||
1505                             scenario == 'gcstress15_pri1r2r' ||
1506                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1507                                 crossgenStr = 'crossgen'
1508                                 runcrossgentestsStr = 'runcrossgentests'
1509
1510                                 if (scenario == 'r2r_jitstress1'){
1511                                     runjitstressStr = 'jitstress 1'
1512                                 }
1513                                 else if (scenario == 'r2r_jitstress2') {
1514                                     runjitstressStr = 'jitstress 2'
1515                                 }
1516                                 else if (scenario == 'r2r_jitstressregs1'){
1517                                     runjitstressregsStr = 'jitstressregs 1'
1518                                 }
1519                                 else if (scenario == 'r2r_jitstressregs2') {
1520                                     runjitstressregsStr = 'jitstressregs 2'
1521                                 }
1522                                 else if (scenario == 'r2r_jitstressregs3') {
1523                                     runjitstressregsStr = 'jitstressregs 3'
1524                                 }
1525                                 else if (scenario == 'r2r_jitstressregs4') {
1526                                     runjitstressregsStr = 'jitstressregs 4'
1527                                 }
1528                                 else if (scenario == 'r2r_jitstressregs8') {
1529                                     runjitstressregsStr = 'jitstressregs 8'
1530                                 }
1531                                 else if (scenario == 'r2r_jitstressregs0x10') {
1532                                     runjitstressregsStr = 'jitstressregs 0x10'
1533                                 }
1534                                 else if (scenario == 'r2r_jitstressregs0x80') {
1535                                     runjitstressregsStr = 'jitstressregs 0x80'
1536                                 }
1537                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1538                                     runjitstressregsStr = 'jitstressregs 0x1000'
1539                                 }
1540                                 else if (scenario == 'r2r_jitminopts') {
1541                                     runjitmioptsStr = 'jitminopts'
1542                                 }
1543                                 else if (scenario == 'r2r_jitforcerelocs') {
1544                                     runjitforcerelocsStr = 'jitforcerelocs'
1545                                 }
1546                         }
1547                         if (scenario == 'gcstress15_pri1r2r')
1548                         {
1549                             gcstressStr = 'gcstresslevel 0xF'
1550                         }
1551
1552                         if (scenario == 'jitdiff')
1553                         {
1554                             runjitdisasmStr = 'jitdisasm crossgen'
1555                         }
1556
1557                         if (scenario == 'ilrt')
1558                         {
1559                             runilasmroundtripStr = 'ilasmroundtrip'
1560                         }
1561
1562                         if (isLongGc(scenario)) {
1563                             gcTestArguments = "${scenario} sequential"
1564                         }
1565
1566                         if (scenario == 'illink')
1567                         {
1568                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1569                         }
1570
1571                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1572
1573                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1574                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1575                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1576
1577                             if (enableCorefxTesting) {
1578                                 def workspaceRelativeFxRoot = "_/fx"
1579                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1580
1581                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1582
1583                                 setTestJobTimeOut(newJob, scenario)
1584
1585                                 // Archive and process (only) the test results
1586                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1587                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1588                             }
1589                             else {
1590                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1591                             }
1592                         }
1593                         else if (isGcReliabilityFramework(scenario)) {
1594                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1595                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1596                         }
1597                         else if (architecture == 'x64' || architecture == 'x86') {
1598                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1599                         }
1600                         else if (architecture == 'x86lb') {
1601                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd"
1602                         }
1603                     }
1604
1605                     if (!enableCorefxTesting) {
1606                         // Run the rest of the build
1607                         // Build the mscorlib for the other OS's
1608                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1609                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1610                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1611
1612                         if (arch == "x64") {
1613                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1614                         }
1615
1616                         // Zip up the tests directory so that we don't use so much space/time copying
1617                         // 10s of thousands of files around.
1618                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1619
1620                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1621                             // For windows, pull full test results and test drops for x86/x64.
1622                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1623                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1624                         }
1625
1626                         if (scenario == 'jitdiff') {
1627                             // retrive jit-dasm output for base commit, and run jit-diff
1628                             if (!isBuildOnly) {
1629                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1630                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1631                             }
1632                         }
1633
1634                         if (!isBuildOnly) {
1635                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1636                             setTestJobTimeOut(newJob, scenario)
1637                         }
1638                     }
1639                     break
1640                 case 'arm':
1641                     def validArmWindowsScenarios = [ "default",
1642                                                      "pri1r2r",
1643                                                      "zapdisable",
1644                                                      "minopts",
1645                                                      "tieredcompilation",
1646                                                      "tailcallstress",
1647                                                      "jitstress1",
1648                                                      "jitstress2",
1649                                                      "gcstress0x3",
1650                                                      "gcstress0xc",
1651                                                      "jitstressregs1",
1652                                                      "jitstressregs2",
1653                                                      "gcstress0xc_jitstress1",
1654                                                      "gcstress0xc_jitstress2"]
1655
1656                     assert validArmWindowsScenarios.contains(scenario)
1657
1658                     // Set time out
1659                     setTestJobTimeOut(newJob, scenario)
1660
1661                     if ( lowerConfiguration == "debug" ) {
1662                         // For Debug builds, we will do a P1 test build
1663                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1664                     }
1665                     else if (lowerConfiguration == "checked") {
1666
1667                         if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1668                         {
1669                            // Up the timeout for arm checked testing only.
1670                            // Keep the longer timeout for gcstress.
1671                            Utilities.setJobTimeout(newJob, 240)
1672                         }
1673
1674                         def machineAffinityOptions = ['use_arm64_build_machine' : true]
1675                         setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1676                         // For checked runs we will also run testing.
1677                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1678                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1679                     }
1680                     else if (lowerConfiguration == "release") {
1681                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1682                     }
1683                     // Add archival.
1684                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1685                     break
1686                 case 'arm64':
1687                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1688
1689                     // Set time out
1690                     setTestJobTimeOut(newJob, scenario)
1691
1692                     // Debug runs take too long to run. So build job only.
1693                     if (lowerConfiguration == "debug") {
1694                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1695                     }
1696                     else {
1697                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1698                        {
1699                            // Up the timeout for arm64 checked testing only.
1700                            // Keep the longer timeout for gcstress.
1701                            Utilities.setJobTimeout(newJob, 240)
1702                        }
1703
1704                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=1"
1705                        // Test build and run are launched together.
1706                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1707                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1708                     }
1709
1710                     // Add archival.
1711                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1712                     break
1713                 default:
1714                     println("Unknown architecture: ${architecture}");
1715                     assert false
1716                     break
1717             }
1718             break
1719         // editor brace matching: }
1720         case 'Ubuntu':
1721         case 'Ubuntu16.04':
1722         case 'Ubuntu16.10':
1723         case 'Debian8.4':
1724         case 'OSX10.12':
1725         case 'FreeBSD':
1726         case 'CentOS7.1':
1727         case 'RHEL7.2':
1728         case 'OpenSUSE42.1':
1729         case 'Tizen':
1730         case 'Fedora24': // editor brace matching: {
1731             switch (architecture) {
1732                 case 'x64':
1733                 case 'x86':
1734                     if (architecture == 'x86' && os == 'Ubuntu') {
1735                         // build and PAL test
1736                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1737                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1738                         break;
1739                     }
1740
1741                     if (scenario == 'formatting') {
1742                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1743                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1744                         break
1745                     }
1746
1747                     if (scenario == 'illink') {
1748                         assert(os == 'Ubuntu')
1749                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1750                     }
1751
1752                     def standaloneGc = ''
1753                     if (scenario == 'standalone_gc') {
1754                         standaloneGc = 'buildstandalonegc'
1755                     }
1756
1757                     if (!enableCorefxTesting) {
1758                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1759                         // only on supported OS platforms.
1760                         if (os == 'FreeBSD')
1761                         {
1762                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1763                         }
1764                         else
1765                         {
1766                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1767                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1768                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1769                         }
1770                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1771
1772                         // Set time out
1773                         setTestJobTimeOut(newJob, scenario)
1774                         // Basic archiving of the build
1775                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1776                         // And pal tests
1777                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1778                     }
1779                     else {
1780                         // Corefx stress testing
1781                         assert os == 'Ubuntu'
1782                         assert architecture == 'x64'
1783                         assert lowerConfiguration == 'checked'
1784                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1785
1786                         // Build coreclr
1787                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1788
1789                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1790                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1791
1792                         // Build and text corefx
1793                         def workspaceRelativeFxRoot = "_/fx"
1794                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1795
1796                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1797
1798                         setTestJobTimeOut(newJob, scenario)
1799
1800                         // Archive and process (only) the test results
1801                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1802                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1803                     }
1804                     break
1805                 case 'arm64':
1806                     def standaloneGc = ''
1807                     if (scenario == 'standalone_gc') {
1808                         standaloneGc = 'buildstandalonegc'
1809                     }
1810
1811                     if (!enableCorefxTesting) {
1812                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8 ${standaloneGc}"
1813
1814                         // HACK -- Arm64 does not have corefx jobs yet.
1815                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1816                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1817                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1818                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1819
1820                         // Set time out
1821                         setTestJobTimeOut(newJob, scenario)
1822                         // Basic archiving of the build
1823                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1824                     }
1825                     break
1826                 case 'arm':
1827                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1828                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1829
1830                     // default values for Ubuntu
1831                     def arm_abi="arm"
1832                     def linuxCodeName="trusty"
1833                     if (os == 'Ubuntu16.04') {
1834                         linuxCodeName="xenial"
1835                     }
1836                     else if (os == 'Tizen') {
1837                         arm_abi="armel"
1838                         linuxCodeName="tizen"
1839                     }
1840
1841                     // Unzip the Windows test binaries first. Exit with 0
1842                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1843
1844                     // Unpack the corefx binaries
1845                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1846                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1847                     if (os != 'Tizen') {
1848                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1849                     }
1850                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1851                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1852                     if (os == 'Ubuntu') {
1853                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1854                     }
1855
1856                     // Call the ARM CI script to cross build and test using docker
1857                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1858                     --mode=docker \\
1859                     --${arm_abi} \\
1860                     --linuxCodeName=${linuxCodeName} \\
1861                     --buildConfig=${lowerConfiguration} \\
1862                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1863                     --coreFxBinDir=./bin/CoreFxBinDir \\
1864                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1865
1866                     // Basic archiving of the build, no pal tests
1867                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1868                     break
1869                 default:
1870                     println("Unknown architecture: ${architecture}");
1871                     assert false
1872                     break
1873             }
1874             break
1875         // editor brace matching: }
1876         default:
1877             println("Unknown os: ${os}");
1878             assert false
1879             break
1880     } // os
1881
1882     return buildCommands
1883 }
1884
1885 // Additional scenario which can alter behavior
1886
1887 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1888 combinedScenarios.each { scenario ->
1889     [true, false].each { isPR ->
1890         Constants.architectureList.each { architecture ->
1891             Constants.configurationList.each { configuration ->
1892                 Constants.osList.each { os ->
1893                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1894                     // and reset the os to Windows_NT
1895                     def isBuildOnly = false
1896                     if (os == 'Windows_NT_BuildOnly') {
1897                         isBuildOnly = true
1898                         os = 'Windows_NT'
1899                     }
1900
1901                     // Tizen is only supported for arm architecture
1902                     if (os == 'Tizen' && architecture != 'arm') {
1903                         return
1904                     }
1905
1906                     // Skip totally unimplemented (in CI) configurations.
1907                     switch (architecture) {
1908                         case 'arm64':
1909                             if (os == 'Ubuntu16.04') {
1910                                 os = 'Ubuntu'
1911                             }
1912
1913                             // Windows and Ubuntu only
1914                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1915                                 return
1916                             }
1917                             break
1918                         case 'arm':
1919                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1920                                 return
1921                             }
1922                             break
1923                         case 'x86':
1924                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1925                                 return
1926                             }
1927                             break
1928                         case 'x86lb':
1929                             if (os != 'Windows_NT') {
1930                                 return
1931                             }
1932                             break
1933                         case 'x64':
1934                             // Everything implemented
1935                             break
1936                         default:
1937                             println("Unknown architecture: ${architecture}")
1938                             assert false
1939                             break
1940                     }
1941
1942                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1943                     // with checked builds
1944                     def enableCorefxTesting = false
1945                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1946                         if (configuration != 'Checked') {
1947                             return
1948                         }
1949
1950                         enableCorefxTesting = isCorefxTesting(scenario)
1951
1952                         // Since these are just execution time differences,
1953                         // skip platforms that don't execute the tests here (Windows_NT only)
1954                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1955                         if (!isEnabledOS || isBuildOnly) {
1956                             return
1957                         }
1958
1959                         switch (architecture) {
1960                             case 'arm':
1961                                 if ((scenario != 'gcstress0x3') &&
1962                                     (scenario != 'gcstress0xc') &&
1963                                     (scenario != 'jitstress1') &&
1964                                     (scenario != 'jitstress2') &&
1965                                     (scenario != 'jitstressregs1') &&
1966                                     (scenario != 'jitstressregs2') &&
1967                                     (scenario != 'gcstress0xc_jitstress1') &&
1968                                     (scenario != 'gcstress0xc_jitstress2') &&
1969                                     (scenario != 'minopts') &&
1970                                     (scenario != 'tieredcompilation') &&
1971                                     (scenario != 'tailcallstress') &&
1972                                     (scenario != 'zapdisable')) {
1973                                         return
1974                                     }
1975                                     break
1976                             case 'arm64':
1977                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1978                                     return
1979                                 }
1980                                 break
1981                             case 'x64':
1982                             case 'x86':
1983                                 // x86 ubuntu: default only
1984                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1985                                     return
1986                                 }
1987                                 // Windows: Everything implemented
1988                                 break
1989                             case 'x86lb':
1990                                 // No stress modes for legacy jit.
1991                                 // (There's no technical reason we couldn't allow these.)
1992                                 return
1993                             default:
1994                                 return
1995                         }
1996                     }
1997                     else {
1998                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
1999                         // and configuration is not Checked, bail out.
2000                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2001                             return;
2002                         }
2003
2004                         // Skip scenarios
2005                         switch (scenario) {
2006                             case 'pri1':
2007                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
2008                                 // the default scenario build
2009                                 if (os != 'Windows_NT') {
2010                                     return
2011                                 }
2012                                 // Only x64 for now
2013                                 if (architecture != 'x64') {
2014                                     return
2015                                 }
2016                                 break
2017                             case 'ilrt':
2018                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2019                                 // the default scenario build
2020                                 if (os != 'Windows_NT') {
2021                                     return
2022                                 }
2023                                 // Only x64 for now
2024                                 if (architecture != 'x64') {
2025                                     return
2026                                 }
2027                                 // Release only
2028                                 if (configuration != 'Release') {
2029                                     return
2030                                 }
2031                                 break
2032                             case 'jitdiff':
2033                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2034                                     return
2035                                 }
2036                                 if (architecture != 'x64') {
2037                                     return
2038                                 }
2039                                 if (configuration != 'Checked') {
2040                                     return
2041                                 }
2042                                 break
2043                             case 'r2r':
2044                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2045                                 // the default scenario build
2046                                 if (os != 'Windows_NT') {
2047                                     return
2048                                 }
2049                                 if (architecture != 'x64') {
2050                                     return
2051                                 }
2052                                 break
2053                             case 'pri1r2r':
2054                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2055                                 // the default scenario build
2056                                 if (os != 'Windows_NT') {
2057                                     return
2058                                 }
2059                                 if (architecture != 'x64') {
2060                                     if ((architecture != 'arm64' && architecture != 'arm') || (configuration == 'Debug')) {
2061                                         return
2062                                     }
2063                                 }
2064                                 break
2065                             case 'gcstress15_pri1r2r':
2066                             case 'r2r_jitstress1':
2067                             case 'r2r_jitstress2':
2068                             case 'r2r_jitstressregs1':
2069                             case 'r2r_jitstressregs2':
2070                             case 'r2r_jitstressregs3':
2071                             case 'r2r_jitstressregs4':
2072                             case 'r2r_jitstressregs8':
2073                             case 'r2r_jitstressregs0x10':
2074                             case 'r2r_jitstressregs0x80':
2075                             case 'r2r_jitstressregs0x1000':
2076                             case 'r2r_jitminopts':
2077                             case 'r2r_jitforcerelocs':
2078                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
2079                                 // the default scenario build
2080                                 if (os != 'Windows_NT') {
2081                                     return
2082                                 }
2083                                 if (architecture != 'x64' && architecture != 'x86') {
2084                                     return
2085                                 }
2086                                 break
2087                             case 'longgc':
2088                             case 'gcsimulator':
2089                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2090                                     return
2091                                 }
2092                                 if (architecture != 'x64') {
2093                                     return
2094                                 }
2095                                 if (configuration != 'Release') {
2096                                     return
2097                                 }
2098                                 break
2099                             case 'gc_reliability_framework':
2100                             case 'standalone_gc':
2101                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2102                                     return
2103                                 }
2104
2105                                 if (architecture != 'x64') {
2106                                     return
2107                                 }
2108
2109                                 if (configuration != 'Release' && configuration != 'Checked') {
2110                                     return
2111                                 }
2112                                 break
2113                             // We need Windows x64 Release bits for the code coverage build
2114                             case 'coverage':
2115                                 if (os != 'Windows_NT') {
2116                                     return
2117                                 }
2118                                 if (architecture != 'x64') {
2119                                     return
2120                                 }
2121                                 if (configuration != 'Release') {
2122                                     return
2123                                 }
2124                                 break
2125                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2126                             case 'formatting':
2127                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2128                                     return
2129                                 }
2130                                 if (architecture != 'x64') {
2131                                     return
2132                                 }
2133                                 if (configuration != 'Checked') {
2134                                     return
2135                                 }
2136                                 if (isBuildOnly) {
2137                                     return
2138                                 }
2139                                 break
2140                             case 'illink':
2141                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2142                                     return
2143                                 }
2144                                 if (architecture != 'x64' && architecture != 'x86') {
2145                                     return
2146                                 }
2147                                 if (isBuildOnly) {
2148                                     return
2149                                 }
2150                                 break
2151                             case 'default':
2152                                 // Nothing skipped
2153                                 break
2154                             default:
2155                                 println("Unknown scenario: ${scenario}")
2156                                 assert false
2157                                 break
2158                         }
2159                     }
2160
2161                     // Calculate names
2162                     def lowerConfiguration = configuration.toLowerCase()
2163                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2164                     def folderName = getJobFolder(scenario)
2165
2166                     // Create the new job
2167                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2168
2169                     def machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2170                     machineAffinityOptions = architecture == 'arm' ? ['use_arm64_build_machine': false] : machineAffinityOptions
2171                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2172
2173                     // Add all the standard options
2174                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2175                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2176
2177                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
2178                     def osGroup = getOSGroup(os)
2179
2180                     newJob.with {
2181                         steps {
2182                             if (os == 'Windows_NT') {
2183                                 buildCommands.each { buildCommand ->
2184                                     batchFile(buildCommand)
2185                                 }
2186                             }
2187                             else {
2188                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2189                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2190                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2191                                     // Define the Windows Tests and Corefx build job names
2192                                     def WindowTestsName = projectFolder + '/' +
2193                                                           Utilities.getFullJobName(project,
2194                                                                                    getJobName(lowerConfiguration,
2195                                                                                               'x64' ,
2196                                                                                               'windows_nt',
2197                                                                                               'default',
2198                                                                                               true),
2199                                                                                    false)
2200                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2201                                                        Utilities.getFolderName(branch)
2202
2203                                     // Copy the Windows test binaries and the Corefx build binaries
2204                                     copyArtifacts(WindowTestsName) {
2205                                         includePatterns('bin/tests/tests.zip')
2206                                         buildSelector {
2207                                             latestSuccessful(true)
2208                                         }
2209                                     }
2210
2211                                     def arm_abi = 'arm'
2212                                     def corefx_os = 'linux'
2213                                     if (os == 'Tizen') {
2214                                         arm_abi = 'armel'
2215                                         corefx_os = 'tizen'
2216                                     }
2217
2218                                     // Let's use release CoreFX to test checked CoreCLR,
2219                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2220                                     def corefx_lowerConfiguration = lowerConfiguration
2221                                     if ( lowerConfiguration == 'checked' ) {
2222                                         corefx_lowerConfiguration='release'
2223                                     }
2224
2225                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2226                                         includePatterns('bin/build.tar.gz')
2227                                         buildSelector {
2228                                             latestSuccessful(true)
2229                                         }
2230                                     }
2231                                 }
2232
2233                                 buildCommands.each { buildCommand ->
2234                                     shell(buildCommand)
2235                                 }
2236                             }
2237                         }
2238                     } // newJob.with
2239
2240                 } // os
2241             } // configuration
2242         } // architecture
2243     } // isPR
2244 } // scenario
2245
2246
2247 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2248 combinedScenarios.each { scenario ->
2249     [true, false].each { isPR ->
2250         // Architectures.  x64 only at this point
2251         ['x64', 'arm64'].each { architecture ->
2252             // Put the OS's supported for coreclr cross testing here
2253             Constants.crossList.each { os ->
2254                 if (architecture == 'arm64') {
2255                     if (os != "Ubuntu") {
2256                         return
2257                     }
2258                 }
2259
2260                 Constants.configurationList.each { configuration ->
2261
2262                     if (architecture == 'arm64') {
2263                         if (scenario != 'default' && scenario != 'pri1r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2264                             return
2265                         }
2266                     }
2267
2268                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2269                         if (configuration != 'Checked') {
2270                             return
2271                         }
2272                         if (isCorefxTesting(scenario)) {
2273                             return
2274                         }
2275                         //Skip stress modes for these scenarios
2276                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2277                             return
2278                         }
2279                     }
2280                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2281                     // and configuration is not Checked, bail out.
2282                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2283                         return;
2284                     }
2285                     // For CentOS, we only want Checked/Release pri1 builds.
2286                     else if (os == 'CentOS7.1') {
2287                         if (scenario != 'pri1' &&
2288                             scenario != 'r2r' &&
2289                             scenario != 'pri1r2r' &&
2290                             scenario != 'gcstress15_pri1r2r' &&
2291                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2292                             return
2293                         }
2294                         if (configuration != 'Checked' && configuration != 'Release') {
2295                             return
2296                         }
2297                     }
2298                     // For RedHat and Debian, we only do Release pri1 builds.
2299                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2300                         if (scenario != 'pri1') {
2301                             return
2302                         }
2303                         if (configuration != 'Release') {
2304                             return
2305                         }
2306                     }
2307                     else {
2308                         // Skip scenarios
2309                         switch (scenario) {
2310                             case 'pri1':
2311                                 // Nothing skipped
2312                                 break
2313                             case 'ilrt':
2314                                 // Release only
2315                                 if (configuration != 'Release') {
2316                                     return
2317                                 }
2318                                 break
2319                             case 'jitdiff':
2320                                 if (configuration != 'Checked') {
2321                                     return;
2322                                 }
2323                             case 'r2r':
2324                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2325                                 if (configuration != 'Checked' && configuration != 'Release') {
2326                                     return
2327                                 }
2328                                 break
2329                             case 'pri1r2r':
2330                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2331                                 if (configuration != 'Checked' && configuration != 'Release') {
2332                                     return
2333                                 }
2334                                 break
2335                             case 'gcstress15_pri1r2r':
2336                             case 'r2r_jitstress1':
2337                             case 'r2r_jitstress2':
2338                             case 'r2r_jitstressregs1':
2339                             case 'r2r_jitstressregs2':
2340                             case 'r2r_jitstressregs3':
2341                             case 'r2r_jitstressregs4':
2342                             case 'r2r_jitstressregs8':
2343                             case 'r2r_jitstressregs0x10':
2344                             case 'r2r_jitstressregs0x80':
2345                             case 'r2r_jitstressregs0x1000':
2346                             case 'r2r_jitminopts':
2347                             case 'r2r_jitforcerelocs':
2348                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2349                                 if (configuration != 'Checked' && configuration != 'Release') {
2350                                     return
2351                                 }
2352                                 break
2353                             case 'longgc':
2354                             case 'gcsimulator':
2355                                 // Long GC tests take a long time on non-Release builds
2356                                 if (configuration != 'Release') {
2357                                     return
2358                                 }
2359                                 break
2360                             case 'gc_reliability_framework':
2361                             case 'standalone_gc':
2362                                 if (configuration != 'Release' && configuration != 'Checked') {
2363                                     return
2364                                 }
2365                                 break
2366                             case 'coverage':
2367                                 //We only want Ubuntu Release for coverage
2368                                 if (os != 'Ubuntu') {
2369                                     return
2370                                 }
2371                                 if (configuration != 'Release') {
2372                                     return
2373                                 }
2374                             case 'formatting':
2375                                 return
2376                             case 'illink':
2377                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2378                                     return
2379                                 }
2380                                 break
2381                             case 'default':
2382                                 // Nothing skipped
2383                                 break
2384                             default:
2385                                 println("Unknown scenario: ${scenario}")
2386                                 assert false
2387                                 break
2388                         }
2389                     }
2390
2391                     def lowerConfiguration = configuration.toLowerCase()
2392                     def osGroup = getOSGroup(os)
2393                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2394
2395                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2396                     def inputScenario = 'default'
2397                     if (scenario == 'coverage') {
2398                         inputScenario = 'coverage'
2399                     }
2400                     def inputCoreCLRBuildName = projectFolder + '/' +
2401                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2402                     // If this is a stress scenario, there isn't any difference in the build job
2403                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2404                     // from the default scenario
2405                     def testBuildScenario = scenario
2406                     if (scenarioNeedsPri1Build(scenario)) {
2407                         testBuildScenario = 'pri1'
2408                     }
2409                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2410                         testBuildScenario = 'default'
2411                     }
2412                     def inputWindowTestsBuildName = ''
2413                     def inputWindowsTestBuildArch = architecture
2414                     if (architecture == "arm64") {
2415                         // Use the x64 test build for arm64 unix
2416                         inputWindowsTestBuildArch = "x64"
2417                     }
2418                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2419                         inputWindowTestsBuildName = projectFolder + '/' +
2420                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', 'default', true), isPR)
2421                     }
2422                     else {
2423                         inputWindowTestsBuildName = projectFolder + '/' +
2424                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2425                     }
2426                     // Enable Server GC for Ubuntu PR builds
2427                     def serverGCString = ''
2428
2429                     // Whether or not this test run should be run sequentially instead
2430                     // of in parallel. Only used for long GC tests.
2431                     def sequentialString = ''
2432
2433                     // Whether or not this test run should run a specific playlist.
2434                     // Only used for long GC tests.
2435
2436                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2437                     // for running long GC and GCSimulator tests, respectively. We don't use them
2438                     // here because using a playlist file produces much more readable output on the CI machines
2439                     // and reduces running time.
2440                     def playlistString = ''
2441
2442                     if (os == 'Ubuntu' && isPR){
2443                         serverGCString = '--useServerGC'
2444                     }
2445
2446                     // pass --crossgen to runtest.sh for crossgen builds
2447                     def crossgenStr = ''
2448                     def runcrossgentestsStr = ''
2449                     def runjitstressStr = ''
2450                     def runjitstressregsStr = ''
2451                     def runjitmioptsStr = ''
2452                     def runjitforcerelocsStr = ''
2453                     def runjitdisasmStr = ''
2454                     def runilasmroundtripStr = ''
2455                     def gcstressStr = ''
2456                     def illinkStr = ''
2457                     def layoutOnlyStr =''
2458
2459                     if (scenario == 'r2r' ||
2460                         scenario == 'pri1r2r' ||
2461                         scenario == 'gcstress15_pri1r2r' ||
2462                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2463                             crossgenStr = '--crossgen'
2464                             runcrossgentestsStr = '--runcrossgentests'
2465
2466                             if (scenario == 'r2r_jitstress1'){
2467                                 runjitstressStr = '--jitstress=1'
2468                             }
2469                             else if (scenario == 'r2r_jitstress2') {
2470                                 runjitstressStr = '--jitstress=2'
2471                             }
2472                             else if (scenario == 'r2r_jitstressregs1'){
2473                                 runjitstressregsStr = '--jitstressregs=1'
2474                             }
2475                             else if (scenario == 'r2r_jitstressregs2') {
2476                                 runjitstressregsStr = '--jitstressregs=2'
2477                             }
2478                             else if (scenario == 'r2r_jitstressregs3') {
2479                                 runjitstressregsStr = '--jitstressregs=3'
2480                             }
2481                             else if (scenario == 'r2r_jitstressregs4') {
2482                                 runjitstressregsStr = '--jitstressregs=4'
2483                             }
2484                             else if (scenario == 'r2r_jitstressregs8') {
2485                                 runjitstressregsStr = '--jitstressregs=8'
2486                             }
2487                             else if (scenario == 'r2r_jitstressregs0x10') {
2488                                 runjitstressregsStr = '--jitstressregs=0x10'
2489                             }
2490                             else if (scenario == 'r2r_jitstressregs0x80') {
2491                                 runjitstressregsStr = '--jitstressregs=0x80'
2492                             }
2493                             else if (scenario == 'r2r_jitstressregs0x1000') {
2494                                 runjitstressregsStr = '--jitstressregs=0x1000'
2495                             }
2496                             else if (scenario == 'r2r_jitminopts') {
2497                                 runjitmioptsStr = '--jitminopts'
2498                             }
2499                             else if (scenario == 'r2r_jitforcerelocs') {
2500                                 runjitforcerelocsStr = '--jitforcerelocs'
2501                             }
2502                     }
2503                     if  (scenario == 'gcstress15_pri1r2r')
2504                     {
2505                         gcstressStr = '--gcstresslevel=0xF'
2506                     }
2507
2508                     if (scenario == 'jitdiff')
2509                     {
2510                         runjitdisasmStr = '--jitdisasm --crossgen'
2511                     }
2512
2513                     if (scenario == 'illink')
2514                     {
2515                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2516                     }
2517
2518                     if (isLongGc(scenario)) {
2519                         // Long GC tests behave very poorly when they are not
2520                         // the only test running (many of them allocate until OOM).
2521                         sequentialString = '--sequential'
2522
2523                         // The Long GC playlist contains all of the tests that are
2524                         // going to be run. The GCSimulator playlist contains all of
2525                         // the GC simulator tests.
2526                         if (scenario == 'longgc') {
2527                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2528                         }
2529                         else if (scenario == 'gcsimulator') {
2530                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2531                         }
2532                     }
2533
2534                     if (isGcReliabilityFramework(scenario)) {
2535                         layoutOnlyStr = '--build-overlay-only'
2536                     }
2537
2538                     def folder = getJobFolder(scenario)
2539                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2540                         // Add parameters for the inputs
2541
2542                         parameters {
2543                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2544                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2545                         }
2546
2547                         steps {
2548                             // Set up the copies
2549
2550                             // Coreclr build containing the tests and mscorlib
2551
2552                             copyArtifacts(inputWindowTestsBuildName) {
2553                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2554                                 buildSelector {
2555                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2556                                 }
2557                             }
2558
2559                             if (scenario == 'coverage') {
2560
2561                                 // Move coreclr to clr directory
2562                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2563
2564                                 // Build coreclr
2565                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2566
2567                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2568                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2569                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2570                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2571                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2572                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2573                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2574                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2575
2576                                 // Run PAL tests
2577                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2578
2579                                 // Remove obj files for PAL tests so they're not included in coverage results
2580                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2581
2582                                 // Unzip the tests first.  Exit with 0
2583                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2584
2585                                 // Get corefx
2586                                 shell("git clone https://github.com/dotnet/corefx fx")
2587
2588                                 // Build Linux corefx
2589                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2590                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2591
2592                                 def testEnvOpt = ""
2593                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2594                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2595                                 shell("${createScriptCmds}")
2596                                 testEnvOpt = "--test-env=" + scriptFileName
2597
2598                                 // Run corefx tests
2599                                 shell("""./fx/run-test.sh \\
2600                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2601                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2602                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2603                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2604                 --configurationGroup Release""")
2605
2606
2607                                 // Run coreclr tests w/ workstation GC
2608                                 shell("""./clr/tests/runtest.sh \\
2609                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2610                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2611                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2612                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2613                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2614                 --crossgen --runcrossgentests""")
2615
2616                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2617                                 shell("""./clr/tests/runtest.sh \\
2618                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2619                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2620                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2621                 --useServerGC ${testEnvOpt}""")
2622
2623                                  // Run long-running coreclr GC tests & produce coverage reports
2624                                 shell("""./clr/tests/runtest.sh \\
2625                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2626                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2627                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2628                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2629                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2630                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2631                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2632
2633                             }
2634                             else {
2635
2636                                 // Coreclr build we are trying to test
2637
2638                                 copyArtifacts(inputCoreCLRBuildName) {
2639                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2640                                     buildSelector {
2641                                         buildNumber('${CORECLR_BUILD}')
2642                                     }
2643                                 }
2644
2645                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2646
2647                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2648                                 def osJobName
2649                                 if (os == 'Ubuntu') {
2650                                     osJobName = 'ubuntu14.04'
2651                                 }
2652                                 else {
2653                                     osJobName = os.toLowerCase()
2654                                 }
2655                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2656                                     includePatterns('bin/build.tar.gz')
2657                                     buildSelector {
2658                                         latestSuccessful(true)
2659                                     }
2660                                 }
2661
2662                                 shell ("mkdir ./bin/CoreFxBinDir")
2663                                 // Unpack the corefx binaries
2664                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2665
2666                                 // HACK -- Arm64 does not have corefx jobs yet.
2667                                 // Clone corefx and build the native packages overwriting the x64 packages.
2668                                 if (architecture == 'arm64') {
2669                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2670                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2671                                 }
2672
2673                                 // Unzip the tests first.  Exit with 0
2674                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2675
2676                                 // Execute the tests
2677                                 // If we are running a stress mode, we'll set those variables first
2678                                 def testEnvOpt = ""
2679                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2680                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2681                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2682                                     shell("${createScriptCmds}")
2683                                     testEnvOpt = "--test-env=" + scriptFileName
2684                                 }
2685
2686                                 if (isGCStressRelatedTesting(scenario)) {
2687                                     shell('./init-tools.sh')
2688                                 }
2689
2690                                 shell("""./tests/runtest.sh \\
2691                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2692                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2693                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2694                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2695                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2696                 --limitedDumpGeneration \\
2697                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
2698                 ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
2699                 ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
2700
2701                                 if (isGcReliabilityFramework(scenario)) {
2702                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2703                                     if (serverGCString != '') {
2704                                         shell("export COMPlus_gcServer=1")
2705                                     }
2706
2707                                     shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2708                                 }
2709                             }
2710                         }
2711                     }
2712
2713                     if (scenario == 'coverage') {
2714                         // Publish coverage reports
2715                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2716                         // TODO: Add once external email sending is available again
2717                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2718                     }
2719
2720                     if (scenario == 'jitdiff') {
2721                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2722                     }
2723
2724                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2725                     if (os in ['Ubuntu']) {
2726                         SummaryBuilder summaries = new SummaryBuilder()
2727                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2728                         summaries.emit(newJob)
2729                     }
2730
2731                     setMachineAffinity(newJob, os, architecture)
2732                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2733                     // Set timeouts to 240.
2734                     setTestJobTimeOut(newJob, scenario)
2735
2736                     if (architecture == 'arm64') {
2737                         Utilities.setJobTimeout(newJob, 240)
2738                     }
2739
2740                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2741
2742                     // Create a build flow to join together the build and tests required to run this
2743                     // test.
2744                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2745                     // Linux CoreCLR test
2746                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2747                     def fullTestJobName = projectFolder + '/' + newJob.name
2748                     // Add a reference to the input jobs for report purposes
2749                     JobReport.Report.addReference(inputCoreCLRBuildName)
2750                     JobReport.Report.addReference(inputWindowTestsBuildName)
2751                     JobReport.Report.addReference(fullTestJobName)
2752                     def newFlowJob;
2753
2754                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2755                     // so that coverage data can be preserved.
2756                     if (scenario == 'coverage') {
2757                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2758                         buildFlow("""
2759 // Build the input Windows job
2760 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2761
2762 // And then build the test build
2763 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2764 """)
2765                         }
2766                     // Normal jobs copy a Windows build & a non-Windows build
2767                     } else {
2768                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2769                         buildFlow("""
2770 // Build the input jobs in parallel
2771 parallel (
2772     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2773     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2774 )
2775
2776 // And then build the test build
2777 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2778                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2779 """)
2780                         }
2781                     }
2782
2783                     setMachineAffinity(newFlowJob, os, architecture)
2784                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2785                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2786                 } // configuration
2787             } // os
2788         } // architecture
2789     } // isPR
2790 } // scenario
2791
2792 JobReport.Report.generateJobReport(out)
2793
2794 // Make the call to generate the help job
2795 Utilities.createHelperJob(this, project, branch,
2796     "Welcome to the ${project} Repository",  // This is prepended to the help message
2797     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2798
2799 Utilities.addCROSSCheck(this, project, branch)