netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'FreeBSD':'FreeBSD',
  29         'CentOS7.1': 'Linux',
  30         'OpenSUSE42.1': 'Linux',
  31         'Tizen': 'Linux']
  32     def osGroup = osGroupMap.get(os, null)
  33     assert osGroup != null : "Could not find os group for ${os}"
  34     return osGroupMap[os]
  35 }
  36
  37 // We use this class (vs variables) so that the static functions can access data here.
  38 class Constants {
  39
  40     // Innerloop build OS's
  41     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  42     // test execution in the build flow runs.  It generates the exact same build
  43     // as Windows_NT but without the tests.
  44     def static osList = [
  45                'Ubuntu',
  46                'Debian8.4',
  47                'OSX10.12',
  48                'Windows_NT',
  49                'Windows_NT_BuildOnly',
  50                'FreeBSD',
  51                'CentOS7.1',
  52                'OpenSUSE42.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'corefx_baseline'                : [ : ], // corefx baseline
  89                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  90                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  91                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  92                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  93                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  94                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  95                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  96                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  97                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  98                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  99                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 100                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 101                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 102                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 103                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 104                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 105                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 106                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 107                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 108                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 109                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 110                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 111                ]
 112
 113     // This is a set of r2r jit stress scenarios
 114     def static r2rJitStressScenarios = [
 115                'r2r_jitstress1',
 116                'r2r_jitstress2',
 117                'r2r_jitstressregs1',
 118                'r2r_jitstressregs2',
 119                'r2r_jitstressregs3',
 120                'r2r_jitstressregs4',
 121                'r2r_jitstressregs8',
 122                'r2r_jitstressregs0x10',
 123                'r2r_jitstressregs0x80',
 124                'r2r_jitstressregs0x1000',
 125                'r2r_jitminopts',
 126                'r2r_jitforcerelocs']
 127
 128     // This is the basic set of scenarios
 129     def static basicScenarios = [
 130                'default',
 131                'pri1',
 132                'ilrt',
 133                'r2r',
 134                'pri1r2r',
 135                'gcstress15_pri1r2r',
 136                'longgc',
 137                'coverage',
 138                'formatting',
 139                'gcsimulator',
 140                'jitdiff',
 141                'standalone_gc',
 142                'gc_reliability_framework',
 143                'illink'] + r2rJitStressScenarios
 144
 145     def static configurationList = ['Debug', 'Checked', 'Release']
 146
 147     // This is the set of architectures
 148     def static architectureList = ['arm', 'arm64', 'x64', 'x86', 'x86lb']
 149 }
 150
 151 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 152     if (architecture == 'arm64' && os == 'Windows_NT') {
 153         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 154     } else if (architecture == 'arm64' && os != 'Windows_NT' && options == null) {
 155         Utilities.setMachineAffinity(job, os, 'arm64-small-page-size');
 156     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['large_pages'] == true) {
 157         Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size');
 158     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['is_build_only'] == true) {
 159         Utilities.setMachineAffinity(job, os, 'arm64-cross-latest');
 160     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 161         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 162     } else if ((architecture == 'arm') && (os == 'Windows_NT') && options['use_arm64_build_machine'] == true) {
 163         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 164     }else {
 165         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 166     }
 167 }
 168
 169 def static isJITStressJob(def scenario) {
 170     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 171            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 172 }
 173
 174 def static isGCStressRelatedTesting(def scenario) {
 175     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 176     // Detect it and make it a GCStress related.
 177     if (scenario == 'gcstress15_pri1r2r')
 178     {
 179         return true;
 180     }
 181
 182     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 183     def scenarioName = scenario.toLowerCase()
 184     def isGCStressTesting = false
 185     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 186         if (k in gcStressTestEnvVars) {
 187             isGCStressTesting = true;
 188         }
 189     }
 190     return isGCStressTesting
 191 }
 192
 193 def static isCorefxTesting(def scenario) {
 194     def corefx_prefix = 'corefx_'
 195     if (scenario.length() < corefx_prefix.length()) {
 196         return false
 197     }
 198     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 199 }
 200
 201 def static isR2R(def scenario) {
 202     return (scenario == 'r2r' || scenario == 'pri1r2r')
 203 }
 204
 205 def static isCoverage(def scenario) {
 206     return (scenario == 'coverage')
 207 }
 208
 209 def static isLongGc(def scenario) {
 210     return (scenario == 'longgc' || scenario == 'gcsimulator')
 211 }
 212
 213 def static isJitDiff(def scenario) {
 214     return (scenario == 'jitdiff')
 215 }
 216
 217 def static isGcReliabilityFramework(def scenario) {
 218     return (scenario == 'gc_reliability_framework')
 219 }
 220
 221 def static scenarioNeedsPri1Build(def scenario) {
 222     return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario))
 223 }
 224
 225 def static setTestJobTimeOut(newJob, scenario) {
 226     if (isGCStressRelatedTesting(scenario)) {
 227         Utilities.setJobTimeout(newJob, 4320)
 228     }
 229     else if (isCorefxTesting(scenario)) {
 230         Utilities.setJobTimeout(newJob, 360)
 231     }
 232     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 233         Utilities.setJobTimeout(newJob, 240)
 234     }
 235     else if (isR2R(scenario)) {
 236         Utilities.setJobTimeout(newJob, 240)
 237     }
 238     else if (isCoverage(scenario)) {
 239         Utilities.setJobTimeout(newJob, 1440)
 240     }
 241     else if (isLongGc(scenario)) {
 242         Utilities.setJobTimeout(newJob, 1440)
 243     }
 244     else if (isJitDiff(scenario)) {
 245         Utilities.setJobTimeout(newJob, 240)
 246     }
 247     else if (isGcReliabilityFramework(scenario)) {
 248         Utilities.setJobTimeout(newJob, 1440)
 249     }
 250     // Non-test jobs use the default timeout value.
 251 }
 252
 253 def static getJobFolder(def scenario) {
 254     if (isJITStressJob(scenario)) {
 255         return 'jitstress'
 256     }
 257     if (scenario == 'illink') {
 258         return 'illink'
 259     }
 260     return ''
 261 }
 262
 263 def static getStressModeDisplayName(def scenario) {
 264     def displayStr = ''
 265     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 266         def prefixLength = 'COMPlus_'.length()
 267         if (k.length() >= prefixLength) {
 268             def modeName = k.substring(prefixLength, k.length())
 269             displayStr += ' ' + modeName + '=' + v
 270         }
 271     }
 272     return displayStr
 273 }
 274
 275 def static getR2RStressModeDisplayName(def scenario) {
 276     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 277     // "r2r_" prefix.
 278     def displayStr = scenario
 279     def prefixLength = 'r2r_'.length()
 280     if (displayStr.length() >= prefixLength) {
 281         displayStr = displayStr.substring(prefixLength, displayStr.length())
 282     }
 283     return displayStr
 284 }
 285
 286 // Generates the string for creating a file that sets environment variables
 287 // that makes it possible to run stress modes.  Writes the script to the file
 288 // specified by the stepScriptLocation parameter.
 289 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 290     def stepScript = ''
 291     if (os == 'Windows_NT') {
 292         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 293         stepScript += "del ${stepScriptLocation}\r\n"
 294
 295         // Timeout in ms, default is 10 minutes. For stress
 296         // modes up this to 30 minutes
 297         def timeout = 1800000
 298
 299         // Set the Timeout
 300         stepScript += "set __TestTimeout=${timeout}\r\n"
 301         stepScript += "echo. > ${stepScriptLocation}\r\n"
 302         stressModeVars.each{ k, v ->
 303             // Write out what we are writing to the script file
 304             stepScript += "echo Setting ${k}=${v}\r\n"
 305             // Write out the set itself to the script file`
 306             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 307         }
 308     }
 309     else {
 310         stepScript += "echo Setting variables for ${stressModeName}\n"
 311         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 312         stressModeVars.each{ k, v ->
 313             // Write out what we are writing to the script file
 314             stepScript += "echo Setting ${k}=${v}\n"
 315             // Write out the set itself to the script file`
 316             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 317         }
 318         stepScript += "chmod +x ${stepScriptLocation}\n"
 319     }
 320     return stepScript
 321 }
 322
 323 // Calculates the name of the build job based on some typical parameters.
 324 //
 325 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 326     // If the architecture is x64, do not add that info into the build name.
 327     // Need to change around some systems and other builds to pick up the right builds
 328     // to do that.
 329
 330     def suffix = scenario != 'default' ? "_${scenario}" : '';
 331     if (isBuildOnly) {
 332         suffix += '_bld'
 333     }
 334     def baseName = ''
 335     switch (architecture) {
 336         case 'x64':
 337             if (scenario == 'default') {
 338                 // For now we leave x64 off of the name for compatibility with other jobs
 339                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 340             }
 341             else if (scenario == 'formatting') {
 342                 // we don't care about the configuration for the formatting job. It runs all configs
 343                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 344             }
 345             else {
 346                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 347             }
 348             break
 349         case 'arm64':
 350             if (os.toLowerCase() == "windows_nt") {
 351                 // These are cross builds
 352                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 353             }
 354             else {
 355                 // Defaults to a small page size set of machines.
 356                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 357             }
 358             break
 359         case 'arm':
 360             // These are cross builds
 361             if (os == 'Tizen') {
 362                 // ABI: softfp
 363                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 364             }
 365             else {
 366                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 367             }
 368             break
 369         case 'x86':
 370         case 'x86lb':
 371             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 372             break
 373         default:
 374             println("Unknown architecture: ${architecture}");
 375             assert false
 376             break
 377     }
 378
 379     return baseName + suffix
 380 }
 381
 382 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 383     // Check scenario.
 384     switch (scenario) {
 385         case 'default':
 386             switch (architecture) {
 387                 case 'x64':
 388                 case 'x86':
 389                 case 'x86lb':
 390                     if (architecture == 'x86' && os == 'Ubuntu') {
 391                         Utilities.addPeriodicTrigger(job, '@daily')
 392                     }
 393                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 394                         Utilities.addGithubPushTrigger(job)
 395                     }
 396                     break
 397                 case 'arm':
 398                     Utilities.addGithubPushTrigger(job)
 399                     break
 400                 case 'arm64':
 401                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 402                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 403                     break
 404                 default:
 405                     println("Unknown architecture: ${architecture}");
 406                     assert false
 407                     break
 408             }
 409             break
 410         case 'pri1':
 411             // Pri one gets a push trigger, and only for release
 412             if (architecture == 'x64') {
 413                 if (configuration == 'Release') {
 414                     // We expect release jobs to be Windows, or in the cross list
 415                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 416                     if (!os in bidailyCrossList) {
 417                         if (isFlowJob || os == 'Windows_NT') {
 418                             Utilities.addGithubPushTrigger(job)
 419                         }
 420                     }
 421                     else {
 422                         if (isFlowJob) {
 423                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 424                         }
 425                     }
 426                 }
 427             }
 428             break
 429         case 'r2r':
 430             //r2r jobs that aren't pri1 can only be triggered by phrase
 431             break
 432         case 'pri1r2r':
 433             assert !(os in bidailyCrossList)
 434             //pri1 r2r gets a push trigger for checked/release
 435             if (configuration == 'Checked' || configuration == 'Release') {
 436                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 437                 if (architecture == 'x64' && os != 'OSX10.12') {
 438                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 439                     if (isFlowJob || os == 'Windows_NT') {
 440                         Utilities.addGithubPushTrigger(job)
 441                     }
 442                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 443                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 444                     if (isFlowJob) {
 445                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 446                     }
 447                 }
 448                 // For x86, only add per-commit jobs for Windows
 449                 else if (architecture == 'x86' || architecture == 'x86lb') {
 450                     if (os == 'Windows_NT') {
 451                         Utilities.addGithubPushTrigger(job)
 452                     }
 453                 }
 454                 // arm64 pri1r2r jobs should only run every 12 hours.
 455                 else if (architecture == 'arm64') {
 456                     if (os == 'Windows_NT') {
 457                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 458                         // TODO: Add once external email sending is available again
 459                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 460                     }
 461                 }
 462             }
 463             break
 464         case 'r2r_jitstress1':
 465         case 'r2r_jitstress2':
 466         case 'r2r_jitstressregs1':
 467         case 'r2r_jitstressregs2':
 468         case 'r2r_jitstressregs3':
 469         case 'r2r_jitstressregs4':
 470         case 'r2r_jitstressregs8':
 471         case 'r2r_jitstressregs0x10':
 472         case 'r2r_jitstressregs0x80':
 473         case 'r2r_jitstressregs0x1000':
 474         case 'r2r_jitminopts':
 475         case 'r2r_jitforcerelocs':
 476         case 'gcstress15_pri1r2r':
 477             assert !(os in bidailyCrossList)
 478
 479             // GCStress=C is currently not supported on OS X
 480             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 481                 break
 482             }
 483
 484             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 485             if (configuration == 'Checked' || configuration == 'Release') {
 486                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 487                 if (architecture == 'x64') {
 488                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 489                     if (isFlowJob || os == 'Windows_NT') {
 490                         // Add a weekly periodic trigger
 491                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 492                     }
 493                 }
 494                 // For x86, only add per-commit jobs for Windows
 495                 else if (architecture == 'x86') {
 496                     if (os == 'Windows_NT') {
 497                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 498                     }
 499                 }
 500             }
 501             break
 502         case 'longgc':
 503             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 504             assert configuration == 'Release'
 505             assert architecture == 'x64'
 506             Utilities.addPeriodicTrigger(job, '@daily')
 507             // TODO: Add once external email sending is available again
 508             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 509             break
 510         case 'gcsimulator':
 511             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 512             assert configuration == 'Release'
 513             assert architecture == 'x64'
 514             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 515             // TODO: Add once external email sending is available again
 516             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 517             break
 518         case 'standalone_gc':
 519             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 520             assert (configuration == 'Release' || configuration == 'Checked')
 521             // TODO: Add once external email sending is available again
 522             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 523             Utilities.addPeriodicTrigger(job, '@weekly')
 524             break
 525         case 'gc_reliability_framework':
 526             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 527             assert (configuration == 'Release' || configuration == 'Checked')
 528             // Only triggered by phrase.
 529             break
 530         case 'ilrt':
 531             assert !(os in bidailyCrossList)
 532             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 533             if (architecture == 'x64' && configuration == 'Release') {
 534                 // We don't expect to see a job generated except in these scenarios
 535                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 536                 if (isFlowJob || os == 'Windows_NT') {
 537                     Utilities.addPeriodicTrigger(job, '@daily')
 538                 }
 539             }
 540             break
 541         case 'jitdiff':
 542             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 543             assert configuration == 'Checked'
 544             assert (architecture == 'x64' || architecture == 'x86')
 545             Utilities.addGithubPushTrigger(job)
 546             break
 547         case 'coverage':
 548             assert (os == 'Ubuntu' || os == 'Windows_NT')
 549             assert configuration == 'Release'
 550             assert architecture == 'x64'
 551             Utilities.addPeriodicTrigger(job, '@weekly')
 552             break
 553         case 'formatting':
 554             assert (os == 'Windows_NT' || os == "Ubuntu")
 555             assert architecture == 'x64'
 556             Utilities.addGithubPushTrigger(job)
 557             break
 558         case 'jitstressregs1':
 559         case 'jitstressregs2':
 560         case 'jitstressregs3':
 561         case 'jitstressregs4':
 562         case 'jitstressregs8':
 563         case 'jitstressregs0x10':
 564         case 'jitstressregs0x80':
 565         case 'jitstressregs0x1000':
 566         case 'minopts':
 567         case 'forcerelocs':
 568         case 'jitstress1':
 569         case 'jitstress2':
 570         case 'jitstress2_jitstressregs1':
 571         case 'jitstress2_jitstressregs2':
 572         case 'jitstress2_jitstressregs3':
 573         case 'jitstress2_jitstressregs4':
 574         case 'jitstress2_jitstressregs8':
 575         case 'jitstress2_jitstressregs0x10':
 576         case 'jitstress2_jitstressregs0x80':
 577         case 'jitstress2_jitstressregs0x1000':
 578         case 'tailcallstress':
 579         case 'jitsse2only':
 580         case 'corefx_baseline':
 581         case 'corefx_minopts':
 582         case 'corefx_jitstress1':
 583         case 'corefx_jitstress2':
 584         case 'corefx_jitstressregs1':
 585         case 'corefx_jitstressregs2':
 586         case 'corefx_jitstressregs3':
 587         case 'corefx_jitstressregs4':
 588         case 'corefx_jitstressregs8':
 589         case 'corefx_jitstressregs0x10':
 590         case 'corefx_jitstressregs0x80':
 591         case 'corefx_jitstressregs0x1000':
 592         case 'zapdisable':
 593             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 594             assert (os == 'Windows_NT') || (os in Constants.crossList)
 595             Utilities.addPeriodicTrigger(job, '@daily')
 596         }
 597         break
 598         case 'heapverify1':
 599         case 'gcstress0x3':
 600             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 601                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 602                 if (architecture == 'arm64') {
 603                     if (os == 'Windows_NT') {
 604                         Utilities.addPeriodicTrigger(job, '@daily')
 605                     }
 606                     // TODO: Add once external email sending is available again
 607                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 608                 }
 609                 else {
 610                     Utilities.addPeriodicTrigger(job, '@weekly')
 611                 }
 612             }
 613             break
 614         case 'gcstress0xc':
 615         case 'gcstress0xc_zapdisable':
 616         case 'gcstress0xc_zapdisable_jitstress2':
 617         case 'gcstress0xc_zapdisable_heapverify1':
 618         case 'gcstress0xc_jitstress1':
 619         case 'gcstress0xc_jitstress2':
 620         case 'gcstress0xc_minopts_heapverify1':
 621             // GCStress=C is currently not supported on OS X
 622             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 623                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 624                 if (architecture == 'arm64') {
 625                     // TODO: Enable a periodic trigger after tests are updated.
 626                     // Utilities.addPeriodicTrigger(job, '@daily')
 627                     // TODO: Add once external email sending is available again
 628                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 629                 }
 630                 else {
 631                     Utilities.addPeriodicTrigger(job, '@weekly')
 632                 }
 633             }
 634             break
 635
 636         case 'illink':
 637             // Testing on other operating systems TBD
 638             assert (os == 'Windows_NT' || os == 'Ubuntu')
 639             if (architecture == 'x64' || architecture == 'x86') {
 640                 if (configuration == 'Checked') {
 641                     Utilities.addPeriodicTrigger(job, '@daily')
 642                 }
 643             }
 644             break
 645
 646         case 'tieredcompilation':
 647         case 'corefx_tieredcompilation':
 648             // No periodic jobs just yet, still testing
 649             break
 650
 651         default:
 652             println("Unknown scenario: ${scenario}");
 653             assert false
 654             break
 655     }
 656     return
 657 }
 658
 659 // **************************
 660 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 661 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 662 // builds will do a couple extra steps.
 663 // **************************
 664
 665 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 666 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 667 // it's just used for internal builds
 668 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 669 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 670     if (isWindowsBuildOnlyJob) {
 671         return
 672     }
 673
 674     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 675     // Non pull request builds.
 676     if (!isPR) {
 677         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 678         return
 679     }
 680
 681      def arm64Users = [
 682         'adiaaida',
 683         'AndyAyersMS',
 684         'briansull',
 685         'BruceForstall',
 686         'CarolEidt',
 687         'cmckinsey',
 688         'erozenfeld',
 689         'jashook',
 690         'JosephTremoulet',
 691         'pgavlin',
 692         'russellhadley',
 693         'RussKeldorph',
 694         'sandreenko',
 695         'sdmaclea',
 696         'sivarv',
 697         'swaroop-sridhar',
 698         'gkhanna79',
 699         'jkotas',
 700         'markwilkie',
 701         'rahku',
 702         'ramarag',
 703         'tzwlai',
 704         'weshaggard'
 705     ]
 706
 707     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 708     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 709     def osGroup = getOSGroup(os)
 710     switch (architecture) {
 711         case 'x64': // editor brace matching: {
 712             if (scenario == 'coverage') {
 713                 assert configuration == 'Release'
 714                 if (os == 'Ubuntu') {
 715                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 716                 }
 717                 break
 718             }
 719
 720             if (scenario == 'formatting') {
 721                 assert configuration == 'Checked'
 722                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 723                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 724                 }
 725                 break
 726             }
 727
 728             switch (os) {
 729                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 730                 case 'Debian8.4':
 731                 case 'RHEL7.2':
 732                     if (scenario == 'default') {
 733                         assert !isFlowJob
 734                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 735                     }
 736                     else if (scenario == 'pri1' && isFlowJob) {
 737                         assert (configuration == 'Release')
 738                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 739                     }
 740                     break
 741                 case 'Ubuntu16.04':
 742                     assert !isFlowJob
 743                     assert scenario == 'default'
 744                     // Distinguish with the other architectures (arm and x86)
 745                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 746                     break
 747                 case 'Fedora24':
 748                 case 'Ubuntu16.10':
 749                 case 'OpenSUSE42.1':
 750                     assert !isFlowJob
 751                     assert scenario == 'default'
 752                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 753                     break
 754                 case 'Ubuntu':
 755                     if (scenario == 'illink') {
 756                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 757                         break
 758                     }
 759                     // fall through
 760                 case 'OSX10.12':
 761                     // Triggers on the non-flow jobs aren't necessary here
 762                     // Corefx testing uses non-flow jobs.
 763                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 764                         break
 765                     }
 766                     switch (scenario) {
 767                         case 'default':
 768                             // Ubuntu uses checked for default PR tests
 769                             if (configuration == 'Checked') {
 770                                 // Default trigger
 771                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 772                             }
 773                             break
 774                         case 'pri1':
 775                             if (configuration == 'Release') {
 776                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 777                             }
 778                             break
 779                         case 'jitdiff':
 780                             if (configuration == 'Checked') {
 781                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 782                             }
 783                             break
 784                         case 'ilrt':
 785                             if (configuration == 'Release') {
 786                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 787                             }
 788                             break
 789                         case 'r2r':
 790                             if (configuration == 'Release' || configuration == 'Checked') {
 791                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 792                             }
 793                             break
 794                         case 'pri1r2r':
 795                             if (configuration == 'Release' || configuration == 'Checked') {
 796                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 797                             }
 798                             break
 799                         case 'gcstress15_pri1r2r':
 800                             if (configuration == 'Release' || configuration == 'Checked') {
 801                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 802                             }
 803                             break
 804                         case 'r2r_jitstress1':
 805                         case 'r2r_jitstress2':
 806                         case 'r2r_jitstressregs1':
 807                         case 'r2r_jitstressregs2':
 808                         case 'r2r_jitstressregs3':
 809                         case 'r2r_jitstressregs4':
 810                         case 'r2r_jitstressregs8':
 811                         case 'r2r_jitstressregs0x10':
 812                         case 'r2r_jitstressregs0x80':
 813                         case 'r2r_jitstressregs0x1000':
 814                         case 'r2r_jitminopts':
 815                         case 'r2r_jitforcerelocs':
 816                             if (configuration == 'Release' || configuration == 'Checked') {
 817                                 def displayStr = getR2RStressModeDisplayName(scenario)
 818                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 819                             }
 820                             break
 821                         case 'longgc':
 822                             if (configuration == 'Release') {
 823                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 824                             }
 825                             break
 826                         case 'gcsimulator':
 827                             if (configuration == 'Release') {
 828                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 829                             }
 830                             break
 831                         case 'standalone_gc':
 832                             if (configuration == 'Release' || configuration == 'Checked') {
 833                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 834                             }
 835                             break
 836                         case 'gc_reliability_framework':
 837                             if (configuration == 'Release' || configuration == 'Checked') {
 838                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 839                             }
 840                             break
 841                         case 'minopts':
 842                         case 'tieredcompilation':
 843                         case 'forcerelocs':
 844                         case 'jitstress1':
 845                         case 'jitstress2':
 846                         case 'jitstressregs1':
 847                         case 'jitstressregs2':
 848                         case 'jitstressregs3':
 849                         case 'jitstressregs4':
 850                         case 'jitstressregs8':
 851                         case 'jitstressregs0x10':
 852                         case 'jitstressregs0x80':
 853                         case 'jitstressregs0x1000':
 854                         case 'jitstress2_jitstressregs1':
 855                         case 'jitstress2_jitstressregs2':
 856                         case 'jitstress2_jitstressregs3':
 857                         case 'jitstress2_jitstressregs4':
 858                         case 'jitstress2_jitstressregs8':
 859                         case 'jitstress2_jitstressregs0x10':
 860                         case 'jitstress2_jitstressregs0x80':
 861                         case 'jitstress2_jitstressregs0x1000':
 862                         case 'tailcallstress':
 863                         case 'jitsse2only':
 864                         case 'gcstress0x3':
 865                         case 'gcstress0xc':
 866                         case 'zapdisable':
 867                         case 'heapverify1':
 868                         case 'gcstress0xc_zapdisable':
 869                         case 'gcstress0xc_zapdisable_jitstress2':
 870                         case 'gcstress0xc_zapdisable_heapverify1':
 871                         case 'gcstress0xc_jitstress1':
 872                         case 'gcstress0xc_jitstress2':
 873                         case 'gcstress0xc_minopts_heapverify1':
 874                             def displayStr = getStressModeDisplayName(scenario)
 875                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 876                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 877                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 878                             break
 879                         case 'corefx_baseline':
 880                         case 'corefx_minopts':
 881                         case 'corefx_tieredcompilation':
 882                         case 'corefx_jitstress1':
 883                         case 'corefx_jitstress2':
 884                         case 'corefx_jitstressregs1':
 885                         case 'corefx_jitstressregs2':
 886                         case 'corefx_jitstressregs3':
 887                         case 'corefx_jitstressregs4':
 888                         case 'corefx_jitstressregs8':
 889                         case 'corefx_jitstressregs0x10':
 890                         case 'corefx_jitstressregs0x80':
 891                         case 'corefx_jitstressregs0x1000':
 892                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 893                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 894                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 895                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 896                             break
 897                         default:
 898                             println("Unknown scenario: ${scenario}");
 899                             assert false
 900                             break
 901                     }
 902                     break
 903                 case 'CentOS7.1':
 904                     switch (scenario) {
 905                         case 'pri1':
 906                             if (configuration == 'Release') {
 907                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 908                             }
 909                             break
 910                         case 'r2r':
 911                             if (configuration == 'Checked' || configuration == 'Release') {
 912                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 913                             }
 914                             break
 915                         case 'pri1r2r':
 916                             if (configuration == 'Checked' || configuration == 'Release') {
 917                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 918                             }
 919                             break
 920                         case 'gcstress15_pri1r2r':
 921                             if (configuration == 'Release' || configuration == 'Checked') {
 922                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 923                             }
 924                             break
 925                         case 'r2r_jitstress1':
 926                         case 'r2r_jitstress2':
 927                         case 'r2r_jitstressregs1':
 928                         case 'r2r_jitstressregs2':
 929                         case 'r2r_jitstressregs3':
 930                         case 'r2r_jitstressregs4':
 931                         case 'r2r_jitstressregs8':
 932                         case 'r2r_jitstressregs0x10':
 933                         case 'r2r_jitstressregs0x80':
 934                         case 'r2r_jitstressregs0x1000':
 935                         case 'r2r_jitminopts':
 936                         case 'r2r_jitforcerelocs':
 937                             if (configuration == 'Release' || configuration == 'Checked') {
 938                                 def displayStr = getR2RStressModeDisplayName(scenario)
 939                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 940                             }
 941                             break
 942                         default:
 943                             break
 944                     }
 945                 case 'Windows_NT':
 946                     switch (scenario) {
 947                         case 'default':
 948                             // Default trigger
 949                             if (configuration == 'Debug') {
 950                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 951                             }
 952                             break
 953                         case 'pri1':
 954                             // Default trigger
 955                             if (configuration == 'Release') {
 956                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 957                             }
 958                             break
 959                         case 'jitdiff':
 960                             if (configuration == 'Checked') {
 961                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 962                             }
 963                             break
 964                         case 'ilrt':
 965                             if (configuration == 'Release') {
 966                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 967                             }
 968                             break
 969                         case 'r2r':
 970                             if (configuration == 'Checked' || configuration == 'Release') {
 971                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 972                             }
 973                             break
 974                         case 'pri1r2r':
 975                             if (configuration == 'Checked' || configuration == 'Release') {
 976                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 977                             }
 978                             break
 979                         case 'gcstress15_pri1r2r':
 980                             if (configuration == 'Release' || configuration == 'Checked') {
 981                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 982                             }
 983                             break
 984                         case 'r2r_jitstress1':
 985                         case 'r2r_jitstress2':
 986                         case 'r2r_jitstressregs1':
 987                         case 'r2r_jitstressregs2':
 988                         case 'r2r_jitstressregs3':
 989                         case 'r2r_jitstressregs4':
 990                         case 'r2r_jitstressregs8':
 991                         case 'r2r_jitstressregs0x10':
 992                         case 'r2r_jitstressregs0x80':
 993                         case 'r2r_jitstressregs0x1000':
 994                         case 'r2r_jitminopts':
 995                         case 'r2r_jitforcerelocs':
 996                             if (configuration == 'Release' || configuration == 'Checked') {
 997                                 def displayStr = getR2RStressModeDisplayName(scenario)
 998                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 999                             }
1000                             break
1001                         case 'longgc':
1002                             if (configuration == 'Release') {
1003                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1004                             }
1005                             break
1006                         case 'gcsimulator':
1007                             if (configuration == 'Release') {
1008                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1009                             }
1010                             break
1011                         case 'standalone_gc':
1012                             if (configuration == 'Release' || configuration == 'Checked') {
1013                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1014                             }
1015                             break
1016                         case 'gc_reliability_framework':
1017                             if (configuration == 'Release' || configuration == 'Checked') {
1018                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1019                             }
1020                             break
1021                         case 'minopts':
1022                         case 'tieredcompilation':
1023                         case 'forcerelocs':
1024                         case 'jitstress1':
1025                         case 'jitstress2':
1026                         case 'jitstressregs1':
1027                         case 'jitstressregs2':
1028                         case 'jitstressregs3':
1029                         case 'jitstressregs4':
1030                         case 'jitstressregs8':
1031                         case 'jitstressregs0x10':
1032                         case 'jitstressregs0x80':
1033                         case 'jitstressregs0x1000':
1034                         case 'jitstress2_jitstressregs1':
1035                         case 'jitstress2_jitstressregs2':
1036                         case 'jitstress2_jitstressregs3':
1037                         case 'jitstress2_jitstressregs4':
1038                         case 'jitstress2_jitstressregs8':
1039                         case 'jitstress2_jitstressregs0x10':
1040                         case 'jitstress2_jitstressregs0x80':
1041                         case 'jitstress2_jitstressregs0x1000':
1042                         case 'tailcallstress':
1043                         case 'jitsse2only':
1044                         case 'gcstress0x3':
1045                         case 'gcstress0xc':
1046                         case 'zapdisable':
1047                         case 'heapverify1':
1048                         case 'gcstress0xc_zapdisable':
1049                         case 'gcstress0xc_zapdisable_jitstress2':
1050                         case 'gcstress0xc_zapdisable_heapverify1':
1051                         case 'gcstress0xc_jitstress1':
1052                         case 'gcstress0xc_jitstress2':
1053                         case 'gcstress0xc_minopts_heapverify1':
1054                             def displayStr = getStressModeDisplayName(scenario)
1055                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1056                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1057                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1058                             break
1059                         case 'corefx_baseline':
1060                         case 'corefx_minopts':
1061                         case 'corefx_tieredcompilation':
1062                         case 'corefx_jitstress1':
1063                         case 'corefx_jitstress2':
1064                         case 'corefx_jitstressregs1':
1065                         case 'corefx_jitstressregs2':
1066                         case 'corefx_jitstressregs3':
1067                         case 'corefx_jitstressregs4':
1068                         case 'corefx_jitstressregs8':
1069                         case 'corefx_jitstressregs0x10':
1070                         case 'corefx_jitstressregs0x80':
1071                         case 'corefx_jitstressregs0x1000':
1072                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1073                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1074                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1075                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1076                             break
1077                         case 'illink':
1078                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1079                                     break
1080                         default:
1081                             println("Unknown scenario: ${scenario}");
1082                             assert false
1083                             break
1084                     }
1085                     break
1086                 case 'FreeBSD':
1087                     assert scenario == 'default'
1088                     if (configuration == 'Checked') {
1089                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1090                     }
1091                     break
1092                 default:
1093                     println("Unknown os: ${os}");
1094                     assert false
1095                     break
1096             }
1097             break
1098         // editor brace matching: }
1099         case 'arm': // editor brace matching: {
1100             switch (os) {
1101                 case 'Ubuntu':
1102                 case 'Ubuntu16.04':
1103                     assert scenario == 'default'
1104                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1105                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1106                     }
1107                     else {
1108                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1109                     }
1110                     break;
1111                 case 'Tizen':
1112                     architecture='armel'
1113                     // Removing the regex will cause this to run on each PR.
1114                     if (configuration == 'Release' || configuration == 'Debug') {
1115                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1116                     }
1117                     else {
1118                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1119                     }
1120                     break;
1121                 case 'Windows_NT':
1122                     // Set up a private trigger
1123                     def contextString = "${os} ${architecture} Cross ${configuration}"
1124                     if (scenario != 'default')
1125                         contextString += " ${scenario}"
1126                     contextString += " Build"
1127                     // Debug builds only.
1128                     if (configuration != 'Debug') {
1129                         contextString += " and Test"
1130                     }
1131                     switch (scenario) {
1132                         case 'default':
1133                             // For now only run Debug and Release build jobs on PR Trigger. Note this is not a private trigger.
1134                             if (configuration == 'Debug' || configuration == 'Release')
1135                             {
1136                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1137                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1138                             }
1139                             else
1140                             {
1141                                 // Checked jobs will run on private trigger and run tests.
1142                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1143                             }
1144                             break
1145                         case 'pri1r2r':
1146                         case 'gcstress0x3':
1147                         case 'gcstress0xc':
1148                             // Stress jobs will will run on private trigger and run tests.
1149                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1150                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1151                             break
1152                     }
1153                     break
1154                 default:
1155                     println("NYI os: ${os}");
1156                     assert false
1157                     break
1158             }
1159             break
1160         // editor brace matching: }
1161         case 'arm64': // editor brace matching: {
1162             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1163
1164             // Set up a private trigger
1165             def contextString = "${os} ${architecture} Cross ${configuration}"
1166             if (scenario != 'default')
1167                 contextString += " ${scenario}"
1168             contextString += " Build"
1169             // Debug builds only.
1170             if (configuration != 'Debug') {
1171                contextString += " and Test"
1172             }
1173
1174             switch (os) {
1175                 case 'Ubuntu':
1176                 case 'Ubuntu16.04':
1177                     switch (scenario) {
1178                         case 'pri1':
1179                             if (configuration == 'Release') {
1180                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1181                             }
1182                             break
1183                         case 'r2r':
1184                             if (configuration == 'Checked' || configuration == 'Release') {
1185                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1186                             }
1187                             break
1188                         case 'pri1r2r':
1189                             if (configuration == 'Checked' || configuration == 'Release') {
1190                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1191                             }
1192                             break
1193                         case 'gcstress15_pri1r2r':
1194                             if (configuration == 'Release' || configuration == 'Checked') {
1195                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1196                             }
1197                             break
1198                         case 'r2r_jitstress1':
1199                         case 'r2r_jitstress2':
1200                         case 'r2r_jitstressregs1':
1201                         case 'r2r_jitstressregs2':
1202                         case 'r2r_jitstressregs3':
1203                         case 'r2r_jitstressregs4':
1204                         case 'r2r_jitstressregs8':
1205                         case 'r2r_jitstressregs0x10':
1206                         case 'r2r_jitstressregs0x80':
1207                         case 'r2r_jitstressregs0x1000':
1208                         case 'r2r_jitminopts':
1209                         case 'r2r_jitforcerelocs':
1210                             if (configuration == 'Release' || configuration == 'Checked') {
1211                                 def displayStr = getR2RStressModeDisplayName(scenario)
1212                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1213                             }
1214                             break
1215                         default:
1216                             break
1217                     }
1218                 case 'Windows_NT':
1219                     switch (scenario) {
1220                         case 'default':
1221                             if (isFlowJob == true) {
1222                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration}", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1223                             }
1224
1225                             // For now only run Debug jobs on PR Trigger.
1226                             else if (configuration != 'Debug') {
1227                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1228                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1229                             }
1230                             else {
1231                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1232                                 // are at higher risk of ARM64-breaking changes.
1233                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1234                             }
1235                             break
1236                         case 'pri1r2r':
1237                         case 'gcstress0x3':
1238                         case 'gcstress0xc':
1239                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1240                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1241                             break
1242                     }
1243                     break
1244                 default:
1245                     println("NYI os: ${os}");
1246                     assert false
1247                     break
1248             }
1249             break
1250         // editor brace matching: }
1251         case 'x86': // editor brace matching: {
1252             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1253             if (os == 'Ubuntu') {
1254                 // on-demand only for ubuntu x86
1255                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1256                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1257                 break
1258             }
1259             switch (scenario) {
1260                 case 'default':
1261                     if (configuration == 'Checked') {
1262                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1263                     }
1264                     else if (configuration == 'Release') {
1265                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1266                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1267                     }
1268                     break
1269                 case 'pri1':
1270                     if (configuration == 'Release') {
1271                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1272                     }
1273                     break
1274                 case 'ilrt':
1275                     if (configuration == 'Release') {
1276                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1277                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1278                     }
1279                     break
1280                 case 'r2r':
1281                     if (configuration == 'Checked' || configuration == 'Release') {
1282                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1283                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1284                     }
1285                     break
1286                 case 'pri1r2r':
1287                     if (configuration == 'Checked' || configuration == 'Release') {
1288                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1289                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1290                     }
1291                     break
1292                 case 'gcstress15_pri1r2r':
1293                     if (configuration == 'Release' || configuration == 'Checked') {
1294                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1295                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1296                     }
1297                     break
1298                 case 'r2r_jitstress1':
1299                 case 'r2r_jitstress2':
1300                 case 'r2r_jitstressregs1':
1301                 case 'r2r_jitstressregs2':
1302                 case 'r2r_jitstressregs3':
1303                 case 'r2r_jitstressregs4':
1304                 case 'r2r_jitstressregs8':
1305                 case 'r2r_jitstressregs0x10':
1306                 case 'r2r_jitstressregs0x80':
1307                 case 'r2r_jitstressregs0x1000':
1308                 case 'r2r_jitminopts':
1309                 case 'r2r_jitforcerelocs':
1310                     if (configuration == 'Release' || configuration == 'Checked') {
1311                         def displayStr = getR2RStressModeDisplayName(scenario)
1312                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1313                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1314                     }
1315                     break
1316                 case 'longgc':
1317                     if (configuration == 'Release') {
1318                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1319                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1320                     }
1321                     break
1322                 case 'gcsimulator':
1323                     if (configuration == 'Release') {
1324                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1325                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1326                     }
1327                     break
1328                 case 'standalone_gc':
1329                     if (configuration == 'Release' || configuration == 'Checked') {
1330                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1331                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1332                     }
1333                     break
1334                 case 'minopts':
1335                 case 'tieredcompilation':
1336                 case 'forcerelocs':
1337                 case 'jitstress1':
1338                 case 'jitstress2':
1339                 case 'jitstressregs1':
1340                 case 'jitstressregs2':
1341                 case 'jitstressregs3':
1342                 case 'jitstressregs4':
1343                 case 'jitstressregs8':
1344                 case 'jitstressregs0x10':
1345                 case 'jitstressregs0x80':
1346                 case 'jitstressregs0x1000':
1347                 case 'jitstress2_jitstressregs1':
1348                 case 'jitstress2_jitstressregs2':
1349                 case 'jitstress2_jitstressregs3':
1350                 case 'jitstress2_jitstressregs4':
1351                 case 'jitstress2_jitstressregs8':
1352                 case 'jitstress2_jitstressregs0x10':
1353                 case 'jitstress2_jitstressregs0x80':
1354                 case 'jitstress2_jitstressregs0x1000':
1355                 case 'tailcallstress':
1356                 case 'jitsse2only':
1357                 case 'gcstress0x3':
1358                 case 'gcstress0xc':
1359                 case 'zapdisable':
1360                 case 'heapverify1':
1361                 case 'gcstress0xc_zapdisable':
1362                 case 'gcstress0xc_zapdisable_jitstress2':
1363                 case 'gcstress0xc_zapdisable_heapverify1':
1364                 case 'gcstress0xc_jitstress1':
1365                 case 'gcstress0xc_jitstress2':
1366                 case 'gcstress0xc_minopts_heapverify1':
1367                     def displayStr = getStressModeDisplayName(scenario)
1368                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1369                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1370                     break
1371                 case 'corefx_baseline':
1372                 case 'corefx_minopts':
1373                 case 'corefx_tieredcompilation':
1374                 case 'corefx_jitstress1':
1375                 case 'corefx_jitstress2':
1376                 case 'corefx_jitstressregs1':
1377                 case 'corefx_jitstressregs2':
1378                 case 'corefx_jitstressregs3':
1379                 case 'corefx_jitstressregs4':
1380                 case 'corefx_jitstressregs8':
1381                 case 'corefx_jitstressregs0x10':
1382                 case 'corefx_jitstressregs0x80':
1383                 case 'corefx_jitstressregs0x1000':
1384                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1385                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1386                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1387                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1388                     break
1389                 case 'illink':
1390                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1391                     break
1392                 default:
1393                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1394                     assert false
1395                     break
1396             }
1397             break
1398          // editor brace matching: }
1399         case 'x86lb': // editor brace matching: {
1400             assert (os == 'Windows_NT')
1401             assert (scenario == 'default' || Constants.r2rJitStressScenarios.indexOf(scenario) !=1)
1402
1403             def arch = 'x86'
1404             def jit = 'legacy_backend'
1405             switch (scenario) {
1406                 case 'default':
1407                     if (configuration == 'Checked') {
1408                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1409                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1410                     }
1411                     break
1412                 default:
1413                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1414                     assert false
1415                     break
1416             }
1417             break
1418         // editor brace matching: }
1419         default:
1420             println("Unknown architecture: ${architecture}");
1421             assert false
1422             break
1423     }
1424 }
1425
1426 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1427     def buildCommands = [];
1428     def osGroup = getOSGroup(os)
1429     def lowerConfiguration = configuration.toLowerCase()
1430
1431     // Calculate the build steps, archival, and xunit results
1432     switch (os) {
1433         case 'Windows_NT': // editor brace matching: {
1434             switch (architecture) {
1435                 case 'x64':
1436                 case 'x86':
1437                 case 'x86lb':
1438                     def arch = architecture
1439                     def buildOpts = ''
1440                     if (architecture == 'x86lb') {
1441                         arch = 'x86'
1442                     }
1443
1444                     if (scenario == 'illink') {
1445                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1446                     }
1447
1448                     // If it is a release build for windows, ensure PGO is used, else fail the build
1449                     if ((lowerConfiguration == 'release') && (scenario in Constants.basicScenarios) && (architecture != 'x86lb')) {
1450                         buildOpts += ' enforcepgo'
1451                     }
1452
1453                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1454                             scenario == 'default' ||
1455                             scenario == 'r2r' ||
1456                             scenario == 'jitdiff' ||
1457                             scenario == 'ilrt' ||
1458                             scenario == 'illink' ||
1459                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1460                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1461                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1462                     }
1463
1464                     // For Pri 1 tests, we must shorten the output test binary path names.
1465                     // if __TestIntermediateDir is already set, build-test.cmd will
1466                     // output test binaries to that directory. If it is not set, the
1467                     // binaries are sent to a default directory whose name is about
1468                     // 35 characters long.
1469
1470                     else if (scenarioNeedsPri1Build(scenario)) {
1471                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1472                     }
1473                     else if (isLongGc(scenario)) {
1474                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1475                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1476                     }
1477                     else if (scenario == 'standalone_gc') {
1478                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1479                     }
1480                     else if (scenario == 'formatting') {
1481                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1482                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1483                         break
1484                     }
1485                     else {
1486                         println("Unknown scenario: ${scenario}")
1487                         assert false
1488                     }
1489
1490                     // If we are running a stress mode, we should write out the set of key
1491                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1492
1493                     if (!isBuildOnly) {
1494                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1495                         def crossgenStr = ''
1496                         def runcrossgentestsStr = ''
1497                         def runjitstressStr = ''
1498                         def runjitstressregsStr = ''
1499                         def runjitmioptsStr = ''
1500                         def runjitforcerelocsStr = ''
1501                         def runjitdisasmStr = ''
1502                         def runilasmroundtripStr = ''
1503                         def gcstressStr = ''
1504                         def runtestArguments = ''
1505                         def gcTestArguments = ''
1506                         def illinkArguments = ''
1507
1508                         if (scenario == 'r2r' ||
1509                             scenario == 'pri1r2r' ||
1510                             scenario == 'gcstress15_pri1r2r' ||
1511                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1512                                 crossgenStr = 'crossgen'
1513                                 runcrossgentestsStr = 'runcrossgentests'
1514
1515                                 if (scenario == 'r2r_jitstress1'){
1516                                     runjitstressStr = 'jitstress 1'
1517                                 }
1518                                 else if (scenario == 'r2r_jitstress2') {
1519                                     runjitstressStr = 'jitstress 2'
1520                                 }
1521                                 else if (scenario == 'r2r_jitstressregs1'){
1522                                     runjitstressregsStr = 'jitstressregs 1'
1523                                 }
1524                                 else if (scenario == 'r2r_jitstressregs2') {
1525                                     runjitstressregsStr = 'jitstressregs 2'
1526                                 }
1527                                 else if (scenario == 'r2r_jitstressregs3') {
1528                                     runjitstressregsStr = 'jitstressregs 3'
1529                                 }
1530                                 else if (scenario == 'r2r_jitstressregs4') {
1531                                     runjitstressregsStr = 'jitstressregs 4'
1532                                 }
1533                                 else if (scenario == 'r2r_jitstressregs8') {
1534                                     runjitstressregsStr = 'jitstressregs 8'
1535                                 }
1536                                 else if (scenario == 'r2r_jitstressregs0x10') {
1537                                     runjitstressregsStr = 'jitstressregs 0x10'
1538                                 }
1539                                 else if (scenario == 'r2r_jitstressregs0x80') {
1540                                     runjitstressregsStr = 'jitstressregs 0x80'
1541                                 }
1542                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1543                                     runjitstressregsStr = 'jitstressregs 0x1000'
1544                                 }
1545                                 else if (scenario == 'r2r_jitminopts') {
1546                                     runjitmioptsStr = 'jitminopts'
1547                                 }
1548                                 else if (scenario == 'r2r_jitforcerelocs') {
1549                                     runjitforcerelocsStr = 'jitforcerelocs'
1550                                 }
1551                         }
1552                         if (scenario == 'gcstress15_pri1r2r')
1553                         {
1554                             gcstressStr = 'gcstresslevel 0xF'
1555                         }
1556
1557                         if (scenario == 'jitdiff')
1558                         {
1559                             runjitdisasmStr = 'jitdisasm crossgen'
1560                         }
1561
1562                         if (scenario == 'ilrt')
1563                         {
1564                             runilasmroundtripStr = 'ilasmroundtrip'
1565                         }
1566
1567                         if (isLongGc(scenario)) {
1568                             gcTestArguments = "${scenario} sequential"
1569                         }
1570
1571                         if (scenario == 'illink')
1572                         {
1573                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1574                         }
1575
1576                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1577
1578                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1579                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1580                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1581
1582                             if (enableCorefxTesting) {
1583                                 def workspaceRelativeFxRoot = "_/fx"
1584                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1585
1586                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1587
1588                                 setTestJobTimeOut(newJob, scenario)
1589
1590                                 // Archive and process (only) the test results
1591                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1592                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1593
1594                                 //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1595                                 Utilities.addArchival(newJob, "SetStressModes.bat")
1596                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**")
1597                             }
1598                             else {
1599                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1600                             }
1601                         }
1602                         else if (isGcReliabilityFramework(scenario)) {
1603                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1604                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1605                         }
1606                         else if (architecture == 'x64' || architecture == 'x86') {
1607                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1608                         }
1609                         else if (architecture == 'x86lb') {
1610                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd"
1611                         }
1612                     }
1613
1614                     if (!enableCorefxTesting) {
1615                         // Run the rest of the build
1616                         // Build the mscorlib for the other OS's
1617                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1618                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1619                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1620
1621                         if (arch == "x64") {
1622                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1623                         }
1624
1625                         // Zip up the tests directory so that we don't use so much space/time copying
1626                         // 10s of thousands of files around.
1627                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1628
1629                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1630                             // For windows, pull full test results and test drops for x86/x64.
1631                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1632                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1633                         }
1634
1635                         if (scenario == 'jitdiff') {
1636                             // retrive jit-dasm output for base commit, and run jit-diff
1637                             if (!isBuildOnly) {
1638                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1639                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1640                             }
1641                         }
1642
1643                         if (!isBuildOnly) {
1644                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1645                             setTestJobTimeOut(newJob, scenario)
1646                         }
1647                     }
1648                     break
1649                 case 'arm':
1650                     def validArmWindowsScenarios = [ "default",
1651                                                      "pri1r2r",
1652                                                      "zapdisable",
1653                                                      "minopts",
1654                                                      "tieredcompilation",
1655                                                      "tailcallstress",
1656                                                      "jitstress1",
1657                                                      "jitstress2",
1658                                                      "gcstress0x3",
1659                                                      "gcstress0xc",
1660                                                      "jitstressregs1",
1661                                                      "jitstressregs2",
1662                                                      "gcstress0xc_jitstress1",
1663                                                      "gcstress0xc_jitstress2"]
1664
1665                     assert validArmWindowsScenarios.contains(scenario)
1666
1667                     // Set time out
1668                     setTestJobTimeOut(newJob, scenario)
1669
1670                     if ( lowerConfiguration == "debug" ) {
1671                         // For Debug builds, we will do a P1 test build
1672                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1673                     }
1674                     else if (lowerConfiguration == "checked") {
1675
1676                         if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1677                         {
1678                            // Up the timeout for arm checked testing only.
1679                            // Keep the longer timeout for gcstress.
1680                            Utilities.setJobTimeout(newJob, 240)
1681                         }
1682
1683                         def machineAffinityOptions = ['use_arm64_build_machine' : true]
1684                         setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1685                         // For checked runs we will also run testing.
1686                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1687                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1688                     }
1689                     else if (lowerConfiguration == "release") {
1690                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1691                     }
1692                     // Add archival.
1693                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1694                     break
1695                 case 'arm64':
1696                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1697
1698                     // Set time out
1699                     setTestJobTimeOut(newJob, scenario)
1700
1701                     // Debug runs take too long to run. So build job only.
1702                     if (lowerConfiguration == "debug") {
1703                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1704                     }
1705                     else {
1706                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1707                        {
1708                            // Up the timeout for arm64 checked testing only.
1709                            // Keep the longer timeout for gcstress.
1710                            Utilities.setJobTimeout(newJob, 240)
1711                        }
1712
1713                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=1"
1714                        // Test build and run are launched together.
1715                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1716                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1717                     }
1718
1719                     // Add archival.
1720                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1721                     break
1722                 default:
1723                     println("Unknown architecture: ${architecture}");
1724                     assert false
1725                     break
1726             }
1727             break
1728         // editor brace matching: }
1729         case 'Ubuntu':
1730         case 'Ubuntu16.04':
1731         case 'Ubuntu16.10':
1732         case 'Debian8.4':
1733         case 'OSX10.12':
1734         case 'FreeBSD':
1735         case 'CentOS7.1':
1736         case 'RHEL7.2':
1737         case 'OpenSUSE42.1':
1738         case 'Tizen':
1739         case 'Fedora24': // editor brace matching: {
1740             switch (architecture) {
1741                 case 'x64':
1742                 case 'x86':
1743                     if (architecture == 'x86' && os == 'Ubuntu') {
1744                         // build and PAL test
1745                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1746                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1747                         break;
1748                     }
1749
1750                     if (scenario == 'formatting') {
1751                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1752                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1753                         break
1754                     }
1755
1756                     if (scenario == 'illink') {
1757                         assert(os == 'Ubuntu')
1758                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1759                     }
1760
1761                     def standaloneGc = ''
1762                     if (scenario == 'standalone_gc') {
1763                         standaloneGc = 'buildstandalonegc'
1764                     }
1765
1766                     if (!enableCorefxTesting) {
1767                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1768                         // only on supported OS platforms.
1769                         if (os == 'FreeBSD')
1770                         {
1771                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1772                         }
1773                         else
1774                         {
1775                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1776                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1777                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1778                         }
1779                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1780
1781                         // Set time out
1782                         setTestJobTimeOut(newJob, scenario)
1783                         // Basic archiving of the build
1784                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1785                         // And pal tests
1786                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1787                     }
1788                     else {
1789                         // Corefx stress testing
1790                         assert os == 'Ubuntu'
1791                         assert architecture == 'x64'
1792                         assert lowerConfiguration == 'checked'
1793                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1794
1795                         // Build coreclr
1796                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1797
1798                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1799                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1800
1801                         // Build and text corefx
1802                         def workspaceRelativeFxRoot = "_/fx"
1803                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1804
1805                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1806
1807                         setTestJobTimeOut(newJob, scenario)
1808
1809                         // Archive and process (only) the test results
1810                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1811                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1812                     }
1813                     break
1814                 case 'arm64':
1815                     def standaloneGc = ''
1816                     if (scenario == 'standalone_gc') {
1817                         standaloneGc = 'buildstandalonegc'
1818                     }
1819
1820                     if (!enableCorefxTesting) {
1821                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8 ${standaloneGc}"
1822
1823                         // HACK -- Arm64 does not have corefx jobs yet.
1824                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1825                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1826                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1827                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1828
1829                         // Set time out
1830                         setTestJobTimeOut(newJob, scenario)
1831                         // Basic archiving of the build
1832                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1833                     }
1834                     break
1835                 case 'arm':
1836                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1837                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1838
1839                     // default values for Ubuntu
1840                     def arm_abi="arm"
1841                     def linuxCodeName="trusty"
1842                     if (os == 'Ubuntu16.04') {
1843                         linuxCodeName="xenial"
1844                     }
1845                     else if (os == 'Tizen') {
1846                         arm_abi="armel"
1847                         linuxCodeName="tizen"
1848                     }
1849
1850                     // Unzip the Windows test binaries first. Exit with 0
1851                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1852
1853                     // Unpack the corefx binaries
1854                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1855                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1856                     if (os != 'Tizen') {
1857                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1858                     }
1859                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1860                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1861                     if (os == 'Ubuntu') {
1862                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1863                     }
1864
1865                     // Call the ARM CI script to cross build and test using docker
1866                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1867                     --mode=docker \\
1868                     --${arm_abi} \\
1869                     --linuxCodeName=${linuxCodeName} \\
1870                     --buildConfig=${lowerConfiguration} \\
1871                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1872                     --coreFxBinDir=./bin/CoreFxBinDir \\
1873                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1874
1875                     // Basic archiving of the build, no pal tests
1876                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1877                     break
1878                 default:
1879                     println("Unknown architecture: ${architecture}");
1880                     assert false
1881                     break
1882             }
1883             break
1884         // editor brace matching: }
1885         default:
1886             println("Unknown os: ${os}");
1887             assert false
1888             break
1889     } // os
1890
1891     return buildCommands
1892 }
1893
1894 // Additional scenario which can alter behavior
1895
1896 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1897 combinedScenarios.each { scenario ->
1898     [true, false].each { isPR ->
1899         Constants.architectureList.each { architecture ->
1900             Constants.configurationList.each { configuration ->
1901                 Constants.osList.each { os ->
1902                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1903                     // and reset the os to Windows_NT
1904                     def isBuildOnly = false
1905                     if (os == 'Windows_NT_BuildOnly') {
1906                         isBuildOnly = true
1907                         os = 'Windows_NT'
1908                     }
1909
1910                     // Tizen is only supported for arm architecture
1911                     if (os == 'Tizen' && architecture != 'arm') {
1912                         return
1913                     }
1914
1915                     // Skip totally unimplemented (in CI) configurations.
1916                     switch (architecture) {
1917                         case 'arm64':
1918                             if (os == 'Ubuntu16.04') {
1919                                 os = 'Ubuntu'
1920                             }
1921
1922                             // Windows and Ubuntu only
1923                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1924                                 return
1925                             }
1926                             break
1927                         case 'arm':
1928                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1929                                 return
1930                             }
1931                             break
1932                         case 'x86':
1933                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1934                                 return
1935                             }
1936                             break
1937                         case 'x86lb':
1938                             if (os != 'Windows_NT') {
1939                                 return
1940                             }
1941                             break
1942                         case 'x64':
1943                             // Everything implemented
1944                             break
1945                         default:
1946                             println("Unknown architecture: ${architecture}")
1947                             assert false
1948                             break
1949                     }
1950
1951                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1952                     // with checked builds
1953                     def enableCorefxTesting = false
1954                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1955                         if (configuration != 'Checked') {
1956                             return
1957                         }
1958
1959                         enableCorefxTesting = isCorefxTesting(scenario)
1960
1961                         // Since these are just execution time differences,
1962                         // skip platforms that don't execute the tests here (Windows_NT only)
1963                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1964                         if (!isEnabledOS || isBuildOnly) {
1965                             return
1966                         }
1967
1968                         switch (architecture) {
1969                             case 'arm':
1970                                 if ((scenario != 'gcstress0x3') &&
1971                                     (scenario != 'gcstress0xc') &&
1972                                     (scenario != 'jitstress1') &&
1973                                     (scenario != 'jitstress2') &&
1974                                     (scenario != 'jitstressregs1') &&
1975                                     (scenario != 'jitstressregs2') &&
1976                                     (scenario != 'gcstress0xc_jitstress1') &&
1977                                     (scenario != 'gcstress0xc_jitstress2') &&
1978                                     (scenario != 'minopts') &&
1979                                     (scenario != 'tieredcompilation') &&
1980                                     (scenario != 'tailcallstress') &&
1981                                     (scenario != 'zapdisable')) {
1982                                         return
1983                                     }
1984                                     break
1985                             case 'arm64':
1986                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1987                                     return
1988                                 }
1989                                 break
1990                             case 'x64':
1991                             case 'x86':
1992                                 // x86 ubuntu: default only
1993                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1994                                     return
1995                                 }
1996                                 // Windows: Everything implemented
1997                                 break
1998                             case 'x86lb':
1999                                 // No stress modes for legacy jit.
2000                                 // (There's no technical reason we couldn't allow these.)
2001                                 return
2002                             default:
2003                                 return
2004                         }
2005                     }
2006                     else {
2007                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
2008                         // and configuration is not Checked, bail out.
2009                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2010                             return;
2011                         }
2012
2013                         // Skip scenarios
2014                         switch (scenario) {
2015                             case 'pri1':
2016                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
2017                                 // the default scenario build
2018                                 if (os != 'Windows_NT') {
2019                                     return
2020                                 }
2021                                 // Only x64 for now
2022                                 if (architecture != 'x64') {
2023                                     return
2024                                 }
2025                                 break
2026                             case 'ilrt':
2027                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2028                                 // the default scenario build
2029                                 if (os != 'Windows_NT') {
2030                                     return
2031                                 }
2032                                 // Only x64 for now
2033                                 if (architecture != 'x64') {
2034                                     return
2035                                 }
2036                                 // Release only
2037                                 if (configuration != 'Release') {
2038                                     return
2039                                 }
2040                                 break
2041                             case 'jitdiff':
2042                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2043                                     return
2044                                 }
2045                                 if (architecture != 'x64') {
2046                                     return
2047                                 }
2048                                 if (configuration != 'Checked') {
2049                                     return
2050                                 }
2051                                 break
2052                             case 'r2r':
2053                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2054                                 // the default scenario build
2055                                 if (os != 'Windows_NT') {
2056                                     return
2057                                 }
2058                                 if (architecture != 'x64') {
2059                                     return
2060                                 }
2061                                 break
2062                             case 'pri1r2r':
2063                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2064                                 // the default scenario build
2065                                 if (os != 'Windows_NT') {
2066                                     return
2067                                 }
2068                                 if (architecture != 'x64') {
2069                                     if ((architecture != 'arm64' && architecture != 'arm') || (configuration == 'Debug')) {
2070                                         return
2071                                     }
2072                                 }
2073                                 break
2074                             case 'gcstress15_pri1r2r':
2075                             case 'r2r_jitstress1':
2076                             case 'r2r_jitstress2':
2077                             case 'r2r_jitstressregs1':
2078                             case 'r2r_jitstressregs2':
2079                             case 'r2r_jitstressregs3':
2080                             case 'r2r_jitstressregs4':
2081                             case 'r2r_jitstressregs8':
2082                             case 'r2r_jitstressregs0x10':
2083                             case 'r2r_jitstressregs0x80':
2084                             case 'r2r_jitstressregs0x1000':
2085                             case 'r2r_jitminopts':
2086                             case 'r2r_jitforcerelocs':
2087                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
2088                                 // the default scenario build
2089                                 if (os != 'Windows_NT') {
2090                                     return
2091                                 }
2092                                 if (architecture != 'x64' && architecture != 'x86') {
2093                                     return
2094                                 }
2095                                 break
2096                             case 'longgc':
2097                             case 'gcsimulator':
2098                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2099                                     return
2100                                 }
2101                                 if (architecture != 'x64') {
2102                                     return
2103                                 }
2104                                 if (configuration != 'Release') {
2105                                     return
2106                                 }
2107                                 break
2108                             case 'gc_reliability_framework':
2109                             case 'standalone_gc':
2110                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2111                                     return
2112                                 }
2113
2114                                 if (architecture != 'x64') {
2115                                     return
2116                                 }
2117
2118                                 if (configuration != 'Release' && configuration != 'Checked') {
2119                                     return
2120                                 }
2121                                 break
2122                             // We need Windows x64 Release bits for the code coverage build
2123                             case 'coverage':
2124                                 if (os != 'Windows_NT') {
2125                                     return
2126                                 }
2127                                 if (architecture != 'x64') {
2128                                     return
2129                                 }
2130                                 if (configuration != 'Release') {
2131                                     return
2132                                 }
2133                                 break
2134                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2135                             case 'formatting':
2136                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2137                                     return
2138                                 }
2139                                 if (architecture != 'x64') {
2140                                     return
2141                                 }
2142                                 if (configuration != 'Checked') {
2143                                     return
2144                                 }
2145                                 if (isBuildOnly) {
2146                                     return
2147                                 }
2148                                 break
2149                             case 'illink':
2150                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2151                                     return
2152                                 }
2153                                 if (architecture != 'x64' && architecture != 'x86') {
2154                                     return
2155                                 }
2156                                 if (isBuildOnly) {
2157                                     return
2158                                 }
2159                                 break
2160                             case 'default':
2161                                 // Nothing skipped
2162                                 break
2163                             default:
2164                                 println("Unknown scenario: ${scenario}")
2165                                 assert false
2166                                 break
2167                         }
2168                     }
2169
2170                     // Calculate names
2171                     def lowerConfiguration = configuration.toLowerCase()
2172                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2173                     def folderName = getJobFolder(scenario)
2174
2175                     // Create the new job
2176                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2177
2178                     def machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2179                     machineAffinityOptions = architecture == 'arm' ? ['use_arm64_build_machine': false] : machineAffinityOptions
2180                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2181
2182                     // Add all the standard options
2183                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2184                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2185
2186                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
2187                     def osGroup = getOSGroup(os)
2188
2189                     newJob.with {
2190                         steps {
2191                             if (os == 'Windows_NT') {
2192                                 buildCommands.each { buildCommand ->
2193                                     batchFile(buildCommand)
2194                                 }
2195                             }
2196                             else {
2197                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2198                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2199                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2200                                     // Define the Windows Tests and Corefx build job names
2201                                     def WindowTestsName = projectFolder + '/' +
2202                                                           Utilities.getFullJobName(project,
2203                                                                                    getJobName(lowerConfiguration,
2204                                                                                               'x64' ,
2205                                                                                               'windows_nt',
2206                                                                                               'default',
2207                                                                                               true),
2208                                                                                    false)
2209                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2210                                                        Utilities.getFolderName(branch)
2211
2212                                     // Copy the Windows test binaries and the Corefx build binaries
2213                                     copyArtifacts(WindowTestsName) {
2214                                         includePatterns('bin/tests/tests.zip')
2215                                         buildSelector {
2216                                             latestSuccessful(true)
2217                                         }
2218                                     }
2219
2220                                     def arm_abi = 'arm'
2221                                     def corefx_os = 'linux'
2222                                     if (os == 'Tizen') {
2223                                         arm_abi = 'armel'
2224                                         corefx_os = 'tizen'
2225                                     }
2226
2227                                     // Let's use release CoreFX to test checked CoreCLR,
2228                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2229                                     def corefx_lowerConfiguration = lowerConfiguration
2230                                     if ( lowerConfiguration == 'checked' ) {
2231                                         corefx_lowerConfiguration='release'
2232                                     }
2233
2234                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2235                                         includePatterns('bin/build.tar.gz')
2236                                         buildSelector {
2237                                             latestSuccessful(true)
2238                                         }
2239                                     }
2240                                 }
2241
2242                                 buildCommands.each { buildCommand ->
2243                                     shell(buildCommand)
2244                                 }
2245                             }
2246                         }
2247                     } // newJob.with
2248
2249                 } // os
2250             } // configuration
2251         } // architecture
2252     } // isPR
2253 } // scenario
2254
2255
2256 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2257 combinedScenarios.each { scenario ->
2258     [true, false].each { isPR ->
2259         // Architectures.  x64 only at this point
2260         ['x64', 'arm64'].each { architecture ->
2261             // Put the OS's supported for coreclr cross testing here
2262             Constants.crossList.each { os ->
2263                 if (architecture == 'arm64') {
2264                     if (os != "Ubuntu") {
2265                         return
2266                     }
2267                 }
2268
2269                 Constants.configurationList.each { configuration ->
2270
2271                     if (architecture == 'arm64') {
2272                         if (scenario != 'default' && scenario != 'pri1r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2273                             return
2274                         }
2275                     }
2276
2277                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2278                         if (configuration != 'Checked') {
2279                             return
2280                         }
2281                         if (isCorefxTesting(scenario)) {
2282                             return
2283                         }
2284                         //Skip stress modes for these scenarios
2285                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2286                             return
2287                         }
2288                     }
2289                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2290                     // and configuration is not Checked, bail out.
2291                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2292                         return;
2293                     }
2294                     // For CentOS, we only want Checked/Release pri1 builds.
2295                     else if (os == 'CentOS7.1') {
2296                         if (scenario != 'pri1' &&
2297                             scenario != 'r2r' &&
2298                             scenario != 'pri1r2r' &&
2299                             scenario != 'gcstress15_pri1r2r' &&
2300                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2301                             return
2302                         }
2303                         if (configuration != 'Checked' && configuration != 'Release') {
2304                             return
2305                         }
2306                     }
2307                     // For RedHat and Debian, we only do Release pri1 builds.
2308                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2309                         if (scenario != 'pri1') {
2310                             return
2311                         }
2312                         if (configuration != 'Release') {
2313                             return
2314                         }
2315                     }
2316                     else {
2317                         // Skip scenarios
2318                         switch (scenario) {
2319                             case 'pri1':
2320                                 // Nothing skipped
2321                                 break
2322                             case 'ilrt':
2323                                 // Release only
2324                                 if (configuration != 'Release') {
2325                                     return
2326                                 }
2327                                 break
2328                             case 'jitdiff':
2329                                 if (configuration != 'Checked') {
2330                                     return;
2331                                 }
2332                             case 'r2r':
2333                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2334                                 if (configuration != 'Checked' && configuration != 'Release') {
2335                                     return
2336                                 }
2337                                 break
2338                             case 'pri1r2r':
2339                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2340                                 if (configuration != 'Checked' && configuration != 'Release') {
2341                                     return
2342                                 }
2343                                 break
2344                             case 'gcstress15_pri1r2r':
2345                             case 'r2r_jitstress1':
2346                             case 'r2r_jitstress2':
2347                             case 'r2r_jitstressregs1':
2348                             case 'r2r_jitstressregs2':
2349                             case 'r2r_jitstressregs3':
2350                             case 'r2r_jitstressregs4':
2351                             case 'r2r_jitstressregs8':
2352                             case 'r2r_jitstressregs0x10':
2353                             case 'r2r_jitstressregs0x80':
2354                             case 'r2r_jitstressregs0x1000':
2355                             case 'r2r_jitminopts':
2356                             case 'r2r_jitforcerelocs':
2357                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2358                                 if (configuration != 'Checked' && configuration != 'Release') {
2359                                     return
2360                                 }
2361                                 break
2362                             case 'longgc':
2363                             case 'gcsimulator':
2364                                 // Long GC tests take a long time on non-Release builds
2365                                 if (configuration != 'Release') {
2366                                     return
2367                                 }
2368                                 break
2369                             case 'gc_reliability_framework':
2370                             case 'standalone_gc':
2371                                 if (configuration != 'Release' && configuration != 'Checked') {
2372                                     return
2373                                 }
2374                                 break
2375                             case 'coverage':
2376                                 //We only want Ubuntu Release for coverage
2377                                 if (os != 'Ubuntu') {
2378                                     return
2379                                 }
2380                                 if (configuration != 'Release') {
2381                                     return
2382                                 }
2383                             case 'formatting':
2384                                 return
2385                             case 'illink':
2386                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2387                                     return
2388                                 }
2389                                 break
2390                             case 'default':
2391                                 // Nothing skipped
2392                                 break
2393                             default:
2394                                 println("Unknown scenario: ${scenario}")
2395                                 assert false
2396                                 break
2397                         }
2398                     }
2399
2400                     def lowerConfiguration = configuration.toLowerCase()
2401                     def osGroup = getOSGroup(os)
2402                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2403
2404                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2405                     def inputScenario = 'default'
2406                     if (scenario == 'coverage') {
2407                         inputScenario = 'coverage'
2408                     }
2409                     def inputCoreCLRBuildName = projectFolder + '/' +
2410                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2411                     // If this is a stress scenario, there isn't any difference in the build job
2412                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2413                     // from the default scenario
2414                     def testBuildScenario = scenario
2415                     if (scenarioNeedsPri1Build(scenario)) {
2416                         testBuildScenario = 'pri1'
2417                     }
2418                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2419                         testBuildScenario = 'default'
2420                     }
2421                     def inputWindowTestsBuildName = ''
2422                     def inputWindowsTestBuildArch = architecture
2423                     if (architecture == "arm64") {
2424                         // Use the x64 test build for arm64 unix
2425                         inputWindowsTestBuildArch = "x64"
2426                     }
2427                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2428                         inputWindowTestsBuildName = projectFolder + '/' +
2429                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', 'default', true), isPR)
2430                     }
2431                     else {
2432                         inputWindowTestsBuildName = projectFolder + '/' +
2433                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2434                     }
2435                     // Enable Server GC for Ubuntu PR builds
2436                     def serverGCString = ''
2437
2438                     // Whether or not this test run should be run sequentially instead
2439                     // of in parallel. Only used for long GC tests.
2440                     def sequentialString = ''
2441
2442                     // Whether or not this test run should run a specific playlist.
2443                     // Only used for long GC tests.
2444
2445                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2446                     // for running long GC and GCSimulator tests, respectively. We don't use them
2447                     // here because using a playlist file produces much more readable output on the CI machines
2448                     // and reduces running time.
2449                     def playlistString = ''
2450
2451                     if (os == 'Ubuntu' && isPR){
2452                         serverGCString = '--useServerGC'
2453                     }
2454
2455                     // pass --crossgen to runtest.sh for crossgen builds
2456                     def crossgenStr = ''
2457                     def runcrossgentestsStr = ''
2458                     def runjitstressStr = ''
2459                     def runjitstressregsStr = ''
2460                     def runjitmioptsStr = ''
2461                     def runjitforcerelocsStr = ''
2462                     def runjitdisasmStr = ''
2463                     def runilasmroundtripStr = ''
2464                     def gcstressStr = ''
2465                     def illinkStr = ''
2466                     def layoutOnlyStr =''
2467
2468                     if (scenario == 'r2r' ||
2469                         scenario == 'pri1r2r' ||
2470                         scenario == 'gcstress15_pri1r2r' ||
2471                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2472                             crossgenStr = '--crossgen'
2473                             runcrossgentestsStr = '--runcrossgentests'
2474
2475                             if (scenario == 'r2r_jitstress1'){
2476                                 runjitstressStr = '--jitstress=1'
2477                             }
2478                             else if (scenario == 'r2r_jitstress2') {
2479                                 runjitstressStr = '--jitstress=2'
2480                             }
2481                             else if (scenario == 'r2r_jitstressregs1'){
2482                                 runjitstressregsStr = '--jitstressregs=1'
2483                             }
2484                             else if (scenario == 'r2r_jitstressregs2') {
2485                                 runjitstressregsStr = '--jitstressregs=2'
2486                             }
2487                             else if (scenario == 'r2r_jitstressregs3') {
2488                                 runjitstressregsStr = '--jitstressregs=3'
2489                             }
2490                             else if (scenario == 'r2r_jitstressregs4') {
2491                                 runjitstressregsStr = '--jitstressregs=4'
2492                             }
2493                             else if (scenario == 'r2r_jitstressregs8') {
2494                                 runjitstressregsStr = '--jitstressregs=8'
2495                             }
2496                             else if (scenario == 'r2r_jitstressregs0x10') {
2497                                 runjitstressregsStr = '--jitstressregs=0x10'
2498                             }
2499                             else if (scenario == 'r2r_jitstressregs0x80') {
2500                                 runjitstressregsStr = '--jitstressregs=0x80'
2501                             }
2502                             else if (scenario == 'r2r_jitstressregs0x1000') {
2503                                 runjitstressregsStr = '--jitstressregs=0x1000'
2504                             }
2505                             else if (scenario == 'r2r_jitminopts') {
2506                                 runjitmioptsStr = '--jitminopts'
2507                             }
2508                             else if (scenario == 'r2r_jitforcerelocs') {
2509                                 runjitforcerelocsStr = '--jitforcerelocs'
2510                             }
2511                     }
2512                     if  (scenario == 'gcstress15_pri1r2r')
2513                     {
2514                         gcstressStr = '--gcstresslevel=0xF'
2515                     }
2516
2517                     if (scenario == 'jitdiff')
2518                     {
2519                         runjitdisasmStr = '--jitdisasm --crossgen'
2520                     }
2521
2522                     if (scenario == 'illink')
2523                     {
2524                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2525                     }
2526
2527                     if (isLongGc(scenario)) {
2528                         // Long GC tests behave very poorly when they are not
2529                         // the only test running (many of them allocate until OOM).
2530                         sequentialString = '--sequential'
2531
2532                         // The Long GC playlist contains all of the tests that are
2533                         // going to be run. The GCSimulator playlist contains all of
2534                         // the GC simulator tests.
2535                         if (scenario == 'longgc') {
2536                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2537                         }
2538                         else if (scenario == 'gcsimulator') {
2539                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2540                         }
2541                     }
2542
2543                     if (isGcReliabilityFramework(scenario)) {
2544                         layoutOnlyStr = '--build-overlay-only'
2545                     }
2546
2547                     def folder = getJobFolder(scenario)
2548                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2549                         // Add parameters for the inputs
2550
2551                         parameters {
2552                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2553                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2554                         }
2555
2556                         steps {
2557                             // Set up the copies
2558
2559                             // Coreclr build containing the tests and mscorlib
2560
2561                             copyArtifacts(inputWindowTestsBuildName) {
2562                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2563                                 buildSelector {
2564                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2565                                 }
2566                             }
2567
2568                             if (scenario == 'coverage') {
2569
2570                                 // Move coreclr to clr directory
2571                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2572
2573                                 // Build coreclr
2574                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2575
2576                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2577                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2578                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2579                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2580                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2581                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2582                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2583                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2584
2585                                 // Run PAL tests
2586                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2587
2588                                 // Remove obj files for PAL tests so they're not included in coverage results
2589                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2590
2591                                 // Unzip the tests first.  Exit with 0
2592                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2593
2594                                 // Get corefx
2595                                 shell("git clone https://github.com/dotnet/corefx fx")
2596
2597                                 // Build Linux corefx
2598                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2599                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2600
2601                                 def testEnvOpt = ""
2602                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2603                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2604                                 shell("${createScriptCmds}")
2605                                 testEnvOpt = "--test-env=" + scriptFileName
2606
2607                                 // Run corefx tests
2608                                 shell("""./fx/run-test.sh \\
2609                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2610                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2611                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2612                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2613                 --configurationGroup Release""")
2614
2615
2616                                 // Run coreclr tests w/ workstation GC
2617                                 shell("""./clr/tests/runtest.sh \\
2618                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2619                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2620                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2621                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2622                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2623                 --crossgen --runcrossgentests""")
2624
2625                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2626                                 shell("""./clr/tests/runtest.sh \\
2627                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2628                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2629                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2630                 --useServerGC ${testEnvOpt}""")
2631
2632                                  // Run long-running coreclr GC tests & produce coverage reports
2633                                 shell("""./clr/tests/runtest.sh \\
2634                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2635                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2636                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2637                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2638                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2639                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2640                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2641
2642                             }
2643                             else {
2644
2645                                 // Coreclr build we are trying to test
2646
2647                                 copyArtifacts(inputCoreCLRBuildName) {
2648                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2649                                     buildSelector {
2650                                         buildNumber('${CORECLR_BUILD}')
2651                                     }
2652                                 }
2653
2654                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2655
2656                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2657                                 def osJobName
2658                                 if (os == 'Ubuntu') {
2659                                     osJobName = 'ubuntu14.04'
2660                                 }
2661                                 else {
2662                                     osJobName = os.toLowerCase()
2663                                 }
2664                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2665                                     includePatterns('bin/build.tar.gz')
2666                                     buildSelector {
2667                                         latestSuccessful(true)
2668                                     }
2669                                 }
2670
2671                                 shell ("mkdir ./bin/CoreFxBinDir")
2672                                 // Unpack the corefx binaries
2673                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2674
2675                                 // HACK -- Arm64 does not have corefx jobs yet.
2676                                 // Clone corefx and build the native packages overwriting the x64 packages.
2677                                 if (architecture == 'arm64') {
2678                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2679                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2680                                 }
2681
2682                                 // Unzip the tests first.  Exit with 0
2683                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2684
2685                                 // Execute the tests
2686                                 // If we are running a stress mode, we'll set those variables first
2687                                 def testEnvOpt = ""
2688                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2689                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2690                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2691                                     shell("${createScriptCmds}")
2692                                     testEnvOpt = "--test-env=" + scriptFileName
2693                                 }
2694
2695                                 if (isGCStressRelatedTesting(scenario)) {
2696                                     shell('./init-tools.sh')
2697                                 }
2698
2699                                 shell("""./tests/runtest.sh \\
2700                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2701                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2702                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2703                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2704                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2705                 --limitedDumpGeneration \\
2706                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
2707                 ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
2708                 ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
2709
2710                                 if (isGcReliabilityFramework(scenario)) {
2711                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2712                                     if (serverGCString != '') {
2713                                         shell("export COMPlus_gcServer=1")
2714                                     }
2715
2716                                     shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2717                                 }
2718                             }
2719                         }
2720                     }
2721
2722                     if (scenario == 'coverage') {
2723                         // Publish coverage reports
2724                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2725                         // TODO: Add once external email sending is available again
2726                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2727                     }
2728
2729                     if (scenario == 'jitdiff') {
2730                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2731                     }
2732
2733                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2734                     if (os in ['Ubuntu']) {
2735                         SummaryBuilder summaries = new SummaryBuilder()
2736                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2737                         summaries.emit(newJob)
2738                     }
2739
2740                     setMachineAffinity(newJob, os, architecture)
2741                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2742                     // Set timeouts to 240.
2743                     setTestJobTimeOut(newJob, scenario)
2744
2745                     if (architecture == 'arm64') {
2746                         Utilities.setJobTimeout(newJob, 240)
2747                     }
2748
2749                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2750
2751                     // Create a build flow to join together the build and tests required to run this
2752                     // test.
2753                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2754                     // Linux CoreCLR test
2755                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2756                     def fullTestJobName = projectFolder + '/' + newJob.name
2757                     // Add a reference to the input jobs for report purposes
2758                     JobReport.Report.addReference(inputCoreCLRBuildName)
2759                     JobReport.Report.addReference(inputWindowTestsBuildName)
2760                     JobReport.Report.addReference(fullTestJobName)
2761                     def newFlowJob;
2762
2763                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2764                     // so that coverage data can be preserved.
2765                     if (scenario == 'coverage') {
2766                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2767                         buildFlow("""
2768 // Build the input Windows job
2769 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2770
2771 // And then build the test build
2772 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2773 """)
2774                         }
2775                     // Normal jobs copy a Windows build & a non-Windows build
2776                     } else {
2777                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2778                         buildFlow("""
2779 // Build the input jobs in parallel
2780 parallel (
2781     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2782     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2783 )
2784
2785 // And then build the test build
2786 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2787                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2788 """)
2789                         }
2790                     }
2791
2792                     setMachineAffinity(newFlowJob, os, architecture)
2793                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2794                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2795                 } // configuration
2796             } // os
2797         } // architecture
2798     } // isPR
2799 } // scenario
2800
2801 JobReport.Report.generateJobReport(out)
2802
2803 // Make the call to generate the help job
2804 Utilities.createHelperJob(this, project, branch,
2805     "Welcome to the ${project} Repository",  // This is prepended to the help message
2806     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2807
2808 Utilities.addCROSSCheck(this, project, branch)