netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'FreeBSD':'FreeBSD',
  29         'CentOS7.1': 'Linux',
  30         'OpenSUSE42.1': 'Linux',
  31         'Tizen': 'Linux']
  32     def osGroup = osGroupMap.get(os, null)
  33     assert osGroup != null : "Could not find os group for ${os}"
  34     return osGroupMap[os]
  35 }
  36
  37 // We use this class (vs variables) so that the static functions can access data here.
  38 class Constants {
  39
  40     // Innerloop build OS's
  41     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  42     // test execution in the build flow runs.  It generates the exact same build
  43     // as Windows_NT but without the tests.
  44     def static osList = [
  45                'Ubuntu',
  46                'Debian8.4',
  47                'OSX10.12',
  48                'Windows_NT',
  49                'Windows_NT_BuildOnly',
  50                'FreeBSD',
  51                'CentOS7.1',
  52                'OpenSUSE42.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  67                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  68                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  69                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  70                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  71                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  72                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  73                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  74                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  75                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  76                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  77                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  78                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  79                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  80                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  81                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  82                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  83                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  84                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  85                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  86                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  87                'corefx_baseline'                : [ : ], // corefx baseline
  88                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  89                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  90                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  91                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  92                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  93                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  94                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  95                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  96                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  97                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  98                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  99                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 100                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 101                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 103                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 104                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 105                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 107                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 109                ]
 110
 111     // This is a set of r2r jit stress scenarios
 112     def static r2rJitStressScenarios = [
 113                'r2r_jitstress1',
 114                'r2r_jitstress2',
 115                'r2r_jitstressregs1',
 116                'r2r_jitstressregs2',
 117                'r2r_jitstressregs3',
 118                'r2r_jitstressregs4',
 119                'r2r_jitstressregs8',
 120                'r2r_jitstressregs0x10',
 121                'r2r_jitstressregs0x80',
 122                'r2r_jitstressregs0x1000',
 123                'r2r_jitminopts',
 124                'r2r_jitforcerelocs']
 125
 126     // This is the basic set of scenarios
 127     def static basicScenarios = [
 128                'default',
 129                'pri1',
 130                'ilrt',
 131                'r2r',
 132                'pri1r2r',
 133                'gcstress15_pri1r2r',
 134                'longgc',
 135                'coverage',
 136                'formatting',
 137                'gcsimulator',
 138                'jitdiff',
 139                'standalone_gc',
 140                'gc_reliability_framework',
 141                'illink'] + r2rJitStressScenarios
 142
 143     def static configurationList = ['Debug', 'Checked', 'Release']
 144
 145     // This is the set of architectures
 146     def static architectureList = ['arm', 'arm64', 'x64', 'x86', 'x86lb']
 147 }
 148
 149 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 150     if (architecture == 'arm64' && os == 'Windows_NT') {
 151         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 152     } else if (architecture == 'arm64' && os != 'Windows_NT' && options == null) {
 153         Utilities.setMachineAffinity(job, os, 'arm64-small-page-size');
 154     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['large_pages'] == true) {
 155         Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size');
 156     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['is_build_only'] == true) {
 157         Utilities.setMachineAffinity(job, os, 'arm64-cross-latest');
 158     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 159         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 160     } else if ((architecture == 'arm') && (os == 'Windows_NT') && options['use_arm64_build_machine'] == true) {
 161         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 162     }else {
 163         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 164     }
 165 }
 166
 167 def static isJITStressJob(def scenario) {
 168     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 169            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 170 }
 171
 172 def static isGCStressRelatedTesting(def scenario) {
 173     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 174     // Detect it and make it a GCStress related.
 175     if (scenario == 'gcstress15_pri1r2r')
 176     {
 177         return true;
 178     }
 179
 180     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 181     def scenarioName = scenario.toLowerCase()
 182     def isGCStressTesting = false
 183     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 184         if (k in gcStressTestEnvVars) {
 185             isGCStressTesting = true;
 186         }
 187     }
 188     return isGCStressTesting
 189 }
 190
 191 def static isCorefxTesting(def scenario) {
 192     def corefx_prefix = 'corefx_'
 193     if (scenario.length() < corefx_prefix.length()) {
 194         return false
 195     }
 196     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 197 }
 198
 199 def static isR2R(def scenario) {
 200     return (scenario == 'r2r' || scenario == 'pri1r2r')
 201 }
 202
 203 def static isCoverage(def scenario) {
 204     return (scenario == 'coverage')
 205 }
 206
 207 def static isLongGc(def scenario) {
 208     return (scenario == 'longgc' || scenario == 'gcsimulator')
 209 }
 210
 211 def static isJitDiff(def scenario) {
 212     return (scenario == 'jitdiff')
 213 }
 214
 215 def static isGcReliabilityFramework(def scenario) {
 216     return (scenario == 'gc_reliability_framework')
 217 }
 218
 219 def static scenarioNeedsPri1Build(def scenario) {
 220     return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario))
 221 }
 222
 223 def static setTestJobTimeOut(newJob, scenario) {
 224     if (isGCStressRelatedTesting(scenario)) {
 225         Utilities.setJobTimeout(newJob, 4320)
 226     }
 227     else if (isCorefxTesting(scenario)) {
 228         Utilities.setJobTimeout(newJob, 360)
 229     }
 230     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 231         Utilities.setJobTimeout(newJob, 240)
 232     }
 233     else if (isR2R(scenario)) {
 234         Utilities.setJobTimeout(newJob, 240)
 235     }
 236     else if (isCoverage(scenario)) {
 237         Utilities.setJobTimeout(newJob, 1440)
 238     }
 239     else if (isLongGc(scenario)) {
 240         Utilities.setJobTimeout(newJob, 1440)
 241     }
 242     else if (isJitDiff(scenario)) {
 243         Utilities.setJobTimeout(newJob, 240)
 244     }
 245     else if (isGcReliabilityFramework(scenario)) {
 246         Utilities.setJobTimeout(newJob, 1440)
 247     }
 248     // Non-test jobs use the default timeout value.
 249 }
 250
 251 def static getJobFolder(def scenario) {
 252     if (isJITStressJob(scenario)) {
 253         return 'jitstress'
 254     }
 255     if (scenario == 'illink') {
 256         return 'illink'
 257     }
 258     return ''
 259 }
 260
 261 def static getStressModeDisplayName(def scenario) {
 262     def displayStr = ''
 263     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 264         def prefixLength = 'COMPlus_'.length()
 265         if (k.length() >= prefixLength) {
 266             def modeName = k.substring(prefixLength, k.length())
 267             displayStr += ' ' + modeName + '=' + v
 268         }
 269     }
 270     return displayStr
 271 }
 272
 273 def static getR2RStressModeDisplayName(def scenario) {
 274     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 275     // "r2r_" prefix.
 276     def displayStr = scenario
 277     def prefixLength = 'r2r_'.length()
 278     if (displayStr.length() >= prefixLength) {
 279         displayStr = displayStr.substring(prefixLength, displayStr.length())
 280     }
 281     return displayStr
 282 }
 283
 284 // Generates the string for creating a file that sets environment variables
 285 // that makes it possible to run stress modes.  Writes the script to the file
 286 // specified by the stepScriptLocation parameter.
 287 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 288     def stepScript = ''
 289     if (os == 'Windows_NT') {
 290         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 291         stepScript += "del ${stepScriptLocation}\r\n"
 292
 293         // Timeout in ms, default is 10 minutes. For stress
 294         // modes up this to 30 minutes
 295         def timeout = 1800000
 296
 297         // Set the Timeout
 298         stepScript += "set __TestTimeout=${timeout}\r\n"
 299         stepScript += "echo. > ${stepScriptLocation}\r\n"
 300         stressModeVars.each{ k, v ->
 301             // Write out what we are writing to the script file
 302             stepScript += "echo Setting ${k}=${v}\r\n"
 303             // Write out the set itself to the script file`
 304             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 305         }
 306     }
 307     else {
 308         stepScript += "echo Setting variables for ${stressModeName}\n"
 309         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 310         stressModeVars.each{ k, v ->
 311             // Write out what we are writing to the script file
 312             stepScript += "echo Setting ${k}=${v}\n"
 313             // Write out the set itself to the script file`
 314             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 315         }
 316         stepScript += "chmod +x ${stepScriptLocation}\n"
 317     }
 318     return stepScript
 319 }
 320
 321 // Calculates the name of the build job based on some typical parameters.
 322 //
 323 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 324     // If the architecture is x64, do not add that info into the build name.
 325     // Need to change around some systems and other builds to pick up the right builds
 326     // to do that.
 327
 328     def suffix = scenario != 'default' ? "_${scenario}" : '';
 329     if (isBuildOnly) {
 330         suffix += '_bld'
 331     }
 332     def baseName = ''
 333     switch (architecture) {
 334         case 'x64':
 335             if (scenario == 'default') {
 336                 // For now we leave x64 off of the name for compatibility with other jobs
 337                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 338             }
 339             else if (scenario == 'formatting') {
 340                 // we don't care about the configuration for the formatting job. It runs all configs
 341                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 342             }
 343             else {
 344                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 345             }
 346             break
 347         case 'arm64':
 348             if (os.toLowerCase() == "windows_nt") {
 349                 // These are cross builds
 350                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 351             }
 352             else {
 353                 // Defaults to a small page size set of machines.
 354                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 355             }
 356             break
 357         case 'arm':
 358             // These are cross builds
 359             if (os == 'Tizen') {
 360                 // ABI: softfp
 361                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 362             }
 363             else {
 364                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 365             }
 366             break
 367         case 'x86':
 368         case 'x86lb':
 369             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 370             break
 371         default:
 372             println("Unknown architecture: ${architecture}");
 373             assert false
 374             break
 375     }
 376
 377     return baseName + suffix
 378 }
 379
 380 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 381     // Check scenario.
 382     switch (scenario) {
 383         case 'default':
 384             switch (architecture) {
 385                 case 'x64':
 386                 case 'x86':
 387                 case 'x86lb':
 388                     if (architecture == 'x86' && os == 'Ubuntu') {
 389                         Utilities.addPeriodicTrigger(job, '@daily')
 390                     }
 391                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 392                         Utilities.addGithubPushTrigger(job)
 393                     }
 394                     break
 395                 case 'arm':
 396                     Utilities.addGithubPushTrigger(job)
 397                     break
 398                 case 'arm64':
 399                     Utilities.addGithubPushTrigger(job)
 400                     break
 401                 default:
 402                     println("Unknown architecture: ${architecture}");
 403                     assert false
 404                     break
 405             }
 406             break
 407         case 'pri1':
 408             // Pri one gets a push trigger, and only for release
 409             if (architecture == 'x64') {
 410                 if (configuration == 'Release') {
 411                     // We expect release jobs to be Windows, or in the cross list
 412                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 413                     if (!os in bidailyCrossList) {
 414                         if (isFlowJob || os == 'Windows_NT') {
 415                             Utilities.addGithubPushTrigger(job)
 416                         }
 417                     }
 418                     else {
 419                         if (isFlowJob) {
 420                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 421                         }
 422                     }
 423                 }
 424             }
 425             break
 426         case 'r2r':
 427             //r2r jobs that aren't pri1 can only be triggered by phrase
 428             break
 429         case 'pri1r2r':
 430             assert !(os in bidailyCrossList)
 431             //pri1 r2r gets a push trigger for checked/release
 432             if (configuration == 'Checked' || configuration == 'Release') {
 433                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 434                 if (architecture == 'x64' && os != 'OSX10.12') {
 435                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 436                     if (isFlowJob || os == 'Windows_NT') {
 437                         Utilities.addGithubPushTrigger(job)
 438                     }
 439                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 440                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 441                     if (isFlowJob) {
 442                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 443                     }
 444                 }
 445                 // For x86, only add per-commit jobs for Windows
 446                 else if (architecture == 'x86' || architecture == 'x86lb') {
 447                     if (os == 'Windows_NT') {
 448                         Utilities.addGithubPushTrigger(job)
 449                     }
 450                 }
 451                 // arm64 pri1r2r jobs should only run every 12 hours.
 452                 else if (architecture == 'arm64') {
 453                     if (os == 'Windows_NT') {
 454                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 455                         // TODO: Add once external email sending is available again
 456                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 457                     }
 458                 }
 459             }
 460             break
 461         case 'r2r_jitstress1':
 462         case 'r2r_jitstress2':
 463         case 'r2r_jitstressregs1':
 464         case 'r2r_jitstressregs2':
 465         case 'r2r_jitstressregs3':
 466         case 'r2r_jitstressregs4':
 467         case 'r2r_jitstressregs8':
 468         case 'r2r_jitstressregs0x10':
 469         case 'r2r_jitstressregs0x80':
 470         case 'r2r_jitstressregs0x1000':
 471         case 'r2r_jitminopts':
 472         case 'r2r_jitforcerelocs':
 473         case 'gcstress15_pri1r2r':
 474             assert !(os in bidailyCrossList)
 475
 476             // GCStress=C is currently not supported on OS X
 477             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 478                 break
 479             }
 480
 481             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 482             if (configuration == 'Checked' || configuration == 'Release') {
 483                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 484                 if (architecture == 'x64') {
 485                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 486                     if (isFlowJob || os == 'Windows_NT') {
 487                         // Add a weekly periodic trigger
 488                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 489                     }
 490                 }
 491                 // For x86, only add per-commit jobs for Windows
 492                 else if (architecture == 'x86') {
 493                     if (os == 'Windows_NT') {
 494                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 495                     }
 496                 }
 497             }
 498             break
 499         case 'longgc':
 500             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 501             assert configuration == 'Release'
 502             assert architecture == 'x64'
 503             Utilities.addPeriodicTrigger(job, '@daily')
 504             // TODO: Add once external email sending is available again
 505             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 506             break
 507         case 'gcsimulator':
 508             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 509             assert configuration == 'Release'
 510             assert architecture == 'x64'
 511             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 512             // TODO: Add once external email sending is available again
 513             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 514             break
 515         case 'standalone_gc':
 516             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 517             assert (configuration == 'Release' || configuration == 'Checked')
 518             // TODO: Add once external email sending is available again
 519             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 520             Utilities.addPeriodicTrigger(job, '@weekly')
 521             break
 522         case 'gc_reliability_framework':
 523             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 524             assert (configuration == 'Release' || configuration == 'Checked')
 525             // Only triggered by phrase.
 526             break
 527         case 'ilrt':
 528             assert !(os in bidailyCrossList)
 529             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 530             if (architecture == 'x64' && configuration == 'Release') {
 531                 // We don't expect to see a job generated except in these scenarios
 532                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 533                 if (isFlowJob || os == 'Windows_NT') {
 534                     Utilities.addPeriodicTrigger(job, '@daily')
 535                 }
 536             }
 537             break
 538         case 'jitdiff':
 539             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 540             assert configuration == 'Checked'
 541             assert (architecture == 'x64' || architecture == 'x86')
 542             Utilities.addGithubPushTrigger(job)
 543             break
 544         case 'coverage':
 545             assert (os == 'Ubuntu' || os == 'Windows_NT')
 546             assert configuration == 'Release'
 547             assert architecture == 'x64'
 548             Utilities.addPeriodicTrigger(job, '@weekly')
 549             break
 550         case 'formatting':
 551             assert (os == 'Windows_NT' || os == "Ubuntu")
 552             assert architecture == 'x64'
 553             Utilities.addGithubPushTrigger(job)
 554             break
 555         case 'jitstressregs1':
 556         case 'jitstressregs2':
 557         case 'jitstressregs3':
 558         case 'jitstressregs4':
 559         case 'jitstressregs8':
 560         case 'jitstressregs0x10':
 561         case 'jitstressregs0x80':
 562         case 'jitstressregs0x1000':
 563         case 'minopts':
 564         case 'forcerelocs':
 565         case 'jitstress1':
 566         case 'jitstress2':
 567         case 'jitstress2_jitstressregs1':
 568         case 'jitstress2_jitstressregs2':
 569         case 'jitstress2_jitstressregs3':
 570         case 'jitstress2_jitstressregs4':
 571         case 'jitstress2_jitstressregs8':
 572         case 'jitstress2_jitstressregs0x10':
 573         case 'jitstress2_jitstressregs0x80':
 574         case 'jitstress2_jitstressregs0x1000':
 575         case 'tailcallstress':
 576         case 'jitsse2only':
 577         case 'corefx_baseline':
 578         case 'corefx_minopts':
 579         case 'corefx_jitstress1':
 580         case 'corefx_jitstress2':
 581         case 'corefx_jitstressregs1':
 582         case 'corefx_jitstressregs2':
 583         case 'corefx_jitstressregs3':
 584         case 'corefx_jitstressregs4':
 585         case 'corefx_jitstressregs8':
 586         case 'corefx_jitstressregs0x10':
 587         case 'corefx_jitstressregs0x80':
 588         case 'corefx_jitstressregs0x1000':
 589         case 'zapdisable':
 590             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 591             assert (os == 'Windows_NT') || (os in Constants.crossList)
 592             Utilities.addPeriodicTrigger(job, '@daily')
 593         }
 594         break
 595         case 'heapverify1':
 596         case 'gcstress0x3':
 597             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 598                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 599                 if (architecture == 'arm64') {
 600                     if (os == 'Windows_NT') {
 601                         Utilities.addPeriodicTrigger(job, '@daily')
 602                     }
 603                     // TODO: Add once external email sending is available again
 604                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 605                 }
 606                 else {
 607                     Utilities.addPeriodicTrigger(job, '@weekly')
 608                 }
 609             }
 610             break
 611         case 'gcstress0xc':
 612         case 'gcstress0xc_zapdisable':
 613         case 'gcstress0xc_zapdisable_jitstress2':
 614         case 'gcstress0xc_zapdisable_heapverify1':
 615         case 'gcstress0xc_jitstress1':
 616         case 'gcstress0xc_jitstress2':
 617         case 'gcstress0xc_minopts_heapverify1':
 618             // GCStress=C is currently not supported on OS X
 619             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 620                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 621                 if (architecture == 'arm64') {
 622                     // TODO: Enable a periodic trigger after tests are updated.
 623                     // Utilities.addPeriodicTrigger(job, '@daily')
 624                     // TODO: Add once external email sending is available again
 625                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 626                 }
 627                 else {
 628                     Utilities.addPeriodicTrigger(job, '@weekly')
 629                 }
 630             }
 631             break
 632
 633         case 'illink':
 634             // Testing on other operating systems TBD
 635             assert (os == 'Windows_NT' || os == 'Ubuntu')
 636             if (architecture == 'x64' || architecture == 'x86') {
 637                 if (configuration == 'Checked') {
 638                     Utilities.addPeriodicTrigger(job, '@daily')
 639                 }
 640             }
 641             break
 642
 643         default:
 644             println("Unknown scenario: ${scenario}");
 645             assert false
 646             break
 647     }
 648     return
 649 }
 650
 651 // **************************
 652 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 653 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 654 // builds will do a couple extra steps.
 655 // **************************
 656
 657 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 658 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 659 // it's just used for internal builds
 660 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 661 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 662     if (isWindowsBuildOnlyJob) {
 663         return
 664     }
 665
 666     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 667     // Non pull request builds.
 668     if (!isPR) {
 669         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 670         return
 671     }
 672
 673      def arm64Users = [
 674         'adiaaida',
 675         'AndyAyersMS',
 676         'briansull',
 677         'BruceForstall',
 678         'CarolEidt',
 679         'cmckinsey',
 680         'erozenfeld',
 681         'jashook',
 682         'JosephTremoulet',
 683         'pgavlin',
 684         'russellhadley',
 685         'RussKeldorph',
 686         'sandreenko',
 687         'sdmaclea',
 688         'sivarv',
 689         'swaroop-sridhar',
 690         'gkhanna79',
 691         'jkotas',
 692         'markwilkie',
 693         'rahku',
 694         'ramarag',
 695         'tzwlai',
 696         'weshaggard'
 697     ]
 698
 699     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 700     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 701     def osGroup = getOSGroup(os)
 702     switch (architecture) {
 703         case 'x64': // editor brace matching: {
 704             if (scenario == 'coverage') {
 705                 assert configuration == 'Release'
 706                 if (os == 'Ubuntu') {
 707                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 708                 }
 709                 break
 710             }
 711
 712             if (scenario == 'formatting') {
 713                 assert configuration == 'Checked'
 714                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 715                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 716                 }
 717                 break
 718             }
 719
 720             switch (os) {
 721                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 722                 case 'Debian8.4':
 723                 case 'RHEL7.2':
 724                     if (scenario == 'default') {
 725                         assert !isFlowJob
 726                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 727                     }
 728                     else if (scenario == 'pri1' && isFlowJob) {
 729                         assert (configuration == 'Release')
 730                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 731                     }
 732                     break
 733                 case 'Ubuntu16.04':
 734                     assert !isFlowJob
 735                     assert scenario == 'default'
 736                     // Distinguish with the other architectures (arm and x86)
 737                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 738                     break
 739                 case 'Fedora24':
 740                 case 'Ubuntu16.10':
 741                 case 'OpenSUSE42.1':
 742                     assert !isFlowJob
 743                     assert scenario == 'default'
 744                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 745                     break
 746                 case 'Ubuntu':
 747                     if (scenario == 'illink') {
 748                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 749                         break
 750                     }
 751                     // fall through
 752                 case 'OSX10.12':
 753                     // Triggers on the non-flow jobs aren't necessary here
 754                     // Corefx testing uses non-flow jobs.
 755                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 756                         break
 757                     }
 758                     switch (scenario) {
 759                         case 'default':
 760                             // Ubuntu uses checked for default PR tests
 761                             if (configuration == 'Checked') {
 762                                 // Default trigger
 763                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 764                             }
 765                             break
 766                         case 'pri1':
 767                             if (configuration == 'Release') {
 768                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 769                             }
 770                             break
 771                         case 'jitdiff':
 772                             if (configuration == 'Checked') {
 773                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 774                             }
 775                             break
 776                         case 'ilrt':
 777                             if (configuration == 'Release') {
 778                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 779                             }
 780                             break
 781                         case 'r2r':
 782                             if (configuration == 'Release' || configuration == 'Checked') {
 783                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 784                             }
 785                             break
 786                         case 'pri1r2r':
 787                             if (configuration == 'Release' || configuration == 'Checked') {
 788                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 789                             }
 790                             break
 791                         case 'gcstress15_pri1r2r':
 792                             if (configuration == 'Release' || configuration == 'Checked') {
 793                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 794                             }
 795                             break
 796                         case 'r2r_jitstress1':
 797                         case 'r2r_jitstress2':
 798                         case 'r2r_jitstressregs1':
 799                         case 'r2r_jitstressregs2':
 800                         case 'r2r_jitstressregs3':
 801                         case 'r2r_jitstressregs4':
 802                         case 'r2r_jitstressregs8':
 803                         case 'r2r_jitstressregs0x10':
 804                         case 'r2r_jitstressregs0x80':
 805                         case 'r2r_jitstressregs0x1000':
 806                         case 'r2r_jitminopts':
 807                         case 'r2r_jitforcerelocs':
 808                             if (configuration == 'Release' || configuration == 'Checked') {
 809                                 def displayStr = getR2RStressModeDisplayName(scenario)
 810                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 811                             }
 812                             break
 813                         case 'longgc':
 814                             if (configuration == 'Release') {
 815                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 816                             }
 817                             break
 818                         case 'gcsimulator':
 819                             if (configuration == 'Release') {
 820                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 821                             }
 822                             break
 823                         case 'standalone_gc':
 824                             if (configuration == 'Release' || configuration == 'Checked') {
 825                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 826                             }
 827                             break
 828                         case 'gc_reliability_framework':
 829                             if (configuration == 'Release' || configuration == 'Checked') {
 830                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 831                             }
 832                             break
 833                         case 'minopts':
 834                         case 'forcerelocs':
 835                         case 'jitstress1':
 836                         case 'jitstress2':
 837                         case 'jitstressregs1':
 838                         case 'jitstressregs2':
 839                         case 'jitstressregs3':
 840                         case 'jitstressregs4':
 841                         case 'jitstressregs8':
 842                         case 'jitstressregs0x10':
 843                         case 'jitstressregs0x80':
 844                         case 'jitstressregs0x1000':
 845                         case 'jitstress2_jitstressregs1':
 846                         case 'jitstress2_jitstressregs2':
 847                         case 'jitstress2_jitstressregs3':
 848                         case 'jitstress2_jitstressregs4':
 849                         case 'jitstress2_jitstressregs8':
 850                         case 'jitstress2_jitstressregs0x10':
 851                         case 'jitstress2_jitstressregs0x80':
 852                         case 'jitstress2_jitstressregs0x1000':
 853                         case 'tailcallstress':
 854                         case 'jitsse2only':
 855                         case 'gcstress0x3':
 856                         case 'gcstress0xc':
 857                         case 'zapdisable':
 858                         case 'heapverify1':
 859                         case 'gcstress0xc_zapdisable':
 860                         case 'gcstress0xc_zapdisable_jitstress2':
 861                         case 'gcstress0xc_zapdisable_heapverify1':
 862                         case 'gcstress0xc_jitstress1':
 863                         case 'gcstress0xc_jitstress2':
 864                         case 'gcstress0xc_minopts_heapverify1':
 865                             def displayStr = getStressModeDisplayName(scenario)
 866                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 867                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 868                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 869                             break
 870                         case 'corefx_baseline':
 871                         case 'corefx_minopts':
 872                         case 'corefx_jitstress1':
 873                         case 'corefx_jitstress2':
 874                         case 'corefx_jitstressregs1':
 875                         case 'corefx_jitstressregs2':
 876                         case 'corefx_jitstressregs3':
 877                         case 'corefx_jitstressregs4':
 878                         case 'corefx_jitstressregs8':
 879                         case 'corefx_jitstressregs0x10':
 880                         case 'corefx_jitstressregs0x80':
 881                         case 'corefx_jitstressregs0x1000':
 882                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 883                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 884                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 885                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 886                             break
 887                         default:
 888                             println("Unknown scenario: ${scenario}");
 889                             assert false
 890                             break
 891                     }
 892                     break
 893                 case 'CentOS7.1':
 894                     switch (scenario) {
 895                         case 'pri1':
 896                             if (configuration == 'Release') {
 897                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 898                             }
 899                             break
 900                         case 'r2r':
 901                             if (configuration == 'Checked' || configuration == 'Release') {
 902                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 903                             }
 904                             break
 905                         case 'pri1r2r':
 906                             if (configuration == 'Checked' || configuration == 'Release') {
 907                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 908                             }
 909                             break
 910                         case 'gcstress15_pri1r2r':
 911                             if (configuration == 'Release' || configuration == 'Checked') {
 912                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 913                             }
 914                             break
 915                         case 'r2r_jitstress1':
 916                         case 'r2r_jitstress2':
 917                         case 'r2r_jitstressregs1':
 918                         case 'r2r_jitstressregs2':
 919                         case 'r2r_jitstressregs3':
 920                         case 'r2r_jitstressregs4':
 921                         case 'r2r_jitstressregs8':
 922                         case 'r2r_jitstressregs0x10':
 923                         case 'r2r_jitstressregs0x80':
 924                         case 'r2r_jitstressregs0x1000':
 925                         case 'r2r_jitminopts':
 926                         case 'r2r_jitforcerelocs':
 927                             if (configuration == 'Release' || configuration == 'Checked') {
 928                                 def displayStr = getR2RStressModeDisplayName(scenario)
 929                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 930                             }
 931                             break
 932                         default:
 933                             break
 934                     }
 935                 case 'Windows_NT':
 936                     switch (scenario) {
 937                         case 'default':
 938                             // Default trigger
 939                             if (configuration == 'Debug') {
 940                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 941                             }
 942                             break
 943                         case 'pri1':
 944                             // Default trigger
 945                             if (configuration == 'Release') {
 946                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 947                             }
 948                             break
 949                         case 'jitdiff':
 950                             if (configuration == 'Checked') {
 951                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 952                             }
 953                             break
 954                         case 'ilrt':
 955                             if (configuration == 'Release') {
 956                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 957                             }
 958                             break
 959                         case 'r2r':
 960                             if (configuration == 'Checked' || configuration == 'Release') {
 961                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 962                             }
 963                             break
 964                         case 'pri1r2r':
 965                             if (configuration == 'Checked' || configuration == 'Release') {
 966                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 967                             }
 968                             break
 969                         case 'gcstress15_pri1r2r':
 970                             if (configuration == 'Release' || configuration == 'Checked') {
 971                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 972                             }
 973                             break
 974                         case 'r2r_jitstress1':
 975                         case 'r2r_jitstress2':
 976                         case 'r2r_jitstressregs1':
 977                         case 'r2r_jitstressregs2':
 978                         case 'r2r_jitstressregs3':
 979                         case 'r2r_jitstressregs4':
 980                         case 'r2r_jitstressregs8':
 981                         case 'r2r_jitstressregs0x10':
 982                         case 'r2r_jitstressregs0x80':
 983                         case 'r2r_jitstressregs0x1000':
 984                         case 'r2r_jitminopts':
 985                         case 'r2r_jitforcerelocs':
 986                             if (configuration == 'Release' || configuration == 'Checked') {
 987                                 def displayStr = getR2RStressModeDisplayName(scenario)
 988                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 989                             }
 990                             break
 991                         case 'longgc':
 992                             if (configuration == 'Release') {
 993                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 994                             }
 995                             break
 996                         case 'gcsimulator':
 997                             if (configuration == 'Release') {
 998                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 999                             }
1000                             break
1001                         case 'standalone_gc':
1002                             if (configuration == 'Release' || configuration == 'Checked') {
1003                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1004                             }
1005                             break
1006                         case 'gc_reliability_framework':
1007                             if (configuration == 'Release' || configuration == 'Checked') {
1008                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1009                             }
1010                             break
1011                         case 'minopts':
1012                         case 'forcerelocs':
1013                         case 'jitstress1':
1014                         case 'jitstress2':
1015                         case 'jitstressregs1':
1016                         case 'jitstressregs2':
1017                         case 'jitstressregs3':
1018                         case 'jitstressregs4':
1019                         case 'jitstressregs8':
1020                         case 'jitstressregs0x10':
1021                         case 'jitstressregs0x80':
1022                         case 'jitstressregs0x1000':
1023                         case 'jitstress2_jitstressregs1':
1024                         case 'jitstress2_jitstressregs2':
1025                         case 'jitstress2_jitstressregs3':
1026                         case 'jitstress2_jitstressregs4':
1027                         case 'jitstress2_jitstressregs8':
1028                         case 'jitstress2_jitstressregs0x10':
1029                         case 'jitstress2_jitstressregs0x80':
1030                         case 'jitstress2_jitstressregs0x1000':
1031                         case 'tailcallstress':
1032                         case 'jitsse2only':
1033                         case 'gcstress0x3':
1034                         case 'gcstress0xc':
1035                         case 'zapdisable':
1036                         case 'heapverify1':
1037                         case 'gcstress0xc_zapdisable':
1038                         case 'gcstress0xc_zapdisable_jitstress2':
1039                         case 'gcstress0xc_zapdisable_heapverify1':
1040                         case 'gcstress0xc_jitstress1':
1041                         case 'gcstress0xc_jitstress2':
1042                         case 'gcstress0xc_minopts_heapverify1':
1043                             def displayStr = getStressModeDisplayName(scenario)
1044                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1045                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1046                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1047                             break
1048                         case 'corefx_baseline':
1049                         case 'corefx_minopts':
1050                         case 'corefx_jitstress1':
1051                         case 'corefx_jitstress2':
1052                         case 'corefx_jitstressregs1':
1053                         case 'corefx_jitstressregs2':
1054                         case 'corefx_jitstressregs3':
1055                         case 'corefx_jitstressregs4':
1056                         case 'corefx_jitstressregs8':
1057                         case 'corefx_jitstressregs0x10':
1058                         case 'corefx_jitstressregs0x80':
1059                         case 'corefx_jitstressregs0x1000':
1060                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1061                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1062                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1063                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1064                             break
1065                         case 'illink':
1066                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1067                                     break
1068                         default:
1069                             println("Unknown scenario: ${scenario}");
1070                             assert false
1071                             break
1072                     }
1073                     break
1074                 case 'FreeBSD':
1075                     assert scenario == 'default'
1076                     if (configuration == 'Checked') {
1077                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1078                     }
1079                     break
1080                 default:
1081                     println("Unknown os: ${os}");
1082                     assert false
1083                     break
1084             }
1085             break
1086         // editor brace matching: }
1087         case 'arm': // editor brace matching: {
1088             switch (os) {
1089                 case 'Ubuntu':
1090                 case 'Ubuntu16.04':
1091                     assert scenario == 'default'
1092                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1093                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1094                     }
1095                     else {
1096                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1097                     }
1098                     break;
1099                 case 'Tizen':
1100                     architecture='armel'
1101                     // Removing the regex will cause this to run on each PR.
1102                     if (configuration == 'Release' || configuration == 'Debug') {
1103                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1104                     }
1105                     else {
1106                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1107                     }
1108                     break;
1109                 case 'Windows_NT':
1110                     // Set up a private trigger
1111                     def contextString = "${os} ${architecture} Cross ${configuration}"
1112                     if (scenario != 'default')
1113                         contextString += " ${scenario}"
1114                     contextString += " Build"
1115                     // Debug builds only.
1116                     if (configuration != 'Debug') {
1117                         contextString += " and Test"
1118                     }
1119                     switch (scenario) {
1120                         case 'default':
1121                             // For now only run Debug and Release build jobs on PR Trigger. Note this is not a private trigger.
1122                             if (configuration == 'Debug' || configuration == 'Release')
1123                             {
1124                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1125                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1126                             }
1127                             else
1128                             {
1129                                 // Checked jobs will run on private trigger and run tests.
1130                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1131                             }
1132                             break
1133                         case 'pri1r2r':
1134                         case 'gcstress0x3':
1135                         case 'gcstress0xc':
1136                             // Stress jobs will will run on private trigger and run tests.
1137                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1138                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1139                             break
1140                     }
1141                     break
1142                 default:
1143                     println("NYI os: ${os}");
1144                     assert false
1145                     break
1146             }
1147             break
1148         // editor brace matching: }
1149         case 'arm64': // editor brace matching: {
1150             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1151
1152             // Set up a private trigger
1153             def contextString = "${os} ${architecture} Cross ${configuration}"
1154             if (scenario != 'default')
1155                 contextString += " ${scenario}"
1156             contextString += " Build"
1157             // Debug builds only.
1158             if (configuration != 'Debug') {
1159                contextString += " and Test"
1160             }
1161
1162             switch (os) {
1163                 case 'Ubuntu':
1164                 case 'Ubuntu16.04':
1165                     switch (scenario) {
1166                         case 'pri1':
1167                             if (configuration == 'Release') {
1168                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1169                             }
1170                             break
1171                         case 'r2r':
1172                             if (configuration == 'Checked' || configuration == 'Release') {
1173                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1174                             }
1175                             break
1176                         case 'pri1r2r':
1177                             if (configuration == 'Checked' || configuration == 'Release') {
1178                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1179                             }
1180                             break
1181                         case 'gcstress15_pri1r2r':
1182                             if (configuration == 'Release' || configuration == 'Checked') {
1183                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1184                             }
1185                             break
1186                         case 'r2r_jitstress1':
1187                         case 'r2r_jitstress2':
1188                         case 'r2r_jitstressregs1':
1189                         case 'r2r_jitstressregs2':
1190                         case 'r2r_jitstressregs3':
1191                         case 'r2r_jitstressregs4':
1192                         case 'r2r_jitstressregs8':
1193                         case 'r2r_jitstressregs0x10':
1194                         case 'r2r_jitstressregs0x80':
1195                         case 'r2r_jitstressregs0x1000':
1196                         case 'r2r_jitminopts':
1197                         case 'r2r_jitforcerelocs':
1198                             if (configuration == 'Release' || configuration == 'Checked') {
1199                                 def displayStr = getR2RStressModeDisplayName(scenario)
1200                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1201                             }
1202                             break
1203                         default:
1204                             break
1205                     }
1206                 case 'Windows_NT':
1207                     switch (scenario) {
1208                         case 'default':
1209                             // For now only run Debug jobs on PR Trigger.
1210                             if (configuration != 'Debug') {
1211                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1212                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1213                             }
1214                             else {
1215                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1216                                 // are at higher risk of ARM64-breaking changes.
1217                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1218                             }
1219                             break
1220                         case 'pri1r2r':
1221                         case 'gcstress0x3':
1222                         case 'gcstress0xc':
1223                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1224                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1225                             break
1226                     }
1227                     break
1228                 default:
1229                     println("NYI os: ${os}");
1230                     assert false
1231                     break
1232             }
1233             break
1234         // editor brace matching: }
1235         case 'x86': // editor brace matching: {
1236             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1237             if (os == 'Ubuntu') {
1238                 // on-demand only for ubuntu x86
1239                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1240                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1241                 break
1242             }
1243             switch (scenario) {
1244                 case 'default':
1245                     if (configuration == 'Checked') {
1246                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1247                     }
1248                     else if (configuration == 'Release') {
1249                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1250                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1251                     }
1252                     break
1253                 case 'pri1':
1254                     if (configuration == 'Release') {
1255                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1256                     }
1257                     break
1258                 case 'ilrt':
1259                     if (configuration == 'Release') {
1260                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1261                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1262                     }
1263                     break
1264                 case 'r2r':
1265                     if (configuration == 'Checked' || configuration == 'Release') {
1266                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1267                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1268                     }
1269                     break
1270                 case 'pri1r2r':
1271                     if (configuration == 'Checked' || configuration == 'Release') {
1272                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1273                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1274                     }
1275                     break
1276                 case 'gcstress15_pri1r2r':
1277                     if (configuration == 'Release' || configuration == 'Checked') {
1278                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1279                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1280                     }
1281                     break
1282                 case 'r2r_jitstress1':
1283                 case 'r2r_jitstress2':
1284                 case 'r2r_jitstressregs1':
1285                 case 'r2r_jitstressregs2':
1286                 case 'r2r_jitstressregs3':
1287                 case 'r2r_jitstressregs4':
1288                 case 'r2r_jitstressregs8':
1289                 case 'r2r_jitstressregs0x10':
1290                 case 'r2r_jitstressregs0x80':
1291                 case 'r2r_jitstressregs0x1000':
1292                 case 'r2r_jitminopts':
1293                 case 'r2r_jitforcerelocs':
1294                     if (configuration == 'Release' || configuration == 'Checked') {
1295                         def displayStr = getR2RStressModeDisplayName(scenario)
1296                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1297                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1298                     }
1299                     break
1300                 case 'longgc':
1301                     if (configuration == 'Release') {
1302                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1303                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1304                     }
1305                     break
1306                 case 'gcsimulator':
1307                     if (configuration == 'Release') {
1308                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1309                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1310                     }
1311                     break
1312                 case 'standalone_gc':
1313                     if (configuration == 'Release' || configuration == 'Checked') {
1314                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1315                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1316                     }
1317                     break
1318                 case 'minopts':
1319                 case 'forcerelocs':
1320                 case 'jitstress1':
1321                 case 'jitstress2':
1322                 case 'jitstressregs1':
1323                 case 'jitstressregs2':
1324                 case 'jitstressregs3':
1325                 case 'jitstressregs4':
1326                 case 'jitstressregs8':
1327                 case 'jitstressregs0x10':
1328                 case 'jitstressregs0x80':
1329                 case 'jitstressregs0x1000':
1330                 case 'jitstress2_jitstressregs1':
1331                 case 'jitstress2_jitstressregs2':
1332                 case 'jitstress2_jitstressregs3':
1333                 case 'jitstress2_jitstressregs4':
1334                 case 'jitstress2_jitstressregs8':
1335                 case 'jitstress2_jitstressregs0x10':
1336                 case 'jitstress2_jitstressregs0x80':
1337                 case 'jitstress2_jitstressregs0x1000':
1338                 case 'tailcallstress':
1339                 case 'jitsse2only':
1340                 case 'gcstress0x3':
1341                 case 'gcstress0xc':
1342                 case 'zapdisable':
1343                 case 'heapverify1':
1344                 case 'gcstress0xc_zapdisable':
1345                 case 'gcstress0xc_zapdisable_jitstress2':
1346                 case 'gcstress0xc_zapdisable_heapverify1':
1347                 case 'gcstress0xc_jitstress1':
1348                 case 'gcstress0xc_jitstress2':
1349                 case 'gcstress0xc_minopts_heapverify1':
1350                     def displayStr = getStressModeDisplayName(scenario)
1351                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1352                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1353                     break
1354                 case 'corefx_baseline':
1355                 case 'corefx_minopts':
1356                 case 'corefx_jitstress1':
1357                 case 'corefx_jitstress2':
1358                 case 'corefx_jitstressregs1':
1359                 case 'corefx_jitstressregs2':
1360                 case 'corefx_jitstressregs3':
1361                 case 'corefx_jitstressregs4':
1362                 case 'corefx_jitstressregs8':
1363                 case 'corefx_jitstressregs0x10':
1364                 case 'corefx_jitstressregs0x80':
1365                 case 'corefx_jitstressregs0x1000':
1366                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1367                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1368                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1369                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1370                     break
1371                 case 'illink':
1372                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1373                     break
1374                 default:
1375                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1376                     assert false
1377                     break
1378             }
1379             break
1380          // editor brace matching: }
1381         case 'x86lb': // editor brace matching: {
1382             assert (os == 'Windows_NT')
1383             assert (scenario == 'default' || Constants.r2rJitStressScenarios.indexOf(scenario) !=1)
1384
1385             def arch = 'x86'
1386             def jit = 'legacy_backend'
1387             switch (scenario) {
1388                 case 'default':
1389                     if (configuration == 'Checked') {
1390                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1391                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1392                     }
1393                     break
1394                 default:
1395                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1396                     assert false
1397                     break
1398             }
1399             break
1400         // editor brace matching: }
1401         default:
1402             println("Unknown architecture: ${architecture}");
1403             assert false
1404             break
1405     }
1406 }
1407
1408 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1409     def buildCommands = [];
1410     def osGroup = getOSGroup(os)
1411     def lowerConfiguration = configuration.toLowerCase()
1412
1413     // Calculate the build steps, archival, and xunit results
1414     switch (os) {
1415         case 'Windows_NT': // editor brace matching: {
1416             switch (architecture) {
1417                 case 'x64':
1418                 case 'x86':
1419                 case 'x86lb':
1420                     def arch = architecture
1421                     def buildOpts = ''
1422                     if (architecture == 'x86lb') {
1423                         arch = 'x86'
1424                     }
1425
1426                     if (scenario == 'illink') {
1427                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1428                     }
1429
1430                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1431                             scenario == 'default' ||
1432                             scenario == 'r2r' ||
1433                             scenario == 'jitdiff' ||
1434                             scenario == 'ilrt' ||
1435                             scenario == 'illink' ||
1436                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1437                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1438                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1439                     }
1440
1441                     // For Pri 1 tests, we must shorten the output test binary path names.
1442                     // if __TestIntermediateDir is already set, build-test.cmd will
1443                     // output test binaries to that directory. If it is not set, the
1444                     // binaries are sent to a default directory whose name is about
1445                     // 35 characters long.
1446
1447                     else if (scenarioNeedsPri1Build(scenario)) {
1448                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1449                     }
1450                     else if (isLongGc(scenario)) {
1451                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1452                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1453                     }
1454                     else if (scenario == 'standalone_gc') {
1455                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1456                     }
1457                     else if (scenario == 'formatting') {
1458                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1459                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1460                         break
1461                     }
1462                     else {
1463                         println("Unknown scenario: ${scenario}")
1464                         assert false
1465                     }
1466
1467                     // If we are running a stress mode, we should write out the set of key
1468                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1469
1470                     if (!isBuildOnly) {
1471                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1472                         def crossgenStr = ''
1473                         def runcrossgentestsStr = ''
1474                         def runjitstressStr = ''
1475                         def runjitstressregsStr = ''
1476                         def runjitmioptsStr = ''
1477                         def runjitforcerelocsStr = ''
1478                         def runjitdisasmStr = ''
1479                         def runilasmroundtripStr = ''
1480                         def gcstressStr = ''
1481                         def runtestArguments = ''
1482                         def gcTestArguments = ''
1483                         def illinkArguments = ''
1484
1485                         if (scenario == 'r2r' ||
1486                             scenario == 'pri1r2r' ||
1487                             scenario == 'gcstress15_pri1r2r' ||
1488                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1489                                 crossgenStr = 'crossgen'
1490                                 runcrossgentestsStr = 'runcrossgentests'
1491
1492                                 if (scenario == 'r2r_jitstress1'){
1493                                     runjitstressStr = 'jitstress 1'
1494                                 }
1495                                 else if (scenario == 'r2r_jitstress2') {
1496                                     runjitstressStr = 'jitstress 2'
1497                                 }
1498                                 else if (scenario == 'r2r_jitstressregs1'){
1499                                     runjitstressregsStr = 'jitstressregs 1'
1500                                 }
1501                                 else if (scenario == 'r2r_jitstressregs2') {
1502                                     runjitstressregsStr = 'jitstressregs 2'
1503                                 }
1504                                 else if (scenario == 'r2r_jitstressregs3') {
1505                                     runjitstressregsStr = 'jitstressregs 3'
1506                                 }
1507                                 else if (scenario == 'r2r_jitstressregs4') {
1508                                     runjitstressregsStr = 'jitstressregs 4'
1509                                 }
1510                                 else if (scenario == 'r2r_jitstressregs8') {
1511                                     runjitstressregsStr = 'jitstressregs 8'
1512                                 }
1513                                 else if (scenario == 'r2r_jitstressregs0x10') {
1514                                     runjitstressregsStr = 'jitstressregs 0x10'
1515                                 }
1516                                 else if (scenario == 'r2r_jitstressregs0x80') {
1517                                     runjitstressregsStr = 'jitstressregs 0x80'
1518                                 }
1519                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1520                                     runjitstressregsStr = 'jitstressregs 0x1000'
1521                                 }
1522                                 else if (scenario == 'r2r_jitminopts') {
1523                                     runjitmioptsStr = 'jitminopts'
1524                                 }
1525                                 else if (scenario == 'r2r_jitforcerelocs') {
1526                                     runjitforcerelocsStr = 'jitforcerelocs'
1527                                 }
1528                         }
1529                         if (scenario == 'gcstress15_pri1r2r')
1530                         {
1531                             gcstressStr = 'gcstresslevel 0xF'
1532                         }
1533
1534                         if (scenario == 'jitdiff')
1535                         {
1536                             runjitdisasmStr = 'jitdisasm crossgen'
1537                         }
1538
1539                         if (scenario == 'ilrt')
1540                         {
1541                             runilasmroundtripStr = 'ilasmroundtrip'
1542                         }
1543
1544                         if (isLongGc(scenario)) {
1545                             gcTestArguments = "${scenario} sequential"
1546                         }
1547
1548                         if (scenario == 'illink')
1549                         {
1550                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1551                         }
1552
1553                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1554
1555                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1556                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1557                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1558
1559                             if (enableCorefxTesting) {
1560                                 def workspaceRelativeFxRoot = "_/fx"
1561                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1562
1563                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1564
1565                                 setTestJobTimeOut(newJob, scenario)
1566
1567                                 // Archive and process (only) the test results
1568                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1569                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1570                             }
1571                             else {
1572                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1573                             }
1574                         }
1575                         else if (isGcReliabilityFramework(scenario)) {
1576                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1577                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1578                         }
1579                         else if (architecture == 'x64' || architecture == 'x86') {
1580                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1581                         }
1582                         else if (architecture == 'x86lb') {
1583                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd"
1584                         }
1585                     }
1586
1587                     if (!enableCorefxTesting) {
1588                         // Run the rest of the build
1589                         // Build the mscorlib for the other OS's
1590                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1591                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1592                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1593
1594                         if (arch == "x64") {
1595                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1596                         }
1597
1598                         // Zip up the tests directory so that we don't use so much space/time copying
1599                         // 10s of thousands of files around.
1600                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1601
1602                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1603                             // For windows, pull full test results and test drops for x86/x64.
1604                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1605                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1606                         }
1607
1608                         if (scenario == 'jitdiff') {
1609                             // retrive jit-dasm output for base commit, and run jit-diff
1610                             if (!isBuildOnly) {
1611                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1612                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1613                             }
1614                         }
1615
1616                         if (!isBuildOnly) {
1617                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1618                             setTestJobTimeOut(newJob, scenario)
1619                         }
1620                     }
1621                     break
1622                 case 'arm':
1623                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1624
1625                     // Set time out
1626                     setTestJobTimeOut(newJob, scenario)
1627
1628                     if ( lowerConfiguration == "debug" ) {
1629                         // For Debug builds, we will do a P1 test build
1630                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1631                     }
1632                     else if (lowerConfiguration == "checked") {
1633                         def machineAffinityOptions = ['use_arm64_build_machine' : true]
1634                         setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1635                         // For checked runs we will also run testing.
1636                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1637                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1638                     }
1639                     else if (lowerConfiguration == "release") {
1640                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1641                     }
1642                     // Add archival.
1643                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1644                     break
1645                 case 'arm64':
1646                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1647
1648                     // Set time out
1649                     setTestJobTimeOut(newJob, scenario)
1650
1651                     // Debug runs take too long to run. So build job only.
1652                     if (lowerConfiguration == "debug") {
1653                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1654                     }
1655                     else {
1656                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1657                        {
1658                            // Up the timeout for arm64 checked testing only.
1659                            // Keep the longer timeout for gcstress.
1660                            Utilities.setJobTimeout(newJob, 240)
1661                        }
1662
1663                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=1"
1664                        // Test build and run are launched together.
1665                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1666                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1667                     }
1668
1669                     // Add archival.
1670                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1671                     break
1672                 default:
1673                     println("Unknown architecture: ${architecture}");
1674                     assert false
1675                     break
1676             }
1677             break
1678         // editor brace matching: }
1679         case 'Ubuntu':
1680         case 'Ubuntu16.04':
1681         case 'Ubuntu16.10':
1682         case 'Debian8.4':
1683         case 'OSX10.12':
1684         case 'FreeBSD':
1685         case 'CentOS7.1':
1686         case 'RHEL7.2':
1687         case 'OpenSUSE42.1':
1688         case 'Tizen':
1689         case 'Fedora24': // editor brace matching: {
1690             switch (architecture) {
1691                 case 'x64':
1692                 case 'x86':
1693                     if (architecture == 'x86' && os == 'Ubuntu') {
1694                         // build and PAL test
1695                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1696                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1697                         break;
1698                     }
1699
1700                     if (scenario == 'formatting') {
1701                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1702                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1703                         break
1704                     }
1705
1706                     if (scenario == 'illink') {
1707                         assert(os == 'Ubuntu')
1708                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1709                     }
1710
1711                     def standaloneGc = ''
1712                     if (scenario == 'standalone_gc') {
1713                         standaloneGc = 'buildstandalonegc'
1714                     }
1715
1716                     if (!enableCorefxTesting) {
1717                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1718                         // only on supported OS platforms.
1719                         if (os == 'FreeBSD')
1720                         {
1721                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1722                         }
1723                         else
1724                         {
1725                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1726                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1727                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1728                         }
1729                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1730
1731                         // Set time out
1732                         setTestJobTimeOut(newJob, scenario)
1733                         // Basic archiving of the build
1734                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1735                         // And pal tests
1736                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1737                     }
1738                     else {
1739                         // Corefx stress testing
1740                         assert os == 'Ubuntu'
1741                         assert architecture == 'x64'
1742                         assert lowerConfiguration == 'checked'
1743                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1744
1745                         // Build coreclr
1746                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1747
1748                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1749                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1750
1751                         // Build and text corefx
1752                         def workspaceRelativeFxRoot = "_/fx"
1753                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1754
1755                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1756
1757                         setTestJobTimeOut(newJob, scenario)
1758
1759                         // Archive and process (only) the test results
1760                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1761                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1762                     }
1763                     break
1764                 case 'arm64':
1765                     def standaloneGc = ''
1766                     if (scenario == 'standalone_gc') {
1767                         standaloneGc = 'buildstandalonegc'
1768                     }
1769
1770                     if (!enableCorefxTesting) {
1771                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} cross clang3.8 skipnuget ${standaloneGc}"
1772
1773                         // HACK -- Arm64 does not have corefx jobs yet.
1774                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1775                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1776                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1777                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1778
1779                         // Set time out
1780                         setTestJobTimeOut(newJob, scenario)
1781                         // Basic archiving of the build
1782                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1783                     }
1784                     break
1785                 case 'arm':
1786                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1787                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1788
1789                     // default values for Ubuntu
1790                     def arm_abi="arm"
1791                     def linuxCodeName="trusty"
1792                     if (os == 'Ubuntu16.04') {
1793                         linuxCodeName="xenial"
1794                     }
1795                     else if (os == 'Tizen') {
1796                         arm_abi="armel"
1797                         linuxCodeName="tizen"
1798                     }
1799
1800                     // Unzip the Windows test binaries first. Exit with 0
1801                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1802
1803                     // Unpack the corefx binaries
1804                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1805                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1806                     if (os != 'Tizen') {
1807                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1808                     }
1809                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1810                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1811                     if (os == 'Ubuntu') {
1812                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1813                     }
1814
1815                     // Call the ARM CI script to cross build and test using docker
1816                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1817                     --mode=docker \\
1818                     --${arm_abi} \\
1819                     --linuxCodeName=${linuxCodeName} \\
1820                     --buildConfig=${lowerConfiguration} \\
1821                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1822                     --coreFxBinDir=./bin/CoreFxBinDir \\
1823                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1824
1825                     // Basic archiving of the build, no pal tests
1826                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1827                     break
1828                 default:
1829                     println("Unknown architecture: ${architecture}");
1830                     assert false
1831                     break
1832             }
1833             break
1834         // editor brace matching: }
1835         default:
1836             println("Unknown os: ${os}");
1837             assert false
1838             break
1839     } // os
1840
1841     return buildCommands
1842 }
1843
1844 // Additional scenario which can alter behavior
1845
1846 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1847 combinedScenarios.each { scenario ->
1848     [true, false].each { isPR ->
1849         Constants.architectureList.each { architecture ->
1850             Constants.configurationList.each { configuration ->
1851                 Constants.osList.each { os ->
1852                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1853                     // and reset the os to Windows_NT
1854                     def isBuildOnly = false
1855                     if (os == 'Windows_NT_BuildOnly') {
1856                         isBuildOnly = true
1857                         os = 'Windows_NT'
1858                     }
1859
1860                     // Tizen is only supported for arm architecture
1861                     if (os == 'Tizen' && architecture != 'arm') {
1862                         return
1863                     }
1864
1865                     // Skip totally unimplemented (in CI) configurations.
1866                     switch (architecture) {
1867                         case 'arm64':
1868                             if (os == 'Ubuntu16.04') {
1869                                 os = 'Ubuntu'
1870                             }
1871
1872                             // Windows and Ubuntu only
1873                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1874                                 return
1875                             }
1876                             break
1877                         case 'arm':
1878                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1879                                 return
1880                             }
1881                             break
1882                         case 'x86':
1883                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1884                                 return
1885                             }
1886                             break
1887                         case 'x86lb':
1888                             if (os != 'Windows_NT') {
1889                                 return
1890                             }
1891                             break
1892                         case 'x64':
1893                             // Everything implemented
1894                             break
1895                         default:
1896                             println("Unknown architecture: ${architecture}")
1897                             assert false
1898                             break
1899                     }
1900
1901                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1902                     // with checked builds
1903                     def enableCorefxTesting = false
1904                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1905                         if (configuration != 'Checked') {
1906                             return
1907                         }
1908
1909                         enableCorefxTesting = isCorefxTesting(scenario)
1910
1911                         // Since these are just execution time differences,
1912                         // skip platforms that don't execute the tests here (Windows_NT only)
1913                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1914                         if (!isEnabledOS || isBuildOnly) {
1915                             return
1916                         }
1917
1918                         switch (architecture) {
1919                             case 'arm':
1920                             case 'arm64':
1921                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1922                                     return
1923                                 }
1924                                 break
1925                             case 'x64':
1926                             case 'x86':
1927                                 // x86 ubuntu: default only
1928                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1929                                     return
1930                                 }
1931                                 // Windows: Everything implemented
1932                                 break
1933                             case 'x86lb':
1934                                 // No stress modes for legacy jit.
1935                                 // (There's no technical reason we couldn't allow these.)
1936                                 return
1937                             default:
1938                                 return
1939                         }
1940                     }
1941                     else {
1942                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
1943                         // and configuration is not Checked, bail out.
1944                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1945                             return;
1946                         }
1947
1948                         // Skip scenarios
1949                         switch (scenario) {
1950                             case 'pri1':
1951                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1952                                 // the default scenario build
1953                                 if (os != 'Windows_NT') {
1954                                     return
1955                                 }
1956                                 // Only x64 for now
1957                                 if (architecture != 'x64') {
1958                                     return
1959                                 }
1960                                 break
1961                             case 'ilrt':
1962                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1963                                 // the default scenario build
1964                                 if (os != 'Windows_NT') {
1965                                     return
1966                                 }
1967                                 // Only x64 for now
1968                                 if (architecture != 'x64') {
1969                                     return
1970                                 }
1971                                 // Release only
1972                                 if (configuration != 'Release') {
1973                                     return
1974                                 }
1975                                 break
1976                             case 'jitdiff':
1977                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1978                                     return
1979                                 }
1980                                 if (architecture != 'x64') {
1981                                     return
1982                                 }
1983                                 if (configuration != 'Checked') {
1984                                     return
1985                                 }
1986                                 break
1987                             case 'r2r':
1988                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1989                                 // the default scenario build
1990                                 if (os != 'Windows_NT') {
1991                                     return
1992                                 }
1993                                 if (architecture != 'x64') {
1994                                     return
1995                                 }
1996                                 break
1997                             case 'pri1r2r':
1998                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1999                                 // the default scenario build
2000                                 if (os != 'Windows_NT') {
2001                                     return
2002                                 }
2003                                 if (architecture != 'x64') {
2004                                     if ((architecture != 'arm64' && architecture != 'arm') || (configuration == 'Debug')) {
2005                                         return
2006                                     }
2007                                 }
2008                                 break
2009                             case 'gcstress15_pri1r2r':
2010                             case 'r2r_jitstress1':
2011                             case 'r2r_jitstress2':
2012                             case 'r2r_jitstressregs1':
2013                             case 'r2r_jitstressregs2':
2014                             case 'r2r_jitstressregs3':
2015                             case 'r2r_jitstressregs4':
2016                             case 'r2r_jitstressregs8':
2017                             case 'r2r_jitstressregs0x10':
2018                             case 'r2r_jitstressregs0x80':
2019                             case 'r2r_jitstressregs0x1000':
2020                             case 'r2r_jitminopts':
2021                             case 'r2r_jitforcerelocs':
2022                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
2023                                 // the default scenario build
2024                                 if (os != 'Windows_NT') {
2025                                     return
2026                                 }
2027                                 if (architecture != 'x64' && architecture != 'x86') {
2028                                     return
2029                                 }
2030                                 break
2031                             case 'longgc':
2032                             case 'gcsimulator':
2033                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2034                                     return
2035                                 }
2036                                 if (architecture != 'x64') {
2037                                     return
2038                                 }
2039                                 if (configuration != 'Release') {
2040                                     return
2041                                 }
2042                                 break
2043                             case 'gc_reliability_framework':
2044                             case 'standalone_gc':
2045                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2046                                     return
2047                                 }
2048
2049                                 if (architecture != 'x64') {
2050                                     return
2051                                 }
2052
2053                                 if (configuration != 'Release' && configuration != 'Checked') {
2054                                     return
2055                                 }
2056                                 break
2057                             // We need Windows x64 Release bits for the code coverage build
2058                             case 'coverage':
2059                                 if (os != 'Windows_NT') {
2060                                     return
2061                                 }
2062                                 if (architecture != 'x64') {
2063                                     return
2064                                 }
2065                                 if (configuration != 'Release') {
2066                                     return
2067                                 }
2068                                 break
2069                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2070                             case 'formatting':
2071                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2072                                     return
2073                                 }
2074                                 if (architecture != 'x64') {
2075                                     return
2076                                 }
2077                                 if (configuration != 'Checked') {
2078                                     return
2079                                 }
2080                                 if (isBuildOnly) {
2081                                     return
2082                                 }
2083                                 break
2084                             case 'illink':
2085                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2086                                     return
2087                                 }
2088                                 if (architecture != 'x64' && architecture != 'x86') {
2089                                     return
2090                                 }
2091                                 if (isBuildOnly) {
2092                                     return
2093                                 }
2094                                 break
2095                             case 'default':
2096                                 // Nothing skipped
2097                                 break
2098                             default:
2099                                 println("Unknown scenario: ${scenario}")
2100                                 assert false
2101                                 break
2102                         }
2103                     }
2104
2105                     // Calculate names
2106                     def lowerConfiguration = configuration.toLowerCase()
2107                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2108                     def folderName = getJobFolder(scenario)
2109
2110                     // Create the new job
2111                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2112
2113                     def machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2114                     machineAffinityOptions = architecture == 'arm' ? ['use_arm64_build_machine': false] : machineAffinityOptions
2115                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2116
2117                     // Add all the standard options
2118                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2119                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2120
2121                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
2122                     def osGroup = getOSGroup(os)
2123
2124                     newJob.with {
2125                         steps {
2126                             if (os == 'Windows_NT') {
2127                                 buildCommands.each { buildCommand ->
2128                                     batchFile(buildCommand)
2129                                 }
2130                             }
2131                             else {
2132                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2133                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2134                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2135                                     // Define the Windows Tests and Corefx build job names
2136                                     def WindowTestsName = projectFolder + '/' +
2137                                                           Utilities.getFullJobName(project,
2138                                                                                    getJobName(lowerConfiguration,
2139                                                                                               'x64' ,
2140                                                                                               'windows_nt',
2141                                                                                               'default',
2142                                                                                               true),
2143                                                                                    false)
2144                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2145                                                        Utilities.getFolderName(branch)
2146
2147                                     // Copy the Windows test binaries and the Corefx build binaries
2148                                     copyArtifacts(WindowTestsName) {
2149                                         includePatterns('bin/tests/tests.zip')
2150                                         buildSelector {
2151                                             latestSuccessful(true)
2152                                         }
2153                                     }
2154
2155                                     def arm_abi = 'arm'
2156                                     def corefx_os = 'linux'
2157                                     if (os == 'Tizen') {
2158                                         arm_abi = 'armel'
2159                                         corefx_os = 'tizen'
2160                                     }
2161
2162                                     // Let's use release CoreFX to test checked CoreCLR,
2163                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2164                                     def corefx_lowerConfiguration = lowerConfiguration
2165                                     if ( lowerConfiguration == 'checked' ) {
2166                                         corefx_lowerConfiguration='release'
2167                                     }
2168
2169                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2170                                         includePatterns('bin/build.tar.gz')
2171                                         buildSelector {
2172                                             latestSuccessful(true)
2173                                         }
2174                                     }
2175                                 }
2176
2177                                 buildCommands.each { buildCommand ->
2178                                     shell(buildCommand)
2179                                 }
2180                             }
2181                         }
2182                     } // newJob.with
2183
2184                 } // os
2185             } // configuration
2186         } // architecture
2187     } // isPR
2188 } // scenario
2189
2190
2191 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2192 combinedScenarios.each { scenario ->
2193     [true, false].each { isPR ->
2194         // Architectures.  x64 only at this point
2195         ['x64', 'arm64'].each { architecture ->
2196             // Put the OS's supported for coreclr cross testing here
2197             Constants.crossList.each { os ->
2198                 if (architecture == 'arm64') {
2199                     if (os != "Ubuntu") {
2200                         return
2201                     }
2202                 }
2203
2204                 Constants.configurationList.each { configuration ->
2205
2206                     if (architecture == 'arm64') {
2207                         if (scenario != 'default' && scenario != 'pri1r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2208                             return
2209                         }
2210                     }
2211
2212                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2213                         if (configuration != 'Checked') {
2214                             return
2215                         }
2216                         if (isCorefxTesting(scenario)) {
2217                             return
2218                         }
2219                         //Skip stress modes for these scenarios
2220                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2221                             return
2222                         }
2223                     }
2224                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2225                     // and configuration is not Checked, bail out.
2226                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2227                         return;
2228                     }
2229                     // For CentOS, we only want Checked/Release pri1 builds.
2230                     else if (os == 'CentOS7.1') {
2231                         if (scenario != 'pri1' &&
2232                             scenario != 'r2r' &&
2233                             scenario != 'pri1r2r' &&
2234                             scenario != 'gcstress15_pri1r2r' &&
2235                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2236                             return
2237                         }
2238                         if (configuration != 'Checked' && configuration != 'Release') {
2239                             return
2240                         }
2241                     }
2242                     // For RedHat and Debian, we only do Release pri1 builds.
2243                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2244                         if (scenario != 'pri1') {
2245                             return
2246                         }
2247                         if (configuration != 'Release') {
2248                             return
2249                         }
2250                     }
2251                     else {
2252                         // Skip scenarios
2253                         switch (scenario) {
2254                             case 'pri1':
2255                                 // Nothing skipped
2256                                 break
2257                             case 'ilrt':
2258                                 // Release only
2259                                 if (configuration != 'Release') {
2260                                     return
2261                                 }
2262                                 break
2263                             case 'jitdiff':
2264                                 if (configuration != 'Checked') {
2265                                     return;
2266                                 }
2267                             case 'r2r':
2268                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2269                                 if (configuration != 'Checked' && configuration != 'Release') {
2270                                     return
2271                                 }
2272                                 break
2273                             case 'pri1r2r':
2274                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2275                                 if (configuration != 'Checked' && configuration != 'Release') {
2276                                     return
2277                                 }
2278                                 break
2279                             case 'gcstress15_pri1r2r':
2280                             case 'r2r_jitstress1':
2281                             case 'r2r_jitstress2':
2282                             case 'r2r_jitstressregs1':
2283                             case 'r2r_jitstressregs2':
2284                             case 'r2r_jitstressregs3':
2285                             case 'r2r_jitstressregs4':
2286                             case 'r2r_jitstressregs8':
2287                             case 'r2r_jitstressregs0x10':
2288                             case 'r2r_jitstressregs0x80':
2289                             case 'r2r_jitstressregs0x1000':
2290                             case 'r2r_jitminopts':
2291                             case 'r2r_jitforcerelocs':
2292                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2293                                 if (configuration != 'Checked' && configuration != 'Release') {
2294                                     return
2295                                 }
2296                                 break
2297                             case 'longgc':
2298                             case 'gcsimulator':
2299                                 // Long GC tests take a long time on non-Release builds
2300                                 if (configuration != 'Release') {
2301                                     return
2302                                 }
2303                                 break
2304                             case 'gc_reliability_framework':
2305                             case 'standalone_gc':
2306                                 if (configuration != 'Release' && configuration != 'Checked') {
2307                                     return
2308                                 }
2309                                 break
2310                             case 'coverage':
2311                                 //We only want Ubuntu Release for coverage
2312                                 if (os != 'Ubuntu') {
2313                                     return
2314                                 }
2315                                 if (configuration != 'Release') {
2316                                     return
2317                                 }
2318                             case 'formatting':
2319                                 return
2320                             case 'illink':
2321                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2322                                     return
2323                                 }
2324                                 break
2325                             case 'default':
2326                                 // Nothing skipped
2327                                 break
2328                             default:
2329                                 println("Unknown scenario: ${scenario}")
2330                                 assert false
2331                                 break
2332                         }
2333                     }
2334
2335                     def lowerConfiguration = configuration.toLowerCase()
2336                     def osGroup = getOSGroup(os)
2337                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2338
2339                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2340                     def inputScenario = 'default'
2341                     if (scenario == 'coverage') {
2342                         inputScenario = 'coverage'
2343                     }
2344                     def inputCoreCLRBuildName = projectFolder + '/' +
2345                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2346                     // If this is a stress scenario, there isn't any difference in the build job
2347                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2348                     // from the default scenario
2349                     def testBuildScenario = scenario
2350                     if (scenarioNeedsPri1Build(scenario)) {
2351                         testBuildScenario = 'pri1'
2352                     }
2353                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2354                         testBuildScenario = 'default'
2355                     }
2356                     def inputWindowTestsBuildName = ''
2357                     def inputWindowsTestBuildArch = architecture
2358                     if (architecture == "arm64") {
2359                         // Use the x64 test build for arm64 unix
2360                         inputWindowsTestBuildArch = "x64"
2361                     }
2362                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2363                         inputWindowTestsBuildName = projectFolder + '/' +
2364                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', 'default', true), isPR)
2365                     }
2366                     else {
2367                         inputWindowTestsBuildName = projectFolder + '/' +
2368                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2369                     }
2370                     // Enable Server GC for Ubuntu PR builds
2371                     def serverGCString = ''
2372
2373                     // Whether or not this test run should be run sequentially instead
2374                     // of in parallel. Only used for long GC tests.
2375                     def sequentialString = ''
2376
2377                     // Whether or not this test run should run a specific playlist.
2378                     // Only used for long GC tests.
2379
2380                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2381                     // for running long GC and GCSimulator tests, respectively. We don't use them
2382                     // here because using a playlist file produces much more readable output on the CI machines
2383                     // and reduces running time.
2384                     def playlistString = ''
2385
2386                     if (os == 'Ubuntu' && isPR){
2387                         serverGCString = '--useServerGC'
2388                     }
2389
2390                     // pass --crossgen to runtest.sh for crossgen builds
2391                     def crossgenStr = ''
2392                     def runcrossgentestsStr = ''
2393                     def runjitstressStr = ''
2394                     def runjitstressregsStr = ''
2395                     def runjitmioptsStr = ''
2396                     def runjitforcerelocsStr = ''
2397                     def runjitdisasmStr = ''
2398                     def runilasmroundtripStr = ''
2399                     def gcstressStr = ''
2400                     def illinkStr = ''
2401                     def layoutOnlyStr =''
2402
2403                     if (scenario == 'r2r' ||
2404                         scenario == 'pri1r2r' ||
2405                         scenario == 'gcstress15_pri1r2r' ||
2406                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2407                             crossgenStr = '--crossgen'
2408                             runcrossgentestsStr = '--runcrossgentests'
2409
2410                             if (scenario == 'r2r_jitstress1'){
2411                                 runjitstressStr = '--jitstress=1'
2412                             }
2413                             else if (scenario == 'r2r_jitstress2') {
2414                                 runjitstressStr = '--jitstress=2'
2415                             }
2416                             else if (scenario == 'r2r_jitstressregs1'){
2417                                 runjitstressregsStr = '--jitstressregs=1'
2418                             }
2419                             else if (scenario == 'r2r_jitstressregs2') {
2420                                 runjitstressregsStr = '--jitstressregs=2'
2421                             }
2422                             else if (scenario == 'r2r_jitstressregs3') {
2423                                 runjitstressregsStr = '--jitstressregs=3'
2424                             }
2425                             else if (scenario == 'r2r_jitstressregs4') {
2426                                 runjitstressregsStr = '--jitstressregs=4'
2427                             }
2428                             else if (scenario == 'r2r_jitstressregs8') {
2429                                 runjitstressregsStr = '--jitstressregs=8'
2430                             }
2431                             else if (scenario == 'r2r_jitstressregs0x10') {
2432                                 runjitstressregsStr = '--jitstressregs=0x10'
2433                             }
2434                             else if (scenario == 'r2r_jitstressregs0x80') {
2435                                 runjitstressregsStr = '--jitstressregs=0x80'
2436                             }
2437                             else if (scenario == 'r2r_jitstressregs0x1000') {
2438                                 runjitstressregsStr = '--jitstressregs=0x1000'
2439                             }
2440                             else if (scenario == 'r2r_jitminopts') {
2441                                 runjitmioptsStr = '--jitminopts'
2442                             }
2443                             else if (scenario == 'r2r_jitforcerelocs') {
2444                                 runjitforcerelocsStr = '--jitforcerelocs'
2445                             }
2446                     }
2447                     if  (scenario == 'gcstress15_pri1r2r')
2448                     {
2449                         gcstressStr = '--gcstresslevel=0xF'
2450                     }
2451
2452                     if (scenario == 'jitdiff')
2453                     {
2454                         runjitdisasmStr = '--jitdisasm --crossgen'
2455                     }
2456
2457                     if (scenario == 'illink')
2458                     {
2459                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2460                     }
2461
2462                     if (isLongGc(scenario)) {
2463                         // Long GC tests behave very poorly when they are not
2464                         // the only test running (many of them allocate until OOM).
2465                         sequentialString = '--sequential'
2466
2467                         // The Long GC playlist contains all of the tests that are
2468                         // going to be run. The GCSimulator playlist contains all of
2469                         // the GC simulator tests.
2470                         if (scenario == 'longgc') {
2471                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2472                         }
2473                         else if (scenario == 'gcsimulator') {
2474                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2475                         }
2476                     }
2477
2478                     if (isGcReliabilityFramework(scenario)) {
2479                         layoutOnlyStr = '--build-overlay-only'
2480                     }
2481
2482                     def folder = getJobFolder(scenario)
2483                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2484                         // Add parameters for the inputs
2485
2486                         parameters {
2487                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2488                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2489                         }
2490
2491                         steps {
2492                             // Set up the copies
2493
2494                             // Coreclr build containing the tests and mscorlib
2495
2496                             copyArtifacts(inputWindowTestsBuildName) {
2497                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2498                                 buildSelector {
2499                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2500                                 }
2501                             }
2502
2503                             if (scenario == 'coverage') {
2504
2505                                 // Move coreclr to clr directory
2506                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2507
2508                                 // Build coreclr
2509                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2510
2511                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2512                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2513                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2514                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2515                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2516                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2517                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2518                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2519
2520                                 // Run PAL tests
2521                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2522
2523                                 // Remove obj files for PAL tests so they're not included in coverage results
2524                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2525
2526                                 // Unzip the tests first.  Exit with 0
2527                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2528
2529                                 // Get corefx
2530                                 shell("git clone https://github.com/dotnet/corefx fx")
2531
2532                                 // Build Linux corefx
2533                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2534                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2535
2536                                 def testEnvOpt = ""
2537                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2538                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2539                                 shell("${createScriptCmds}")
2540                                 testEnvOpt = "--test-env=" + scriptFileName
2541
2542                                 // Run corefx tests
2543                                 shell("""./fx/run-test.sh \\
2544                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2545                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2546                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2547                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2548                 --configurationGroup Release""")
2549
2550
2551                                 // Run coreclr tests w/ workstation GC
2552                                 shell("""./clr/tests/runtest.sh \\
2553                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2554                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2555                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2556                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2557                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2558                 --crossgen --runcrossgentests""")
2559
2560                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2561                                 shell("""./clr/tests/runtest.sh \\
2562                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2563                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2564                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2565                 --useServerGC ${testEnvOpt}""")
2566
2567                                  // Run long-running coreclr GC tests & produce coverage reports
2568                                 shell("""./clr/tests/runtest.sh \\
2569                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2570                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2571                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2572                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2573                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2574                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2575                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2576
2577                             }
2578                             else {
2579
2580                                 // Coreclr build we are trying to test
2581
2582                                 copyArtifacts(inputCoreCLRBuildName) {
2583                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2584                                     buildSelector {
2585                                         buildNumber('${CORECLR_BUILD}')
2586                                     }
2587                                 }
2588
2589                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2590
2591                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2592                                 def osJobName
2593                                 if (os == 'Ubuntu') {
2594                                     osJobName = 'ubuntu14.04'
2595                                 }
2596                                 else {
2597                                     osJobName = os.toLowerCase()
2598                                 }
2599                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2600                                     includePatterns('bin/build.tar.gz')
2601                                     buildSelector {
2602                                         latestSuccessful(true)
2603                                     }
2604                                 }
2605
2606                                 shell ("mkdir ./bin/CoreFxBinDir")
2607                                 // Unpack the corefx binaries
2608                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2609
2610                                 // HACK -- Arm64 does not have corefx jobs yet.
2611                                 // Clone corefx and build the native packages overwriting the x64 packages.
2612                                 if (architecture == 'arm64') {
2613                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2614                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2615                                 }
2616
2617                                 // Unzip the tests first.  Exit with 0
2618                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2619
2620                                 // Execute the tests
2621                                 // If we are running a stress mode, we'll set those variables first
2622                                 def testEnvOpt = ""
2623                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2624                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2625                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2626                                     shell("${createScriptCmds}")
2627                                     testEnvOpt = "--test-env=" + scriptFileName
2628                                 }
2629
2630                                 if (isGCStressRelatedTesting(scenario)) {
2631                                     shell('./init-tools.sh')
2632                                 }
2633
2634                                 shell("""./tests/runtest.sh \\
2635                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2636                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2637                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2638                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2639                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2640                 --limitedDumpGeneration \\
2641                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
2642                 ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
2643                 ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
2644
2645                                 if (isGcReliabilityFramework(scenario)) {
2646                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2647                                     if (serverGCString != '') {
2648                                         shell("export COMPlus_gcServer=1")
2649                                     }
2650
2651                                     shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2652                                 }
2653                             }
2654                         }
2655                     }
2656
2657                     if (scenario == 'coverage') {
2658                         // Publish coverage reports
2659                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2660                         // TODO: Add once external email sending is available again
2661                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2662                     }
2663
2664                     if (scenario == 'jitdiff') {
2665                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2666                     }
2667
2668                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2669                     if (os in ['Ubuntu']) {
2670                         SummaryBuilder summaries = new SummaryBuilder()
2671                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2672                         summaries.emit(newJob)
2673                     }
2674
2675                     setMachineAffinity(newJob, os, architecture)
2676                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2677                     // Set timeouts to 240.
2678                     setTestJobTimeOut(newJob, scenario)
2679
2680                     if (architecture == 'arm64') {
2681                         Utilities.setJobTimeout(newJob, 480)
2682                     }
2683
2684                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2685
2686                     // Create a build flow to join together the build and tests required to run this
2687                     // test.
2688                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2689                     // Linux CoreCLR test
2690                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2691                     def fullTestJobName = projectFolder + '/' + newJob.name
2692                     // Add a reference to the input jobs for report purposes
2693                     JobReport.Report.addReference(inputCoreCLRBuildName)
2694                     JobReport.Report.addReference(inputWindowTestsBuildName)
2695                     JobReport.Report.addReference(fullTestJobName)
2696                     def newFlowJob;
2697
2698                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2699                     // so that coverage data can be preserved.
2700                     if (scenario == 'coverage') {
2701                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2702                         buildFlow("""
2703 // Build the input Windows job
2704 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2705
2706 // And then build the test build
2707 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2708 """)
2709                         }
2710                     // Normal jobs copy a Windows build & a non-Windows build
2711                     } else {
2712                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2713                         buildFlow("""
2714 // Build the input jobs in parallel
2715 parallel (
2716     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2717     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2718 )
2719
2720 // And then build the test build
2721 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2722                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2723 """)
2724                         }
2725                     }
2726
2727                     setMachineAffinity(newFlowJob, os, architecture)
2728                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2729                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2730                 } // configuration
2731             } // os
2732         } // architecture
2733     } // isPR
2734 } // scenario
2735
2736 JobReport.Report.generateJobReport(out)
2737
2738 // Make the call to generate the help job
2739 Utilities.createHelperJob(this, project, branch,
2740     "Welcome to the ${project} Repository",  // This is prepended to the help message
2741     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2742
2743 Utilities.addCROSSCheck(this, project, branch)