netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'CentOS7.1',
  49                'RHEL7.2',
  50                'Ubuntu16.04',
  51                'Ubuntu16.10',
  52                'Fedora24',
  53                'Tizen']
  54
  55     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  56
  57     // This is a set of JIT stress modes combined with the set of variables that
  58     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  59     // the values are the environment variables
  60     def static jitStressModeScenarios = [
  61                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  62                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  63                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  64                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  65                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  66                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  67                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  68                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  69                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  70                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  71                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  72                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  73                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  74                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  75                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  76                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  77                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  78                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  79                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  80                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  81                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  82                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  83                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  84                'corefx_baseline'                : [ : ], // corefx baseline
  85                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  86                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  87                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  88                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  89                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  90                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  91                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  92                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  93                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  94                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  95                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  96                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  97                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  98                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
  99                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 100                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 101                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 103                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 104                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 105                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 106                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 107                ]
 108
 109     // This is a set of r2r jit stress scenarios
 110     def static r2rJitStressScenarios = [
 111                'r2r_jitstress1',
 112                'r2r_jitstress2',
 113                'r2r_jitstressregs1',
 114                'r2r_jitstressregs2',
 115                'r2r_jitstressregs3',
 116                'r2r_jitstressregs4',
 117                'r2r_jitstressregs8',
 118                'r2r_jitstressregs0x10',
 119                'r2r_jitstressregs0x80',
 120                'r2r_jitstressregs0x1000',
 121                'r2r_jitminopts',
 122                'r2r_jitforcerelocs']
 123
 124     // This is the basic set of scenarios
 125     def static basicScenarios = [
 126                'default',
 127                'pri1',
 128                'ilrt',
 129                'r2r',
 130                'pri1r2r',
 131                'gcstress15_pri1r2r',
 132                'longgc',
 133                'coverage',
 134                'formatting',
 135                'gcsimulator',
 136                'jitdiff',
 137                'standalone_gc',
 138                'gc_reliability_framework',
 139                'illink'] + r2rJitStressScenarios
 140
 141     // Knowledge of the "validArmWindowsScenarios" scenario names is embedded in the ARM64 CI code, so when adding any,
 142     // make sure that code knows how to process them.
 143     def static validArmWindowsScenarios = [
 144                'default',
 145                'pri1r2r',
 146                'zapdisable',
 147                'minopts',
 148                'tailcallstress',
 149                'jitstress1',
 150                'jitstress2',
 151                'gcstress0x3',
 152                'gcstress0xc',
 153                'jitstressregs1',
 154                'jitstressregs2',
 155                'jitstressregs3',
 156                'jitstressregs4',
 157                'jitstressregs8',
 158                'jitstressregs0x10',
 159                'jitstressregs0x80',
 160                'jitstressregs0x1000',
 161                'gcstress0xc_jitstress1',
 162                'gcstress0xc_jitstress2',
 163                'minopts_zapdisable',
 164                'gcstress0x3_jitstress1',
 165                'gcstress0x3_jitstress2',
 166                'gcstress0xc_jitstress1',
 167                'gcstress0xc_jitstress2',
 168                'gcstress0x3_jitstressregs1',
 169                'gcstress0x3_jitstressregs2',
 170                'gcstress0x3_jitstressregs3',
 171                'gcstress0x3_jitstressregs4',
 172                'gcstress0x3_jitstressregs8',
 173                'gcstress0x3_jitstressregs0x10',
 174                'gcstress0x3_jitstressregs0x80',
 175                'gcstress0x3_jitstressregs0x1000',
 176                'gcstress0xc_jitstressregs1',
 177                'gcstress0xc_jitstressregs2',
 178                'gcstress0xc_jitstressregs3',
 179                'gcstress0xc_jitstressregs4',
 180                'gcstress0xc_jitstressregs8',
 181                'gcstress0xc_jitstressregs0x10',
 182                'gcstress0xc_jitstressregs0x80',
 183                'gcstress0xc_jitstressregs0x1000']
 184
 185     def static configurationList = ['Debug', 'Checked', 'Release']
 186
 187     // This is the set of architectures
 188     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86', 'x86lb']
 189 }
 190
 191 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 192     if (architecture == 'arm64' && os == 'Windows_NT') {
 193         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 194     } else if (architecture == 'arm64' && os != 'Windows_NT' && options == null) {
 195         Utilities.setMachineAffinity(job, os, 'arm64-small-page-size');
 196     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['large_pages'] == true) {
 197         Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size');
 198     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['is_build_only'] == true) {
 199         Utilities.setMachineAffinity(job, os, 'arm64-cross-latest');
 200     } else if ((architecture == 'armlb') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 201         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 202     } else if ((architecture == 'arm' || architecture == 'armlb') && (os == 'Windows_NT') && options['use_arm64_build_machine'] == true) {
 203         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 204     }else {
 205         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 206     }
 207 }
 208
 209 def static isJITStressJob(def scenario) {
 210     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 211            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 212 }
 213
 214 def static isGCStressRelatedTesting(def scenario) {
 215     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 216     // Detect it and make it a GCStress related.
 217     if (scenario == 'gcstress15_pri1r2r')
 218     {
 219         return true;
 220     }
 221
 222     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 223     def scenarioName = scenario.toLowerCase()
 224     def isGCStressTesting = false
 225     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 226         if (k in gcStressTestEnvVars) {
 227             isGCStressTesting = true;
 228         }
 229     }
 230     return isGCStressTesting
 231 }
 232
 233 def static isCorefxTesting(def scenario) {
 234     def corefx_prefix = 'corefx_'
 235     if (scenario.length() < corefx_prefix.length()) {
 236         return false
 237     }
 238     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 239 }
 240
 241 def static isR2R(def scenario) {
 242     return (scenario == 'r2r' || scenario == 'pri1r2r')
 243 }
 244
 245 def static isCoverage(def scenario) {
 246     return (scenario == 'coverage')
 247 }
 248
 249 def static isLongGc(def scenario) {
 250     return (scenario == 'longgc' || scenario == 'gcsimulator')
 251 }
 252
 253 def static isJitDiff(def scenario) {
 254     return (scenario == 'jitdiff')
 255 }
 256
 257 def static isGcReliabilityFramework(def scenario) {
 258     return (scenario == 'gc_reliability_framework')
 259 }
 260
 261 def static scenarioNeedsPri1Build(def scenario) {
 262     return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario) || isLongGc(scenario))
 263 }
 264
 265 def static setTestJobTimeOut(newJob, scenario) {
 266     if (isGCStressRelatedTesting(scenario)) {
 267         Utilities.setJobTimeout(newJob, 4320)
 268     }
 269     else if (isCorefxTesting(scenario)) {
 270         Utilities.setJobTimeout(newJob, 360)
 271     }
 272     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 273         Utilities.setJobTimeout(newJob, 240)
 274     }
 275     else if (isR2R(scenario)) {
 276         Utilities.setJobTimeout(newJob, 240)
 277     }
 278     else if (isCoverage(scenario)) {
 279         Utilities.setJobTimeout(newJob, 1440)
 280     }
 281     else if (isLongGc(scenario)) {
 282         Utilities.setJobTimeout(newJob, 1440)
 283     }
 284     else if (isJitDiff(scenario)) {
 285         Utilities.setJobTimeout(newJob, 240)
 286     }
 287     else if (isGcReliabilityFramework(scenario)) {
 288         Utilities.setJobTimeout(newJob, 1440)
 289     }
 290     // Non-test jobs use the default timeout value.
 291 }
 292
 293 def static getJobFolder(def scenario) {
 294     if (isJITStressJob(scenario)) {
 295         return 'jitstress'
 296     }
 297     if (scenario == 'illink') {
 298         return 'illink'
 299     }
 300     return ''
 301 }
 302
 303 def static getStressModeDisplayName(def scenario) {
 304     def displayStr = ''
 305     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 306         def prefixLength = 'COMPlus_'.length()
 307         if (k.length() >= prefixLength) {
 308             def modeName = k.substring(prefixLength, k.length())
 309             displayStr += ' ' + modeName + '=' + v
 310         }
 311     }
 312     return displayStr
 313 }
 314
 315 def static getR2RStressModeDisplayName(def scenario) {
 316     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 317     // "r2r_" prefix.
 318     def displayStr = scenario
 319     def prefixLength = 'r2r_'.length()
 320     if (displayStr.length() >= prefixLength) {
 321         displayStr = displayStr.substring(prefixLength, displayStr.length())
 322     }
 323     return displayStr
 324 }
 325
 326 // Generates the string for creating a file that sets environment variables
 327 // that makes it possible to run stress modes.  Writes the script to the file
 328 // specified by the stepScriptLocation parameter.
 329 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 330     def stepScript = ''
 331     if (os == 'Windows_NT') {
 332         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 333         stepScript += "del ${stepScriptLocation}\r\n"
 334
 335         // Timeout in ms, default is 10 minutes. For stress
 336         // modes up this to 30 minutes
 337         def timeout = 1800000
 338
 339         // Set the Timeout
 340         stepScript += "set __TestTimeout=${timeout}\r\n"
 341         stepScript += "echo. > ${stepScriptLocation}\r\n"
 342         stressModeVars.each{ k, v ->
 343             // Write out what we are writing to the script file
 344             stepScript += "echo Setting ${k}=${v}\r\n"
 345             // Write out the set itself to the script file`
 346             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 347         }
 348     }
 349     else {
 350         stepScript += "echo Setting variables for ${stressModeName}\n"
 351         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 352         stressModeVars.each{ k, v ->
 353             // Write out what we are writing to the script file
 354             stepScript += "echo Setting ${k}=${v}\n"
 355             // Write out the set itself to the script file`
 356             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 357         }
 358         stepScript += "chmod +x ${stepScriptLocation}\n"
 359     }
 360     return stepScript
 361 }
 362
 363 // Append an existing script to a stress mode script already created by genStressModeScriptStep().
 364 // Returns string of commands to do this.
 365 def static appendStressModeScriptStep(def os, def appendScript, def stepScriptLocation) {
 366     assert (os == 'Windows_NT')
 367     def stepScript = ''
 368     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 369     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 370     return stepScript
 371 }
 372
 373 // Calculates the name of the build job based on some typical parameters.
 374 //
 375 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 376     // If the architecture is x64, do not add that info into the build name.
 377     // Need to change around some systems and other builds to pick up the right builds
 378     // to do that.
 379
 380     def suffix = scenario != 'default' ? "_${scenario}" : '';
 381     if (isBuildOnly) {
 382         suffix += '_bld'
 383     }
 384     def baseName = ''
 385     switch (architecture) {
 386         case 'x64':
 387             if (scenario == 'default') {
 388                 // For now we leave x64 off of the name for compatibility with other jobs
 389                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 390             }
 391             else if (scenario == 'formatting') {
 392                 // we don't care about the configuration for the formatting job. It runs all configs
 393                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 394             }
 395             else {
 396                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 397             }
 398             break
 399         case 'arm64':
 400             if (os.toLowerCase() == "windows_nt") {
 401                 // These are cross builds
 402                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 403             }
 404             else {
 405                 // Defaults to a small page size set of machines.
 406                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 407             }
 408             break
 409         case 'arm':
 410             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 411             break
 412         case 'armlb':
 413             // These are cross builds
 414             if (os == 'Tizen') {
 415                 // ABI: softfp
 416                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 417             }
 418             else {
 419                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 420             }
 421             break
 422         case 'x86':
 423         case 'x86lb':
 424         case 'x86_arm_altjit':
 425         case 'x64_arm64_altjit':
 426             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 427             break
 428         default:
 429             println("Unknown architecture: ${architecture}");
 430             assert false
 431             break
 432     }
 433
 434     return baseName + suffix
 435 }
 436
 437 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 438
 439     // Only PR triggers for altjit jobs.
 440     switch (architecture) {
 441         case 'x86_arm_altjit':
 442         case 'x64_arm64_altjit':
 443             return
 444         default:
 445             break
 446     }
 447
 448     // Check scenario.
 449     switch (scenario) {
 450         case 'default':
 451             switch (architecture) {
 452                 case 'x64':
 453                 case 'x86':
 454                 case 'x86lb':
 455                     if (architecture == 'x86' && os == 'Ubuntu') {
 456                         Utilities.addPeriodicTrigger(job, '@daily')
 457                     }
 458                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 459                         Utilities.addGithubPushTrigger(job)
 460                     }
 461                     break
 462                 case 'arm':
 463                 case 'armlb':
 464                     Utilities.addGithubPushTrigger(job)
 465                     break
 466                 case 'arm64':
 467                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 468                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 469                     break
 470                 default:
 471                     println("Unknown architecture: ${architecture}");
 472                     assert false
 473                     break
 474             }
 475             break
 476         case 'pri1':
 477             // Pri one gets a push trigger, and only for release
 478             if (architecture == 'x64') {
 479                 if (configuration == 'Release') {
 480                     // We expect release jobs to be Windows, or in the cross list
 481                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 482                     if (!os in bidailyCrossList) {
 483                         if (isFlowJob || os == 'Windows_NT') {
 484                             Utilities.addGithubPushTrigger(job)
 485                         }
 486                     }
 487                     else {
 488                         if (isFlowJob) {
 489                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 490                         }
 491                     }
 492                 }
 493             }
 494             break
 495         case 'r2r':
 496             //r2r jobs that aren't pri1 can only be triggered by phrase
 497             break
 498         case 'pri1r2r':
 499             assert !(os in bidailyCrossList)
 500             //pri1 r2r gets a push trigger for checked/release
 501             if (configuration == 'Checked' || configuration == 'Release') {
 502                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 503                 if (architecture == 'x64' && os != 'OSX10.12') {
 504                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 505                     if (isFlowJob || os == 'Windows_NT') {
 506                         Utilities.addGithubPushTrigger(job)
 507                     }
 508                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 509                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 510                     if (isFlowJob) {
 511                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 512                     }
 513                 }
 514                 // For x86, only add per-commit jobs for Windows
 515                 else if (architecture == 'x86' || architecture == 'x86lb') {
 516                     if (os == 'Windows_NT') {
 517                         Utilities.addGithubPushTrigger(job)
 518                     }
 519                 }
 520                 // arm64 pri1r2r jobs should only run every 12 hours.
 521                 else if (architecture == 'arm64') {
 522                     if (os == 'Windows_NT') {
 523                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 524                         // TODO: Add once external email sending is available again
 525                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 526                     }
 527                 }
 528             }
 529             break
 530         case 'r2r_jitstress1':
 531         case 'r2r_jitstress2':
 532         case 'r2r_jitstressregs1':
 533         case 'r2r_jitstressregs2':
 534         case 'r2r_jitstressregs3':
 535         case 'r2r_jitstressregs4':
 536         case 'r2r_jitstressregs8':
 537         case 'r2r_jitstressregs0x10':
 538         case 'r2r_jitstressregs0x80':
 539         case 'r2r_jitstressregs0x1000':
 540         case 'r2r_jitminopts':
 541         case 'r2r_jitforcerelocs':
 542         case 'gcstress15_pri1r2r':
 543             assert !(os in bidailyCrossList)
 544
 545             // GCStress=C is currently not supported on OS X
 546             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 547                 break
 548             }
 549
 550             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 551             if (configuration == 'Checked' || configuration == 'Release') {
 552                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 553                 if (architecture == 'x64') {
 554                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 555                     if (isFlowJob || os == 'Windows_NT') {
 556                         // Add a weekly periodic trigger
 557                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 558                     }
 559                 }
 560                 // For x86, only add per-commit jobs for Windows
 561                 else if (architecture == 'x86') {
 562                     if (os == 'Windows_NT') {
 563                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 564                     }
 565                 }
 566             }
 567             break
 568         case 'longgc':
 569             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 570             assert configuration == 'Release'
 571             assert architecture == 'x64'
 572             Utilities.addPeriodicTrigger(job, '@daily')
 573             // TODO: Add once external email sending is available again
 574             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 575             break
 576         case 'gcsimulator':
 577             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 578             assert configuration == 'Release'
 579             assert architecture == 'x64'
 580             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 581             // TODO: Add once external email sending is available again
 582             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 583             break
 584         case 'standalone_gc':
 585             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 586             assert (configuration == 'Release' || configuration == 'Checked')
 587             // TODO: Add once external email sending is available again
 588             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 589             Utilities.addPeriodicTrigger(job, '@weekly')
 590             break
 591         case 'gc_reliability_framework':
 592             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 593             assert (configuration == 'Release' || configuration == 'Checked')
 594             // Only triggered by phrase.
 595             break
 596         case 'ilrt':
 597             assert !(os in bidailyCrossList)
 598             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 599             if (architecture == 'x64' && configuration == 'Release') {
 600                 // We don't expect to see a job generated except in these scenarios
 601                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 602                 if (isFlowJob || os == 'Windows_NT') {
 603                     Utilities.addPeriodicTrigger(job, '@daily')
 604                 }
 605             }
 606             break
 607         case 'jitdiff':
 608             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 609             assert configuration == 'Checked'
 610             assert (architecture == 'x64' || architecture == 'x86')
 611             Utilities.addGithubPushTrigger(job)
 612             break
 613         case 'coverage':
 614             assert (os == 'Ubuntu' || os == 'Windows_NT')
 615             assert configuration == 'Release'
 616             assert architecture == 'x64'
 617             Utilities.addPeriodicTrigger(job, '@weekly')
 618             break
 619         case 'formatting':
 620             assert (os == 'Windows_NT' || os == "Ubuntu")
 621             assert architecture == 'x64'
 622             Utilities.addGithubPushTrigger(job)
 623             break
 624         case 'jitstressregs1':
 625         case 'jitstressregs2':
 626         case 'jitstressregs3':
 627         case 'jitstressregs4':
 628         case 'jitstressregs8':
 629         case 'jitstressregs0x10':
 630         case 'jitstressregs0x80':
 631         case 'jitstressregs0x1000':
 632         case 'minopts':
 633         case 'forcerelocs':
 634         case 'jitstress1':
 635         case 'jitstress2':
 636         case 'jitstress2_jitstressregs1':
 637         case 'jitstress2_jitstressregs2':
 638         case 'jitstress2_jitstressregs3':
 639         case 'jitstress2_jitstressregs4':
 640         case 'jitstress2_jitstressregs8':
 641         case 'jitstress2_jitstressregs0x10':
 642         case 'jitstress2_jitstressregs0x80':
 643         case 'jitstress2_jitstressregs0x1000':
 644         case 'tailcallstress':
 645         case 'jitsse2only':
 646         case 'corefx_baseline':
 647         case 'corefx_minopts':
 648         case 'corefx_jitstress1':
 649         case 'corefx_jitstress2':
 650         case 'corefx_jitstressregs1':
 651         case 'corefx_jitstressregs2':
 652         case 'corefx_jitstressregs3':
 653         case 'corefx_jitstressregs4':
 654         case 'corefx_jitstressregs8':
 655         case 'corefx_jitstressregs0x10':
 656         case 'corefx_jitstressregs0x80':
 657         case 'corefx_jitstressregs0x1000':
 658         case 'zapdisable':
 659             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 660                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 661                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 662                     if (os == 'Windows_NT') {
 663                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 664                         Utilities.addPeriodicTrigger(job, '@weekly')
 665                     }
 666                 }
 667                 else {
 668                     Utilities.addPeriodicTrigger(job, '@daily')
 669                 }
 670             }
 671             break
 672         case 'heapverify1':
 673         case 'gcstress0x3':
 674             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 675                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 676                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 677                     if (os == 'Windows_NT') {
 678                         Utilities.addPeriodicTrigger(job, '@daily')
 679                     }
 680                     // TODO: Add once external email sending is available again
 681                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 682                 }
 683                 else {
 684                     Utilities.addPeriodicTrigger(job, '@weekly')
 685                 }
 686             }
 687             break
 688         case 'gcstress0xc':
 689         case 'gcstress0xc_zapdisable':
 690         case 'gcstress0xc_zapdisable_jitstress2':
 691         case 'gcstress0xc_zapdisable_heapverify1':
 692         case 'gcstress0xc_jitstress1':
 693         case 'gcstress0xc_jitstress2':
 694         case 'gcstress0xc_minopts_heapverify1':
 695             // GCStress=C is currently not supported on OS X
 696             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 697                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 698                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 699                     // TODO: Enable a periodic trigger after tests are updated.
 700                     // Utilities.addPeriodicTrigger(job, '@daily')
 701                     // TODO: Add once external email sending is available again
 702                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 703                 }
 704                 else {
 705                     Utilities.addPeriodicTrigger(job, '@weekly')
 706                 }
 707             }
 708             break
 709
 710         case 'illink':
 711             // Testing on other operating systems TBD
 712             assert (os == 'Windows_NT' || os == 'Ubuntu')
 713             if (architecture == 'x64' || architecture == 'x86') {
 714                 if (configuration == 'Checked') {
 715                     Utilities.addPeriodicTrigger(job, '@daily')
 716                 }
 717             }
 718             break
 719
 720         case 'tieredcompilation':
 721         case 'corefx_tieredcompilation':
 722             // No periodic jobs just yet, still testing
 723             break
 724
 725         default:
 726             println("Unknown scenario: ${scenario}");
 727             assert false
 728             break
 729     }
 730     return
 731 }
 732
 733 // **************************
 734 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 735 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 736 // builds will do a couple extra steps.
 737 // **************************
 738
 739 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 740 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 741 // it's just used for internal builds
 742 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 743 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 744     if (isWindowsBuildOnlyJob) {
 745         return
 746     }
 747
 748     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 749     // Non pull request builds.
 750     if (!isPR) {
 751         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 752         return
 753     }
 754
 755      def arm64Users = [
 756         'adiaaida',
 757         'AndyAyersMS',
 758         'briansull',
 759         'BruceForstall',
 760         'CarolEidt',
 761         'cmckinsey',
 762         'erozenfeld',
 763         'jashook',
 764         'JosephTremoulet',
 765         'pgodeq',
 766         'pgavlin',
 767         'rartemev',
 768         'russellhadley',
 769         'RussKeldorph',
 770         'sandreenko',
 771         'sdmaclea',
 772         'swaroop-sridhar',
 773         'jkotas',
 774         'markwilkie',
 775         'weshaggard'
 776     ]
 777
 778     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 779     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 780     def osGroup = getOSGroup(os)
 781     switch (architecture) {
 782         case 'x64': // editor brace matching: {
 783             if (scenario == 'coverage') {
 784                 assert configuration == 'Release'
 785                 if (os == 'Ubuntu') {
 786                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 787                 }
 788                 break
 789             }
 790
 791             if (scenario == 'formatting') {
 792                 assert configuration == 'Checked'
 793                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 794                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 795                 }
 796                 break
 797             }
 798
 799             switch (os) {
 800                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 801                 case 'Debian8.4':
 802                 case 'RHEL7.2':
 803                     if (scenario == 'default') {
 804                         assert !isFlowJob
 805                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 806                     }
 807                     else if (scenario == 'pri1' && isFlowJob) {
 808                         assert (configuration == 'Release')
 809                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 810                     }
 811                     break
 812                 case 'Ubuntu16.04':
 813                     assert !isFlowJob
 814                     assert scenario == 'default'
 815                     // Distinguish with the other architectures (arm and x86)
 816                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 817                     break
 818                 case 'Fedora24':
 819                 case 'Ubuntu16.10':
 820                     assert !isFlowJob
 821                     assert scenario == 'default'
 822                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 823                     break
 824                 case 'Ubuntu':
 825                     if (scenario == 'illink') {
 826                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 827                         break
 828                     }
 829                     // fall through
 830                 case 'OSX10.12':
 831                     // Triggers on the non-flow jobs aren't necessary here
 832                     // Corefx testing uses non-flow jobs.
 833                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 834                         break
 835                     }
 836                     switch (scenario) {
 837                         case 'default':
 838                             // Ubuntu uses checked for default PR tests
 839                             if (configuration == 'Checked') {
 840                                 // Default trigger
 841                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 842                             }
 843                             break
 844                         case 'pri1':
 845                             if (configuration == 'Release') {
 846                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 847                             }
 848                             break
 849                         case 'jitdiff':
 850                             if (configuration == 'Checked') {
 851                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 852                             }
 853                             break
 854                         case 'ilrt':
 855                             if (configuration == 'Release') {
 856                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 857                             }
 858                             break
 859                         case 'r2r':
 860                             if (configuration == 'Release' || configuration == 'Checked') {
 861                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 862                             }
 863                             break
 864                         case 'pri1r2r':
 865                             if (configuration == 'Release' || configuration == 'Checked') {
 866                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 867                             }
 868                             break
 869                         case 'gcstress15_pri1r2r':
 870                             if (configuration == 'Release' || configuration == 'Checked') {
 871                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 872                             }
 873                             break
 874                         case 'r2r_jitstress1':
 875                         case 'r2r_jitstress2':
 876                         case 'r2r_jitstressregs1':
 877                         case 'r2r_jitstressregs2':
 878                         case 'r2r_jitstressregs3':
 879                         case 'r2r_jitstressregs4':
 880                         case 'r2r_jitstressregs8':
 881                         case 'r2r_jitstressregs0x10':
 882                         case 'r2r_jitstressregs0x80':
 883                         case 'r2r_jitstressregs0x1000':
 884                         case 'r2r_jitminopts':
 885                         case 'r2r_jitforcerelocs':
 886                             if (configuration == 'Release' || configuration == 'Checked') {
 887                                 def displayStr = getR2RStressModeDisplayName(scenario)
 888                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 889                             }
 890                             break
 891                         case 'longgc':
 892                             if (configuration == 'Release') {
 893                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 894                             }
 895                             break
 896                         case 'gcsimulator':
 897                             if (configuration == 'Release') {
 898                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 899                             }
 900                             break
 901                         case 'standalone_gc':
 902                             if (configuration == 'Release' || configuration == 'Checked') {
 903                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 904                             }
 905                             break
 906                         case 'gc_reliability_framework':
 907                             if (configuration == 'Release' || configuration == 'Checked') {
 908                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 909                             }
 910                             break
 911                         case 'minopts':
 912                         case 'tieredcompilation':
 913                         case 'forcerelocs':
 914                         case 'jitstress1':
 915                         case 'jitstress2':
 916                         case 'jitstressregs1':
 917                         case 'jitstressregs2':
 918                         case 'jitstressregs3':
 919                         case 'jitstressregs4':
 920                         case 'jitstressregs8':
 921                         case 'jitstressregs0x10':
 922                         case 'jitstressregs0x80':
 923                         case 'jitstressregs0x1000':
 924                         case 'jitstress2_jitstressregs1':
 925                         case 'jitstress2_jitstressregs2':
 926                         case 'jitstress2_jitstressregs3':
 927                         case 'jitstress2_jitstressregs4':
 928                         case 'jitstress2_jitstressregs8':
 929                         case 'jitstress2_jitstressregs0x10':
 930                         case 'jitstress2_jitstressregs0x80':
 931                         case 'jitstress2_jitstressregs0x1000':
 932                         case 'tailcallstress':
 933                         case 'jitsse2only':
 934                         case 'gcstress0x3':
 935                         case 'gcstress0xc':
 936                         case 'zapdisable':
 937                         case 'heapverify1':
 938                         case 'gcstress0xc_zapdisable':
 939                         case 'gcstress0xc_zapdisable_jitstress2':
 940                         case 'gcstress0xc_zapdisable_heapverify1':
 941                         case 'gcstress0xc_jitstress1':
 942                         case 'gcstress0xc_jitstress2':
 943                         case 'gcstress0xc_minopts_heapverify1':
 944                             def displayStr = getStressModeDisplayName(scenario)
 945                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 946                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 947                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 948                             break
 949                         case 'corefx_baseline':
 950                         case 'corefx_minopts':
 951                         case 'corefx_tieredcompilation':
 952                         case 'corefx_jitstress1':
 953                         case 'corefx_jitstress2':
 954                         case 'corefx_jitstressregs1':
 955                         case 'corefx_jitstressregs2':
 956                         case 'corefx_jitstressregs3':
 957                         case 'corefx_jitstressregs4':
 958                         case 'corefx_jitstressregs8':
 959                         case 'corefx_jitstressregs0x10':
 960                         case 'corefx_jitstressregs0x80':
 961                         case 'corefx_jitstressregs0x1000':
 962                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 963                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 964                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 965                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 966                             break
 967                         default:
 968                             println("Unknown scenario: ${scenario}");
 969                             assert false
 970                             break
 971                     }
 972                     break
 973                 case 'CentOS7.1':
 974                     switch (scenario) {
 975                         case 'pri1':
 976                             if (configuration == 'Release') {
 977                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 978                             }
 979                             break
 980                         case 'r2r':
 981                             if (configuration == 'Checked' || configuration == 'Release') {
 982                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 983                             }
 984                             break
 985                         case 'pri1r2r':
 986                             if (configuration == 'Checked' || configuration == 'Release') {
 987                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 988                             }
 989                             break
 990                         case 'gcstress15_pri1r2r':
 991                             if (configuration == 'Release' || configuration == 'Checked') {
 992                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 993                             }
 994                             break
 995                         case 'r2r_jitstress1':
 996                         case 'r2r_jitstress2':
 997                         case 'r2r_jitstressregs1':
 998                         case 'r2r_jitstressregs2':
 999                         case 'r2r_jitstressregs3':
1000                         case 'r2r_jitstressregs4':
1001                         case 'r2r_jitstressregs8':
1002                         case 'r2r_jitstressregs0x10':
1003                         case 'r2r_jitstressregs0x80':
1004                         case 'r2r_jitstressregs0x1000':
1005                         case 'r2r_jitminopts':
1006                         case 'r2r_jitforcerelocs':
1007                             if (configuration == 'Release' || configuration == 'Checked') {
1008                                 def displayStr = getR2RStressModeDisplayName(scenario)
1009                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1010                             }
1011                             break
1012                         default:
1013                             break
1014                     }
1015                 case 'Windows_NT':
1016                     switch (scenario) {
1017                         case 'default':
1018                             // Default trigger
1019                             if (configuration == 'Debug') {
1020                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1021                             }
1022                             break
1023                         case 'pri1':
1024                             // Default trigger
1025                             if (configuration == 'Release') {
1026                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1027                             }
1028                             break
1029                         case 'jitdiff':
1030                             if (configuration == 'Checked') {
1031                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1032                             }
1033                             break
1034                         case 'ilrt':
1035                             if (configuration == 'Release') {
1036                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1037                             }
1038                             break
1039                         case 'r2r':
1040                             if (configuration == 'Checked' || configuration == 'Release') {
1041                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1042                             }
1043                             break
1044                         case 'pri1r2r':
1045                             if (configuration == 'Checked' || configuration == 'Release') {
1046                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1047                             }
1048                             break
1049                         case 'gcstress15_pri1r2r':
1050                             if (configuration == 'Release' || configuration == 'Checked') {
1051                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1052                             }
1053                             break
1054                         case 'r2r_jitstress1':
1055                         case 'r2r_jitstress2':
1056                         case 'r2r_jitstressregs1':
1057                         case 'r2r_jitstressregs2':
1058                         case 'r2r_jitstressregs3':
1059                         case 'r2r_jitstressregs4':
1060                         case 'r2r_jitstressregs8':
1061                         case 'r2r_jitstressregs0x10':
1062                         case 'r2r_jitstressregs0x80':
1063                         case 'r2r_jitstressregs0x1000':
1064                         case 'r2r_jitminopts':
1065                         case 'r2r_jitforcerelocs':
1066                             if (configuration == 'Release' || configuration == 'Checked') {
1067                                 def displayStr = getR2RStressModeDisplayName(scenario)
1068                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1069                             }
1070                             break
1071                         case 'longgc':
1072                             if (configuration == 'Release') {
1073                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1074                             }
1075                             break
1076                         case 'gcsimulator':
1077                             if (configuration == 'Release') {
1078                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1079                             }
1080                             break
1081                         case 'standalone_gc':
1082                             if (configuration == 'Release' || configuration == 'Checked') {
1083                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1084                             }
1085                             break
1086                         case 'gc_reliability_framework':
1087                             if (configuration == 'Release' || configuration == 'Checked') {
1088                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1089                             }
1090                             break
1091                         case 'minopts':
1092                         case 'tieredcompilation':
1093                         case 'forcerelocs':
1094                         case 'jitstress1':
1095                         case 'jitstress2':
1096                         case 'jitstressregs1':
1097                         case 'jitstressregs2':
1098                         case 'jitstressregs3':
1099                         case 'jitstressregs4':
1100                         case 'jitstressregs8':
1101                         case 'jitstressregs0x10':
1102                         case 'jitstressregs0x80':
1103                         case 'jitstressregs0x1000':
1104                         case 'jitstress2_jitstressregs1':
1105                         case 'jitstress2_jitstressregs2':
1106                         case 'jitstress2_jitstressregs3':
1107                         case 'jitstress2_jitstressregs4':
1108                         case 'jitstress2_jitstressregs8':
1109                         case 'jitstress2_jitstressregs0x10':
1110                         case 'jitstress2_jitstressregs0x80':
1111                         case 'jitstress2_jitstressregs0x1000':
1112                         case 'tailcallstress':
1113                         case 'jitsse2only':
1114                         case 'gcstress0x3':
1115                         case 'gcstress0xc':
1116                         case 'zapdisable':
1117                         case 'heapverify1':
1118                         case 'gcstress0xc_zapdisable':
1119                         case 'gcstress0xc_zapdisable_jitstress2':
1120                         case 'gcstress0xc_zapdisable_heapverify1':
1121                         case 'gcstress0xc_jitstress1':
1122                         case 'gcstress0xc_jitstress2':
1123                         case 'gcstress0xc_minopts_heapverify1':
1124                             def displayStr = getStressModeDisplayName(scenario)
1125                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1126                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1127                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1128                             break
1129                         case 'corefx_baseline':
1130                         case 'corefx_minopts':
1131                         case 'corefx_tieredcompilation':
1132                         case 'corefx_jitstress1':
1133                         case 'corefx_jitstress2':
1134                         case 'corefx_jitstressregs1':
1135                         case 'corefx_jitstressregs2':
1136                         case 'corefx_jitstressregs3':
1137                         case 'corefx_jitstressregs4':
1138                         case 'corefx_jitstressregs8':
1139                         case 'corefx_jitstressregs0x10':
1140                         case 'corefx_jitstressregs0x80':
1141                         case 'corefx_jitstressregs0x1000':
1142                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1143                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1144                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1145                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1146                             break
1147                         case 'illink':
1148                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1149                             break
1150                         default:
1151                             println("Unknown scenario: ${scenario}");
1152                             assert false
1153                             break
1154                     }
1155                     break
1156                 default:
1157                     println("Unknown os: ${os}");
1158                     assert false
1159                     break
1160             }
1161             break
1162         // editor brace matching: }
1163         case 'armlb':
1164         case 'arm': // editor brace matching: {
1165             switch (os) {
1166                 case 'Ubuntu':
1167                 case 'Ubuntu16.04':
1168                     if (architecture == 'armlb') { // Ubuntu arm is only for armlb currently
1169                         assert scenario == 'default'
1170                         job.with {
1171                             publishers {
1172                                 azureVMAgentPostBuildAction {
1173                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1174                                 }
1175                             }
1176                         }
1177                         if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1178                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1179                         }
1180                         else {
1181                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1182                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1183                         }
1184                     }
1185                     break
1186                 case 'Tizen':
1187                     if (architecture == 'armlb') { // Ubuntu arm is only for armlb currently
1188                         architecture='armel'
1189                         job.with {
1190                             publishers {
1191                                 azureVMAgentPostBuildAction {
1192                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1193                                 }
1194                             }
1195                         }
1196                         // Removing the regex will cause this to run on each PR.
1197                         if (configuration == 'Release' || configuration == 'Debug') {
1198                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1199                         }
1200                         else {
1201                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1202                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1203                         }
1204                     }
1205                     break
1206                 case 'Windows_NT':
1207                     // Set up a private trigger
1208                     def contextString = "${os} ${architecture} Cross ${configuration}"
1209                     if (scenario != 'default')
1210                         contextString += " ${scenario}"
1211                     contextString += " Build"
1212                     // Debug configuration only does builds, no tests.
1213                     if (configuration != 'Debug') {
1214                         contextString += " and Test"
1215                     }
1216                     switch (scenario) {
1217                         case 'default':
1218                             // Only Checked is a default trigger.
1219                             if (configuration == 'Debug' || configuration == 'Release')
1220                             {
1221                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1222                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}.*", null, arm64Users)
1223                             }
1224                             else
1225                             {
1226                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1227                             }
1228                             break
1229                         default:
1230                             // Stress jobs will use this code path.
1231                             if (Constants.validArmWindowsScenarios.contains(scenario)) {
1232                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1233                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1234                             }
1235                             break
1236                     }
1237                     break
1238                 default:
1239                     println("NYI os: ${os}");
1240                     assert false
1241                     break
1242             }
1243             break
1244         // editor brace matching: }
1245         case 'arm64': // editor brace matching: {
1246             assert Constants.validArmWindowsScenarios.contains(scenario)
1247
1248             // Set up a private trigger
1249             def contextString = "${os} ${architecture} Cross ${configuration}"
1250             if (scenario != 'default')
1251                 contextString += " ${scenario}"
1252             contextString += " Build"
1253             // Debug builds only.
1254             if (configuration != 'Debug') {
1255                contextString += " and Test"
1256             }
1257
1258             switch (os) {
1259                 case 'Ubuntu':
1260                 case 'Ubuntu16.04':
1261                     switch (scenario) {
1262                         case 'pri1':
1263                             if (configuration == 'Release') {
1264                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test",
1265                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1266                             }
1267                             break
1268                         case 'r2r':
1269                             if (configuration == 'Checked' || configuration == 'Release') {
1270                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1271                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1272                             }
1273                             break
1274                         case 'pri1r2r':
1275                             if (configuration == 'Checked' || configuration == 'Release') {
1276                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1277                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1278                             }
1279                             break
1280                         case 'gcstress15_pri1r2r':
1281                             if (configuration == 'Release' || configuration == 'Checked') {
1282                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1283                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1284                             }
1285                             break
1286                         case 'r2r_jitstress1':
1287                         case 'r2r_jitstress2':
1288                         case 'r2r_jitstressregs1':
1289                         case 'r2r_jitstressregs2':
1290                         case 'r2r_jitstressregs3':
1291                         case 'r2r_jitstressregs4':
1292                         case 'r2r_jitstressregs8':
1293                         case 'r2r_jitstressregs0x10':
1294                         case 'r2r_jitstressregs0x80':
1295                         case 'r2r_jitstressregs0x1000':
1296                         case 'r2r_jitminopts':
1297                         case 'r2r_jitforcerelocs':
1298                             if (configuration == 'Release' || configuration == 'Checked') {
1299                                 def displayStr = getR2RStressModeDisplayName(scenario)
1300                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1301                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1302                             }
1303                             break
1304                         default:
1305                             break
1306                     }
1307                 case 'Windows_NT':
1308                     switch (scenario) {
1309                         case 'default':
1310                             if (isFlowJob == true) {
1311                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration}",
1312                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1313                             }
1314                             // For now only run Debug jobs on PR Trigger.
1315                             else if (configuration != 'Debug') {
1316                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1317                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}.*", null, arm64Users)
1318                             }
1319                             else {
1320                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1321                                 // are at higher risk of ARM64-breaking changes.
1322                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1323                             }
1324                             break
1325                         default:
1326                             // Stress jobs will use this code path.
1327                             if (Constants.validArmWindowsScenarios.contains(scenario)) {
1328                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1329                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1330                             }
1331                             break
1332                     }
1333                     break
1334                 default:
1335                     println("NYI os: ${os}");
1336                     assert false
1337                     break
1338             }
1339             break
1340         // editor brace matching: }
1341         case 'x86': // editor brace matching: {
1342             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1343             if (os == 'Ubuntu') {
1344                 // on-demand only for ubuntu x86
1345                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1346                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1347                 break
1348             }
1349             switch (scenario) {
1350                 case 'default':
1351                     if (configuration == 'Checked') {
1352                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1353                     }
1354                     else if (configuration == 'Release') {
1355                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1356                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1357                     }
1358                     break
1359                 case 'pri1':
1360                     if (configuration == 'Release') {
1361                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1362                     }
1363                     break
1364                 case 'ilrt':
1365                     if (configuration == 'Release') {
1366                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1367                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1368                     }
1369                     break
1370                 case 'r2r':
1371                     if (configuration == 'Checked' || configuration == 'Release') {
1372                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1373                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1374                     }
1375                     break
1376                 case 'pri1r2r':
1377                     if (configuration == 'Checked' || configuration == 'Release') {
1378                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1379                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1380                     }
1381                     break
1382                 case 'gcstress15_pri1r2r':
1383                     if (configuration == 'Release' || configuration == 'Checked') {
1384                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1385                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1386                     }
1387                     break
1388                 case 'r2r_jitstress1':
1389                 case 'r2r_jitstress2':
1390                 case 'r2r_jitstressregs1':
1391                 case 'r2r_jitstressregs2':
1392                 case 'r2r_jitstressregs3':
1393                 case 'r2r_jitstressregs4':
1394                 case 'r2r_jitstressregs8':
1395                 case 'r2r_jitstressregs0x10':
1396                 case 'r2r_jitstressregs0x80':
1397                 case 'r2r_jitstressregs0x1000':
1398                 case 'r2r_jitminopts':
1399                 case 'r2r_jitforcerelocs':
1400                     if (configuration == 'Release' || configuration == 'Checked') {
1401                         def displayStr = getR2RStressModeDisplayName(scenario)
1402                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1403                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1404                     }
1405                     break
1406                 case 'longgc':
1407                     if (configuration == 'Release') {
1408                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1409                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1410                     }
1411                     break
1412                 case 'gcsimulator':
1413                     if (configuration == 'Release') {
1414                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1415                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1416                     }
1417                     break
1418                 case 'standalone_gc':
1419                     if (configuration == 'Release' || configuration == 'Checked') {
1420                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1421                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1422                     }
1423                     break
1424                 case 'minopts':
1425                 case 'tieredcompilation':
1426                 case 'forcerelocs':
1427                 case 'jitstress1':
1428                 case 'jitstress2':
1429                 case 'jitstressregs1':
1430                 case 'jitstressregs2':
1431                 case 'jitstressregs3':
1432                 case 'jitstressregs4':
1433                 case 'jitstressregs8':
1434                 case 'jitstressregs0x10':
1435                 case 'jitstressregs0x80':
1436                 case 'jitstressregs0x1000':
1437                 case 'jitstress2_jitstressregs1':
1438                 case 'jitstress2_jitstressregs2':
1439                 case 'jitstress2_jitstressregs3':
1440                 case 'jitstress2_jitstressregs4':
1441                 case 'jitstress2_jitstressregs8':
1442                 case 'jitstress2_jitstressregs0x10':
1443                 case 'jitstress2_jitstressregs0x80':
1444                 case 'jitstress2_jitstressregs0x1000':
1445                 case 'tailcallstress':
1446                 case 'jitsse2only':
1447                 case 'gcstress0x3':
1448                 case 'gcstress0xc':
1449                 case 'zapdisable':
1450                 case 'heapverify1':
1451                 case 'gcstress0xc_zapdisable':
1452                 case 'gcstress0xc_zapdisable_jitstress2':
1453                 case 'gcstress0xc_zapdisable_heapverify1':
1454                 case 'gcstress0xc_jitstress1':
1455                 case 'gcstress0xc_jitstress2':
1456                 case 'gcstress0xc_minopts_heapverify1':
1457                     def displayStr = getStressModeDisplayName(scenario)
1458                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1459                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1460                     break
1461                 case 'corefx_baseline':
1462                 case 'corefx_minopts':
1463                 case 'corefx_tieredcompilation':
1464                 case 'corefx_jitstress1':
1465                 case 'corefx_jitstress2':
1466                 case 'corefx_jitstressregs1':
1467                 case 'corefx_jitstressregs2':
1468                 case 'corefx_jitstressregs3':
1469                 case 'corefx_jitstressregs4':
1470                 case 'corefx_jitstressregs8':
1471                 case 'corefx_jitstressregs0x10':
1472                 case 'corefx_jitstressregs0x80':
1473                 case 'corefx_jitstressregs0x1000':
1474                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1475                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1476                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1477                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1478                     break
1479                 case 'illink':
1480                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1481                     break
1482                 default:
1483                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1484                     assert false
1485                     break
1486             }
1487             break
1488          // editor brace matching: }
1489         case 'x86lb': // editor brace matching: {
1490             assert (os == 'Windows_NT')
1491             assert (scenario == 'default' || Constants.r2rJitStressScenarios.indexOf(scenario) !=1)
1492
1493             def arch = 'x86'
1494             def jit = 'legacy_backend'
1495             switch (scenario) {
1496                 case 'default':
1497                     if (configuration == 'Checked') {
1498                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1499                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1500                     }
1501                     break
1502                 default:
1503                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1504                     assert false
1505                     break
1506             }
1507             break
1508         // editor brace matching: }
1509         case 'x64_arm64_altjit':
1510         case 'x86_arm_altjit': // editor brace matching: {
1511             assert (os == 'Windows_NT')
1512             switch (scenario) {
1513                 case 'default':
1514                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1515                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1516                     break
1517                 default:
1518                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1519                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1520                     break
1521             }
1522             break
1523         // editor brace matching: }
1524         default:
1525             println("Unknown architecture: ${architecture}");
1526             assert false
1527             break
1528     }
1529 }
1530
1531 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1532     def buildCommands = [];
1533     def osGroup = getOSGroup(os)
1534     def lowerConfiguration = configuration.toLowerCase()
1535
1536     // Calculate the build steps, archival, and xunit results
1537     switch (os) {
1538         case 'Windows_NT': // editor brace matching: {
1539             switch (architecture) {
1540                 case 'x64':
1541                 case 'x86':
1542                 case 'x86lb':
1543                 case 'x86_arm_altjit':
1544                 case 'x64_arm64_altjit':
1545                     def arch = architecture
1546                     def buildOpts = ''
1547                     if ((architecture == 'x86lb') || (architecture == 'x86_arm_altjit')) {
1548                         arch = 'x86'
1549                     }
1550                     else if (architecture == 'x64_arm64_altjit') {
1551                         arch = 'x64'
1552                     }
1553
1554                     if (scenario == 'illink') {
1555                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1556                     }
1557
1558                     // If it is a release build for windows, ensure PGO is used, else fail the build
1559                     if ((lowerConfiguration == 'release') &&
1560                         (scenario in Constants.basicScenarios) &&
1561                         (architecture != 'x86lb') &&
1562                         (architecture != 'x86_arm_altjit') &&
1563                         (architecture != 'x64_arm64_altjit')) {
1564
1565                         buildOpts += ' enforcepgo'
1566                     }
1567
1568                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1569                             scenario == 'default' ||
1570                             scenario == 'r2r' ||
1571                             scenario == 'jitdiff' ||
1572                             scenario == 'ilrt' ||
1573                             scenario == 'illink' ||
1574                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1575                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1576                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1577                     }
1578
1579                     // For Pri 1 tests, we must shorten the output test binary path names.
1580                     // if __TestIntermediateDir is already set, build-test.cmd will
1581                     // output test binaries to that directory. If it is not set, the
1582                     // binaries are sent to a default directory whose name is about
1583                     // 35 characters long.
1584
1585                     else if (scenarioNeedsPri1Build(scenario)) {
1586                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1587                     }
1588                     else if (isLongGc(scenario)) {
1589                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1590                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1591                     }
1592                     else if (scenario == 'standalone_gc') {
1593                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1594                     }
1595                     else if (scenario == 'formatting') {
1596                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1597                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1598                         break
1599                     }
1600                     else {
1601                         println("Unknown scenario: ${scenario}")
1602                         assert false
1603                     }
1604
1605                     if (!isBuildOnly) {
1606                         def crossgenStr = ''
1607                         def runcrossgentestsStr = ''
1608                         def runjitstressStr = ''
1609                         def runjitstressregsStr = ''
1610                         def runjitmioptsStr = ''
1611                         def runjitforcerelocsStr = ''
1612                         def runjitdisasmStr = ''
1613                         def runilasmroundtripStr = ''
1614                         def gcstressStr = ''
1615                         def gcTestArguments = ''
1616                         def illinkArguments = ''
1617                         def testEnvStr = ''
1618                         def runtestArguments = ''
1619
1620                         if (scenario == 'r2r' ||
1621                             scenario == 'pri1r2r' ||
1622                             scenario == 'gcstress15_pri1r2r' ||
1623                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1624
1625                             // If this is a crossgen build, pass 'crossgen' to runtest.cmd
1626                             crossgenStr = 'crossgen'
1627                             runcrossgentestsStr = 'runcrossgentests'
1628
1629                             if (scenario == 'r2r_jitstress1'){
1630                                 runjitstressStr = 'jitstress 1'
1631                             }
1632                             else if (scenario == 'r2r_jitstress2') {
1633                                 runjitstressStr = 'jitstress 2'
1634                             }
1635                             else if (scenario == 'r2r_jitstressregs1'){
1636                                 runjitstressregsStr = 'jitstressregs 1'
1637                             }
1638                             else if (scenario == 'r2r_jitstressregs2') {
1639                                 runjitstressregsStr = 'jitstressregs 2'
1640                             }
1641                             else if (scenario == 'r2r_jitstressregs3') {
1642                                 runjitstressregsStr = 'jitstressregs 3'
1643                                }
1644                             else if (scenario == 'r2r_jitstressregs4') {
1645                                 runjitstressregsStr = 'jitstressregs 4'
1646                             }
1647                             else if (scenario == 'r2r_jitstressregs8') {
1648                                 runjitstressregsStr = 'jitstressregs 8'
1649                             }
1650                             else if (scenario == 'r2r_jitstressregs0x10') {
1651                                 runjitstressregsStr = 'jitstressregs 0x10'
1652                             }
1653                             else if (scenario == 'r2r_jitstressregs0x80') {
1654                                 runjitstressregsStr = 'jitstressregs 0x80'
1655                             }
1656                             else if (scenario == 'r2r_jitstressregs0x1000') {
1657                                 runjitstressregsStr = 'jitstressregs 0x1000'
1658                             }
1659                             else if (scenario == 'r2r_jitminopts') {
1660                                 runjitmioptsStr = 'jitminopts'
1661                             }
1662                             else if (scenario == 'r2r_jitforcerelocs') {
1663                                 runjitforcerelocsStr = 'jitforcerelocs'
1664                             }
1665                         }
1666                         else if (scenario == 'gcstress15_pri1r2r') {
1667                             gcstressStr = 'gcstresslevel 0xF'
1668                         }
1669                         else if (scenario == 'jitdiff') {
1670                             runjitdisasmStr = 'jitdisasm crossgen'
1671                         }
1672                         else if (scenario == 'ilrt') {
1673                             runilasmroundtripStr = 'ilasmroundtrip'
1674                         }
1675                         else if (isLongGc(scenario)) {
1676                             gcTestArguments = "${scenario} sequential"
1677                         }
1678                         else if (scenario == 'illink')
1679                         {
1680                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1681                         }
1682
1683                         def envScriptPath = ''
1684                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1685                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1686                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], envScriptPath)
1687                             if (architecture == 'x86lb') {
1688                                 buildCommands += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd", envScriptPath)
1689                             }
1690                             else if (architecture == 'x86_arm_altjit') {
1691                                 buildCommands += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1692                             }
1693                             else if (architecture == 'x64_arm64_altjit') {
1694                                 buildCommands += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1695                             }
1696                         }
1697                         else if (architecture == 'x86lb') {
1698                             envScriptPath = "%WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd"
1699                         }
1700                         else if (architecture == 'x86_arm_altjit') {
1701                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1702                         }
1703                         else if (architecture == 'x64_arm64_altjit') {
1704                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1705                         }
1706                         if (envScriptPath != '') {
1707                             testEnvStr = "TestEnv ${envScriptPath}"
1708                         }
1709
1710                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps ${testEnvStr}"
1711
1712                         // If we are running a stress mode, we should write out the set of key
1713                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1714
1715                         if (Constants.jitStressModeScenarios.containsKey(scenario) && enableCorefxTesting) {
1716                             def workspaceRelativeFxRoot = "_/fx"
1717                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1718
1719                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1720
1721                             setTestJobTimeOut(newJob, scenario)
1722
1723                             // Archive and process (only) the test results
1724                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1725                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1726
1727                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1728                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1729                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1730                         }
1731                         else if (isGcReliabilityFramework(scenario)) {
1732                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1733                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1734                         }
1735                         else {
1736                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1737                         }
1738                     }
1739
1740                     if (!enableCorefxTesting) {
1741                         // Run the rest of the build
1742                         // Build the mscorlib for the other OS's
1743                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1744                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1745
1746                         if (arch == "x64") {
1747                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1748                         }
1749
1750                         // Zip up the tests directory so that we don't use so much space/time copying
1751                         // 10s of thousands of files around.
1752                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1753
1754                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1755                             // For windows, pull full test results and test drops for x86/x64.
1756                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1757                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1758                         }
1759
1760                         if (scenario == 'jitdiff') {
1761                             // retrive jit-dasm output for base commit, and run jit-diff
1762                             if (!isBuildOnly) {
1763                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1764                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1765                             }
1766                         }
1767
1768                         if (!isBuildOnly) {
1769                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1770                             setTestJobTimeOut(newJob, scenario)
1771                         }
1772                     }
1773                     break
1774                 case 'armlb':
1775                 case 'arm':
1776                     assert Constants.validArmWindowsScenarios.contains(scenario)
1777
1778                     // Set time out
1779                     setTestJobTimeOut(newJob, scenario)
1780
1781                     def buildArchitecture = 'arm'
1782
1783                     if ( lowerConfiguration == "debug" ) {
1784                         // For Debug builds, we will do a P1 test build
1785                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=1"
1786                     }
1787                     else {
1788                         def priority = "1"
1789
1790                         if (scenario == "default")
1791                         {
1792                             priority = "0"
1793                         }
1794
1795                         if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1796                         {
1797                            // Up the timeout for arm checked testing only.
1798                            // Keep the longer timeout for gcstress.
1799                            Utilities.setJobTimeout(newJob, 240)
1800                         }
1801
1802                         def machineAffinityOptions = ['use_arm64_build_machine' : true]
1803                         setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1804
1805                         if (priority == "1")
1806                         {
1807                             buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=1"
1808                         }
1809                         else
1810                         {
1811                             buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture}"
1812                         }
1813
1814                         // Also run testing.
1815                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${buildArchitecture} -build_type ${lowerConfiguration} -scenario ${scenario} -testarch ${architecture} -priority ${priority} -key_location C:\\tools\\key.txt"
1816                     }
1817                     // Add archival.
1818                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1819                     break
1820                 case 'arm64':
1821                     assert Constants.validArmWindowsScenarios.contains(scenario)
1822
1823                     def priority = "1"
1824
1825                     if (scenario == "default")
1826                     {
1827                         priority = "0"
1828                     }
1829
1830                     // Set time out
1831                     setTestJobTimeOut(newJob, scenario)
1832
1833                     // Debug runs take too long to run. So build job only.
1834                     if (lowerConfiguration == "debug") {
1835                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1836                     }
1837                     else {
1838                         if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1839                             // Up the timeout for arm64 checked testing only.
1840                             // Keep the longer timeout for gcstress.
1841                             Utilities.setJobTimeout(newJob, 240)
1842                         }
1843
1844                         if (priority == "1") {
1845                             buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=1"
1846                         }
1847                         else{
1848                             buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1849                         }
1850
1851                         // Test build and run are launched together.
1852                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -testarch ${architecture} -priority ${priority} -key_location C:\\tools\\key.txt"
1853                         //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1854                     }
1855
1856                     // Add archival.
1857                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1858                     break
1859                 default:
1860                     println("Unknown architecture: ${architecture}");
1861                     assert false
1862                     break
1863             }
1864             break
1865         // editor brace matching: }
1866         case 'Ubuntu':
1867         case 'Ubuntu16.04':
1868         case 'Ubuntu16.10':
1869         case 'Debian8.4':
1870         case 'OSX10.12':
1871         case 'CentOS7.1':
1872         case 'RHEL7.2':
1873         case 'Tizen':
1874         case 'Fedora24': // editor brace matching: {
1875             switch (architecture) {
1876                 case 'x64':
1877                 case 'x86':
1878                     if (architecture == 'x86' && os == 'Ubuntu') {
1879                         // build and PAL test
1880                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1881                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1882                         break
1883                     }
1884
1885                     if (scenario == 'formatting') {
1886                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1887                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1888                         break
1889                     }
1890
1891                     if (scenario == 'illink') {
1892                         assert(os == 'Ubuntu')
1893                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1894                     }
1895
1896                     def standaloneGc = ''
1897                     if (scenario == 'standalone_gc') {
1898                         standaloneGc = 'buildstandalonegc'
1899                     }
1900
1901                     if (!enableCorefxTesting) {
1902                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1903                         // only on supported OS platforms.
1904                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1905                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1906                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1907                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1908
1909                         // Set time out
1910                         setTestJobTimeOut(newJob, scenario)
1911                         // Basic archiving of the build
1912                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1913                         // And pal tests
1914                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1915                     }
1916                     else {
1917                         // Corefx stress testing
1918                         assert os == 'Ubuntu'
1919                         assert architecture == 'x64'
1920                         assert lowerConfiguration == 'checked'
1921                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1922
1923                         // Build coreclr
1924                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1925
1926                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1927                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1928
1929                         // Build and text corefx
1930                         def workspaceRelativeFxRoot = "_/fx"
1931                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1932
1933                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1934
1935                         setTestJobTimeOut(newJob, scenario)
1936
1937                         // Archive and process (only) the test results
1938                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1939                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1940                     }
1941                     break
1942                 case 'arm64':
1943                     def standaloneGc = ''
1944                     if (scenario == 'standalone_gc') {
1945                         standaloneGc = 'buildstandalonegc'
1946                     }
1947
1948                     if (!enableCorefxTesting) {
1949                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8 ${standaloneGc}"
1950
1951                         // HACK -- Arm64 does not have corefx jobs yet.
1952                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1953                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1954                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1955                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1956
1957                         // Set time out
1958                         setTestJobTimeOut(newJob, scenario)
1959                         // Basic archiving of the build
1960                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1961                     }
1962                     break
1963                 case 'armlb':
1964                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1965                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1966
1967                     // default values for Ubuntu
1968                     def arm_abi="arm"
1969                     def linuxCodeName="trusty"
1970                     if (os == 'Ubuntu16.04') {
1971                         linuxCodeName="xenial"
1972                     }
1973                     else if (os == 'Tizen') {
1974                         arm_abi="armel"
1975                         linuxCodeName="tizen"
1976                     }
1977
1978                     // Unzip the Windows test binaries first. Exit with 0
1979                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1980
1981                     // Unpack the corefx binaries
1982                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1983                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1984                     if (os != 'Tizen') {
1985                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1986                     }
1987                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1988                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1989                     if (os == 'Ubuntu') {
1990                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1991                     }
1992
1993                     // Call the ARM CI script to cross build and test using docker
1994                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1995                     --mode=docker \\
1996                     --${arm_abi} \\
1997                     --linuxCodeName=${linuxCodeName} \\
1998                     --buildConfig=${lowerConfiguration} \\
1999                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2000                     --coreFxBinDir=./bin/CoreFxBinDir \\
2001                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2002
2003                     // Basic archiving of the build, no pal tests
2004                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
2005                     break
2006                 default:
2007                     println("Unknown architecture: ${architecture}");
2008                     assert false
2009                     break
2010             }
2011             break
2012         // editor brace matching: }
2013         default:
2014             println("Unknown os: ${os}");
2015             assert false
2016             break
2017     } // os
2018
2019     return buildCommands
2020 }
2021
2022 // Additional scenario which can alter behavior
2023
2024 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
2025 combinedScenarios.each { scenario ->
2026     [true, false].each { isPR ->
2027         Constants.architectureList.each { architecture ->
2028             Constants.configurationList.each { configuration ->
2029                 Constants.osList.each { os ->
2030                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2031                     // and reset the os to Windows_NT
2032                     def isBuildOnly = false
2033                     if (os == 'Windows_NT_BuildOnly') {
2034                         isBuildOnly = true
2035                         os = 'Windows_NT'
2036                     }
2037
2038                     // Tizen is only supported for arm legacy_backend architecture
2039                     if (os == 'Tizen' && architecture != 'armlb') {
2040                         return
2041                     }
2042
2043                     // Skip totally unimplemented (in CI) configurations.
2044                     switch (architecture) {
2045                         case 'arm64':
2046                             if (os == 'Ubuntu16.04') {
2047                                 os = 'Ubuntu'
2048                             }
2049
2050                             // Windows and Ubuntu only
2051                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2052                                 return
2053                             }
2054                             break
2055                         case 'armlb':
2056                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2057                                 return
2058                             }
2059                             break
2060                         case 'arm':
2061                             if (os != 'Windows_NT') {
2062                                 return
2063                             }
2064                             break
2065                         case 'x86':
2066                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2067                                 return
2068                             }
2069                             break
2070                         case 'x86lb':
2071                         case 'x86_arm_altjit':
2072                         case 'x64_arm64_altjit':
2073                             if (os != 'Windows_NT') {
2074                                 return
2075                             }
2076                             break
2077                         case 'x64':
2078                             // Everything implemented
2079                             break
2080                         default:
2081                             println("Unknown architecture: ${architecture}")
2082                             assert false
2083                             break
2084                     }
2085
2086                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2087                     // with checked builds)
2088                     def enableCorefxTesting = false
2089                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2090                         if (configuration != 'Checked') {
2091                             return
2092                         }
2093
2094                         enableCorefxTesting = isCorefxTesting(scenario)
2095
2096                         // Since these are just execution time differences,
2097                         // skip platforms that don't execute the tests here (Windows_NT only)
2098                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
2099                         if (!isEnabledOS || isBuildOnly) {
2100                             return
2101                         }
2102
2103                         switch (architecture) {
2104                             case 'armlb':
2105                             case 'arm':
2106                             case 'arm64':
2107                                 if (!Constants.validArmWindowsScenarios.contains(scenario)) {
2108                                     return
2109                                 }
2110                                 break
2111                             case 'x64':
2112                             case 'x86':
2113                             case 'x86_arm_altjit':
2114                             case 'x64_arm64_altjit':
2115                                 // x86 ubuntu: default only
2116                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2117                                     return
2118                                 }
2119                                 // Windows: Everything implemented
2120                                 break
2121                             case 'x86lb':
2122                                 // No stress modes for legacy jit.
2123                                 // (There's no technical reason we couldn't allow these.)
2124                                 return
2125                             default:
2126                                 return
2127                         }
2128                     }
2129                     else {
2130                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
2131                         // and configuration is not Checked, bail out.
2132                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2133                             return;
2134                         }
2135
2136                         // Skip scenarios
2137                         switch (scenario) {
2138                             case 'pri1':
2139                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
2140                                 // the default scenario build
2141                                 if (os != 'Windows_NT') {
2142                                     return
2143                                 }
2144                                 // Only x64 for now
2145                                 if (architecture != 'x64') {
2146                                     return
2147                                 }
2148                                 break
2149                             case 'ilrt':
2150                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2151                                 // the default scenario build
2152                                 if (os != 'Windows_NT') {
2153                                     return
2154                                 }
2155                                 // Only x64 for now
2156                                 if (architecture != 'x64') {
2157                                     return
2158                                 }
2159                                 // Release only
2160                                 if (configuration != 'Release') {
2161                                     return
2162                                 }
2163                                 break
2164                             case 'jitdiff':
2165                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2166                                     return
2167                                 }
2168                                 if (architecture != 'x64') {
2169                                     return
2170                                 }
2171                                 if (configuration != 'Checked') {
2172                                     return
2173                                 }
2174                                 break
2175                             case 'r2r':
2176                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2177                                 // the default scenario build
2178                                 if (os != 'Windows_NT') {
2179                                     return
2180                                 }
2181                                 if (architecture != 'x64') {
2182                                     return
2183                                 }
2184                                 break
2185                             case 'pri1r2r':
2186                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2187                                 // the default scenario build
2188                                 if (os != 'Windows_NT') {
2189                                     return
2190                                 }
2191                                 if (architecture != 'x64') {
2192                                     if ((architecture != 'arm64' && architecture != 'armlb' && architecture != 'arm') || (configuration == 'Debug')) {
2193                                         return
2194                                     }
2195                                 }
2196                                 break
2197                             case 'gcstress15_pri1r2r':
2198                             case 'r2r_jitstress1':
2199                             case 'r2r_jitstress2':
2200                             case 'r2r_jitstressregs1':
2201                             case 'r2r_jitstressregs2':
2202                             case 'r2r_jitstressregs3':
2203                             case 'r2r_jitstressregs4':
2204                             case 'r2r_jitstressregs8':
2205                             case 'r2r_jitstressregs0x10':
2206                             case 'r2r_jitstressregs0x80':
2207                             case 'r2r_jitstressregs0x1000':
2208                             case 'r2r_jitminopts':
2209                             case 'r2r_jitforcerelocs':
2210                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
2211                                 // the default scenario build
2212                                 if (os != 'Windows_NT') {
2213                                     return
2214                                 }
2215                                 if (architecture != 'x64' && architecture != 'x86') {
2216                                     return
2217                                 }
2218                                 break
2219                             case 'longgc':
2220                             case 'gcsimulator':
2221                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2222                                     return
2223                                 }
2224                                 if (architecture != 'x64') {
2225                                     return
2226                                 }
2227                                 if (configuration != 'Release') {
2228                                     return
2229                                 }
2230                                 break
2231                             case 'gc_reliability_framework':
2232                             case 'standalone_gc':
2233                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2234                                     return
2235                                 }
2236
2237                                 if (architecture != 'x64') {
2238                                     return
2239                                 }
2240
2241                                 if (configuration != 'Release' && configuration != 'Checked') {
2242                                     return
2243                                 }
2244                                 break
2245                             // We need Windows x64 Release bits for the code coverage build
2246                             case 'coverage':
2247                                 if (os != 'Windows_NT') {
2248                                     return
2249                                 }
2250                                 if (architecture != 'x64') {
2251                                     return
2252                                 }
2253                                 if (configuration != 'Release') {
2254                                     return
2255                                 }
2256                                 break
2257                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2258                             case 'formatting':
2259                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2260                                     return
2261                                 }
2262                                 if (architecture != 'x64') {
2263                                     return
2264                                 }
2265                                 if (configuration != 'Checked') {
2266                                     return
2267                                 }
2268                                 if (isBuildOnly) {
2269                                     return
2270                                 }
2271                                 break
2272                             case 'illink':
2273                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2274                                     return
2275                                 }
2276                                 if (architecture != 'x64' && architecture != 'x86') {
2277                                     return
2278                                 }
2279                                 if (isBuildOnly) {
2280                                     return
2281                                 }
2282                                 break
2283                             case 'default':
2284                                 // Nothing skipped
2285                                 break
2286                             default:
2287                                 println("Unknown scenario: ${scenario}")
2288                                 assert false
2289                                 break
2290                         }
2291                     }
2292
2293                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2294                     // runtime behavior, not compile-time behavior, are not interesting.
2295                     switch (architecture) {
2296                         case 'x86_arm_altjit':
2297                         case 'x64_arm64_altjit':
2298                             if (isGCStressRelatedTesting(scenario)) {
2299                                 return
2300                             }
2301                             break
2302                         default:
2303                             break
2304                     }
2305
2306                     // Calculate names
2307                     def lowerConfiguration = configuration.toLowerCase()
2308                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2309                     def folderName = getJobFolder(scenario)
2310
2311                     // Create the new job
2312                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2313
2314                     def machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2315                     machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb') ? ['use_arm64_build_machine': false] : machineAffinityOptions
2316                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2317
2318                     // Add all the standard options
2319                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2320                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2321
2322                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
2323                     def osGroup = getOSGroup(os)
2324
2325                     newJob.with {
2326                         steps {
2327                             if (os == 'Windows_NT') {
2328                                 buildCommands.each { buildCommand ->
2329                                     batchFile(buildCommand)
2330                                 }
2331                             }
2332                             else {
2333                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2334                                 if ( architecture == 'armlb' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2335                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2336                                     // Define the Windows Tests and Corefx build job names
2337                                     def WindowTestsName = projectFolder + '/' +
2338                                                           Utilities.getFullJobName(project,
2339                                                                                    getJobName(lowerConfiguration,
2340                                                                                               'x64' ,
2341                                                                                               'windows_nt',
2342                                                                                               'default',
2343                                                                                               true),
2344                                                                                    false)
2345                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2346                                                        Utilities.getFolderName(branch)
2347
2348                                     // Copy the Windows test binaries and the Corefx build binaries
2349                                     copyArtifacts(WindowTestsName) {
2350                                         includePatterns('bin/tests/tests.zip')
2351                                         buildSelector {
2352                                             latestSuccessful(true)
2353                                         }
2354                                     }
2355
2356                                     def arm_abi = 'arm'
2357                                     def corefx_os = 'linux'
2358                                     if (os == 'Tizen') {
2359                                         arm_abi = 'armel'
2360                                         corefx_os = 'tizen'
2361                                     }
2362
2363                                     // Let's use release CoreFX to test checked CoreCLR,
2364                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2365                                     def corefx_lowerConfiguration = lowerConfiguration
2366                                     if ( lowerConfiguration == 'checked' ) {
2367                                         corefx_lowerConfiguration='release'
2368                                     }
2369
2370                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2371                                         includePatterns('bin/build.tar.gz')
2372                                         buildSelector {
2373                                             latestSuccessful(true)
2374                                         }
2375                                     }
2376                                 }
2377
2378                                 buildCommands.each { buildCommand ->
2379                                     shell(buildCommand)
2380                                 }
2381                             }
2382                         }
2383                     } // newJob.with
2384
2385                 } // os
2386             } // configuration
2387         } // architecture
2388     } // isPR
2389 } // scenario
2390
2391
2392 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2393 combinedScenarios.each { scenario ->
2394     [true, false].each { isPR ->
2395         // Architectures.  x64 only at this point
2396         ['x64', 'arm64'].each { architecture ->
2397             // Put the OS's supported for coreclr cross testing here
2398             Constants.crossList.each { os ->
2399                 if (architecture == 'arm64') {
2400                     if (os != "Ubuntu") {
2401                         return
2402                     }
2403                 }
2404
2405                 Constants.configurationList.each { configuration ->
2406
2407                     if (architecture == 'arm64') {
2408                         if (scenario != 'default' && scenario != 'pri1r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2409                             return
2410                         }
2411                     }
2412
2413                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2414                         if (configuration != 'Checked') {
2415                             return
2416                         }
2417                         if (isCorefxTesting(scenario)) {
2418                             return
2419                         }
2420                         //Skip stress modes for these scenarios
2421                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2422                             return
2423                         }
2424                     }
2425                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2426                     // and configuration is not Checked, bail out.
2427                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2428                         return;
2429                     }
2430                     // For CentOS, we only want Checked/Release pri1 builds.
2431                     else if (os == 'CentOS7.1') {
2432                         if (scenario != 'pri1' &&
2433                             scenario != 'r2r' &&
2434                             scenario != 'pri1r2r' &&
2435                             scenario != 'gcstress15_pri1r2r' &&
2436                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2437                             return
2438                         }
2439                         if (configuration != 'Checked' && configuration != 'Release') {
2440                             return
2441                         }
2442                     }
2443                     // For RedHat and Debian, we only do Release pri1 builds.
2444                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2445                         if (scenario != 'pri1') {
2446                             return
2447                         }
2448                         if (configuration != 'Release') {
2449                             return
2450                         }
2451                     }
2452                     else {
2453                         // Skip scenarios
2454                         switch (scenario) {
2455                             case 'pri1':
2456                                 // Nothing skipped
2457                                 break
2458                             case 'ilrt':
2459                                 // Release only
2460                                 if (configuration != 'Release') {
2461                                     return
2462                                 }
2463                                 break
2464                             case 'jitdiff':
2465                                 if (configuration != 'Checked') {
2466                                     return;
2467                                 }
2468                             case 'r2r':
2469                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2470                                 if (configuration != 'Checked' && configuration != 'Release') {
2471                                     return
2472                                 }
2473                                 break
2474                             case 'pri1r2r':
2475                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2476                                 if (configuration != 'Checked' && configuration != 'Release') {
2477                                     return
2478                                 }
2479                                 break
2480                             case 'gcstress15_pri1r2r':
2481                             case 'r2r_jitstress1':
2482                             case 'r2r_jitstress2':
2483                             case 'r2r_jitstressregs1':
2484                             case 'r2r_jitstressregs2':
2485                             case 'r2r_jitstressregs3':
2486                             case 'r2r_jitstressregs4':
2487                             case 'r2r_jitstressregs8':
2488                             case 'r2r_jitstressregs0x10':
2489                             case 'r2r_jitstressregs0x80':
2490                             case 'r2r_jitstressregs0x1000':
2491                             case 'r2r_jitminopts':
2492                             case 'r2r_jitforcerelocs':
2493                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2494                                 if (configuration != 'Checked' && configuration != 'Release') {
2495                                     return
2496                                 }
2497                                 break
2498                             case 'longgc':
2499                             case 'gcsimulator':
2500                                 // Long GC tests take a long time on non-Release builds
2501                                 if (configuration != 'Release') {
2502                                     return
2503                                 }
2504                                 break
2505                             case 'gc_reliability_framework':
2506                             case 'standalone_gc':
2507                                 if (configuration != 'Release' && configuration != 'Checked') {
2508                                     return
2509                                 }
2510                                 break
2511                             case 'coverage':
2512                                 //We only want Ubuntu Release for coverage
2513                                 if (os != 'Ubuntu') {
2514                                     return
2515                                 }
2516                                 if (configuration != 'Release') {
2517                                     return
2518                                 }
2519                             case 'formatting':
2520                                 return
2521                             case 'illink':
2522                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2523                                     return
2524                                 }
2525                                 break
2526                             case 'default':
2527                                 // Nothing skipped
2528                                 break
2529                             default:
2530                                 println("Unknown scenario: ${scenario}")
2531                                 assert false
2532                                 break
2533                         }
2534                     }
2535
2536                     def lowerConfiguration = configuration.toLowerCase()
2537                     def osGroup = getOSGroup(os)
2538                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2539
2540                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2541                     def inputScenario = 'default'
2542                     if (scenario == 'coverage') {
2543                         inputScenario = 'coverage'
2544                     }
2545                     def inputCoreCLRBuildName = projectFolder + '/' +
2546                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2547                     // If this is a stress scenario, there isn't any difference in the build job
2548                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2549                     // from the default scenario
2550                     def testBuildScenario = scenario
2551                     if (scenarioNeedsPri1Build(scenario)) {
2552                         testBuildScenario = 'pri1'
2553                     }
2554                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2555                         testBuildScenario = 'default'
2556                     }
2557                     def inputWindowTestsBuildName = ''
2558                     def inputWindowsTestBuildArch = architecture
2559                     if (architecture == "arm64") {
2560                         // Use the x64 test build for arm64 unix
2561                         inputWindowsTestBuildArch = "x64"
2562                     }
2563                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2564                         inputWindowTestsBuildName = projectFolder + '/' +
2565                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', 'default', true), isPR)
2566                     }
2567                     else {
2568                         inputWindowTestsBuildName = projectFolder + '/' +
2569                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2570                     }
2571                     // Enable Server GC for Ubuntu PR builds
2572                     def serverGCString = ''
2573
2574                     // Whether or not this test run should be run sequentially instead
2575                     // of in parallel. Only used for long GC tests.
2576                     def sequentialString = ''
2577
2578                     // Whether or not this test run should run a specific playlist.
2579                     // Only used for long GC tests.
2580
2581                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2582                     // for running long GC and GCSimulator tests, respectively. We don't use them
2583                     // here because using a playlist file produces much more readable output on the CI machines
2584                     // and reduces running time.
2585                     def playlistString = ''
2586
2587                     if (os == 'Ubuntu' && isPR){
2588                         serverGCString = '--useServerGC'
2589                     }
2590
2591                     // pass --crossgen to runtest.sh for crossgen builds
2592                     def crossgenStr = ''
2593                     def runcrossgentestsStr = ''
2594                     def runjitstressStr = ''
2595                     def runjitstressregsStr = ''
2596                     def runjitmioptsStr = ''
2597                     def runjitforcerelocsStr = ''
2598                     def runjitdisasmStr = ''
2599                     def runilasmroundtripStr = ''
2600                     def gcstressStr = ''
2601                     def illinkStr = ''
2602                     def layoutOnlyStr =''
2603
2604                     if (scenario == 'r2r' ||
2605                         scenario == 'pri1r2r' ||
2606                         scenario == 'gcstress15_pri1r2r' ||
2607                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2608                             crossgenStr = '--crossgen'
2609                             runcrossgentestsStr = '--runcrossgentests'
2610
2611                             if (scenario == 'r2r_jitstress1'){
2612                                 runjitstressStr = '--jitstress=1'
2613                             }
2614                             else if (scenario == 'r2r_jitstress2') {
2615                                 runjitstressStr = '--jitstress=2'
2616                             }
2617                             else if (scenario == 'r2r_jitstressregs1'){
2618                                 runjitstressregsStr = '--jitstressregs=1'
2619                             }
2620                             else if (scenario == 'r2r_jitstressregs2') {
2621                                 runjitstressregsStr = '--jitstressregs=2'
2622                             }
2623                             else if (scenario == 'r2r_jitstressregs3') {
2624                                 runjitstressregsStr = '--jitstressregs=3'
2625                             }
2626                             else if (scenario == 'r2r_jitstressregs4') {
2627                                 runjitstressregsStr = '--jitstressregs=4'
2628                             }
2629                             else if (scenario == 'r2r_jitstressregs8') {
2630                                 runjitstressregsStr = '--jitstressregs=8'
2631                             }
2632                             else if (scenario == 'r2r_jitstressregs0x10') {
2633                                 runjitstressregsStr = '--jitstressregs=0x10'
2634                             }
2635                             else if (scenario == 'r2r_jitstressregs0x80') {
2636                                 runjitstressregsStr = '--jitstressregs=0x80'
2637                             }
2638                             else if (scenario == 'r2r_jitstressregs0x1000') {
2639                                 runjitstressregsStr = '--jitstressregs=0x1000'
2640                             }
2641                             else if (scenario == 'r2r_jitminopts') {
2642                                 runjitmioptsStr = '--jitminopts'
2643                             }
2644                             else if (scenario == 'r2r_jitforcerelocs') {
2645                                 runjitforcerelocsStr = '--jitforcerelocs'
2646                             }
2647                     }
2648                     if  (scenario == 'gcstress15_pri1r2r')
2649                     {
2650                         gcstressStr = '--gcstresslevel=0xF'
2651                     }
2652
2653                     if (scenario == 'jitdiff')
2654                     {
2655                         runjitdisasmStr = '--jitdisasm --crossgen'
2656                     }
2657
2658                     if (scenario == 'illink')
2659                     {
2660                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2661                     }
2662
2663                     if (isLongGc(scenario)) {
2664                         // Long GC tests behave very poorly when they are not
2665                         // the only test running (many of them allocate until OOM).
2666                         sequentialString = '--sequential'
2667
2668                         // The Long GC playlist contains all of the tests that are
2669                         // going to be run. The GCSimulator playlist contains all of
2670                         // the GC simulator tests.
2671                         if (scenario == 'longgc') {
2672                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2673                         }
2674                         else if (scenario == 'gcsimulator') {
2675                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2676                         }
2677                     }
2678
2679                     if (isGcReliabilityFramework(scenario)) {
2680                         layoutOnlyStr = '--build-overlay-only'
2681                     }
2682
2683                     def folder = getJobFolder(scenario)
2684                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2685                         // Add parameters for the inputs
2686
2687                         parameters {
2688                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2689                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2690                         }
2691
2692                         steps {
2693                             // Set up the copies
2694
2695                             // Coreclr build containing the tests and mscorlib
2696
2697                             copyArtifacts(inputWindowTestsBuildName) {
2698                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2699                                 buildSelector {
2700                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2701                                 }
2702                             }
2703
2704                             if (scenario == 'coverage') {
2705
2706                                 // Move coreclr to clr directory
2707                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2708
2709                                 // Build coreclr
2710                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2711
2712                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2713                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2714                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2715                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2716                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2717                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2718                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2719                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2720
2721                                 // Run PAL tests
2722                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2723
2724                                 // Remove obj files for PAL tests so they're not included in coverage results
2725                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2726
2727                                 // Unzip the tests first.  Exit with 0
2728                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2729
2730                                 // Get corefx
2731                                 shell("git clone https://github.com/dotnet/corefx fx")
2732
2733                                 // Build Linux corefx
2734                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2735                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2736
2737                                 def testEnvOpt = ""
2738                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2739                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2740                                 shell("${createScriptCmds}")
2741                                 testEnvOpt = "--test-env=" + scriptFileName
2742
2743                                 // Run corefx tests
2744                                 shell("""./fx/run-test.sh \\
2745                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2746                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2747                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2748                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2749                 --configurationGroup Release""")
2750
2751
2752                                 // Run coreclr tests w/ workstation GC
2753                                 shell("""./clr/tests/runtest.sh \\
2754                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2755                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2756                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2757                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2758                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2759                 --crossgen --runcrossgentests""")
2760
2761                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2762                                 shell("""./clr/tests/runtest.sh \\
2763                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2764                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2765                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2766                 --useServerGC ${testEnvOpt}""")
2767
2768                                  // Run long-running coreclr GC tests & produce coverage reports
2769                                 shell("""./clr/tests/runtest.sh \\
2770                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2771                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2772                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2773                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2774                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2775                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2776                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2777
2778                             }
2779                             else {
2780
2781                                 // Coreclr build we are trying to test
2782
2783                                 copyArtifacts(inputCoreCLRBuildName) {
2784                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2785                                     buildSelector {
2786                                         buildNumber('${CORECLR_BUILD}')
2787                                     }
2788                                 }
2789
2790                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2791
2792                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2793                                 def osJobName
2794                                 if (os == 'Ubuntu') {
2795                                     osJobName = 'ubuntu14.04'
2796                                 }
2797                                 else {
2798                                     osJobName = os.toLowerCase()
2799                                 }
2800                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2801                                     includePatterns('bin/build.tar.gz')
2802                                     buildSelector {
2803                                         latestSuccessful(true)
2804                                     }
2805                                 }
2806
2807                                 shell ("mkdir ./bin/CoreFxBinDir")
2808                                 // Unpack the corefx binaries
2809                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2810
2811                                 // HACK -- Arm64 does not have corefx jobs yet.
2812                                 // Clone corefx and build the native packages overwriting the x64 packages.
2813                                 if (architecture == 'arm64') {
2814                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2815                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2816                                 }
2817
2818                                 // Unzip the tests first.  Exit with 0
2819                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2820
2821                                 // Execute the tests
2822                                 // If we are running a stress mode, we'll set those variables first
2823                                 def testEnvOpt = ""
2824                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2825                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2826                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2827                                     shell("${createScriptCmds}")
2828                                     testEnvOpt = "--test-env=" + scriptFileName
2829                                 }
2830
2831                                 if (isGCStressRelatedTesting(scenario)) {
2832                                     shell('./init-tools.sh')
2833                                 }
2834
2835                                 shell("""./tests/runtest.sh \\
2836                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2837                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2838                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2839                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2840                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2841                 --limitedDumpGeneration \\
2842                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
2843                 ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
2844                 ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
2845
2846                                 if (isGcReliabilityFramework(scenario)) {
2847                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2848                                     if (serverGCString != '') {
2849                                         shell("export COMPlus_gcServer=1")
2850                                     }
2851
2852                                     shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2853                                 }
2854                             }
2855                         }
2856                     }
2857
2858                     if (scenario == 'coverage') {
2859                         // Publish coverage reports
2860                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2861                         // TODO: Add once external email sending is available again
2862                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2863                     }
2864
2865                     if (scenario == 'jitdiff') {
2866                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2867                     }
2868
2869                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2870                     if (os in ['Ubuntu']) {
2871                         SummaryBuilder summaries = new SummaryBuilder()
2872                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2873                         summaries.emit(newJob)
2874                     }
2875
2876                     setMachineAffinity(newJob, os, architecture)
2877                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2878                     // Set timeouts to 240.
2879                     setTestJobTimeOut(newJob, scenario)
2880
2881                     if (architecture == 'arm64') {
2882                         Utilities.setJobTimeout(newJob, 240)
2883                     }
2884
2885                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2886
2887                     // Create a build flow to join together the build and tests required to run this
2888                     // test.
2889                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2890                     // Linux CoreCLR test
2891                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2892                     def fullTestJobName = projectFolder + '/' + newJob.name
2893                     // Add a reference to the input jobs for report purposes
2894                     JobReport.Report.addReference(inputCoreCLRBuildName)
2895                     JobReport.Report.addReference(inputWindowTestsBuildName)
2896                     JobReport.Report.addReference(fullTestJobName)
2897                     def newFlowJob;
2898
2899                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2900                     // so that coverage data can be preserved.
2901                     if (scenario == 'coverage') {
2902                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2903                         buildFlow("""
2904 // Build the input Windows job
2905 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2906
2907 // And then build the test build
2908 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2909 """)
2910                         }
2911                     // Normal jobs copy a Windows build & a non-Windows build
2912                     } else {
2913                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2914                         buildFlow("""
2915 // Build the input jobs in parallel
2916 parallel (
2917     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2918     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2919 )
2920
2921 // And then build the test build
2922 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2923                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2924 """)
2925                         }
2926                     }
2927
2928                     setMachineAffinity(newFlowJob, os, architecture)
2929                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2930                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2931                 } // configuration
2932             } // os
2933         } // architecture
2934     } // isPR
2935 } // scenario
2936
2937 JobReport.Report.generateJobReport(out)
2938
2939 // Make the call to generate the help job
2940 Utilities.createHelperJob(this, project, branch,
2941     "Welcome to the ${project} Repository",  // This is prepended to the help message
2942     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2943
2944 Utilities.addCROSSCheck(this, project, branch)