netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'CentOS7.1',
  49                'RHEL7.2',
  50                'Ubuntu16.04',
  51                'Ubuntu16.10',
  52                'Fedora24',
  53                'Tizen']
  54
  55     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  56
  57     // This is a set of JIT stress modes combined with the set of variables that
  58     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  59     // the values are the environment variables
  60     def static jitStressModeScenarios = [
  61                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  62                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  63                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  64                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  65                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  66                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  67                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  68                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  69                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  70                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  71                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  72                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  73                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  74                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  75                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  76                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  77                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  78                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  79                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  80                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  81                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  82                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  83                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  84                'corefx_baseline'                : [ : ], // corefx baseline
  85                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  86                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  87                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  88                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  89                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  90                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  91                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  92                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  93                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  94                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  95                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  96                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  97                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  98                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
  99                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 100                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 101                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 103                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 104                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 105                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 106                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 107                ]
 108
 109     // This is a set of r2r jit stress scenarios
 110     def static r2rJitStressScenarios = [
 111                'r2r_jitstress1',
 112                'r2r_jitstress2',
 113                'r2r_jitstressregs1',
 114                'r2r_jitstressregs2',
 115                'r2r_jitstressregs3',
 116                'r2r_jitstressregs4',
 117                'r2r_jitstressregs8',
 118                'r2r_jitstressregs0x10',
 119                'r2r_jitstressregs0x80',
 120                'r2r_jitstressregs0x1000',
 121                'r2r_jitminopts',
 122                'r2r_jitforcerelocs']
 123
 124     // This is the basic set of scenarios
 125     def static basicScenarios = [
 126                'default',
 127                'pri1',
 128                'ilrt',
 129                'r2r',
 130                'pri1r2r',
 131                'gcstress15_pri1r2r',
 132                'longgc',
 133                'coverage',
 134                'formatting',
 135                'gcsimulator',
 136                'jitdiff',
 137                'standalone_gc',
 138                'gc_reliability_framework',
 139                'illink'] + r2rJitStressScenarios
 140
 141     // Knowledge of the "validArmWindowsScenarios" scenario names is embedded in the ARM64 CI code, so when adding any,
 142     // make sure that code knows how to process them.
 143     def static validArmWindowsScenarios = [
 144                'default',
 145                'pri1r2r',
 146                'zapdisable',
 147                'minopts',
 148                'tailcallstress',
 149                'jitstress1',
 150                'jitstress2',
 151                'gcstress0x3',
 152                'gcstress0xc',
 153                'jitstressregs1',
 154                'jitstressregs2',
 155                'jitstressregs3',
 156                'jitstressregs4',
 157                'jitstressregs8',
 158                'jitstressregs0x10',
 159                'jitstressregs0x80',
 160                'jitstressregs0x1000',
 161                'gcstress0xc_jitstress1',
 162                'gcstress0xc_jitstress2',
 163                'minopts_zapdisable',
 164                'gcstress0x3_jitstress1',
 165                'gcstress0x3_jitstress2',
 166                'gcstress0xc_jitstress1',
 167                'gcstress0xc_jitstress2',
 168                'gcstress0x3_jitstressregs1',
 169                'gcstress0x3_jitstressregs2',
 170                'gcstress0x3_jitstressregs3',
 171                'gcstress0x3_jitstressregs4',
 172                'gcstress0x3_jitstressregs8',
 173                'gcstress0x3_jitstressregs0x10',
 174                'gcstress0x3_jitstressregs0x80',
 175                'gcstress0x3_jitstressregs0x1000',
 176                'gcstress0xc_jitstressregs1',
 177                'gcstress0xc_jitstressregs2',
 178                'gcstress0xc_jitstressregs3',
 179                'gcstress0xc_jitstressregs4',
 180                'gcstress0xc_jitstressregs8',
 181                'gcstress0xc_jitstressregs0x10',
 182                'gcstress0xc_jitstressregs0x80',
 183                'gcstress0xc_jitstressregs0x1000']
 184
 185     def static configurationList = ['Debug', 'Checked', 'Release']
 186
 187     // This is the set of architectures
 188     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86', 'x86lb']
 189 }
 190
 191 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 192     if (architecture == 'arm64' && os == 'Windows_NT') {
 193         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 194     } else if (architecture == 'arm64' && os != 'Windows_NT' && options == null) {
 195         Utilities.setMachineAffinity(job, os, 'arm64-small-page-size');
 196     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['large_pages'] == true) {
 197         Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size');
 198     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['is_build_only'] == true) {
 199         Utilities.setMachineAffinity(job, os, 'arm64-cross-latest');
 200     } else if ((architecture == 'armlb') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 201         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 202     } else if ((architecture == 'arm' || architecture == 'armlb') && (os == 'Windows_NT') && options['use_arm64_build_machine'] == true) {
 203         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 204     }else {
 205         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 206     }
 207 }
 208
 209 def static isJITStressJob(def scenario) {
 210     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 211            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 212 }
 213
 214 def static isGCStressRelatedTesting(def scenario) {
 215     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 216     // Detect it and make it a GCStress related.
 217     if (scenario == 'gcstress15_pri1r2r')
 218     {
 219         return true;
 220     }
 221
 222     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 223     def scenarioName = scenario.toLowerCase()
 224     def isGCStressTesting = false
 225     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 226         if (k in gcStressTestEnvVars) {
 227             isGCStressTesting = true;
 228         }
 229     }
 230     return isGCStressTesting
 231 }
 232
 233 def static isCorefxTesting(def scenario) {
 234     def corefx_prefix = 'corefx_'
 235     if (scenario.length() < corefx_prefix.length()) {
 236         return false
 237     }
 238     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 239 }
 240
 241 def static isR2R(def scenario) {
 242     return (scenario == 'r2r' || scenario == 'pri1r2r')
 243 }
 244
 245 def static isCoverage(def scenario) {
 246     return (scenario == 'coverage')
 247 }
 248
 249 def static isLongGc(def scenario) {
 250     return (scenario == 'longgc' || scenario == 'gcsimulator')
 251 }
 252
 253 def static isJitDiff(def scenario) {
 254     return (scenario == 'jitdiff')
 255 }
 256
 257 def static isGcReliabilityFramework(def scenario) {
 258     return (scenario == 'gc_reliability_framework')
 259 }
 260
 261 def static scenarioNeedsPri1Build(def scenario) {
 262     return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario) || isLongGc(scenario))
 263 }
 264
 265 def static setTestJobTimeOut(newJob, scenario) {
 266     if (isGCStressRelatedTesting(scenario)) {
 267         Utilities.setJobTimeout(newJob, 4320)
 268     }
 269     else if (isCorefxTesting(scenario)) {
 270         Utilities.setJobTimeout(newJob, 360)
 271     }
 272     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 273         Utilities.setJobTimeout(newJob, 240)
 274     }
 275     else if (isR2R(scenario)) {
 276         Utilities.setJobTimeout(newJob, 240)
 277     }
 278     else if (isCoverage(scenario)) {
 279         Utilities.setJobTimeout(newJob, 1440)
 280     }
 281     else if (isLongGc(scenario)) {
 282         Utilities.setJobTimeout(newJob, 1440)
 283     }
 284     else if (isJitDiff(scenario)) {
 285         Utilities.setJobTimeout(newJob, 240)
 286     }
 287     else if (isGcReliabilityFramework(scenario)) {
 288         Utilities.setJobTimeout(newJob, 1440)
 289     }
 290     // Non-test jobs use the default timeout value.
 291 }
 292
 293 def static getJobFolder(def scenario) {
 294     if (isJITStressJob(scenario)) {
 295         return 'jitstress'
 296     }
 297     if (scenario == 'illink') {
 298         return 'illink'
 299     }
 300     return ''
 301 }
 302
 303 def static getStressModeDisplayName(def scenario) {
 304     def displayStr = ''
 305     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 306         def prefixLength = 'COMPlus_'.length()
 307         if (k.length() >= prefixLength) {
 308             def modeName = k.substring(prefixLength, k.length())
 309             displayStr += ' ' + modeName + '=' + v
 310         }
 311     }
 312     return displayStr
 313 }
 314
 315 def static getR2RStressModeDisplayName(def scenario) {
 316     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 317     // "r2r_" prefix.
 318     def displayStr = scenario
 319     def prefixLength = 'r2r_'.length()
 320     if (displayStr.length() >= prefixLength) {
 321         displayStr = displayStr.substring(prefixLength, displayStr.length())
 322     }
 323     return displayStr
 324 }
 325
 326 // Generates the string for creating a file that sets environment variables
 327 // that makes it possible to run stress modes.  Writes the script to the file
 328 // specified by the stepScriptLocation parameter.
 329 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 330     def stepScript = ''
 331     if (os == 'Windows_NT') {
 332
 333         // Timeout in ms, default is 10 minutes. For stress modes up this to 30 minutes.
 334         // BUG?: it seems this is ignored, as this script will be run in an environment where
 335         //       environment variables will be discarded before this is used.
 336         def timeout = 1800000
 337         stepScript += "set __TestTimeout=${timeout}\r\n"
 338
 339         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 340         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 341
 342         // Create at least an empty script.
 343         stepScript += "echo. > ${stepScriptLocation}\r\n"
 344
 345         stressModeVars.each{ k, v ->
 346             // Write out what we are writing to the script file
 347             stepScript += "echo Setting ${k}=${v}\r\n"
 348             // Write out the set itself to the script file`
 349             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 350         }
 351     }
 352     else {
 353         stepScript += "echo Setting variables for ${stressModeName}\n"
 354         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 355         stressModeVars.each{ k, v ->
 356             // Write out what we are writing to the script file
 357             stepScript += "echo Setting ${k}=${v}\n"
 358             // Write out the set itself to the script file`
 359             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 360         }
 361         stepScript += "chmod +x ${stepScriptLocation}\n"
 362     }
 363     return stepScript
 364 }
 365
 366 // Append an existing script to a stress mode script already created by genStressModeScriptStep().
 367 // Returns string of commands to do this.
 368 def static appendStressModeScriptStep(def os, def appendScript, def stepScriptLocation) {
 369     assert (os == 'Windows_NT')
 370     def stepScript = ''
 371     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 372     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 373
 374     // Display the resulting script. This is useful when looking at the output log file.
 375     stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 376     stepScript += "type ${stepScriptLocation}\r\n"
 377
 378     return stepScript
 379 }
 380
 381 // Calculates the name of the build job based on some typical parameters.
 382 //
 383 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 384     // If the architecture is x64, do not add that info into the build name.
 385     // Need to change around some systems and other builds to pick up the right builds
 386     // to do that.
 387
 388     def suffix = scenario != 'default' ? "_${scenario}" : '';
 389     if (isBuildOnly) {
 390         suffix += '_bld'
 391     }
 392     def baseName = ''
 393     switch (architecture) {
 394         case 'x64':
 395             if (scenario == 'default') {
 396                 // For now we leave x64 off of the name for compatibility with other jobs
 397                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 398             }
 399             else if (scenario == 'formatting') {
 400                 // we don't care about the configuration for the formatting job. It runs all configs
 401                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 402             }
 403             else {
 404                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 405             }
 406             break
 407         case 'arm64':
 408             if (os.toLowerCase() == "windows_nt") {
 409                 // These are cross builds
 410                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 411             }
 412             else {
 413                 // Defaults to a small page size set of machines.
 414                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 415             }
 416             break
 417         case 'arm':
 418             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 419             break
 420         case 'armlb':
 421             // These are cross builds
 422             if (os == 'Tizen') {
 423                 // ABI: softfp
 424                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 425             }
 426             else {
 427                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 428             }
 429             break
 430         case 'x86':
 431         case 'x86lb':
 432         case 'x86_arm_altjit':
 433         case 'x64_arm64_altjit':
 434             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 435             break
 436         default:
 437             println("Unknown architecture: ${architecture}");
 438             assert false
 439             break
 440     }
 441
 442     return baseName + suffix
 443 }
 444
 445 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 446
 447     // Only PR triggers for altjit jobs.
 448     switch (architecture) {
 449         case 'x86_arm_altjit':
 450         case 'x64_arm64_altjit':
 451             return
 452         default:
 453             break
 454     }
 455
 456     // Check scenario.
 457     switch (scenario) {
 458         case 'default':
 459             switch (architecture) {
 460                 case 'x64':
 461                 case 'x86':
 462                 case 'x86lb':
 463                     if (architecture == 'x86' && os == 'Ubuntu') {
 464                         Utilities.addPeriodicTrigger(job, '@daily')
 465                     }
 466                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 467                         Utilities.addGithubPushTrigger(job)
 468                     }
 469                     break
 470                 case 'arm':
 471                 case 'armlb':
 472                     Utilities.addGithubPushTrigger(job)
 473                     break
 474                 case 'arm64':
 475                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 476                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 477                     break
 478                 default:
 479                     println("Unknown architecture: ${architecture}");
 480                     assert false
 481                     break
 482             }
 483             break
 484         case 'pri1':
 485             // Pri one gets a push trigger, and only for release
 486             if (architecture == 'x64') {
 487                 if (configuration == 'Release') {
 488                     // We expect release jobs to be Windows, or in the cross list
 489                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 490                     if (!os in bidailyCrossList) {
 491                         if (isFlowJob || os == 'Windows_NT') {
 492                             Utilities.addGithubPushTrigger(job)
 493                         }
 494                     }
 495                     else {
 496                         if (isFlowJob) {
 497                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 498                         }
 499                     }
 500                 }
 501             }
 502             break
 503         case 'r2r':
 504             //r2r jobs that aren't pri1 can only be triggered by phrase
 505             break
 506         case 'pri1r2r':
 507             assert !(os in bidailyCrossList)
 508             //pri1 r2r gets a push trigger for checked/release
 509             if (configuration == 'Checked' || configuration == 'Release') {
 510                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 511                 if (architecture == 'x64' && os != 'OSX10.12') {
 512                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 513                     if (isFlowJob || os == 'Windows_NT') {
 514                         Utilities.addGithubPushTrigger(job)
 515                     }
 516                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 517                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 518                     if (isFlowJob) {
 519                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 520                     }
 521                 }
 522                 // For x86, only add per-commit jobs for Windows
 523                 else if (architecture == 'x86' || architecture == 'x86lb') {
 524                     if (os == 'Windows_NT') {
 525                         Utilities.addGithubPushTrigger(job)
 526                     }
 527                 }
 528                 // arm64 pri1r2r jobs should only run every 12 hours.
 529                 else if (architecture == 'arm64') {
 530                     if (os == 'Windows_NT') {
 531                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 532                         // TODO: Add once external email sending is available again
 533                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 534                     }
 535                 }
 536             }
 537             break
 538         case 'r2r_jitstress1':
 539         case 'r2r_jitstress2':
 540         case 'r2r_jitstressregs1':
 541         case 'r2r_jitstressregs2':
 542         case 'r2r_jitstressregs3':
 543         case 'r2r_jitstressregs4':
 544         case 'r2r_jitstressregs8':
 545         case 'r2r_jitstressregs0x10':
 546         case 'r2r_jitstressregs0x80':
 547         case 'r2r_jitstressregs0x1000':
 548         case 'r2r_jitminopts':
 549         case 'r2r_jitforcerelocs':
 550         case 'gcstress15_pri1r2r':
 551             assert !(os in bidailyCrossList)
 552
 553             // GCStress=C is currently not supported on OS X
 554             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 555                 break
 556             }
 557
 558             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 559             if (configuration == 'Checked' || configuration == 'Release') {
 560                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 561                 if (architecture == 'x64') {
 562                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 563                     if (isFlowJob || os == 'Windows_NT') {
 564                         // Add a weekly periodic trigger
 565                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 566                     }
 567                 }
 568                 // For x86, only add per-commit jobs for Windows
 569                 else if (architecture == 'x86') {
 570                     if (os == 'Windows_NT') {
 571                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 572                     }
 573                 }
 574             }
 575             break
 576         case 'longgc':
 577             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 578             assert configuration == 'Release'
 579             assert architecture == 'x64'
 580             Utilities.addPeriodicTrigger(job, '@daily')
 581             // TODO: Add once external email sending is available again
 582             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 583             break
 584         case 'gcsimulator':
 585             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 586             assert configuration == 'Release'
 587             assert architecture == 'x64'
 588             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 589             // TODO: Add once external email sending is available again
 590             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 591             break
 592         case 'standalone_gc':
 593             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 594             assert (configuration == 'Release' || configuration == 'Checked')
 595             // TODO: Add once external email sending is available again
 596             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 597             Utilities.addPeriodicTrigger(job, '@weekly')
 598             break
 599         case 'gc_reliability_framework':
 600             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 601             assert (configuration == 'Release' || configuration == 'Checked')
 602             // Only triggered by phrase.
 603             break
 604         case 'ilrt':
 605             assert !(os in bidailyCrossList)
 606             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 607             if (architecture == 'x64' && configuration == 'Release') {
 608                 // We don't expect to see a job generated except in these scenarios
 609                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 610                 if (isFlowJob || os == 'Windows_NT') {
 611                     Utilities.addPeriodicTrigger(job, '@daily')
 612                 }
 613             }
 614             break
 615         case 'jitdiff':
 616             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 617             assert configuration == 'Checked'
 618             assert (architecture == 'x64' || architecture == 'x86')
 619             Utilities.addGithubPushTrigger(job)
 620             break
 621         case 'coverage':
 622             assert (os == 'Ubuntu' || os == 'Windows_NT')
 623             assert configuration == 'Release'
 624             assert architecture == 'x64'
 625             Utilities.addPeriodicTrigger(job, '@weekly')
 626             break
 627         case 'formatting':
 628             assert (os == 'Windows_NT' || os == "Ubuntu")
 629             assert architecture == 'x64'
 630             Utilities.addGithubPushTrigger(job)
 631             break
 632         case 'jitstressregs1':
 633         case 'jitstressregs2':
 634         case 'jitstressregs3':
 635         case 'jitstressregs4':
 636         case 'jitstressregs8':
 637         case 'jitstressregs0x10':
 638         case 'jitstressregs0x80':
 639         case 'jitstressregs0x1000':
 640         case 'minopts':
 641         case 'forcerelocs':
 642         case 'jitstress1':
 643         case 'jitstress2':
 644         case 'jitstress2_jitstressregs1':
 645         case 'jitstress2_jitstressregs2':
 646         case 'jitstress2_jitstressregs3':
 647         case 'jitstress2_jitstressregs4':
 648         case 'jitstress2_jitstressregs8':
 649         case 'jitstress2_jitstressregs0x10':
 650         case 'jitstress2_jitstressregs0x80':
 651         case 'jitstress2_jitstressregs0x1000':
 652         case 'tailcallstress':
 653         case 'jitsse2only':
 654         case 'corefx_baseline':
 655         case 'corefx_minopts':
 656         case 'corefx_jitstress1':
 657         case 'corefx_jitstress2':
 658         case 'corefx_jitstressregs1':
 659         case 'corefx_jitstressregs2':
 660         case 'corefx_jitstressregs3':
 661         case 'corefx_jitstressregs4':
 662         case 'corefx_jitstressregs8':
 663         case 'corefx_jitstressregs0x10':
 664         case 'corefx_jitstressregs0x80':
 665         case 'corefx_jitstressregs0x1000':
 666         case 'zapdisable':
 667             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 668                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 669                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 670                     if (os == 'Windows_NT') {
 671                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 672                         Utilities.addPeriodicTrigger(job, '@weekly')
 673                     }
 674                 }
 675                 else {
 676                     Utilities.addPeriodicTrigger(job, '@daily')
 677                 }
 678             }
 679             break
 680         case 'heapverify1':
 681         case 'gcstress0x3':
 682             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 683                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 684                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 685                     if (os == 'Windows_NT') {
 686                         Utilities.addPeriodicTrigger(job, '@daily')
 687                     }
 688                     // TODO: Add once external email sending is available again
 689                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 690                 }
 691                 else {
 692                     Utilities.addPeriodicTrigger(job, '@weekly')
 693                 }
 694             }
 695             break
 696         case 'gcstress0xc':
 697         case 'gcstress0xc_zapdisable':
 698         case 'gcstress0xc_zapdisable_jitstress2':
 699         case 'gcstress0xc_zapdisable_heapverify1':
 700         case 'gcstress0xc_jitstress1':
 701         case 'gcstress0xc_jitstress2':
 702         case 'gcstress0xc_minopts_heapverify1':
 703             // GCStress=C is currently not supported on OS X
 704             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 705                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 706                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 707                     // TODO: Enable a periodic trigger after tests are updated.
 708                     // Utilities.addPeriodicTrigger(job, '@daily')
 709                     // TODO: Add once external email sending is available again
 710                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 711                 }
 712                 else {
 713                     Utilities.addPeriodicTrigger(job, '@weekly')
 714                 }
 715             }
 716             break
 717
 718         case 'illink':
 719             // Testing on other operating systems TBD
 720             assert (os == 'Windows_NT' || os == 'Ubuntu')
 721             if (architecture == 'x64' || architecture == 'x86') {
 722                 if (configuration == 'Checked') {
 723                     Utilities.addPeriodicTrigger(job, '@daily')
 724                 }
 725             }
 726             break
 727
 728         case 'tieredcompilation':
 729         case 'corefx_tieredcompilation':
 730             // No periodic jobs just yet, still testing
 731             break
 732
 733         default:
 734             println("Unknown scenario: ${scenario}");
 735             assert false
 736             break
 737     }
 738     return
 739 }
 740
 741 // **************************
 742 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 743 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 744 // builds will do a couple extra steps.
 745 // **************************
 746
 747 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 748 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 749 // it's just used for internal builds
 750 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 751 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 752     if (isWindowsBuildOnlyJob) {
 753         return
 754     }
 755
 756     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 757     // Non pull request builds.
 758     if (!isPR) {
 759         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 760         return
 761     }
 762
 763      def arm64Users = [
 764         'adiaaida',
 765         'AndyAyersMS',
 766         'briansull',
 767         'BruceForstall',
 768         'CarolEidt',
 769         'cmckinsey',
 770         'erozenfeld',
 771         'jashook',
 772         'JosephTremoulet',
 773         'pgodeq',
 774         'pgavlin',
 775         'rartemev',
 776         'russellhadley',
 777         'RussKeldorph',
 778         'sandreenko',
 779         'sdmaclea',
 780         'swaroop-sridhar',
 781         'jkotas',
 782         'markwilkie',
 783         'weshaggard'
 784     ]
 785
 786     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 787     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 788     def osGroup = getOSGroup(os)
 789     switch (architecture) {
 790         case 'x64': // editor brace matching: {
 791             if (scenario == 'coverage') {
 792                 assert configuration == 'Release'
 793                 if (os == 'Ubuntu') {
 794                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 795                 }
 796                 break
 797             }
 798
 799             if (scenario == 'formatting') {
 800                 assert configuration == 'Checked'
 801                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 802                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 803                 }
 804                 break
 805             }
 806
 807             switch (os) {
 808                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 809                 case 'Debian8.4':
 810                 case 'RHEL7.2':
 811                     if (scenario == 'default') {
 812                         assert !isFlowJob
 813                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 814                     }
 815                     else if (scenario == 'pri1' && isFlowJob) {
 816                         assert (configuration == 'Release')
 817                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 818                     }
 819                     break
 820                 case 'Ubuntu16.04':
 821                     assert !isFlowJob
 822                     assert scenario == 'default'
 823                     // Distinguish with the other architectures (arm and x86)
 824                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 825                     break
 826                 case 'Fedora24':
 827                 case 'Ubuntu16.10':
 828                     assert !isFlowJob
 829                     assert scenario == 'default'
 830                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 831                     break
 832                 case 'Ubuntu':
 833                     if (scenario == 'illink') {
 834                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 835                         break
 836                     }
 837                     // fall through
 838                 case 'OSX10.12':
 839                     // Triggers on the non-flow jobs aren't necessary here
 840                     // Corefx testing uses non-flow jobs.
 841                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 842                         break
 843                     }
 844                     switch (scenario) {
 845                         case 'default':
 846                             // Ubuntu uses checked for default PR tests
 847                             if (configuration == 'Checked') {
 848                                 // Default trigger
 849                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 850                             }
 851                             break
 852                         case 'pri1':
 853                             if (configuration == 'Release') {
 854                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 855                             }
 856                             break
 857                         case 'jitdiff':
 858                             if (configuration == 'Checked') {
 859                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 860                             }
 861                             break
 862                         case 'ilrt':
 863                             if (configuration == 'Release') {
 864                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 865                             }
 866                             break
 867                         case 'r2r':
 868                             if (configuration == 'Release' || configuration == 'Checked') {
 869                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 870                             }
 871                             break
 872                         case 'pri1r2r':
 873                             if (configuration == 'Release' || configuration == 'Checked') {
 874                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 875                             }
 876                             break
 877                         case 'gcstress15_pri1r2r':
 878                             if (configuration == 'Release' || configuration == 'Checked') {
 879                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 880                             }
 881                             break
 882                         case 'r2r_jitstress1':
 883                         case 'r2r_jitstress2':
 884                         case 'r2r_jitstressregs1':
 885                         case 'r2r_jitstressregs2':
 886                         case 'r2r_jitstressregs3':
 887                         case 'r2r_jitstressregs4':
 888                         case 'r2r_jitstressregs8':
 889                         case 'r2r_jitstressregs0x10':
 890                         case 'r2r_jitstressregs0x80':
 891                         case 'r2r_jitstressregs0x1000':
 892                         case 'r2r_jitminopts':
 893                         case 'r2r_jitforcerelocs':
 894                             if (configuration == 'Release' || configuration == 'Checked') {
 895                                 def displayStr = getR2RStressModeDisplayName(scenario)
 896                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 897                             }
 898                             break
 899                         case 'longgc':
 900                             if (configuration == 'Release') {
 901                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 902                             }
 903                             break
 904                         case 'gcsimulator':
 905                             if (configuration == 'Release') {
 906                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 907                             }
 908                             break
 909                         case 'standalone_gc':
 910                             if (configuration == 'Release' || configuration == 'Checked') {
 911                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 912                             }
 913                             break
 914                         case 'gc_reliability_framework':
 915                             if (configuration == 'Release' || configuration == 'Checked') {
 916                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 917                             }
 918                             break
 919                         case 'minopts':
 920                         case 'tieredcompilation':
 921                         case 'forcerelocs':
 922                         case 'jitstress1':
 923                         case 'jitstress2':
 924                         case 'jitstressregs1':
 925                         case 'jitstressregs2':
 926                         case 'jitstressregs3':
 927                         case 'jitstressregs4':
 928                         case 'jitstressregs8':
 929                         case 'jitstressregs0x10':
 930                         case 'jitstressregs0x80':
 931                         case 'jitstressregs0x1000':
 932                         case 'jitstress2_jitstressregs1':
 933                         case 'jitstress2_jitstressregs2':
 934                         case 'jitstress2_jitstressregs3':
 935                         case 'jitstress2_jitstressregs4':
 936                         case 'jitstress2_jitstressregs8':
 937                         case 'jitstress2_jitstressregs0x10':
 938                         case 'jitstress2_jitstressregs0x80':
 939                         case 'jitstress2_jitstressregs0x1000':
 940                         case 'tailcallstress':
 941                         case 'jitsse2only':
 942                         case 'gcstress0x3':
 943                         case 'gcstress0xc':
 944                         case 'zapdisable':
 945                         case 'heapverify1':
 946                         case 'gcstress0xc_zapdisable':
 947                         case 'gcstress0xc_zapdisable_jitstress2':
 948                         case 'gcstress0xc_zapdisable_heapverify1':
 949                         case 'gcstress0xc_jitstress1':
 950                         case 'gcstress0xc_jitstress2':
 951                         case 'gcstress0xc_minopts_heapverify1':
 952                             def displayStr = getStressModeDisplayName(scenario)
 953                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 954                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 955                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 956                             break
 957                         case 'corefx_baseline':
 958                         case 'corefx_minopts':
 959                         case 'corefx_tieredcompilation':
 960                         case 'corefx_jitstress1':
 961                         case 'corefx_jitstress2':
 962                         case 'corefx_jitstressregs1':
 963                         case 'corefx_jitstressregs2':
 964                         case 'corefx_jitstressregs3':
 965                         case 'corefx_jitstressregs4':
 966                         case 'corefx_jitstressregs8':
 967                         case 'corefx_jitstressregs0x10':
 968                         case 'corefx_jitstressregs0x80':
 969                         case 'corefx_jitstressregs0x1000':
 970                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 971                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 972                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 973                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 974                             break
 975                         default:
 976                             println("Unknown scenario: ${scenario}");
 977                             assert false
 978                             break
 979                     }
 980                     break
 981                 case 'CentOS7.1':
 982                     switch (scenario) {
 983                         case 'pri1':
 984                             if (configuration == 'Release') {
 985                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 986                             }
 987                             break
 988                         case 'r2r':
 989                             if (configuration == 'Checked' || configuration == 'Release') {
 990                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 991                             }
 992                             break
 993                         case 'pri1r2r':
 994                             if (configuration == 'Checked' || configuration == 'Release') {
 995                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 996                             }
 997                             break
 998                         case 'gcstress15_pri1r2r':
 999                             if (configuration == 'Release' || configuration == 'Checked') {
1000                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1001                             }
1002                             break
1003                         case 'r2r_jitstress1':
1004                         case 'r2r_jitstress2':
1005                         case 'r2r_jitstressregs1':
1006                         case 'r2r_jitstressregs2':
1007                         case 'r2r_jitstressregs3':
1008                         case 'r2r_jitstressregs4':
1009                         case 'r2r_jitstressregs8':
1010                         case 'r2r_jitstressregs0x10':
1011                         case 'r2r_jitstressregs0x80':
1012                         case 'r2r_jitstressregs0x1000':
1013                         case 'r2r_jitminopts':
1014                         case 'r2r_jitforcerelocs':
1015                             if (configuration == 'Release' || configuration == 'Checked') {
1016                                 def displayStr = getR2RStressModeDisplayName(scenario)
1017                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1018                             }
1019                             break
1020                         default:
1021                             break
1022                     }
1023                 case 'Windows_NT':
1024                     switch (scenario) {
1025                         case 'default':
1026                             // Default trigger
1027                             if (configuration == 'Debug') {
1028                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1029                             }
1030                             break
1031                         case 'pri1':
1032                             // Default trigger
1033                             if (configuration == 'Release') {
1034                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1035                             }
1036                             break
1037                         case 'jitdiff':
1038                             if (configuration == 'Checked') {
1039                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1040                             }
1041                             break
1042                         case 'ilrt':
1043                             if (configuration == 'Release') {
1044                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1045                             }
1046                             break
1047                         case 'r2r':
1048                             if (configuration == 'Checked' || configuration == 'Release') {
1049                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1050                             }
1051                             break
1052                         case 'pri1r2r':
1053                             if (configuration == 'Checked' || configuration == 'Release') {
1054                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1055                             }
1056                             break
1057                         case 'gcstress15_pri1r2r':
1058                             if (configuration == 'Release' || configuration == 'Checked') {
1059                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1060                             }
1061                             break
1062                         case 'r2r_jitstress1':
1063                         case 'r2r_jitstress2':
1064                         case 'r2r_jitstressregs1':
1065                         case 'r2r_jitstressregs2':
1066                         case 'r2r_jitstressregs3':
1067                         case 'r2r_jitstressregs4':
1068                         case 'r2r_jitstressregs8':
1069                         case 'r2r_jitstressregs0x10':
1070                         case 'r2r_jitstressregs0x80':
1071                         case 'r2r_jitstressregs0x1000':
1072                         case 'r2r_jitminopts':
1073                         case 'r2r_jitforcerelocs':
1074                             if (configuration == 'Release' || configuration == 'Checked') {
1075                                 def displayStr = getR2RStressModeDisplayName(scenario)
1076                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1077                             }
1078                             break
1079                         case 'longgc':
1080                             if (configuration == 'Release') {
1081                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1082                             }
1083                             break
1084                         case 'gcsimulator':
1085                             if (configuration == 'Release') {
1086                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1087                             }
1088                             break
1089                         case 'standalone_gc':
1090                             if (configuration == 'Release' || configuration == 'Checked') {
1091                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1092                             }
1093                             break
1094                         case 'gc_reliability_framework':
1095                             if (configuration == 'Release' || configuration == 'Checked') {
1096                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1097                             }
1098                             break
1099                         case 'minopts':
1100                         case 'tieredcompilation':
1101                         case 'forcerelocs':
1102                         case 'jitstress1':
1103                         case 'jitstress2':
1104                         case 'jitstressregs1':
1105                         case 'jitstressregs2':
1106                         case 'jitstressregs3':
1107                         case 'jitstressregs4':
1108                         case 'jitstressregs8':
1109                         case 'jitstressregs0x10':
1110                         case 'jitstressregs0x80':
1111                         case 'jitstressregs0x1000':
1112                         case 'jitstress2_jitstressregs1':
1113                         case 'jitstress2_jitstressregs2':
1114                         case 'jitstress2_jitstressregs3':
1115                         case 'jitstress2_jitstressregs4':
1116                         case 'jitstress2_jitstressregs8':
1117                         case 'jitstress2_jitstressregs0x10':
1118                         case 'jitstress2_jitstressregs0x80':
1119                         case 'jitstress2_jitstressregs0x1000':
1120                         case 'tailcallstress':
1121                         case 'jitsse2only':
1122                         case 'gcstress0x3':
1123                         case 'gcstress0xc':
1124                         case 'zapdisable':
1125                         case 'heapverify1':
1126                         case 'gcstress0xc_zapdisable':
1127                         case 'gcstress0xc_zapdisable_jitstress2':
1128                         case 'gcstress0xc_zapdisable_heapverify1':
1129                         case 'gcstress0xc_jitstress1':
1130                         case 'gcstress0xc_jitstress2':
1131                         case 'gcstress0xc_minopts_heapverify1':
1132                             def displayStr = getStressModeDisplayName(scenario)
1133                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1134                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1135                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1136                             break
1137                         case 'corefx_baseline':
1138                         case 'corefx_minopts':
1139                         case 'corefx_tieredcompilation':
1140                         case 'corefx_jitstress1':
1141                         case 'corefx_jitstress2':
1142                         case 'corefx_jitstressregs1':
1143                         case 'corefx_jitstressregs2':
1144                         case 'corefx_jitstressregs3':
1145                         case 'corefx_jitstressregs4':
1146                         case 'corefx_jitstressregs8':
1147                         case 'corefx_jitstressregs0x10':
1148                         case 'corefx_jitstressregs0x80':
1149                         case 'corefx_jitstressregs0x1000':
1150                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1151                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1152                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1153                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1154                             break
1155                         case 'illink':
1156                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1157                             break
1158                         default:
1159                             println("Unknown scenario: ${scenario}");
1160                             assert false
1161                             break
1162                     }
1163                     break
1164                 default:
1165                     println("Unknown os: ${os}");
1166                     assert false
1167                     break
1168             }
1169             break
1170         // editor brace matching: }
1171         case 'armlb':
1172         case 'arm': // editor brace matching: {
1173             switch (os) {
1174                 case 'Ubuntu':
1175                 case 'Ubuntu16.04':
1176                     if (architecture == 'armlb') { // Ubuntu arm is only for armlb currently
1177                         assert scenario == 'default'
1178                         job.with {
1179                             publishers {
1180                                 azureVMAgentPostBuildAction {
1181                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1182                                 }
1183                             }
1184                         }
1185                         if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1186                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1187                         }
1188                         else {
1189                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1190                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1191                         }
1192                     }
1193                     break
1194                 case 'Tizen':
1195                     if (architecture == 'armlb') { // Ubuntu arm is only for armlb currently
1196                         architecture='armel'
1197                         job.with {
1198                             publishers {
1199                                 azureVMAgentPostBuildAction {
1200                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1201                                 }
1202                             }
1203                         }
1204                         // Removing the regex will cause this to run on each PR.
1205                         if (configuration == 'Release' || configuration == 'Debug') {
1206                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1207                         }
1208                         else {
1209                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1210                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1211                         }
1212                     }
1213                     break
1214                 case 'Windows_NT':
1215                     // Set up a private trigger
1216                     def contextString = "${os} ${architecture} Cross ${configuration}"
1217                     if (scenario != 'default')
1218                         contextString += " ${scenario}"
1219                     contextString += " Build"
1220                     // Debug configuration only does builds, no tests.
1221                     if (configuration != 'Debug') {
1222                         contextString += " and Test"
1223                     }
1224                     switch (scenario) {
1225                         case 'default':
1226                             // Only Checked is a default trigger.
1227                             if (configuration == 'Debug' || configuration == 'Release')
1228                             {
1229                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1230                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}.*", null, arm64Users)
1231                             }
1232                             else
1233                             {
1234                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1235                             }
1236                             break
1237                         default:
1238                             // Stress jobs will use this code path.
1239                             if (Constants.validArmWindowsScenarios.contains(scenario)) {
1240                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1241                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1242                             }
1243                             break
1244                     }
1245                     break
1246                 default:
1247                     println("NYI os: ${os}");
1248                     assert false
1249                     break
1250             }
1251             break
1252         // editor brace matching: }
1253         case 'arm64': // editor brace matching: {
1254             assert Constants.validArmWindowsScenarios.contains(scenario)
1255
1256             // Set up a private trigger
1257             def contextString = "${os} ${architecture} Cross ${configuration}"
1258             if (scenario != 'default')
1259                 contextString += " ${scenario}"
1260             contextString += " Build"
1261             // Debug builds only.
1262             if (configuration != 'Debug') {
1263                contextString += " and Test"
1264             }
1265
1266             switch (os) {
1267                 case 'Ubuntu':
1268                 case 'Ubuntu16.04':
1269                     switch (scenario) {
1270                         case 'pri1':
1271                             if (configuration == 'Release') {
1272                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test",
1273                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1274                             }
1275                             break
1276                         case 'r2r':
1277                             if (configuration == 'Checked' || configuration == 'Release') {
1278                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1279                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1280                             }
1281                             break
1282                         case 'pri1r2r':
1283                             if (configuration == 'Checked' || configuration == 'Release') {
1284                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1285                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1286                             }
1287                             break
1288                         case 'gcstress15_pri1r2r':
1289                             if (configuration == 'Release' || configuration == 'Checked') {
1290                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1291                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1292                             }
1293                             break
1294                         case 'r2r_jitstress1':
1295                         case 'r2r_jitstress2':
1296                         case 'r2r_jitstressregs1':
1297                         case 'r2r_jitstressregs2':
1298                         case 'r2r_jitstressregs3':
1299                         case 'r2r_jitstressregs4':
1300                         case 'r2r_jitstressregs8':
1301                         case 'r2r_jitstressregs0x10':
1302                         case 'r2r_jitstressregs0x80':
1303                         case 'r2r_jitstressregs0x1000':
1304                         case 'r2r_jitminopts':
1305                         case 'r2r_jitforcerelocs':
1306                             if (configuration == 'Release' || configuration == 'Checked') {
1307                                 def displayStr = getR2RStressModeDisplayName(scenario)
1308                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1309                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1310                             }
1311                             break
1312                         default:
1313                             break
1314                     }
1315                 case 'Windows_NT':
1316                     switch (scenario) {
1317                         case 'default':
1318                             if (isFlowJob == true) {
1319                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration}",
1320                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1321                             }
1322                             // For now only run Debug jobs on PR Trigger.
1323                             else if (configuration != 'Debug') {
1324                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1325                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}.*", null, arm64Users)
1326                             }
1327                             else {
1328                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1329                                 // are at higher risk of ARM64-breaking changes.
1330                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1331                             }
1332                             break
1333                         default:
1334                             // Stress jobs will use this code path.
1335                             if (Constants.validArmWindowsScenarios.contains(scenario)) {
1336                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1337                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1338                             }
1339                             break
1340                     }
1341                     break
1342                 default:
1343                     println("NYI os: ${os}");
1344                     assert false
1345                     break
1346             }
1347             break
1348         // editor brace matching: }
1349         case 'x86': // editor brace matching: {
1350             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1351             if (os == 'Ubuntu') {
1352                 // on-demand only for ubuntu x86
1353                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1354                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1355                 break
1356             }
1357             switch (scenario) {
1358                 case 'default':
1359                     if (configuration == 'Checked') {
1360                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1361                     }
1362                     else if (configuration == 'Release') {
1363                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1364                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1365                     }
1366                     break
1367                 case 'pri1':
1368                     if (configuration == 'Release') {
1369                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1370                     }
1371                     break
1372                 case 'ilrt':
1373                     if (configuration == 'Release') {
1374                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1375                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1376                     }
1377                     break
1378                 case 'r2r':
1379                     if (configuration == 'Checked' || configuration == 'Release') {
1380                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1381                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1382                     }
1383                     break
1384                 case 'pri1r2r':
1385                     if (configuration == 'Checked' || configuration == 'Release') {
1386                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1387                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1388                     }
1389                     break
1390                 case 'gcstress15_pri1r2r':
1391                     if (configuration == 'Release' || configuration == 'Checked') {
1392                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1393                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1394                     }
1395                     break
1396                 case 'r2r_jitstress1':
1397                 case 'r2r_jitstress2':
1398                 case 'r2r_jitstressregs1':
1399                 case 'r2r_jitstressregs2':
1400                 case 'r2r_jitstressregs3':
1401                 case 'r2r_jitstressregs4':
1402                 case 'r2r_jitstressregs8':
1403                 case 'r2r_jitstressregs0x10':
1404                 case 'r2r_jitstressregs0x80':
1405                 case 'r2r_jitstressregs0x1000':
1406                 case 'r2r_jitminopts':
1407                 case 'r2r_jitforcerelocs':
1408                     if (configuration == 'Release' || configuration == 'Checked') {
1409                         def displayStr = getR2RStressModeDisplayName(scenario)
1410                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1411                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1412                     }
1413                     break
1414                 case 'longgc':
1415                     if (configuration == 'Release') {
1416                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1417                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1418                     }
1419                     break
1420                 case 'gcsimulator':
1421                     if (configuration == 'Release') {
1422                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1423                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1424                     }
1425                     break
1426                 case 'standalone_gc':
1427                     if (configuration == 'Release' || configuration == 'Checked') {
1428                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1429                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1430                     }
1431                     break
1432                 case 'minopts':
1433                 case 'tieredcompilation':
1434                 case 'forcerelocs':
1435                 case 'jitstress1':
1436                 case 'jitstress2':
1437                 case 'jitstressregs1':
1438                 case 'jitstressregs2':
1439                 case 'jitstressregs3':
1440                 case 'jitstressregs4':
1441                 case 'jitstressregs8':
1442                 case 'jitstressregs0x10':
1443                 case 'jitstressregs0x80':
1444                 case 'jitstressregs0x1000':
1445                 case 'jitstress2_jitstressregs1':
1446                 case 'jitstress2_jitstressregs2':
1447                 case 'jitstress2_jitstressregs3':
1448                 case 'jitstress2_jitstressregs4':
1449                 case 'jitstress2_jitstressregs8':
1450                 case 'jitstress2_jitstressregs0x10':
1451                 case 'jitstress2_jitstressregs0x80':
1452                 case 'jitstress2_jitstressregs0x1000':
1453                 case 'tailcallstress':
1454                 case 'jitsse2only':
1455                 case 'gcstress0x3':
1456                 case 'gcstress0xc':
1457                 case 'zapdisable':
1458                 case 'heapverify1':
1459                 case 'gcstress0xc_zapdisable':
1460                 case 'gcstress0xc_zapdisable_jitstress2':
1461                 case 'gcstress0xc_zapdisable_heapverify1':
1462                 case 'gcstress0xc_jitstress1':
1463                 case 'gcstress0xc_jitstress2':
1464                 case 'gcstress0xc_minopts_heapverify1':
1465                     def displayStr = getStressModeDisplayName(scenario)
1466                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1467                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1468                     break
1469                 case 'corefx_baseline':
1470                 case 'corefx_minopts':
1471                 case 'corefx_tieredcompilation':
1472                 case 'corefx_jitstress1':
1473                 case 'corefx_jitstress2':
1474                 case 'corefx_jitstressregs1':
1475                 case 'corefx_jitstressregs2':
1476                 case 'corefx_jitstressregs3':
1477                 case 'corefx_jitstressregs4':
1478                 case 'corefx_jitstressregs8':
1479                 case 'corefx_jitstressregs0x10':
1480                 case 'corefx_jitstressregs0x80':
1481                 case 'corefx_jitstressregs0x1000':
1482                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1483                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1484                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1485                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1486                     break
1487                 case 'illink':
1488                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1489                     break
1490                 default:
1491                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1492                     assert false
1493                     break
1494             }
1495             break
1496          // editor brace matching: }
1497         case 'x86lb': // editor brace matching: {
1498             assert (os == 'Windows_NT')
1499             assert (scenario == 'default' || Constants.r2rJitStressScenarios.indexOf(scenario) !=1)
1500
1501             def arch = 'x86'
1502             def jit = 'legacy_backend'
1503             switch (scenario) {
1504                 case 'default':
1505                     if (configuration == 'Checked') {
1506                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1507                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1508                     }
1509                     break
1510                 default:
1511                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1512                     assert false
1513                     break
1514             }
1515             break
1516         // editor brace matching: }
1517         case 'x64_arm64_altjit':
1518         case 'x86_arm_altjit': // editor brace matching: {
1519             assert (os == 'Windows_NT')
1520             switch (scenario) {
1521                 case 'default':
1522                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1523                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1524                     break
1525                 default:
1526                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1527                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1528                     break
1529             }
1530             break
1531         // editor brace matching: }
1532         default:
1533             println("Unknown architecture: ${architecture}");
1534             assert false
1535             break
1536     }
1537 }
1538
1539 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1540     def buildCommands = [];
1541     def osGroup = getOSGroup(os)
1542     def lowerConfiguration = configuration.toLowerCase()
1543
1544     // Calculate the build steps, archival, and xunit results
1545     switch (os) {
1546         case 'Windows_NT': // editor brace matching: {
1547             switch (architecture) {
1548                 case 'x64':
1549                 case 'x86':
1550                 case 'x86lb':
1551                 case 'x86_arm_altjit':
1552                 case 'x64_arm64_altjit':
1553                     def arch = architecture
1554                     def buildOpts = ''
1555                     if ((architecture == 'x86lb') || (architecture == 'x86_arm_altjit')) {
1556                         arch = 'x86'
1557                     }
1558                     else if (architecture == 'x64_arm64_altjit') {
1559                         arch = 'x64'
1560                     }
1561
1562                     if (scenario == 'illink') {
1563                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1564                     }
1565
1566                     // If it is a release build for windows, ensure PGO is used, else fail the build
1567                     if ((lowerConfiguration == 'release') &&
1568                         (scenario in Constants.basicScenarios) &&
1569                         (architecture != 'x86lb') &&
1570                         (architecture != 'x86_arm_altjit') &&
1571                         (architecture != 'x64_arm64_altjit')) {
1572
1573                         buildOpts += ' enforcepgo'
1574                     }
1575
1576                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1577                             scenario == 'default' ||
1578                             scenario == 'r2r' ||
1579                             scenario == 'jitdiff' ||
1580                             scenario == 'ilrt' ||
1581                             scenario == 'illink' ||
1582                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1583                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1584                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1585                     }
1586
1587                     // For Pri 1 tests, we must shorten the output test binary path names.
1588                     // if __TestIntermediateDir is already set, build-test.cmd will
1589                     // output test binaries to that directory. If it is not set, the
1590                     // binaries are sent to a default directory whose name is about
1591                     // 35 characters long.
1592
1593                     else if (scenarioNeedsPri1Build(scenario)) {
1594                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1595                     }
1596                     else if (isLongGc(scenario)) {
1597                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1598                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1599                     }
1600                     else if (scenario == 'standalone_gc') {
1601                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1602                     }
1603                     else if (scenario == 'formatting') {
1604                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1605                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1606                         break
1607                     }
1608                     else {
1609                         println("Unknown scenario: ${scenario}")
1610                         assert false
1611                     }
1612
1613                     if (!isBuildOnly) {
1614                         def crossgenStr = ''
1615                         def runcrossgentestsStr = ''
1616                         def runjitstressStr = ''
1617                         def runjitstressregsStr = ''
1618                         def runjitmioptsStr = ''
1619                         def runjitforcerelocsStr = ''
1620                         def runjitdisasmStr = ''
1621                         def runilasmroundtripStr = ''
1622                         def gcstressStr = ''
1623                         def gcTestArguments = ''
1624                         def illinkArguments = ''
1625                         def testEnvStr = ''
1626                         def runtestArguments = ''
1627
1628                         if (scenario == 'r2r' ||
1629                             scenario == 'pri1r2r' ||
1630                             scenario == 'gcstress15_pri1r2r' ||
1631                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1632
1633                             // If this is a crossgen build, pass 'crossgen' to runtest.cmd
1634                             crossgenStr = 'crossgen'
1635                             runcrossgentestsStr = 'runcrossgentests'
1636
1637                             if (scenario == 'r2r_jitstress1'){
1638                                 runjitstressStr = 'jitstress 1'
1639                             }
1640                             else if (scenario == 'r2r_jitstress2') {
1641                                 runjitstressStr = 'jitstress 2'
1642                             }
1643                             else if (scenario == 'r2r_jitstressregs1'){
1644                                 runjitstressregsStr = 'jitstressregs 1'
1645                             }
1646                             else if (scenario == 'r2r_jitstressregs2') {
1647                                 runjitstressregsStr = 'jitstressregs 2'
1648                             }
1649                             else if (scenario == 'r2r_jitstressregs3') {
1650                                 runjitstressregsStr = 'jitstressregs 3'
1651                                }
1652                             else if (scenario == 'r2r_jitstressregs4') {
1653                                 runjitstressregsStr = 'jitstressregs 4'
1654                             }
1655                             else if (scenario == 'r2r_jitstressregs8') {
1656                                 runjitstressregsStr = 'jitstressregs 8'
1657                             }
1658                             else if (scenario == 'r2r_jitstressregs0x10') {
1659                                 runjitstressregsStr = 'jitstressregs 0x10'
1660                             }
1661                             else if (scenario == 'r2r_jitstressregs0x80') {
1662                                 runjitstressregsStr = 'jitstressregs 0x80'
1663                             }
1664                             else if (scenario == 'r2r_jitstressregs0x1000') {
1665                                 runjitstressregsStr = 'jitstressregs 0x1000'
1666                             }
1667                             else if (scenario == 'r2r_jitminopts') {
1668                                 runjitmioptsStr = 'jitminopts'
1669                             }
1670                             else if (scenario == 'r2r_jitforcerelocs') {
1671                                 runjitforcerelocsStr = 'jitforcerelocs'
1672                             }
1673                         }
1674                         else if (scenario == 'gcstress15_pri1r2r') {
1675                             gcstressStr = 'gcstresslevel 0xF'
1676                         }
1677                         else if (scenario == 'jitdiff') {
1678                             runjitdisasmStr = 'jitdisasm crossgen'
1679                         }
1680                         else if (scenario == 'ilrt') {
1681                             runilasmroundtripStr = 'ilasmroundtrip'
1682                         }
1683                         else if (isLongGc(scenario)) {
1684                             gcTestArguments = "${scenario} sequential"
1685                         }
1686                         else if (scenario == 'illink')
1687                         {
1688                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1689                         }
1690
1691                         def envScriptPath = ''
1692                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1693                             def buildCommandsStr = ''
1694                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1695                             buildCommandsStr += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], envScriptPath)
1696                             if (architecture == 'x86lb') {
1697                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd", envScriptPath)
1698                             }
1699                             else if (architecture == 'x86_arm_altjit') {
1700                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1701                             }
1702                             else if (architecture == 'x64_arm64_altjit') {
1703                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1704                             }
1705
1706                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1707                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1708                             // into a single script.
1709                             buildCommands += buildCommandsStr
1710                         }
1711                         else if (architecture == 'x86lb') {
1712                             envScriptPath = "%WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd"
1713                         }
1714                         else if (architecture == 'x86_arm_altjit') {
1715                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1716                         }
1717                         else if (architecture == 'x64_arm64_altjit') {
1718                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1719                         }
1720                         if (envScriptPath != '') {
1721                             testEnvStr = "TestEnv ${envScriptPath}"
1722                         }
1723
1724                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps ${testEnvStr}"
1725
1726                         // If we are running a stress mode, we should write out the set of key
1727                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1728
1729                         if (Constants.jitStressModeScenarios.containsKey(scenario) && enableCorefxTesting) {
1730                             def workspaceRelativeFxRoot = "_/fx"
1731                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1732
1733                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1734
1735                             setTestJobTimeOut(newJob, scenario)
1736
1737                             // Archive and process (only) the test results
1738                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1739                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1740
1741                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1742                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1743                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1744                         }
1745                         else if (isGcReliabilityFramework(scenario)) {
1746                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1747                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1748                         }
1749                         else {
1750                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1751                         }
1752                     }
1753
1754                     if (!enableCorefxTesting) {
1755                         // Run the rest of the build
1756                         // Build the mscorlib for the other OS's
1757                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1758                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1759
1760                         if (arch == "x64") {
1761                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1762                         }
1763
1764                         // Zip up the tests directory so that we don't use so much space/time copying
1765                         // 10s of thousands of files around.
1766                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1767
1768                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1769                             // For windows, pull full test results and test drops for x86/x64.
1770                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1771                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1772                         }
1773
1774                         if (scenario == 'jitdiff') {
1775                             // retrive jit-dasm output for base commit, and run jit-diff
1776                             if (!isBuildOnly) {
1777                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1778                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1779                             }
1780                         }
1781
1782                         if (!isBuildOnly) {
1783                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1784                             setTestJobTimeOut(newJob, scenario)
1785                         }
1786                     }
1787                     break
1788                 case 'armlb':
1789                 case 'arm':
1790                     assert Constants.validArmWindowsScenarios.contains(scenario)
1791
1792                     // Set time out
1793                     setTestJobTimeOut(newJob, scenario)
1794
1795                     def buildArchitecture = 'arm'
1796
1797                     if ( lowerConfiguration == "debug" ) {
1798                         // For Debug builds, we will do a P1 test build
1799                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=1"
1800                     }
1801                     else {
1802                         def priority = "1"
1803
1804                         if (scenario == "default")
1805                         {
1806                             priority = "0"
1807                         }
1808
1809                         if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1810                         {
1811                            // Up the timeout for arm checked testing only.
1812                            // Keep the longer timeout for gcstress.
1813                            Utilities.setJobTimeout(newJob, 240)
1814                         }
1815
1816                         def machineAffinityOptions = ['use_arm64_build_machine' : true]
1817                         setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1818
1819                         if (priority == "1")
1820                         {
1821                             buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=1"
1822                         }
1823                         else
1824                         {
1825                             buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture}"
1826                         }
1827
1828                         // Also run testing.
1829                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${buildArchitecture} -build_type ${lowerConfiguration} -scenario ${scenario} -testarch ${architecture} -priority ${priority} -key_location C:\\tools\\key.txt"
1830                     }
1831                     // Add archival.
1832                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1833                     break
1834                 case 'arm64':
1835                     assert Constants.validArmWindowsScenarios.contains(scenario)
1836
1837                     def priority = "1"
1838
1839                     if (scenario == "default")
1840                     {
1841                         priority = "0"
1842                     }
1843
1844                     // Set time out
1845                     setTestJobTimeOut(newJob, scenario)
1846
1847                     // Debug runs take too long to run. So build job only.
1848                     if (lowerConfiguration == "debug") {
1849                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1850                     }
1851                     else {
1852                         if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1853                             // Up the timeout for arm64 checked testing only.
1854                             // Keep the longer timeout for gcstress.
1855                             Utilities.setJobTimeout(newJob, 240)
1856                         }
1857
1858                         if (priority == "1") {
1859                             buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=1"
1860                         }
1861                         else{
1862                             buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1863                         }
1864
1865                         // Test build and run are launched together.
1866                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -testarch ${architecture} -priority ${priority} -key_location C:\\tools\\key.txt"
1867                         //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1868                     }
1869
1870                     // Add archival.
1871                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1872                     break
1873                 default:
1874                     println("Unknown architecture: ${architecture}");
1875                     assert false
1876                     break
1877             }
1878             break
1879         // editor brace matching: }
1880         case 'Ubuntu':
1881         case 'Ubuntu16.04':
1882         case 'Ubuntu16.10':
1883         case 'Debian8.4':
1884         case 'OSX10.12':
1885         case 'CentOS7.1':
1886         case 'RHEL7.2':
1887         case 'Tizen':
1888         case 'Fedora24': // editor brace matching: {
1889             switch (architecture) {
1890                 case 'x64':
1891                 case 'x86':
1892                     if (architecture == 'x86' && os == 'Ubuntu') {
1893                         // build and PAL test
1894                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1895                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1896                         break
1897                     }
1898
1899                     if (scenario == 'formatting') {
1900                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1901                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1902                         break
1903                     }
1904
1905                     if (scenario == 'illink') {
1906                         assert(os == 'Ubuntu')
1907                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1908                     }
1909
1910                     def standaloneGc = ''
1911                     if (scenario == 'standalone_gc') {
1912                         standaloneGc = 'buildstandalonegc'
1913                     }
1914
1915                     def disablePgo = ''
1916                     if (lowerConfiguration == 'release') {
1917                         disablePgo = 'nopgooptimize'
1918                     }
1919
1920                     if (!enableCorefxTesting) {
1921                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1922                         // only on supported OS platforms.
1923                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1924                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1925                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc} ${disablePgo}"
1926                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1927
1928                         // Set time out
1929                         setTestJobTimeOut(newJob, scenario)
1930                         // Basic archiving of the build
1931                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1932                         // And pal tests
1933                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1934                     }
1935                     else {
1936                         // Corefx stress testing
1937                         assert os == 'Ubuntu'
1938                         assert architecture == 'x64'
1939                         assert lowerConfiguration == 'checked'
1940                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1941
1942                         // Build coreclr
1943                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1944
1945                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1946                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1947
1948                         // Build and text corefx
1949                         def workspaceRelativeFxRoot = "_/fx"
1950                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1951
1952                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1953
1954                         setTestJobTimeOut(newJob, scenario)
1955
1956                         // Archive and process (only) the test results
1957                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1958                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1959                     }
1960                     break
1961                 case 'arm64':
1962                     def standaloneGc = ''
1963                     if (scenario == 'standalone_gc') {
1964                         standaloneGc = 'buildstandalonegc'
1965                     }
1966
1967                     if (!enableCorefxTesting) {
1968                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8 ${standaloneGc}"
1969
1970                         // HACK -- Arm64 does not have corefx jobs yet.
1971                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1972                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1973                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1974                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1975
1976                         // Set time out
1977                         setTestJobTimeOut(newJob, scenario)
1978                         // Basic archiving of the build
1979                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1980                     }
1981                     break
1982                 case 'armlb':
1983                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1984                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1985
1986                     // default values for Ubuntu
1987                     def arm_abi="arm"
1988                     def linuxCodeName="trusty"
1989                     if (os == 'Ubuntu16.04') {
1990                         linuxCodeName="xenial"
1991                     }
1992                     else if (os == 'Tizen') {
1993                         arm_abi="armel"
1994                         linuxCodeName="tizen"
1995                     }
1996
1997                     // Unzip the Windows test binaries first. Exit with 0
1998                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1999
2000                     // Unpack the corefx binaries
2001                     buildCommands += "mkdir ./bin/CoreFxBinDir"
2002                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
2003                     if (os != 'Tizen') {
2004                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
2005                     }
2006                     // Test environment emulation using docker and qemu has some problem to use lttng library.
2007                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
2008                     if (os == 'Ubuntu') {
2009                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
2010                     }
2011
2012                     // Call the ARM CI script to cross build and test using docker
2013                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
2014                     --mode=docker \\
2015                     --${arm_abi} \\
2016                     --linuxCodeName=${linuxCodeName} \\
2017                     --buildConfig=${lowerConfiguration} \\
2018                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
2019                     --coreFxBinDir=./bin/CoreFxBinDir \\
2020                     --testDirFile=./tests/testsRunningInsideARM.txt"""
2021
2022                     // Basic archiving of the build, no pal tests
2023                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
2024                     break
2025                 default:
2026                     println("Unknown architecture: ${architecture}");
2027                     assert false
2028                     break
2029             }
2030             break
2031         // editor brace matching: }
2032         default:
2033             println("Unknown os: ${os}");
2034             assert false
2035             break
2036     } // os
2037
2038     return buildCommands
2039 }
2040
2041 // Additional scenario which can alter behavior
2042
2043 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
2044 combinedScenarios.each { scenario ->
2045     [true, false].each { isPR ->
2046         Constants.architectureList.each { architecture ->
2047             Constants.configurationList.each { configuration ->
2048                 Constants.osList.each { os ->
2049                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
2050                     // and reset the os to Windows_NT
2051                     def isBuildOnly = false
2052                     if (os == 'Windows_NT_BuildOnly') {
2053                         isBuildOnly = true
2054                         os = 'Windows_NT'
2055                     }
2056
2057                     // Tizen is only supported for arm legacy_backend architecture
2058                     if (os == 'Tizen' && architecture != 'armlb') {
2059                         return
2060                     }
2061
2062                     // Skip totally unimplemented (in CI) configurations.
2063                     switch (architecture) {
2064                         case 'arm64':
2065                             if (os == 'Ubuntu16.04') {
2066                                 os = 'Ubuntu'
2067                             }
2068
2069                             // Windows and Ubuntu only
2070                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
2071                                 return
2072                             }
2073                             break
2074                         case 'armlb':
2075                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
2076                                 return
2077                             }
2078                             break
2079                         case 'arm':
2080                             if (os != 'Windows_NT') {
2081                                 return
2082                             }
2083                             break
2084                         case 'x86':
2085                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2086                                 return
2087                             }
2088                             break
2089                         case 'x86lb':
2090                         case 'x86_arm_altjit':
2091                         case 'x64_arm64_altjit':
2092                             if (os != 'Windows_NT') {
2093                                 return
2094                             }
2095                             break
2096                         case 'x64':
2097                             // Everything implemented
2098                             break
2099                         default:
2100                             println("Unknown architecture: ${architecture}")
2101                             assert false
2102                             break
2103                     }
2104
2105                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2106                     // with checked builds)
2107                     def enableCorefxTesting = false
2108                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2109                         if (configuration != 'Checked') {
2110                             return
2111                         }
2112
2113                         enableCorefxTesting = isCorefxTesting(scenario)
2114
2115                         // Since these are just execution time differences,
2116                         // skip platforms that don't execute the tests here (Windows_NT only)
2117                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
2118                         if (!isEnabledOS || isBuildOnly) {
2119                             return
2120                         }
2121
2122                         switch (architecture) {
2123                             case 'armlb':
2124                             case 'arm':
2125                             case 'arm64':
2126                                 if (!Constants.validArmWindowsScenarios.contains(scenario)) {
2127                                     return
2128                                 }
2129                                 break
2130                             case 'x64':
2131                             case 'x86':
2132                             case 'x86_arm_altjit':
2133                             case 'x64_arm64_altjit':
2134                                 // x86 ubuntu: default only
2135                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2136                                     return
2137                                 }
2138                                 // Windows: Everything implemented
2139                                 break
2140                             case 'x86lb':
2141                                 // No stress modes for legacy jit.
2142                                 // (There's no technical reason we couldn't allow these.)
2143                                 return
2144                             default:
2145                                 return
2146                         }
2147                     }
2148                     else {
2149                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
2150                         // and configuration is not Checked, bail out.
2151                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2152                             return;
2153                         }
2154
2155                         // Skip scenarios
2156                         switch (scenario) {
2157                             case 'pri1':
2158                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
2159                                 // the default scenario build
2160                                 if (os != 'Windows_NT') {
2161                                     return
2162                                 }
2163                                 // Only x64 for now
2164                                 if (architecture != 'x64') {
2165                                     return
2166                                 }
2167                                 break
2168                             case 'ilrt':
2169                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2170                                 // the default scenario build
2171                                 if (os != 'Windows_NT') {
2172                                     return
2173                                 }
2174                                 // Only x64 for now
2175                                 if (architecture != 'x64') {
2176                                     return
2177                                 }
2178                                 // Release only
2179                                 if (configuration != 'Release') {
2180                                     return
2181                                 }
2182                                 break
2183                             case 'jitdiff':
2184                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2185                                     return
2186                                 }
2187                                 if (architecture != 'x64') {
2188                                     return
2189                                 }
2190                                 if (configuration != 'Checked') {
2191                                     return
2192                                 }
2193                                 break
2194                             case 'r2r':
2195                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2196                                 // the default scenario build
2197                                 if (os != 'Windows_NT') {
2198                                     return
2199                                 }
2200                                 if (architecture != 'x64') {
2201                                     return
2202                                 }
2203                                 break
2204                             case 'pri1r2r':
2205                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2206                                 // the default scenario build
2207                                 if (os != 'Windows_NT') {
2208                                     return
2209                                 }
2210                                 if (architecture != 'x64') {
2211                                     if ((architecture != 'arm64' && architecture != 'armlb' && architecture != 'arm') || (configuration == 'Debug')) {
2212                                         return
2213                                     }
2214                                 }
2215                                 break
2216                             case 'gcstress15_pri1r2r':
2217                             case 'r2r_jitstress1':
2218                             case 'r2r_jitstress2':
2219                             case 'r2r_jitstressregs1':
2220                             case 'r2r_jitstressregs2':
2221                             case 'r2r_jitstressregs3':
2222                             case 'r2r_jitstressregs4':
2223                             case 'r2r_jitstressregs8':
2224                             case 'r2r_jitstressregs0x10':
2225                             case 'r2r_jitstressregs0x80':
2226                             case 'r2r_jitstressregs0x1000':
2227                             case 'r2r_jitminopts':
2228                             case 'r2r_jitforcerelocs':
2229                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
2230                                 // the default scenario build
2231                                 if (os != 'Windows_NT') {
2232                                     return
2233                                 }
2234                                 if (architecture != 'x64' && architecture != 'x86') {
2235                                     return
2236                                 }
2237                                 break
2238                             case 'longgc':
2239                             case 'gcsimulator':
2240                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2241                                     return
2242                                 }
2243                                 if (architecture != 'x64') {
2244                                     return
2245                                 }
2246                                 if (configuration != 'Release') {
2247                                     return
2248                                 }
2249                                 break
2250                             case 'gc_reliability_framework':
2251                             case 'standalone_gc':
2252                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2253                                     return
2254                                 }
2255
2256                                 if (architecture != 'x64') {
2257                                     return
2258                                 }
2259
2260                                 if (configuration != 'Release' && configuration != 'Checked') {
2261                                     return
2262                                 }
2263                                 break
2264                             // We need Windows x64 Release bits for the code coverage build
2265                             case 'coverage':
2266                                 if (os != 'Windows_NT') {
2267                                     return
2268                                 }
2269                                 if (architecture != 'x64') {
2270                                     return
2271                                 }
2272                                 if (configuration != 'Release') {
2273                                     return
2274                                 }
2275                                 break
2276                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2277                             case 'formatting':
2278                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2279                                     return
2280                                 }
2281                                 if (architecture != 'x64') {
2282                                     return
2283                                 }
2284                                 if (configuration != 'Checked') {
2285                                     return
2286                                 }
2287                                 if (isBuildOnly) {
2288                                     return
2289                                 }
2290                                 break
2291                             case 'illink':
2292                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2293                                     return
2294                                 }
2295                                 if (architecture != 'x64' && architecture != 'x86') {
2296                                     return
2297                                 }
2298                                 if (isBuildOnly) {
2299                                     return
2300                                 }
2301                                 break
2302                             case 'default':
2303                                 // Nothing skipped
2304                                 break
2305                             default:
2306                                 println("Unknown scenario: ${scenario}")
2307                                 assert false
2308                                 break
2309                         }
2310                     }
2311
2312                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2313                     // runtime behavior, not compile-time behavior, are not interesting.
2314                     switch (architecture) {
2315                         case 'x86_arm_altjit':
2316                         case 'x64_arm64_altjit':
2317                             if (isGCStressRelatedTesting(scenario)) {
2318                                 return
2319                             }
2320                             break
2321                         default:
2322                             break
2323                     }
2324
2325                     // Calculate names
2326                     def lowerConfiguration = configuration.toLowerCase()
2327                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2328                     def folderName = getJobFolder(scenario)
2329
2330                     // Create the new job
2331                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2332
2333                     def machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2334                     machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb') ? ['use_arm64_build_machine': false] : machineAffinityOptions
2335                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2336
2337                     // Add all the standard options
2338                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2339                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2340
2341                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
2342                     def osGroup = getOSGroup(os)
2343
2344                     newJob.with {
2345                         steps {
2346                             if (os == 'Windows_NT') {
2347                                 buildCommands.each { buildCommand ->
2348                                     batchFile(buildCommand)
2349                                 }
2350                             }
2351                             else {
2352                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2353                                 if ( architecture == 'armlb' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2354                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2355                                     // Define the Windows Tests and Corefx build job names
2356                                     def WindowTestsName = projectFolder + '/' +
2357                                                           Utilities.getFullJobName(project,
2358                                                                                    getJobName(lowerConfiguration,
2359                                                                                               'x64' ,
2360                                                                                               'windows_nt',
2361                                                                                               'default',
2362                                                                                               true),
2363                                                                                    false)
2364                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2365                                                        Utilities.getFolderName(branch)
2366
2367                                     // Copy the Windows test binaries and the Corefx build binaries
2368                                     copyArtifacts(WindowTestsName) {
2369                                         includePatterns('bin/tests/tests.zip')
2370                                         buildSelector {
2371                                             latestSuccessful(true)
2372                                         }
2373                                     }
2374
2375                                     def arm_abi = 'arm'
2376                                     def corefx_os = 'linux'
2377                                     if (os == 'Tizen') {
2378                                         arm_abi = 'armel'
2379                                         corefx_os = 'tizen'
2380                                     }
2381
2382                                     // Let's use release CoreFX to test checked CoreCLR,
2383                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2384                                     def corefx_lowerConfiguration = lowerConfiguration
2385                                     if ( lowerConfiguration == 'checked' ) {
2386                                         corefx_lowerConfiguration='release'
2387                                     }
2388
2389                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2390                                         includePatterns('bin/build.tar.gz')
2391                                         buildSelector {
2392                                             latestSuccessful(true)
2393                                         }
2394                                     }
2395                                 }
2396
2397                                 buildCommands.each { buildCommand ->
2398                                     shell(buildCommand)
2399                                 }
2400                             }
2401                         }
2402                     } // newJob.with
2403
2404                 } // os
2405             } // configuration
2406         } // architecture
2407     } // isPR
2408 } // scenario
2409
2410
2411 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2412 combinedScenarios.each { scenario ->
2413     [true, false].each { isPR ->
2414         // Architectures.  x64 only at this point
2415         ['x64', 'arm64'].each { architecture ->
2416             // Put the OS's supported for coreclr cross testing here
2417             Constants.crossList.each { os ->
2418                 if (architecture == 'arm64') {
2419                     if (os != "Ubuntu") {
2420                         return
2421                     }
2422                 }
2423
2424                 Constants.configurationList.each { configuration ->
2425
2426                     if (architecture == 'arm64') {
2427                         if (scenario != 'default' && scenario != 'pri1r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2428                             return
2429                         }
2430                     }
2431
2432                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2433                         if (configuration != 'Checked') {
2434                             return
2435                         }
2436                         if (isCorefxTesting(scenario)) {
2437                             return
2438                         }
2439                         //Skip stress modes for these scenarios
2440                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2441                             return
2442                         }
2443                     }
2444                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2445                     // and configuration is not Checked, bail out.
2446                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2447                         return;
2448                     }
2449                     // For CentOS, we only want Checked/Release pri1 builds.
2450                     else if (os == 'CentOS7.1') {
2451                         if (scenario != 'pri1' &&
2452                             scenario != 'r2r' &&
2453                             scenario != 'pri1r2r' &&
2454                             scenario != 'gcstress15_pri1r2r' &&
2455                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2456                             return
2457                         }
2458                         if (configuration != 'Checked' && configuration != 'Release') {
2459                             return
2460                         }
2461                     }
2462                     // For RedHat and Debian, we only do Release pri1 builds.
2463                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2464                         if (scenario != 'pri1') {
2465                             return
2466                         }
2467                         if (configuration != 'Release') {
2468                             return
2469                         }
2470                     }
2471                     else {
2472                         // Skip scenarios
2473                         switch (scenario) {
2474                             case 'pri1':
2475                                 // Nothing skipped
2476                                 break
2477                             case 'ilrt':
2478                                 // Release only
2479                                 if (configuration != 'Release') {
2480                                     return
2481                                 }
2482                                 break
2483                             case 'jitdiff':
2484                                 if (configuration != 'Checked') {
2485                                     return;
2486                                 }
2487                             case 'r2r':
2488                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2489                                 if (configuration != 'Checked' && configuration != 'Release') {
2490                                     return
2491                                 }
2492                                 break
2493                             case 'pri1r2r':
2494                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2495                                 if (configuration != 'Checked' && configuration != 'Release') {
2496                                     return
2497                                 }
2498                                 break
2499                             case 'gcstress15_pri1r2r':
2500                             case 'r2r_jitstress1':
2501                             case 'r2r_jitstress2':
2502                             case 'r2r_jitstressregs1':
2503                             case 'r2r_jitstressregs2':
2504                             case 'r2r_jitstressregs3':
2505                             case 'r2r_jitstressregs4':
2506                             case 'r2r_jitstressregs8':
2507                             case 'r2r_jitstressregs0x10':
2508                             case 'r2r_jitstressregs0x80':
2509                             case 'r2r_jitstressregs0x1000':
2510                             case 'r2r_jitminopts':
2511                             case 'r2r_jitforcerelocs':
2512                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2513                                 if (configuration != 'Checked' && configuration != 'Release') {
2514                                     return
2515                                 }
2516                                 break
2517                             case 'longgc':
2518                             case 'gcsimulator':
2519                                 // Long GC tests take a long time on non-Release builds
2520                                 if (configuration != 'Release') {
2521                                     return
2522                                 }
2523                                 break
2524                             case 'gc_reliability_framework':
2525                             case 'standalone_gc':
2526                                 if (configuration != 'Release' && configuration != 'Checked') {
2527                                     return
2528                                 }
2529                                 break
2530                             case 'coverage':
2531                                 //We only want Ubuntu Release for coverage
2532                                 if (os != 'Ubuntu') {
2533                                     return
2534                                 }
2535                                 if (configuration != 'Release') {
2536                                     return
2537                                 }
2538                             case 'formatting':
2539                                 return
2540                             case 'illink':
2541                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2542                                     return
2543                                 }
2544                                 break
2545                             case 'default':
2546                                 // Nothing skipped
2547                                 break
2548                             default:
2549                                 println("Unknown scenario: ${scenario}")
2550                                 assert false
2551                                 break
2552                         }
2553                     }
2554
2555                     def lowerConfiguration = configuration.toLowerCase()
2556                     def osGroup = getOSGroup(os)
2557                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2558
2559                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2560                     def inputScenario = 'default'
2561                     if (scenario == 'coverage') {
2562                         inputScenario = 'coverage'
2563                     }
2564                     def inputCoreCLRBuildName = projectFolder + '/' +
2565                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2566                     // If this is a stress scenario, there isn't any difference in the build job
2567                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2568                     // from the default scenario
2569                     def testBuildScenario = scenario
2570                     if (scenarioNeedsPri1Build(scenario)) {
2571                         testBuildScenario = 'pri1'
2572                     }
2573                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2574                         testBuildScenario = 'default'
2575                     }
2576                     def inputWindowTestsBuildName = ''
2577                     def inputWindowsTestBuildArch = architecture
2578                     if (architecture == "arm64") {
2579                         // Use the x64 test build for arm64 unix
2580                         inputWindowsTestBuildArch = "x64"
2581                     }
2582                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2583                         inputWindowTestsBuildName = projectFolder + '/' +
2584                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', 'default', true), isPR)
2585                     }
2586                     else {
2587                         inputWindowTestsBuildName = projectFolder + '/' +
2588                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2589                     }
2590                     // Enable Server GC for Ubuntu PR builds
2591                     def serverGCString = ''
2592
2593                     // Whether or not this test run should be run sequentially instead
2594                     // of in parallel. Only used for long GC tests.
2595                     def sequentialString = ''
2596
2597                     // Whether or not this test run should run a specific playlist.
2598                     // Only used for long GC tests.
2599
2600                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2601                     // for running long GC and GCSimulator tests, respectively. We don't use them
2602                     // here because using a playlist file produces much more readable output on the CI machines
2603                     // and reduces running time.
2604                     def playlistString = ''
2605
2606                     if (os == 'Ubuntu' && isPR){
2607                         serverGCString = '--useServerGC'
2608                     }
2609
2610                     // pass --crossgen to runtest.sh for crossgen builds
2611                     def crossgenStr = ''
2612                     def runcrossgentestsStr = ''
2613                     def runjitstressStr = ''
2614                     def runjitstressregsStr = ''
2615                     def runjitmioptsStr = ''
2616                     def runjitforcerelocsStr = ''
2617                     def runjitdisasmStr = ''
2618                     def runilasmroundtripStr = ''
2619                     def gcstressStr = ''
2620                     def illinkStr = ''
2621                     def layoutOnlyStr =''
2622
2623                     if (scenario == 'r2r' ||
2624                         scenario == 'pri1r2r' ||
2625                         scenario == 'gcstress15_pri1r2r' ||
2626                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2627                             crossgenStr = '--crossgen'
2628                             runcrossgentestsStr = '--runcrossgentests'
2629
2630                             if (scenario == 'r2r_jitstress1'){
2631                                 runjitstressStr = '--jitstress=1'
2632                             }
2633                             else if (scenario == 'r2r_jitstress2') {
2634                                 runjitstressStr = '--jitstress=2'
2635                             }
2636                             else if (scenario == 'r2r_jitstressregs1'){
2637                                 runjitstressregsStr = '--jitstressregs=1'
2638                             }
2639                             else if (scenario == 'r2r_jitstressregs2') {
2640                                 runjitstressregsStr = '--jitstressregs=2'
2641                             }
2642                             else if (scenario == 'r2r_jitstressregs3') {
2643                                 runjitstressregsStr = '--jitstressregs=3'
2644                             }
2645                             else if (scenario == 'r2r_jitstressregs4') {
2646                                 runjitstressregsStr = '--jitstressregs=4'
2647                             }
2648                             else if (scenario == 'r2r_jitstressregs8') {
2649                                 runjitstressregsStr = '--jitstressregs=8'
2650                             }
2651                             else if (scenario == 'r2r_jitstressregs0x10') {
2652                                 runjitstressregsStr = '--jitstressregs=0x10'
2653                             }
2654                             else if (scenario == 'r2r_jitstressregs0x80') {
2655                                 runjitstressregsStr = '--jitstressregs=0x80'
2656                             }
2657                             else if (scenario == 'r2r_jitstressregs0x1000') {
2658                                 runjitstressregsStr = '--jitstressregs=0x1000'
2659                             }
2660                             else if (scenario == 'r2r_jitminopts') {
2661                                 runjitmioptsStr = '--jitminopts'
2662                             }
2663                             else if (scenario == 'r2r_jitforcerelocs') {
2664                                 runjitforcerelocsStr = '--jitforcerelocs'
2665                             }
2666                     }
2667                     if  (scenario == 'gcstress15_pri1r2r')
2668                     {
2669                         gcstressStr = '--gcstresslevel=0xF'
2670                     }
2671
2672                     if (scenario == 'jitdiff')
2673                     {
2674                         runjitdisasmStr = '--jitdisasm --crossgen'
2675                     }
2676
2677                     if (scenario == 'illink')
2678                     {
2679                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2680                     }
2681
2682                     if (isLongGc(scenario)) {
2683                         // Long GC tests behave very poorly when they are not
2684                         // the only test running (many of them allocate until OOM).
2685                         sequentialString = '--sequential'
2686
2687                         // The Long GC playlist contains all of the tests that are
2688                         // going to be run. The GCSimulator playlist contains all of
2689                         // the GC simulator tests.
2690                         if (scenario == 'longgc') {
2691                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2692                         }
2693                         else if (scenario == 'gcsimulator') {
2694                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2695                         }
2696                     }
2697
2698                     if (isGcReliabilityFramework(scenario)) {
2699                         layoutOnlyStr = '--build-overlay-only'
2700                     }
2701
2702                     def folder = getJobFolder(scenario)
2703                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2704                         // Add parameters for the inputs
2705
2706                         parameters {
2707                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2708                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2709                         }
2710
2711                         steps {
2712                             // Set up the copies
2713
2714                             // Coreclr build containing the tests and mscorlib
2715
2716                             copyArtifacts(inputWindowTestsBuildName) {
2717                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2718                                 buildSelector {
2719                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2720                                 }
2721                             }
2722
2723                             if (scenario == 'coverage') {
2724
2725                                 // Move coreclr to clr directory
2726                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2727
2728                                 // Build coreclr
2729                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2730
2731                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2732                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2733                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2734                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2735                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2736                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2737                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2738                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2739
2740                                 // Run PAL tests
2741                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2742
2743                                 // Remove obj files for PAL tests so they're not included in coverage results
2744                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2745
2746                                 // Unzip the tests first.  Exit with 0
2747                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2748
2749                                 // Get corefx
2750                                 shell("git clone https://github.com/dotnet/corefx fx")
2751
2752                                 // Build Linux corefx
2753                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2754                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2755
2756                                 def testEnvOpt = ""
2757                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2758                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2759                                 shell("${createScriptCmds}")
2760                                 testEnvOpt = "--test-env=" + scriptFileName
2761
2762                                 // Run corefx tests
2763                                 shell("""./fx/run-test.sh \\
2764                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2765                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2766                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2767                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2768                 --configurationGroup Release""")
2769
2770
2771                                 // Run coreclr tests w/ workstation GC
2772                                 shell("""./clr/tests/runtest.sh \\
2773                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2774                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2775                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2776                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2777                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2778                 --crossgen --runcrossgentests""")
2779
2780                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2781                                 shell("""./clr/tests/runtest.sh \\
2782                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2783                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2784                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2785                 --useServerGC ${testEnvOpt}""")
2786
2787                                  // Run long-running coreclr GC tests & produce coverage reports
2788                                 shell("""./clr/tests/runtest.sh \\
2789                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2790                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2791                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2792                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2793                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2794                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2795                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2796
2797                             }
2798                             else {
2799
2800                                 // Coreclr build we are trying to test
2801
2802                                 copyArtifacts(inputCoreCLRBuildName) {
2803                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2804                                     buildSelector {
2805                                         buildNumber('${CORECLR_BUILD}')
2806                                     }
2807                                 }
2808
2809                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2810
2811                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2812                                 def osJobName
2813                                 if (os == 'Ubuntu') {
2814                                     osJobName = 'ubuntu14.04'
2815                                 }
2816                                 else {
2817                                     osJobName = os.toLowerCase()
2818                                 }
2819                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2820                                     includePatterns('bin/build.tar.gz')
2821                                     buildSelector {
2822                                         latestSuccessful(true)
2823                                     }
2824                                 }
2825
2826                                 shell ("mkdir ./bin/CoreFxBinDir")
2827                                 // Unpack the corefx binaries
2828                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2829
2830                                 // HACK -- Arm64 does not have corefx jobs yet.
2831                                 // Clone corefx and build the native packages overwriting the x64 packages.
2832                                 if (architecture == 'arm64') {
2833                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2834                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2835                                 }
2836
2837                                 // Unzip the tests first.  Exit with 0
2838                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2839
2840                                 // Execute the tests
2841                                 // If we are running a stress mode, we'll set those variables first
2842                                 def testEnvOpt = ""
2843                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2844                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2845                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2846                                     shell("${createScriptCmds}")
2847                                     testEnvOpt = "--test-env=" + scriptFileName
2848                                 }
2849
2850                                 if (isGCStressRelatedTesting(scenario)) {
2851                                     shell('./init-tools.sh')
2852                                 }
2853
2854                                 shell("""./tests/runtest.sh \\
2855                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2856                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2857                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2858                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2859                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2860                 --limitedDumpGeneration \\
2861                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
2862                 ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
2863                 ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
2864
2865                                 if (isGcReliabilityFramework(scenario)) {
2866                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2867                                     if (serverGCString != '') {
2868                                         shell("export COMPlus_gcServer=1")
2869                                     }
2870
2871                                     shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2872                                 }
2873                             }
2874                         }
2875                     }
2876
2877                     if (scenario == 'coverage') {
2878                         // Publish coverage reports
2879                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2880                         // TODO: Add once external email sending is available again
2881                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2882                     }
2883
2884                     if (scenario == 'jitdiff') {
2885                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2886                     }
2887
2888                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2889                     if (os in ['Ubuntu']) {
2890                         SummaryBuilder summaries = new SummaryBuilder()
2891                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2892                         summaries.emit(newJob)
2893                     }
2894
2895                     setMachineAffinity(newJob, os, architecture)
2896                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2897                     // Set timeouts to 240.
2898                     setTestJobTimeOut(newJob, scenario)
2899
2900                     if (architecture == 'arm64') {
2901                         Utilities.setJobTimeout(newJob, 240)
2902                     }
2903
2904                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2905
2906                     // Create a build flow to join together the build and tests required to run this
2907                     // test.
2908                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2909                     // Linux CoreCLR test
2910                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2911                     def fullTestJobName = projectFolder + '/' + newJob.name
2912                     // Add a reference to the input jobs for report purposes
2913                     JobReport.Report.addReference(inputCoreCLRBuildName)
2914                     JobReport.Report.addReference(inputWindowTestsBuildName)
2915                     JobReport.Report.addReference(fullTestJobName)
2916                     def newFlowJob;
2917
2918                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2919                     // so that coverage data can be preserved.
2920                     if (scenario == 'coverage') {
2921                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2922                         buildFlow("""
2923 // Build the input Windows job
2924 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2925
2926 // And then build the test build
2927 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2928 """)
2929                         }
2930                     // Normal jobs copy a Windows build & a non-Windows build
2931                     } else {
2932                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2933                         buildFlow("""
2934 // Build the input jobs in parallel
2935 parallel (
2936     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2937     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2938 )
2939
2940 // And then build the test build
2941 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2942                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2943 """)
2944                         }
2945                     }
2946
2947                     setMachineAffinity(newFlowJob, os, architecture)
2948                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2949                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2950                 } // configuration
2951             } // os
2952         } // architecture
2953     } // isPR
2954 } // scenario
2955
2956 JobReport.Report.generateJobReport(out)
2957
2958 // Make the call to generate the help job
2959 Utilities.createHelperJob(this, project, branch,
2960     "Welcome to the ${project} Repository",  // This is prepended to the help message
2961     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2962
2963 Utilities.addCROSSCheck(this, project, branch)