netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'CentOS7.1',
  49                'RHEL7.2',
  50                'Ubuntu16.04',
  51                'Ubuntu16.10',
  52                'Fedora24',
  53                'Tizen']
  54
  55     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  56
  57     // This is a set of JIT stress modes combined with the set of variables that
  58     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  59     // the values are the environment variables
  60     def static jitStressModeScenarios = [
  61                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  62                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  63                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  64                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  65                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  66                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  67                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  68                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  69                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  70                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  71                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  72                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  73                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  74                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  75                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  76                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  77                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  78                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  79                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  80                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  81                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  82                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  83                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  84                'corefx_baseline'                : [ : ], // corefx baseline
  85                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  86                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  87                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  88                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  89                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  90                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  91                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  92                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  93                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  94                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  95                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  96                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  97                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  98                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
  99                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 100                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 101                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 103                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 104                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 105                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 106                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 107                ]
 108
 109     // This is a set of r2r jit stress scenarios
 110     def static r2rJitStressScenarios = [
 111                'r2r_jitstress1',
 112                'r2r_jitstress2',
 113                'r2r_jitstressregs1',
 114                'r2r_jitstressregs2',
 115                'r2r_jitstressregs3',
 116                'r2r_jitstressregs4',
 117                'r2r_jitstressregs8',
 118                'r2r_jitstressregs0x10',
 119                'r2r_jitstressregs0x80',
 120                'r2r_jitstressregs0x1000',
 121                'r2r_jitminopts',
 122                'r2r_jitforcerelocs']
 123
 124     // This is the basic set of scenarios
 125     def static basicScenarios = [
 126                'default',
 127                'ilrt',
 128                'r2r',
 129                'r2r',
 130                'gcstress15_r2r',
 131                'longgc',
 132                'formatting',
 133                'gcsimulator',
 134                'jitdiff',
 135                'standalone_gc',
 136                'gc_reliability_framework',
 137                'illink'] + r2rJitStressScenarios
 138
 139     // Knowledge of the "validArmWindowsScenarios" scenario names is embedded in the ARM64 CI code, so when adding any,
 140     // make sure that code knows how to process them.
 141     def static validArmWindowsScenarios = [
 142                'default',
 143                'r2r',
 144                'zapdisable',
 145                'minopts',
 146                'tailcallstress',
 147                'jitstress1',
 148                'jitstress2',
 149                'gcstress0x3',
 150                'gcstress0xc',
 151                'jitstressregs1',
 152                'jitstressregs2',
 153                'jitstressregs3',
 154                'jitstressregs4',
 155                'jitstressregs8',
 156                'jitstressregs0x10',
 157                'jitstressregs0x80',
 158                'jitstressregs0x1000',
 159                'gcstress0xc_jitstress1',
 160                'gcstress0xc_jitstress2',
 161                'minopts_zapdisable',
 162                'gcstress0x3_jitstress1',
 163                'gcstress0x3_jitstress2',
 164                'gcstress0xc_jitstress1',
 165                'gcstress0xc_jitstress2',
 166                'gcstress0x3_jitstressregs1',
 167                'gcstress0x3_jitstressregs2',
 168                'gcstress0x3_jitstressregs3',
 169                'gcstress0x3_jitstressregs4',
 170                'gcstress0x3_jitstressregs8',
 171                'gcstress0x3_jitstressregs0x10',
 172                'gcstress0x3_jitstressregs0x80',
 173                'gcstress0x3_jitstressregs0x1000',
 174                'gcstress0xc_jitstressregs1',
 175                'gcstress0xc_jitstressregs2',
 176                'gcstress0xc_jitstressregs3',
 177                'gcstress0xc_jitstressregs4',
 178                'gcstress0xc_jitstressregs8',
 179                'gcstress0xc_jitstressregs0x10',
 180                'gcstress0xc_jitstressregs0x80',
 181                'gcstress0xc_jitstressregs0x1000']
 182
 183     def static configurationList = ['Debug', 'Checked', 'Release']
 184
 185     // This is the set of architectures
 186     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86', 'x86lb']
 187 }
 188
 189 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 190     assert os instanceof String
 191     assert architecture instanceof String
 192
 193     if (architecture == 'arm64' && os == 'Windows_NT') {
 194         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 195     } else if (architecture == 'arm64' && os != 'Windows_NT' && options == null) {
 196         Utilities.setMachineAffinity(job, os, 'arm64-small-page-size');
 197     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['large_pages'] == true) {
 198         Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size');
 199     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['is_build_only'] == true) {
 200         Utilities.setMachineAffinity(job, os, 'arm64-cross-latest');
 201     } else if ((architecture == 'armlb') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 202         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 203     } else if ((architecture == 'arm' || architecture == 'armlb') && (os == 'Windows_NT') && options['use_arm64_build_machine'] == true) {
 204         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 205     } else {
 206         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 207     }
 208 }
 209
 210 def static isJITStressJob(def scenario) {
 211     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 212            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 213 }
 214
 215 def static isGCStressRelatedTesting(def scenario) {
 216     // The 'gcstress15_r2r' scenario is a basic scenario.
 217     // Detect it and make it a GCStress related.
 218     if (scenario == 'gcstress15_r2r')
 219     {
 220         return true;
 221     }
 222
 223     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 224     def scenarioName = scenario.toLowerCase()
 225     def isGCStressTesting = false
 226     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 227         if (k in gcStressTestEnvVars) {
 228             isGCStressTesting = true;
 229         }
 230     }
 231     return isGCStressTesting
 232 }
 233
 234 def static isCorefxTesting(def scenario) {
 235     def corefx_prefix = 'corefx_'
 236     if (scenario.length() < corefx_prefix.length()) {
 237         return false
 238     }
 239     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 240 }
 241
 242 def static isR2R(def scenario) {
 243     return (scenario == 'r2r')
 244 }
 245
 246 def static isLongGc(def scenario) {
 247     return (scenario == 'longgc' || scenario == 'gcsimulator')
 248 }
 249
 250 def static isJitDiff(def scenario) {
 251     return (scenario == 'jitdiff')
 252 }
 253
 254 def static isGcReliabilityFramework(def scenario) {
 255     return (scenario == 'gc_reliability_framework')
 256 }
 257
 258 def static setTestJobTimeOut(newJob, scenario) {
 259     if (isGCStressRelatedTesting(scenario)) {
 260         Utilities.setJobTimeout(newJob, 4320)
 261     }
 262     else if (isCorefxTesting(scenario)) {
 263         Utilities.setJobTimeout(newJob, 360)
 264     }
 265     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 266         Utilities.setJobTimeout(newJob, 240)
 267     }
 268     else if (isR2R(scenario)) {
 269         Utilities.setJobTimeout(newJob, 240)
 270     }
 271     else if (isLongGc(scenario)) {
 272         Utilities.setJobTimeout(newJob, 1440)
 273     }
 274     else if (isJitDiff(scenario)) {
 275         Utilities.setJobTimeout(newJob, 240)
 276     }
 277     else if (isGcReliabilityFramework(scenario)) {
 278         Utilities.setJobTimeout(newJob, 1440)
 279     }
 280     // Non-test jobs use the default timeout value.
 281 }
 282
 283 def static getJobFolder(def scenario) {
 284     if (isJITStressJob(scenario)) {
 285         return 'jitstress'
 286     }
 287     if (scenario == 'illink') {
 288         return 'illink'
 289     }
 290     return ''
 291 }
 292
 293 def static getStressModeDisplayName(def scenario) {
 294     def displayStr = ''
 295     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 296         def prefixLength = 'COMPlus_'.length()
 297         if (k.length() >= prefixLength) {
 298             def modeName = k.substring(prefixLength, k.length())
 299             displayStr += ' ' + modeName + '=' + v
 300         }
 301     }
 302     return displayStr
 303 }
 304
 305 def static getR2RStressModeDisplayName(def scenario) {
 306     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 307     // "r2r_" prefix.
 308     def displayStr = scenario
 309     def prefixLength = 'r2r_'.length()
 310     if (displayStr.length() >= prefixLength) {
 311         displayStr = displayStr.substring(prefixLength, displayStr.length())
 312     }
 313     return displayStr
 314 }
 315
 316 // Generates the string for creating a file that sets environment variables
 317 // that makes it possible to run stress modes.  Writes the script to the file
 318 // specified by the stepScriptLocation parameter.
 319 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 320     def stepScript = ''
 321     if (os == 'Windows_NT') {
 322
 323         // Timeout in ms, default is 10 minutes. For stress modes up this to 30 minutes.
 324         // BUG?: it seems this is ignored, as this script will be run in an environment where
 325         //       environment variables will be discarded before this is used.
 326         def timeout = 1800000
 327         stepScript += "set __TestTimeout=${timeout}\r\n"
 328
 329         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 330         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 331
 332         // Create at least an empty script.
 333         stepScript += "echo. > ${stepScriptLocation}\r\n"
 334
 335         stressModeVars.each{ k, v ->
 336             // Write out what we are writing to the script file
 337             stepScript += "echo Setting ${k}=${v}\r\n"
 338             // Write out the set itself to the script file`
 339             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 340         }
 341     }
 342     else {
 343         stepScript += "echo Setting variables for ${stressModeName}\n"
 344         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 345         stressModeVars.each{ k, v ->
 346             // Write out what we are writing to the script file
 347             stepScript += "echo Setting ${k}=${v}\n"
 348             // Write out the set itself to the script file`
 349             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 350         }
 351         stepScript += "chmod +x ${stepScriptLocation}\n"
 352     }
 353     return stepScript
 354 }
 355
 356 // Append an existing script to a stress mode script already created by genStressModeScriptStep().
 357 // Returns string of commands to do this.
 358 def static appendStressModeScriptStep(def os, def appendScript, def stepScriptLocation) {
 359     assert (os == 'Windows_NT')
 360     def stepScript = ''
 361     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 362     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 363
 364     // Display the resulting script. This is useful when looking at the output log file.
 365     stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 366     stepScript += "type ${stepScriptLocation}\r\n"
 367
 368     return stepScript
 369 }
 370
 371 // Calculates the name of the build job based on some typical parameters.
 372 //
 373 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 374     // If the architecture is x64, do not add that info into the build name.
 375     // Need to change around some systems and other builds to pick up the right builds
 376     // to do that.
 377
 378     def suffix = scenario != 'default' ? "_${scenario}" : '';
 379     if (isBuildOnly) {
 380         suffix += '_bld'
 381     }
 382     def baseName = ''
 383     switch (architecture) {
 384         case 'x64':
 385             if (scenario == 'default') {
 386                 // For now we leave x64 off of the name for compatibility with other jobs
 387                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 388             }
 389             else if (scenario == 'formatting') {
 390                 // we don't care about the configuration for the formatting job. It runs all configs
 391                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 392             }
 393             else {
 394                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 395             }
 396             break
 397         case 'arm64':
 398             if (os.toLowerCase() == "windows_nt") {
 399                 // These are cross builds
 400                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 401             }
 402             else {
 403                 // Defaults to a small page size set of machines.
 404                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 405             }
 406             break
 407         case 'arm':
 408             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 409             break
 410         case 'armlb':
 411             // These are cross builds
 412             if (os == 'Tizen') {
 413                 // ABI: softfp
 414                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 415             }
 416             else {
 417                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 418             }
 419             break
 420         case 'x86':
 421         case 'x86lb':
 422         case 'x86_arm_altjit':
 423         case 'x64_arm64_altjit':
 424             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 425             break
 426         default:
 427             println("Unknown architecture: ${architecture}");
 428             assert false
 429             break
 430     }
 431
 432     return baseName + suffix
 433 }
 434
 435 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 436
 437     // Only PR triggers for altjit jobs.
 438     switch (architecture) {
 439         case 'x86_arm_altjit':
 440         case 'x64_arm64_altjit':
 441             return
 442         default:
 443             break
 444     }
 445
 446     // Check scenario.
 447     switch (scenario) {
 448         case 'default':
 449             switch (architecture) {
 450                 case 'x64':
 451                 case 'x86':
 452                 case 'x86lb':
 453                     if (architecture == 'x86' && os == 'Ubuntu') {
 454                         Utilities.addPeriodicTrigger(job, '@daily')
 455                     }
 456                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 457                         Utilities.addGithubPushTrigger(job)
 458                     }
 459                     break
 460                 case 'arm':
 461                 case 'armlb':
 462                     Utilities.addGithubPushTrigger(job)
 463                     break
 464                 case 'arm64':
 465                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 466                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 467                     break
 468                 default:
 469                     println("Unknown architecture: ${architecture}");
 470                     assert false
 471                     break
 472             }
 473             break
 474         case 'r2r':
 475             assert !(os in bidailyCrossList)
 476             // r2r gets a push trigger for checked/release
 477             if (configuration == 'Checked' || configuration == 'Release') {
 478                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 479                 if (architecture == 'x64' && os != 'OSX10.12') {
 480                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 481                     if (isFlowJob || os == 'Windows_NT') {
 482                         Utilities.addGithubPushTrigger(job)
 483                     }
 484                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 485                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 486                     if (isFlowJob) {
 487                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 488                     }
 489                 }
 490                 // For x86, only add per-commit jobs for Windows
 491                 else if (architecture == 'x86' || architecture == 'x86lb') {
 492                     if (os == 'Windows_NT') {
 493                         Utilities.addGithubPushTrigger(job)
 494                     }
 495                 }
 496                 // arm64 r2r jobs should only run daily.
 497                 else if (architecture == 'arm64') {
 498                     if (os == 'Windows_NT') {
 499                         Utilities.addPeriodicTrigger(job, '@daily')
 500                     }
 501                 }
 502             }
 503             break
 504         case 'r2r_jitstress1':
 505         case 'r2r_jitstress2':
 506         case 'r2r_jitstressregs1':
 507         case 'r2r_jitstressregs2':
 508         case 'r2r_jitstressregs3':
 509         case 'r2r_jitstressregs4':
 510         case 'r2r_jitstressregs8':
 511         case 'r2r_jitstressregs0x10':
 512         case 'r2r_jitstressregs0x80':
 513         case 'r2r_jitstressregs0x1000':
 514         case 'r2r_jitminopts':
 515         case 'r2r_jitforcerelocs':
 516         case 'gcstress15_r2r':
 517             assert !(os in bidailyCrossList)
 518
 519             // GCStress=C is currently not supported on OS X
 520             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 521                 break
 522             }
 523
 524             // GC Stress 15 r2r gets a push trigger for checked/release
 525             if (configuration == 'Checked' || configuration == 'Release') {
 526                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 527                 if (architecture == 'x64') {
 528                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 529                     if (isFlowJob || os == 'Windows_NT') {
 530                         // Add a weekly periodic trigger
 531                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 532                     }
 533                 }
 534                 // For x86, only add per-commit jobs for Windows
 535                 else if (architecture == 'x86') {
 536                     if (os == 'Windows_NT') {
 537                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 538                     }
 539                 }
 540             }
 541             break
 542         case 'longgc':
 543             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 544             assert configuration == 'Release'
 545             assert architecture == 'x64'
 546             Utilities.addPeriodicTrigger(job, '@daily')
 547             // TODO: Add once external email sending is available again
 548             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 549             break
 550         case 'gcsimulator':
 551             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 552             assert configuration == 'Release'
 553             assert architecture == 'x64'
 554             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 555             // TODO: Add once external email sending is available again
 556             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 557             break
 558         case 'standalone_gc':
 559             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 560             assert (configuration == 'Release' || configuration == 'Checked')
 561             // TODO: Add once external email sending is available again
 562             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 563             Utilities.addPeriodicTrigger(job, '@weekly')
 564             break
 565         case 'gc_reliability_framework':
 566             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 567             assert (configuration == 'Release' || configuration == 'Checked')
 568             // Only triggered by phrase.
 569             break
 570         case 'ilrt':
 571             assert !(os in bidailyCrossList)
 572             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 573             if (architecture == 'x64' && configuration == 'Release') {
 574                 // We don't expect to see a job generated except in these scenarios
 575                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 576                 if (isFlowJob || os == 'Windows_NT') {
 577                     Utilities.addPeriodicTrigger(job, '@daily')
 578                 }
 579             }
 580             break
 581         case 'jitdiff':
 582             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 583             assert configuration == 'Checked'
 584             assert (architecture == 'x64' || architecture == 'x86')
 585             Utilities.addGithubPushTrigger(job)
 586             break
 587         case 'formatting':
 588             assert (os == 'Windows_NT' || os == "Ubuntu")
 589             assert architecture == 'x64'
 590             Utilities.addGithubPushTrigger(job)
 591             break
 592         case 'jitstressregs1':
 593         case 'jitstressregs2':
 594         case 'jitstressregs3':
 595         case 'jitstressregs4':
 596         case 'jitstressregs8':
 597         case 'jitstressregs0x10':
 598         case 'jitstressregs0x80':
 599         case 'jitstressregs0x1000':
 600         case 'minopts':
 601         case 'forcerelocs':
 602         case 'jitstress1':
 603         case 'jitstress2':
 604         case 'jitstress2_jitstressregs1':
 605         case 'jitstress2_jitstressregs2':
 606         case 'jitstress2_jitstressregs3':
 607         case 'jitstress2_jitstressregs4':
 608         case 'jitstress2_jitstressregs8':
 609         case 'jitstress2_jitstressregs0x10':
 610         case 'jitstress2_jitstressregs0x80':
 611         case 'jitstress2_jitstressregs0x1000':
 612         case 'tailcallstress':
 613         case 'jitsse2only':
 614         case 'corefx_baseline':
 615         case 'corefx_minopts':
 616         case 'corefx_jitstress1':
 617         case 'corefx_jitstress2':
 618         case 'corefx_jitstressregs1':
 619         case 'corefx_jitstressregs2':
 620         case 'corefx_jitstressregs3':
 621         case 'corefx_jitstressregs4':
 622         case 'corefx_jitstressregs8':
 623         case 'corefx_jitstressregs0x10':
 624         case 'corefx_jitstressregs0x80':
 625         case 'corefx_jitstressregs0x1000':
 626         case 'zapdisable':
 627             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 628                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 629                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 630                     if (os == 'Windows_NT') {
 631                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 632                         Utilities.addPeriodicTrigger(job, '@weekly')
 633                     }
 634                 }
 635                 else {
 636                     Utilities.addPeriodicTrigger(job, '@daily')
 637                 }
 638             }
 639             break
 640         case 'heapverify1':
 641         case 'gcstress0x3':
 642             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 643                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 644                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 645                     if (os == 'Windows_NT') {
 646                         Utilities.addPeriodicTrigger(job, '@daily')
 647                     }
 648                     // TODO: Add once external email sending is available again
 649                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 650                 }
 651                 else {
 652                     Utilities.addPeriodicTrigger(job, '@weekly')
 653                 }
 654             }
 655             break
 656         case 'gcstress0xc':
 657         case 'gcstress0xc_zapdisable':
 658         case 'gcstress0xc_zapdisable_jitstress2':
 659         case 'gcstress0xc_zapdisable_heapverify1':
 660         case 'gcstress0xc_jitstress1':
 661         case 'gcstress0xc_jitstress2':
 662         case 'gcstress0xc_minopts_heapverify1':
 663             // GCStress=C is currently not supported on OS X
 664             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 665                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 666                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 667                     // TODO: Enable a periodic trigger after tests are updated.
 668                     // Utilities.addPeriodicTrigger(job, '@daily')
 669                     // TODO: Add once external email sending is available again
 670                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 671                 }
 672                 else {
 673                     Utilities.addPeriodicTrigger(job, '@weekly')
 674                 }
 675             }
 676             break
 677
 678         case 'illink':
 679             // Testing on other operating systems TBD
 680             assert (os == 'Windows_NT' || os == 'Ubuntu')
 681             if (architecture == 'x64' || architecture == 'x86') {
 682                 if (configuration == 'Checked') {
 683                     Utilities.addPeriodicTrigger(job, '@daily')
 684                 }
 685             }
 686             break
 687
 688         case 'tieredcompilation':
 689         case 'corefx_tieredcompilation':
 690             // No periodic jobs just yet, still testing
 691             break
 692
 693         default:
 694             println("Unknown scenario: ${scenario}");
 695             assert false
 696             break
 697     }
 698     return
 699 }
 700
 701 // **************************
 702 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 703 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 704 // builds will do a couple extra steps.
 705 // **************************
 706
 707 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 708 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 709 // it's just used for internal builds
 710 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 711 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 712     if (isWindowsBuildOnlyJob) {
 713         return
 714     }
 715
 716     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 717     // Non pull request builds.
 718     if (!isPR) {
 719         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 720         return
 721     }
 722
 723      def arm64Users = [
 724         'AndyAyersMS',
 725         'briansull',
 726         'BruceForstall',
 727         'CarolEidt',
 728         'cmckinsey',
 729         'erozenfeld',
 730         'janvorli',
 731         'jashook',
 732         'JosephTremoulet',
 733         'pgodeq',
 734         'pgavlin',
 735         'rartemev',
 736         'russellhadley',
 737         'RussKeldorph',
 738         'sandreenko',
 739         'sdmaclea',
 740         'swaroop-sridhar',
 741         'jkotas',
 742         'markwilkie',
 743         'weshaggard'
 744     ]
 745
 746     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 747     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 748     def osGroup = getOSGroup(os)
 749     switch (architecture) {
 750         case 'x64': // editor brace matching: {
 751             if (scenario == 'formatting') {
 752                 assert configuration == 'Checked'
 753                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 754                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
 755                 }
 756                 break
 757             }
 758
 759             switch (os) {
 760                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 761                 case 'Debian8.4':
 762                 case 'RHEL7.2':
 763                     if (scenario == 'default') {
 764                         assert !isFlowJob
 765                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 766                     }
 767                     break
 768                 case 'Ubuntu16.04':
 769                     assert !isFlowJob
 770                     assert scenario == 'default'
 771                     // Distinguish with the other architectures (arm and x86)
 772                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 773                     break
 774                 case 'Fedora24':
 775                 case 'Ubuntu16.10':
 776                     assert !isFlowJob
 777                     assert scenario == 'default'
 778                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 779                     break
 780                 case 'Ubuntu':
 781                     if (scenario == 'illink') {
 782                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 783                         break
 784                     }
 785                     // fall through
 786                 case 'OSX10.12':
 787                     // Triggers on the non-flow jobs aren't necessary here
 788                     // Corefx testing uses non-flow jobs.
 789                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 790                         break
 791                     }
 792                     switch (scenario) {
 793                         case 'default':
 794                             // OSX uses checked for default PR tests
 795                             if (configuration == 'Checked') {
 796                                 // Default trigger
 797                                 assert !job.name.contains("centos")
 798                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
 799                             }
 800                             break
 801                         case 'jitdiff':
 802                             if (configuration == 'Checked') {
 803                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 804                             }
 805                             break
 806                         case 'ilrt':
 807                             if (configuration == 'Release') {
 808                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 809                             }
 810                             break
 811                         case 'r2r':
 812                             if (configuration == 'Release' || configuration == 'Checked') {
 813                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 814                             }
 815                             break
 816                         case 'gcstress15_r2r':
 817                             if (configuration == 'Release' || configuration == 'Checked') {
 818                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 819                             }
 820                             break
 821                         case 'r2r_jitstress1':
 822                         case 'r2r_jitstress2':
 823                         case 'r2r_jitstressregs1':
 824                         case 'r2r_jitstressregs2':
 825                         case 'r2r_jitstressregs3':
 826                         case 'r2r_jitstressregs4':
 827                         case 'r2r_jitstressregs8':
 828                         case 'r2r_jitstressregs0x10':
 829                         case 'r2r_jitstressregs0x80':
 830                         case 'r2r_jitstressregs0x1000':
 831                         case 'r2r_jitminopts':
 832                         case 'r2r_jitforcerelocs':
 833                             if (configuration == 'Release' || configuration == 'Checked') {
 834                                 def displayStr = getR2RStressModeDisplayName(scenario)
 835                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 836                             }
 837                             break
 838                         case 'longgc':
 839                             if (configuration == 'Release') {
 840                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 841                             }
 842                             break
 843                         case 'gcsimulator':
 844                             if (configuration == 'Release') {
 845                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 846                             }
 847                             break
 848                         case 'standalone_gc':
 849                             if (configuration == 'Release' || configuration == 'Checked') {
 850                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 851                             }
 852                             break
 853                         case 'gc_reliability_framework':
 854                             if (configuration == 'Release' || configuration == 'Checked') {
 855                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 856                             }
 857                             break
 858                         case 'minopts':
 859                         case 'tieredcompilation':
 860                         case 'forcerelocs':
 861                         case 'jitstress1':
 862                         case 'jitstress2':
 863                         case 'jitstressregs1':
 864                         case 'jitstressregs2':
 865                         case 'jitstressregs3':
 866                         case 'jitstressregs4':
 867                         case 'jitstressregs8':
 868                         case 'jitstressregs0x10':
 869                         case 'jitstressregs0x80':
 870                         case 'jitstressregs0x1000':
 871                         case 'jitstress2_jitstressregs1':
 872                         case 'jitstress2_jitstressregs2':
 873                         case 'jitstress2_jitstressregs3':
 874                         case 'jitstress2_jitstressregs4':
 875                         case 'jitstress2_jitstressregs8':
 876                         case 'jitstress2_jitstressregs0x10':
 877                         case 'jitstress2_jitstressregs0x80':
 878                         case 'jitstress2_jitstressregs0x1000':
 879                         case 'tailcallstress':
 880                         case 'jitsse2only':
 881                         case 'gcstress0x3':
 882                         case 'gcstress0xc':
 883                         case 'zapdisable':
 884                         case 'heapverify1':
 885                         case 'gcstress0xc_zapdisable':
 886                         case 'gcstress0xc_zapdisable_jitstress2':
 887                         case 'gcstress0xc_zapdisable_heapverify1':
 888                         case 'gcstress0xc_jitstress1':
 889                         case 'gcstress0xc_jitstress2':
 890                         case 'gcstress0xc_minopts_heapverify1':
 891                             def displayStr = getStressModeDisplayName(scenario)
 892                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 893                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 894                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 895                             break
 896                         case 'corefx_baseline':
 897                         case 'corefx_minopts':
 898                         case 'corefx_tieredcompilation':
 899                         case 'corefx_jitstress1':
 900                         case 'corefx_jitstress2':
 901                         case 'corefx_jitstressregs1':
 902                         case 'corefx_jitstressregs2':
 903                         case 'corefx_jitstressregs3':
 904                         case 'corefx_jitstressregs4':
 905                         case 'corefx_jitstressregs8':
 906                         case 'corefx_jitstressregs0x10':
 907                         case 'corefx_jitstressregs0x80':
 908                         case 'corefx_jitstressregs0x1000':
 909                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 910                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 911                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 912                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 913                             break
 914                         default:
 915                             println("Unknown scenario: ${scenario}");
 916                             assert false
 917                             break
 918                     }
 919                     break
 920                 case 'CentOS7.1':
 921                     switch (scenario) {
 922                         case 'default':
 923                             // CentOS uses checked for default PR tests while debug is build only
 924                             if (configuration == 'Debug') {
 925                                 // Default trigger
 926                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
 927                             }
 928
 929                             // Make sure this is a flow job to get build and test.
 930                             if (configuration == 'Checked' && isFlowJob) {
 931                                 assert job.name.contains("flow")
 932                                 // Default trigger
 933                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
 934                             }
 935                             break
 936                         case 'r2r':
 937                             if (configuration == 'Checked' || configuration == 'Release') {
 938                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 939                             }
 940                             break
 941                         case 'gcstress15_r2r':
 942                             if (configuration == 'Release' || configuration == 'Checked') {
 943                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 944                             }
 945                             break
 946                         case 'r2r_jitstress1':
 947                         case 'r2r_jitstress2':
 948                         case 'r2r_jitstressregs1':
 949                         case 'r2r_jitstressregs2':
 950                         case 'r2r_jitstressregs3':
 951                         case 'r2r_jitstressregs4':
 952                         case 'r2r_jitstressregs8':
 953                         case 'r2r_jitstressregs0x10':
 954                         case 'r2r_jitstressregs0x80':
 955                         case 'r2r_jitstressregs0x1000':
 956                         case 'r2r_jitminopts':
 957                         case 'r2r_jitforcerelocs':
 958                             if (configuration == 'Release' || configuration == 'Checked') {
 959                                 def displayStr = getR2RStressModeDisplayName(scenario)
 960                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 961                             }
 962                             break
 963                         default:
 964                             break
 965                     }
 966
 967                     break
 968                 case 'Windows_NT':
 969                     switch (scenario) {
 970                         case 'default':
 971                             // Default trigger
 972                             if (configuration == 'Checked') {
 973                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
 974                             }
 975                             break
 976                         case 'jitdiff':
 977                             if (configuration == 'Checked') {
 978                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 979                             }
 980                             break
 981                         case 'ilrt':
 982                             if (configuration == 'Release') {
 983                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 984                             }
 985                             break
 986                         case 'r2r':
 987                             if (configuration == 'Checked' || configuration == 'Release') {
 988                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 989                             }
 990                             break
 991                         case 'gcstress15_r2r':
 992                             if (configuration == 'Release' || configuration == 'Checked') {
 993                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 994                             }
 995                             break
 996                         case 'r2r_jitstress1':
 997                         case 'r2r_jitstress2':
 998                         case 'r2r_jitstressregs1':
 999                         case 'r2r_jitstressregs2':
1000                         case 'r2r_jitstressregs3':
1001                         case 'r2r_jitstressregs4':
1002                         case 'r2r_jitstressregs8':
1003                         case 'r2r_jitstressregs0x10':
1004                         case 'r2r_jitstressregs0x80':
1005                         case 'r2r_jitstressregs0x1000':
1006                         case 'r2r_jitminopts':
1007                         case 'r2r_jitforcerelocs':
1008                             if (configuration == 'Release' || configuration == 'Checked') {
1009                                 def displayStr = getR2RStressModeDisplayName(scenario)
1010                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1011                             }
1012                             break
1013                         case 'longgc':
1014                             if (configuration == 'Release') {
1015                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1016                             }
1017                             break
1018                         case 'gcsimulator':
1019                             if (configuration == 'Release') {
1020                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1021                             }
1022                             break
1023                         case 'standalone_gc':
1024                             if (configuration == 'Release' || configuration == 'Checked') {
1025                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1026                             }
1027                             break
1028                         case 'gc_reliability_framework':
1029                             if (configuration == 'Release' || configuration == 'Checked') {
1030                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1031                             }
1032                             break
1033                         case 'minopts':
1034                         case 'tieredcompilation':
1035                         case 'forcerelocs':
1036                         case 'jitstress1':
1037                         case 'jitstress2':
1038                         case 'jitstressregs1':
1039                         case 'jitstressregs2':
1040                         case 'jitstressregs3':
1041                         case 'jitstressregs4':
1042                         case 'jitstressregs8':
1043                         case 'jitstressregs0x10':
1044                         case 'jitstressregs0x80':
1045                         case 'jitstressregs0x1000':
1046                         case 'jitstress2_jitstressregs1':
1047                         case 'jitstress2_jitstressregs2':
1048                         case 'jitstress2_jitstressregs3':
1049                         case 'jitstress2_jitstressregs4':
1050                         case 'jitstress2_jitstressregs8':
1051                         case 'jitstress2_jitstressregs0x10':
1052                         case 'jitstress2_jitstressregs0x80':
1053                         case 'jitstress2_jitstressregs0x1000':
1054                         case 'tailcallstress':
1055                         case 'jitsse2only':
1056                         case 'gcstress0x3':
1057                         case 'gcstress0xc':
1058                         case 'zapdisable':
1059                         case 'heapverify1':
1060                         case 'gcstress0xc_zapdisable':
1061                         case 'gcstress0xc_zapdisable_jitstress2':
1062                         case 'gcstress0xc_zapdisable_heapverify1':
1063                         case 'gcstress0xc_jitstress1':
1064                         case 'gcstress0xc_jitstress2':
1065                         case 'gcstress0xc_minopts_heapverify1':
1066                             def displayStr = getStressModeDisplayName(scenario)
1067                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1068                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1069                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1070                             break
1071                         case 'corefx_baseline':
1072                         case 'corefx_minopts':
1073                         case 'corefx_tieredcompilation':
1074                         case 'corefx_jitstress1':
1075                         case 'corefx_jitstress2':
1076                         case 'corefx_jitstressregs1':
1077                         case 'corefx_jitstressregs2':
1078                         case 'corefx_jitstressregs3':
1079                         case 'corefx_jitstressregs4':
1080                         case 'corefx_jitstressregs8':
1081                         case 'corefx_jitstressregs0x10':
1082                         case 'corefx_jitstressregs0x80':
1083                         case 'corefx_jitstressregs0x1000':
1084                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1085                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1086                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1087                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1088                             break
1089                         case 'illink':
1090                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1091                             break
1092                         default:
1093                             println("Unknown scenario: ${scenario}");
1094                             assert false
1095                             break
1096                     }
1097                     break
1098                 default:
1099                     println("Unknown os: ${os}");
1100                     assert false
1101                     break
1102             }
1103             break
1104         // editor brace matching: }
1105         case 'armlb':
1106         case 'arm': // editor brace matching: {
1107             switch (os) {
1108                 case 'Ubuntu':
1109                 case 'Ubuntu16.04':
1110                     if (architecture == 'armlb') { // Ubuntu arm is only for armlb currently
1111                         assert scenario == 'default'
1112                         job.with {
1113                             publishers {
1114                                 azureVMAgentPostBuildAction {
1115                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1116                                 }
1117                             }
1118                         }
1119                         if ((os == 'Ubuntu' && configuration == 'Debug') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1120                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1121                         }
1122                         else {
1123                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1124                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1125                         }
1126                     }
1127                     break
1128                 case 'Tizen':
1129                     if (architecture == 'armlb') { // Tizen armel is only for armlb currently
1130                         architecture='armel'
1131                         job.with {
1132                             publishers {
1133                                 azureVMAgentPostBuildAction {
1134                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1135                                 }
1136                             }
1137                         }
1138                         // Removing the regex will cause this to run on each PR.
1139                         if (configuration == 'Checked') {
1140                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1141                         }
1142                         else {
1143                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1144                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1145                         }
1146                     }
1147                     break
1148                 case 'Windows_NT':
1149                     // Set up a private trigger
1150                     def contextString = "${os} ${architecture} Cross ${configuration}"
1151                     if (scenario == 'default') {
1152                         contextString += " Innerloop"
1153                     }
1154                     else {
1155                         contextString += " ${scenario}"
1156                     }
1157
1158                     contextString += " Build"
1159                     // Debug configuration only does builds, no tests.
1160                     if (configuration != 'Debug') {
1161                         contextString += " and Test"
1162                     }
1163                     switch (scenario) {
1164                         case 'default':
1165                             // Only Checked is a default trigger.
1166                             if (configuration == 'Checked')
1167                             {
1168                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1169                             }
1170                             else
1171                             {
1172                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1173                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}.*", null, arm64Users)
1174                             }
1175                             break
1176                         default:
1177                             // Stress jobs will use this code path.
1178                             if (Constants.validArmWindowsScenarios.contains(scenario)) {
1179                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1180                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1181                             }
1182                             break
1183                     }
1184                     break
1185                 default:
1186                     println("NYI os: ${os}");
1187                     assert false
1188                     break
1189             }
1190             break
1191         // editor brace matching: }
1192         case 'arm64': // editor brace matching: {
1193             assert Constants.validArmWindowsScenarios.contains(scenario)
1194
1195             // Set up a private trigger
1196             def contextString = "${os} ${architecture} Cross ${configuration}"
1197             if (scenario == 'default') {
1198                 contextString += " Innerloop"
1199             }
1200             else {
1201                 contextString += " ${scenario}"
1202             }
1203
1204             contextString += " Build"
1205             // Debug builds only.
1206             if (configuration != 'Debug') {
1207                contextString += " and Test"
1208             }
1209
1210             switch (os) {
1211                 case 'Ubuntu':
1212                 case 'Ubuntu16.04':
1213                     switch (scenario) {
1214                         case 'default':
1215                             if (configuration == 'Debug' && !isFlowJob) {
1216                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1217                             }
1218                             else {
1219                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build & Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1220                             }
1221
1222                             break
1223                         case 'r2r':
1224                             if (configuration == 'Checked' || configuration == 'Release') {
1225                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R Build & Test",
1226                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1227                             }
1228                             break
1229                         case 'gcstress15_r2r':
1230                             if (configuration == 'Release' || configuration == 'Checked') {
1231                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R Build & Test",
1232                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1233                             }
1234                             break
1235                         case 'r2r_jitstress1':
1236                         case 'r2r_jitstress2':
1237                         case 'r2r_jitstressregs1':
1238                         case 'r2r_jitstressregs2':
1239                         case 'r2r_jitstressregs3':
1240                         case 'r2r_jitstressregs4':
1241                         case 'r2r_jitstressregs8':
1242                         case 'r2r_jitstressregs0x10':
1243                         case 'r2r_jitstressregs0x80':
1244                         case 'r2r_jitstressregs0x1000':
1245                         case 'r2r_jitminopts':
1246                         case 'r2r_jitforcerelocs':
1247                             if (configuration == 'Release' || configuration == 'Checked') {
1248                                 def displayStr = getR2RStressModeDisplayName(scenario)
1249                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1250                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1251                             }
1252                             break
1253                         default:
1254                             break
1255                     }
1256                     break
1257                 case 'Windows_NT':
1258                     switch (scenario) {
1259                         case 'default':
1260                             if (isFlowJob == true) {
1261                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration}",
1262                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1263                             }
1264                             // Only run Checked jobs on PR Trigger.
1265                             else if (configuration == 'Debug' || configuration == 'Checked') {
1266                                 // Add "Debug Build" to the above users' PRs since many of them are at higher risk of ARM64-breaking changes.
1267                                 //
1268                                 // Arm64 Ubuntu only builds at the moment due to insufficient hardware to do testing on pr.
1269                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1270                             }
1271                             else {
1272                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1273                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}.*", null, arm64Users)
1274                             }
1275                             break
1276                         default:
1277                             // Stress jobs will use this code path.
1278                             if (Constants.validArmWindowsScenarios.contains(scenario)) {
1279                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1280                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1281                             }
1282                             break
1283                     }
1284                     break
1285                 default:
1286                     println("NYI os: ${os}");
1287                     assert false
1288                     break
1289             }
1290             break
1291         // editor brace matching: }
1292         case 'x86': // editor brace matching: {
1293             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1294             if (os == 'Ubuntu') {
1295                 // on-demand only for ubuntu x86
1296                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1297                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1298                 break
1299             }
1300             switch (scenario) {
1301                 case 'default':
1302                     if (configuration == 'Checked') {
1303                         assert !job.name.contains("centos")
1304                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1305                     }
1306                     else {
1307                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1308                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1309                     }
1310                     break
1311                 case 'ilrt':
1312                     if (configuration == 'Release') {
1313                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1314                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1315                     }
1316                     break
1317                 case 'r2r':
1318                     if (configuration == 'Checked' || configuration == 'Release') {
1319                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R Build & Test",
1320                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1321                     }
1322                     break
1323                 case 'gcstress15_r2r':
1324                     if (configuration == 'Release' || configuration == 'Checked') {
1325                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R Build & Test",
1326                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1327                     }
1328                     break
1329                 case 'r2r_jitstress1':
1330                 case 'r2r_jitstress2':
1331                 case 'r2r_jitstressregs1':
1332                 case 'r2r_jitstressregs2':
1333                 case 'r2r_jitstressregs3':
1334                 case 'r2r_jitstressregs4':
1335                 case 'r2r_jitstressregs8':
1336                 case 'r2r_jitstressregs0x10':
1337                 case 'r2r_jitstressregs0x80':
1338                 case 'r2r_jitstressregs0x1000':
1339                 case 'r2r_jitminopts':
1340                 case 'r2r_jitforcerelocs':
1341                     if (configuration == 'Release' || configuration == 'Checked') {
1342                         def displayStr = getR2RStressModeDisplayName(scenario)
1343                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1344                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1345                     }
1346                     break
1347                 case 'longgc':
1348                     if (configuration == 'Release') {
1349                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1350                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1351                     }
1352                     break
1353                 case 'gcsimulator':
1354                     if (configuration == 'Release') {
1355                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1356                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1357                     }
1358                     break
1359                 case 'standalone_gc':
1360                     if (configuration == 'Release' || configuration == 'Checked') {
1361                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1362                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1363                     }
1364                     break
1365                 case 'minopts':
1366                 case 'tieredcompilation':
1367                 case 'forcerelocs':
1368                 case 'jitstress1':
1369                 case 'jitstress2':
1370                 case 'jitstressregs1':
1371                 case 'jitstressregs2':
1372                 case 'jitstressregs3':
1373                 case 'jitstressregs4':
1374                 case 'jitstressregs8':
1375                 case 'jitstressregs0x10':
1376                 case 'jitstressregs0x80':
1377                 case 'jitstressregs0x1000':
1378                 case 'jitstress2_jitstressregs1':
1379                 case 'jitstress2_jitstressregs2':
1380                 case 'jitstress2_jitstressregs3':
1381                 case 'jitstress2_jitstressregs4':
1382                 case 'jitstress2_jitstressregs8':
1383                 case 'jitstress2_jitstressregs0x10':
1384                 case 'jitstress2_jitstressregs0x80':
1385                 case 'jitstress2_jitstressregs0x1000':
1386                 case 'tailcallstress':
1387                 case 'jitsse2only':
1388                 case 'gcstress0x3':
1389                 case 'gcstress0xc':
1390                 case 'zapdisable':
1391                 case 'heapverify1':
1392                 case 'gcstress0xc_zapdisable':
1393                 case 'gcstress0xc_zapdisable_jitstress2':
1394                 case 'gcstress0xc_zapdisable_heapverify1':
1395                 case 'gcstress0xc_jitstress1':
1396                 case 'gcstress0xc_jitstress2':
1397                 case 'gcstress0xc_minopts_heapverify1':
1398                     def displayStr = getStressModeDisplayName(scenario)
1399                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1400                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1401                     break
1402                 case 'corefx_baseline':
1403                 case 'corefx_minopts':
1404                 case 'corefx_tieredcompilation':
1405                 case 'corefx_jitstress1':
1406                 case 'corefx_jitstress2':
1407                 case 'corefx_jitstressregs1':
1408                 case 'corefx_jitstressregs2':
1409                 case 'corefx_jitstressregs3':
1410                 case 'corefx_jitstressregs4':
1411                 case 'corefx_jitstressregs8':
1412                 case 'corefx_jitstressregs0x10':
1413                 case 'corefx_jitstressregs0x80':
1414                 case 'corefx_jitstressregs0x1000':
1415                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1416                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1417                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1418                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1419                     break
1420                 case 'illink':
1421                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1422                     break
1423                 default:
1424                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1425                     assert false
1426                     break
1427             }
1428             break
1429          // editor brace matching: }
1430         case 'x86lb': // editor brace matching: {
1431             assert (os == 'Windows_NT')
1432             assert (scenario == 'default' || Constants.r2rJitStressScenarios.indexOf(scenario) !=1)
1433
1434             def arch = 'x86'
1435             def jit = 'legacy_backend'
1436             switch (scenario) {
1437                 case 'default':
1438                     if (configuration == 'Checked') {
1439                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1440                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1441                     }
1442                     break
1443                 default:
1444                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1445                     assert false
1446                     break
1447             }
1448             break
1449         // editor brace matching: }
1450         case 'x64_arm64_altjit':
1451         case 'x86_arm_altjit': // editor brace matching: {
1452             assert (os == 'Windows_NT')
1453             switch (scenario) {
1454                 case 'default':
1455                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1456                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1457                     break
1458                 default:
1459                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1460                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1461                     break
1462             }
1463             break
1464         // editor brace matching: }
1465         default:
1466             println("Unknown architecture: ${architecture}");
1467             assert false
1468             break
1469     }
1470 }
1471
1472 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1473     def buildCommands = [];
1474     def osGroup = getOSGroup(os)
1475     def lowerConfiguration = configuration.toLowerCase()
1476
1477     def priority = '1'
1478     if (scenario == 'default' && isPR == true) {
1479         priority = '0'
1480     }
1481
1482     // Calculate the build steps, archival, and xunit results
1483     switch (os) {
1484         case 'Windows_NT': // editor brace matching: {
1485             switch (architecture) {
1486                 case 'x64':
1487                 case 'x86':
1488                 case 'x86lb':
1489                 case 'x86_arm_altjit':
1490                 case 'x64_arm64_altjit':
1491                     def arch = architecture
1492                     def buildOpts = ''
1493                     if ((architecture == 'x86lb') || (architecture == 'x86_arm_altjit')) {
1494                         arch = 'x86'
1495                     }
1496                     else if (architecture == 'x64_arm64_altjit') {
1497                         arch = 'x64'
1498                     }
1499
1500                     if (scenario == 'illink') {
1501                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1502                     }
1503
1504                     // If it is a release build for windows, ensure PGO is used, else fail the build
1505                     if ((lowerConfiguration == 'release') &&
1506                         (scenario in Constants.basicScenarios) &&
1507                         (architecture != 'x86lb') &&
1508                         (architecture != 'x86_arm_altjit') &&
1509                         (architecture != 'x64_arm64_altjit')) {
1510
1511                         buildOpts += ' -enforcepgo'
1512                     }
1513
1514                     if (enableCorefxTesting) {
1515                         buildOpts += ' skiptests';
1516                     } else {
1517                         // Note: currently, build.cmd will pass through all arguments starting with an argument it doesn't recognize.
1518                         // Since it doesn't process/recognize '-priority', make sure this is the last argument passed.
1519                         buildOpts += " -priority=${priority}"
1520                     }
1521
1522                     // Set __TestIntermediateDir for pri1 test builds.
1523                     // if __TestIntermediateDir is already set, build-test.cmd will
1524                     // output test binaries to that directory. If it is not set, the
1525                     // binaries are sent to a default directory whose name is about
1526                     // 35 characters long.
1527
1528                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1529                             scenario == 'default' ||
1530                             scenario == 'r2r' ||
1531                             scenario == 'gcstress15_r2r' ||
1532                             isGcReliabilityFramework(scenario) ||
1533                             scenario == 'jitdiff' ||
1534                             scenario == 'ilrt' ||
1535                             scenario == 'illink' ||
1536                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1537                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1538                     }
1539                     else if (isLongGc(scenario)) {
1540                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1541                     }
1542                     else if (scenario == 'standalone_gc') {
1543                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1544                     }
1545                     else if (scenario == 'formatting') {
1546                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1547                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1548                         break
1549                     }
1550                     else {
1551                         println("Unknown scenario: ${scenario}")
1552                         assert false
1553                     }
1554
1555                     if (!isBuildOnly) {
1556                         def crossgenStr = ''
1557                         def runcrossgentestsStr = ''
1558                         def runjitstressStr = ''
1559                         def runjitstressregsStr = ''
1560                         def runjitmioptsStr = ''
1561                         def runjitforcerelocsStr = ''
1562                         def runjitdisasmStr = ''
1563                         def runilasmroundtripStr = ''
1564                         def gcstressStr = ''
1565                         def gcTestArguments = ''
1566                         def illinkArguments = ''
1567                         def testEnvStr = ''
1568                         def runtestArguments = ''
1569
1570                         if (scenario == 'r2r' ||
1571                             scenario == 'gcstress15_r2r' ||
1572                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1573
1574                             // If this is a crossgen build, pass 'crossgen' to runtest.cmd
1575                             crossgenStr = 'crossgen'
1576                             runcrossgentestsStr = 'runcrossgentests'
1577
1578                             if (scenario == 'r2r_jitstress1'){
1579                                 runjitstressStr = 'jitstress 1'
1580                             }
1581                             else if (scenario == 'r2r_jitstress2') {
1582                                 runjitstressStr = 'jitstress 2'
1583                             }
1584                             else if (scenario == 'r2r_jitstressregs1'){
1585                                 runjitstressregsStr = 'jitstressregs 1'
1586                             }
1587                             else if (scenario == 'r2r_jitstressregs2') {
1588                                 runjitstressregsStr = 'jitstressregs 2'
1589                             }
1590                             else if (scenario == 'r2r_jitstressregs3') {
1591                                 runjitstressregsStr = 'jitstressregs 3'
1592                                }
1593                             else if (scenario == 'r2r_jitstressregs4') {
1594                                 runjitstressregsStr = 'jitstressregs 4'
1595                             }
1596                             else if (scenario == 'r2r_jitstressregs8') {
1597                                 runjitstressregsStr = 'jitstressregs 8'
1598                             }
1599                             else if (scenario == 'r2r_jitstressregs0x10') {
1600                                 runjitstressregsStr = 'jitstressregs 0x10'
1601                             }
1602                             else if (scenario == 'r2r_jitstressregs0x80') {
1603                                 runjitstressregsStr = 'jitstressregs 0x80'
1604                             }
1605                             else if (scenario == 'r2r_jitstressregs0x1000') {
1606                                 runjitstressregsStr = 'jitstressregs 0x1000'
1607                             }
1608                             else if (scenario == 'r2r_jitminopts') {
1609                                 runjitmioptsStr = 'jitminopts'
1610                             }
1611                             else if (scenario == 'r2r_jitforcerelocs') {
1612                                 runjitforcerelocsStr = 'jitforcerelocs'
1613                             }
1614                         }
1615                         else if (scenario == 'gcstress15_r2r') {
1616                             gcstressStr = 'gcstresslevel 0xF'
1617                         }
1618                         else if (scenario == 'jitdiff') {
1619                             runjitdisasmStr = 'jitdisasm crossgen'
1620                         }
1621                         else if (scenario == 'ilrt') {
1622                             runilasmroundtripStr = 'ilasmroundtrip'
1623                         }
1624                         else if (isLongGc(scenario)) {
1625                             gcTestArguments = "${scenario} sequential"
1626                         }
1627                         else if (scenario == 'illink')
1628                         {
1629                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1630                         }
1631
1632                         def envScriptPath = ''
1633                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1634                             def buildCommandsStr = ''
1635                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1636                             buildCommandsStr += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], envScriptPath)
1637                             if (architecture == 'x86lb') {
1638                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd", envScriptPath)
1639                             }
1640                             else if (architecture == 'x86_arm_altjit') {
1641                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1642                             }
1643                             else if (architecture == 'x64_arm64_altjit') {
1644                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1645                             }
1646
1647                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1648                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1649                             // into a single script.
1650                             buildCommands += buildCommandsStr
1651                         }
1652                         else if (architecture == 'x86lb') {
1653                             envScriptPath = "%WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd"
1654                         }
1655                         else if (architecture == 'x86_arm_altjit') {
1656                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1657                         }
1658                         else if (architecture == 'x64_arm64_altjit') {
1659                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1660                         }
1661                         if (envScriptPath != '') {
1662                             testEnvStr = "TestEnv ${envScriptPath}"
1663                         }
1664
1665                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps ${testEnvStr}"
1666
1667                         // If we are running a stress mode, we should write out the set of key
1668                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1669
1670                         if (Constants.jitStressModeScenarios.containsKey(scenario) && enableCorefxTesting) {
1671                             def workspaceRelativeFxRoot = "_/fx"
1672                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1673
1674                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1675
1676                             setTestJobTimeOut(newJob, scenario)
1677
1678                             // Archive and process (only) the test results
1679                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1680                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1681
1682                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1683                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1684                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1685                         }
1686                         else if (isGcReliabilityFramework(scenario)) {
1687                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1688                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1689                         }
1690                         else {
1691                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1692                         }
1693                     }
1694
1695                     if (!enableCorefxTesting) {
1696                         // Run the rest of the build
1697                         // Build the mscorlib for the other OS's
1698                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1699                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1700
1701                         if (arch == "x64") {
1702                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1703                         }
1704
1705                         // Zip up the tests directory so that we don't use so much space/time copying
1706                         // 10s of thousands of files around.
1707                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1708
1709                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1710                             // For windows, pull full test results and test drops for x86/x64.
1711                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1712                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1713                         }
1714
1715                         if (scenario == 'jitdiff') {
1716                             // retrive jit-dasm output for base commit, and run jit-diff
1717                             if (!isBuildOnly) {
1718                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1719                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1720                             }
1721                         }
1722
1723                         if (!isBuildOnly) {
1724                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1725                             setTestJobTimeOut(newJob, scenario)
1726                         }
1727                     }
1728                     break
1729                 case 'armlb':
1730                 case 'arm':
1731                     assert Constants.validArmWindowsScenarios.contains(scenario)
1732
1733                     // Set time out
1734                     setTestJobTimeOut(newJob, scenario)
1735
1736                     def buildArchitecture = 'arm'
1737
1738                     if ( lowerConfiguration == "debug" ) {
1739                         // For Debug builds, we will do a priority 0 test build
1740                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority}"
1741                     }
1742                     else {
1743                         if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1744                         {
1745                            // Up the timeout for arm checked testing only.
1746                            // Keep the longer timeout for gcstress.
1747                            Utilities.setJobTimeout(newJob, 240)
1748                         }
1749
1750                         def machineAffinityOptions = ['use_arm64_build_machine' : true]
1751                         setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1752
1753                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority}"
1754
1755                         // Also run testing.
1756                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${buildArchitecture} -build_type ${lowerConfiguration} -scenario ${scenario} -testarch ${architecture} -priority ${priority} -key_location C:\\tools\\key.txt"
1757                     }
1758                     // Add archival.
1759                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1760                     break
1761                 case 'arm64':
1762                     assert Constants.validArmWindowsScenarios.contains(scenario)
1763
1764                     // Set time out
1765                     setTestJobTimeOut(newJob, scenario)
1766
1767                     // Debug runs take too long to run. So build job only.
1768                     if (lowerConfiguration == "debug") {
1769                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1770                     }
1771                     else {
1772                         if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1773                             // Up the timeout for arm64 checked testing only.
1774                             // Keep the longer timeout for gcstress.
1775                             Utilities.setJobTimeout(newJob, 240)
1776                         }
1777
1778                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1779
1780                         // Test build and run are launched together.
1781                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -testarch ${architecture} -priority ${priority} -key_location C:\\tools\\key.txt"
1782                         //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1783                     }
1784
1785                     // Add archival.
1786                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1787                     break
1788                 default:
1789                     println("Unknown architecture: ${architecture}");
1790                     assert false
1791                     break
1792             }
1793             break
1794         // editor brace matching: }
1795         case 'Ubuntu':
1796         case 'Ubuntu16.04':
1797         case 'Ubuntu16.10':
1798         case 'Debian8.4':
1799         case 'OSX10.12':
1800         case 'CentOS7.1':
1801         case 'RHEL7.2':
1802         case 'Tizen':
1803         case 'Fedora24': // editor brace matching: {
1804             switch (architecture) {
1805                 case 'x64':
1806                 case 'x86':
1807                     if (architecture == 'x86' && os == 'Ubuntu') {
1808                         // build and PAL test
1809                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1810                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1811                         break
1812                     }
1813
1814                     if (scenario == 'formatting') {
1815                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1816                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1817                         break
1818                     }
1819
1820                     if (scenario == 'illink') {
1821                         assert(os == 'Ubuntu')
1822                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1823                     }
1824
1825                     def standaloneGc = ''
1826                     if (scenario == 'standalone_gc') {
1827                         standaloneGc = 'buildstandalonegc'
1828                     }
1829
1830                     def disablePgo = ''
1831                     if (lowerConfiguration == 'release') {
1832                         disablePgo = 'nopgooptimize'
1833                     }
1834
1835                     if (!enableCorefxTesting) {
1836                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1837                         // only on supported OS platforms.
1838                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1839                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1840                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc} ${disablePgo}"
1841                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1842
1843                         // Set time out
1844                         setTestJobTimeOut(newJob, scenario)
1845                         // Basic archiving of the build
1846                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1847                         // And pal tests
1848                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1849                     }
1850                     else {
1851                         // Corefx stress testing
1852                         assert os == 'Ubuntu'
1853                         assert architecture == 'x64'
1854                         assert lowerConfiguration == 'checked'
1855                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1856
1857                         // Build coreclr
1858                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1859
1860                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1861                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1862
1863                         // Build and text corefx
1864                         def workspaceRelativeFxRoot = "_/fx"
1865                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1866
1867                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1868
1869                         setTestJobTimeOut(newJob, scenario)
1870
1871                         // Archive and process (only) the test results
1872                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1873                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1874                     }
1875                     break
1876                 case 'arm64':
1877                     def standaloneGc = ''
1878                     if (scenario == 'standalone_gc') {
1879                         standaloneGc = 'buildstandalonegc'
1880                     }
1881
1882                     if (!enableCorefxTesting) {
1883                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8 ${standaloneGc}"
1884
1885                         // HACK -- Arm64 does not have corefx jobs yet.
1886                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1887                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1888                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1889                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1890
1891                         // Set time out
1892                         setTestJobTimeOut(newJob, scenario)
1893                         // Basic archiving of the build
1894                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1895                     }
1896                     break
1897                 case 'armlb':
1898                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1899                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1900
1901                     // default values for Ubuntu
1902                     def arm_abi="arm"
1903                     def linuxCodeName="trusty"
1904                     if (os == 'Ubuntu16.04') {
1905                         linuxCodeName="xenial"
1906                     }
1907                     else if (os == 'Tizen') {
1908                         arm_abi="armel"
1909                         linuxCodeName="tizen"
1910                     }
1911
1912                     // Unzip the Windows test binaries first. Exit with 0
1913                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1914
1915                     // Unpack the corefx binaries
1916                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1917                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1918                     if (os != 'Tizen') {
1919                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1920                     }
1921                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1922                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1923                     if (os == 'Ubuntu') {
1924                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1925                     }
1926
1927                     // Call the ARM CI script to cross build and test using docker
1928                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1929                     --mode=docker \\
1930                     --${arm_abi} \\
1931                     --linuxCodeName=${linuxCodeName} \\
1932                     --buildConfig=${lowerConfiguration} \\
1933                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1934                     --coreFxBinDir=./bin/CoreFxBinDir \\
1935                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1936
1937                     // Basic archiving of the build, no pal tests
1938                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1939                     break
1940                 default:
1941                     println("Unknown architecture: ${architecture}");
1942                     assert false
1943                     break
1944             }
1945             break
1946         // editor brace matching: }
1947         default:
1948             println("Unknown os: ${os}");
1949             assert false
1950             break
1951     } // os
1952
1953     return buildCommands
1954 }
1955
1956 // Additional scenario which can alter behavior
1957
1958 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1959 combinedScenarios.each { scenario ->
1960     [true, false].each { isPR ->
1961         Constants.architectureList.each { architecture ->
1962             Constants.configurationList.each { configuration ->
1963                 Constants.osList.each { os ->
1964                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1965                     // and reset the os to Windows_NT
1966                     def isBuildOnly = false
1967                     if (os == 'Windows_NT_BuildOnly') {
1968                         isBuildOnly = true
1969                         os = 'Windows_NT'
1970                     }
1971
1972                     // Tizen is only supported for arm legacy_backend architecture
1973                     if (os == 'Tizen' && architecture != 'armlb') {
1974                         return
1975                     }
1976
1977                     // Skip totally unimplemented (in CI) configurations.
1978                     switch (architecture) {
1979                         case 'arm64':
1980                             if (os == 'Ubuntu16.04') {
1981                                 os = 'Ubuntu'
1982                             }
1983
1984                             // Windows and Ubuntu only
1985                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1986                                 return
1987                             }
1988                             break
1989                         case 'armlb':
1990                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1991                                 return
1992                             }
1993                             break
1994                         case 'arm':
1995                             if (os != 'Windows_NT') {
1996                                 return
1997                             }
1998                             break
1999                         case 'x86':
2000                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
2001                                 return
2002                             }
2003                             break
2004                         case 'x86lb':
2005                         case 'x86_arm_altjit':
2006                         case 'x64_arm64_altjit':
2007                             if (os != 'Windows_NT') {
2008                                 return
2009                             }
2010                             break
2011                         case 'x64':
2012                             // Everything implemented
2013                             break
2014                         default:
2015                             println("Unknown architecture: ${architecture}")
2016                             assert false
2017                             break
2018                     }
2019
2020                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
2021                     // with checked builds)
2022                     def enableCorefxTesting = false
2023                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2024                         if (configuration != 'Checked') {
2025                             return
2026                         }
2027
2028                         enableCorefxTesting = isCorefxTesting(scenario)
2029
2030                         // Since these are just execution time differences,
2031                         // skip platforms that don't execute the tests here (Windows_NT only)
2032                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
2033                         if (!isEnabledOS || isBuildOnly) {
2034                             return
2035                         }
2036
2037                         switch (architecture) {
2038                             case 'armlb':
2039                             case 'arm':
2040                             case 'arm64':
2041                                 if (!Constants.validArmWindowsScenarios.contains(scenario)) {
2042                                     return
2043                                 }
2044                                 break
2045                             case 'x64':
2046                             case 'x86':
2047                             case 'x86_arm_altjit':
2048                             case 'x64_arm64_altjit':
2049                                 // x86 ubuntu: default only
2050                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
2051                                     return
2052                                 }
2053                                 // Windows: Everything implemented
2054                                 break
2055                             case 'x86lb':
2056                                 // No stress modes for legacy jit.
2057                                 // (There's no technical reason we couldn't allow these.)
2058                                 return
2059                             default:
2060                                 return
2061                         }
2062                     }
2063                     else {
2064                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
2065                         // and configuration is not Checked, bail out.
2066                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2067                             return;
2068                         }
2069
2070                         // Skip scenarios
2071                         switch (scenario) {
2072                             case 'ilrt':
2073                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2074                                 // the default scenario build
2075                                 if (os != 'Windows_NT') {
2076                                     return
2077                                 }
2078                                 // Only x64 for now
2079                                 if (architecture != 'x64') {
2080                                     return
2081                                 }
2082                                 // Release only
2083                                 if (configuration != 'Release') {
2084                                     return
2085                                 }
2086                                 break
2087                             case 'jitdiff':
2088                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2089                                     return
2090                                 }
2091                                 if (architecture != 'x64') {
2092                                     return
2093                                 }
2094                                 if (configuration != 'Checked') {
2095                                     return
2096                                 }
2097                                 break
2098                             case 'r2r':
2099                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2100                                 // the default scenario build
2101                                 if (os != 'Windows_NT') {
2102                                     return
2103                                 }
2104                                 if (architecture != 'x64') {
2105                                     if ((architecture != 'arm64' && architecture != 'armlb' && architecture != 'arm') || (configuration == 'Debug')) {
2106                                         return
2107                                     }
2108                                 }
2109                                 break
2110                             case 'gcstress15_r2r':
2111                             case 'r2r_jitstress1':
2112                             case 'r2r_jitstress2':
2113                             case 'r2r_jitstressregs1':
2114                             case 'r2r_jitstressregs2':
2115                             case 'r2r_jitstressregs3':
2116                             case 'r2r_jitstressregs4':
2117                             case 'r2r_jitstressregs8':
2118                             case 'r2r_jitstressregs0x10':
2119                             case 'r2r_jitstressregs0x80':
2120                             case 'r2r_jitstressregs0x1000':
2121                             case 'r2r_jitminopts':
2122                             case 'r2r_jitforcerelocs':
2123                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
2124                                 // the default scenario build
2125                                 if (os != 'Windows_NT') {
2126                                     return
2127                                 }
2128                                 if (architecture != 'x64' && architecture != 'x86') {
2129                                     return
2130                                 }
2131                                 break
2132                             case 'longgc':
2133                             case 'gcsimulator':
2134                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2135                                     return
2136                                 }
2137                                 if (architecture != 'x64') {
2138                                     return
2139                                 }
2140                                 if (configuration != 'Release') {
2141                                     return
2142                                 }
2143                                 break
2144                             case 'gc_reliability_framework':
2145                             case 'standalone_gc':
2146                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2147                                     return
2148                                 }
2149
2150                                 if (architecture != 'x64') {
2151                                     return
2152                                 }
2153
2154                                 if (configuration != 'Release' && configuration != 'Checked') {
2155                                     return
2156                                 }
2157                                 break
2158                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2159                             case 'formatting':
2160                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2161                                     return
2162                                 }
2163                                 if (architecture != 'x64') {
2164                                     return
2165                                 }
2166                                 if (configuration != 'Checked') {
2167                                     return
2168                                 }
2169                                 if (isBuildOnly) {
2170                                     return
2171                                 }
2172                                 break
2173                             case 'illink':
2174                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2175                                     return
2176                                 }
2177                                 if (architecture != 'x64' && architecture != 'x86') {
2178                                     return
2179                                 }
2180                                 if (isBuildOnly) {
2181                                     return
2182                                 }
2183                                 break
2184                             case 'default':
2185                                 // Nothing skipped
2186                                 break
2187                             default:
2188                                 println("Unknown scenario: ${scenario}")
2189                                 assert false
2190                                 break
2191                         }
2192                     }
2193
2194                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2195                     // runtime behavior, not compile-time behavior, are not interesting.
2196                     switch (architecture) {
2197                         case 'x86_arm_altjit':
2198                         case 'x64_arm64_altjit':
2199                             if (isGCStressRelatedTesting(scenario)) {
2200                                 return
2201                             }
2202                             break
2203                         default:
2204                             break
2205                     }
2206
2207                     // Calculate names
2208                     def lowerConfiguration = configuration.toLowerCase()
2209                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2210                     def folderName = getJobFolder(scenario)
2211
2212                     // Create the new job
2213                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2214
2215                     def machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2216                     machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb') ? ['use_arm64_build_machine': false] : machineAffinityOptions
2217                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2218
2219                     // Add all the standard options
2220                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2221                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2222
2223                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
2224                     def osGroup = getOSGroup(os)
2225
2226                     newJob.with {
2227                         steps {
2228                             if (os == 'Windows_NT') {
2229                                 buildCommands.each { buildCommand ->
2230                                     batchFile(buildCommand)
2231                                 }
2232                             }
2233                             else {
2234                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2235                                 if ( architecture == 'armlb' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2236                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2237                                     // Define the Windows Tests and Corefx build job names
2238                                     def WindowTestsName = projectFolder + '/' +
2239                                                           Utilities.getFullJobName(project,
2240                                                                                    getJobName(lowerConfiguration,
2241                                                                                               'x64' ,
2242                                                                                               'windows_nt',
2243                                                                                               'default',
2244                                                                                               true),
2245                                                                                    false)
2246                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2247                                                        Utilities.getFolderName(branch)
2248
2249                                     // Copy the Windows test binaries and the Corefx build binaries
2250                                     copyArtifacts(WindowTestsName) {
2251                                         includePatterns('bin/tests/tests.zip')
2252                                         buildSelector {
2253                                             latestSuccessful(true)
2254                                         }
2255                                     }
2256
2257                                     def arm_abi = 'arm'
2258                                     def corefx_os = 'linux'
2259                                     if (os == 'Tizen') {
2260                                         arm_abi = 'armel'
2261                                         corefx_os = 'tizen'
2262                                     }
2263
2264                                     // Let's use release CoreFX to test checked CoreCLR,
2265                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2266                                     def corefx_lowerConfiguration = lowerConfiguration
2267                                     if ( lowerConfiguration == 'checked' ) {
2268                                         corefx_lowerConfiguration='release'
2269                                     }
2270
2271                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2272                                         includePatterns('bin/build.tar.gz')
2273                                         buildSelector {
2274                                             latestSuccessful(true)
2275                                         }
2276                                     }
2277                                 }
2278
2279                                 buildCommands.each { buildCommand ->
2280                                     shell(buildCommand)
2281                                 }
2282                             }
2283                         }
2284                     } // newJob.with
2285
2286                 } // os
2287             } // configuration
2288         } // architecture
2289     } // isPR
2290 } // scenario
2291
2292
2293 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2294 combinedScenarios.each { scenario ->
2295     [true, false].each { isPR ->
2296         // Architectures.  x64 only at this point
2297         ['x64', 'arm64'].each { architecture ->
2298             // Put the OS's supported for coreclr cross testing here
2299             Constants.crossList.each { os ->
2300                 if (architecture == 'arm64') {
2301                     if (os != "Ubuntu") {
2302                         return
2303                     }
2304                 }
2305
2306                 Constants.configurationList.each { configuration ->
2307
2308                     if (architecture == 'arm64') {
2309                         if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2310                             return
2311                         }
2312                     }
2313
2314                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2315                         if (configuration != 'Checked') {
2316                             return
2317                         }
2318                         if (isCorefxTesting(scenario)) {
2319                             return
2320                         }
2321                         //Skip stress modes for these scenarios
2322                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2323                             return
2324                         }
2325                     }
2326                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2327                     // and configuration is not Checked, bail out.
2328                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2329                         return;
2330                     }
2331                     // For CentOS, we only want Checked/Release builds.
2332                     else if (os == 'CentOS7.1') {
2333                         if (scenario != 'default' &&
2334                             scenario != 'r2r' &&
2335                             scenario != 'gcstress15_r2r' &&
2336                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2337                             return
2338                         }
2339                         if (configuration != 'Checked' && configuration != 'Release') {
2340                             return
2341                         }
2342                     }
2343                     // For RedHat and Debian, we only do Release builds.
2344                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2345
2346                         if (configuration != 'Release' || scenario != 'default') {
2347                             return
2348                         }
2349                     }
2350                     else {
2351                         // Skip scenarios
2352                         switch (scenario) {
2353                             case 'ilrt':
2354                                 // Release only
2355                                 if (configuration != 'Release') {
2356                                     return
2357                                 }
2358                                 break
2359                             case 'jitdiff':
2360                                 if (configuration != 'Checked') {
2361                                     return;
2362                                 }
2363                             case 'r2r':
2364                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2365                                 if (configuration != 'Checked' && configuration != 'Release') {
2366                                     return
2367                                 }
2368                                 break
2369                             case 'r2r':
2370                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2371                                 if (configuration != 'Checked' && configuration != 'Release') {
2372                                     return
2373                                 }
2374                                 break
2375                             case 'gcstress15_r2r':
2376                             case 'r2r_jitstress1':
2377                             case 'r2r_jitstress2':
2378                             case 'r2r_jitstressregs1':
2379                             case 'r2r_jitstressregs2':
2380                             case 'r2r_jitstressregs3':
2381                             case 'r2r_jitstressregs4':
2382                             case 'r2r_jitstressregs8':
2383                             case 'r2r_jitstressregs0x10':
2384                             case 'r2r_jitstressregs0x80':
2385                             case 'r2r_jitstressregs0x1000':
2386                             case 'r2r_jitminopts':
2387                             case 'r2r_jitforcerelocs':
2388                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2389                                 if (configuration != 'Checked' && configuration != 'Release') {
2390                                     return
2391                                 }
2392                                 break
2393                             case 'longgc':
2394                             case 'gcsimulator':
2395                                 // Long GC tests take a long time on non-Release builds
2396                                 if (configuration != 'Release') {
2397                                     return
2398                                 }
2399                                 break
2400                             case 'gc_reliability_framework':
2401                             case 'standalone_gc':
2402                                 if (configuration != 'Release' && configuration != 'Checked') {
2403                                     return
2404                                 }
2405                                 break
2406                             case 'formatting':
2407                                 return
2408                             case 'illink':
2409                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2410                                     return
2411                                 }
2412                                 break
2413                             case 'default':
2414                                 // Nothing skipped
2415                                 break
2416                             default:
2417                                 println("Unknown scenario: ${scenario}")
2418                                 assert false
2419                                 break
2420                         }
2421                     }
2422
2423                     def lowerConfiguration = configuration.toLowerCase()
2424                     def osGroup = getOSGroup(os)
2425                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2426
2427                     def inputScenario = 'default'
2428                     def inputCoreCLRBuildName = projectFolder + '/' +
2429                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2430                     // If this is a stress scenario, there isn't any difference in the build job
2431                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2432                     // from the default scenario
2433                     def testBuildScenario = scenario
2434                     if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2435                         testBuildScenario = 'default'
2436                     }
2437                     def inputWindowTestsBuildName = ''
2438                     def inputWindowsTestBuildArch = architecture
2439                     if (architecture == "arm64") {
2440                         // Use the x64 test build for arm64 unix
2441                         inputWindowsTestBuildArch = "x64"
2442                     }
2443                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2444                         inputWindowTestsBuildName = projectFolder + '/' +
2445                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', 'default', true), isPR)
2446                     }
2447                     else {
2448                         inputWindowTestsBuildName = projectFolder + '/' +
2449                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2450                     }
2451                     // Enable Server GC for Ubuntu PR builds
2452                     def serverGCString = ''
2453
2454                     // Whether or not this test run should be run sequentially instead
2455                     // of in parallel. Only used for long GC tests.
2456                     def sequentialString = ''
2457
2458                     // Whether or not this test run should run a specific playlist.
2459                     // Only used for long GC tests.
2460
2461                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2462                     // for running long GC and GCSimulator tests, respectively. We don't use them
2463                     // here because using a playlist file produces much more readable output on the CI machines
2464                     // and reduces running time.
2465                     def playlistString = ''
2466
2467                     if (os == 'Ubuntu' && isPR){
2468                         serverGCString = '--useServerGC'
2469                     }
2470
2471                     // pass --crossgen to runtest.sh for crossgen builds
2472                     def crossgenStr = ''
2473                     def runcrossgentestsStr = ''
2474                     def runjitstressStr = ''
2475                     def runjitstressregsStr = ''
2476                     def runjitmioptsStr = ''
2477                     def runjitforcerelocsStr = ''
2478                     def runjitdisasmStr = ''
2479                     def runilasmroundtripStr = ''
2480                     def gcstressStr = ''
2481                     def illinkStr = ''
2482                     def layoutOnlyStr =''
2483
2484                     if (scenario == 'r2r' ||
2485                         scenario == 'gcstress15_r2r' ||
2486                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2487                             crossgenStr = '--crossgen'
2488                             runcrossgentestsStr = '--runcrossgentests'
2489
2490                             if (scenario == 'r2r_jitstress1'){
2491                                 runjitstressStr = '--jitstress=1'
2492                             }
2493                             else if (scenario == 'r2r_jitstress2') {
2494                                 runjitstressStr = '--jitstress=2'
2495                             }
2496                             else if (scenario == 'r2r_jitstressregs1'){
2497                                 runjitstressregsStr = '--jitstressregs=1'
2498                             }
2499                             else if (scenario == 'r2r_jitstressregs2') {
2500                                 runjitstressregsStr = '--jitstressregs=2'
2501                             }
2502                             else if (scenario == 'r2r_jitstressregs3') {
2503                                 runjitstressregsStr = '--jitstressregs=3'
2504                             }
2505                             else if (scenario == 'r2r_jitstressregs4') {
2506                                 runjitstressregsStr = '--jitstressregs=4'
2507                             }
2508                             else if (scenario == 'r2r_jitstressregs8') {
2509                                 runjitstressregsStr = '--jitstressregs=8'
2510                             }
2511                             else if (scenario == 'r2r_jitstressregs0x10') {
2512                                 runjitstressregsStr = '--jitstressregs=0x10'
2513                             }
2514                             else if (scenario == 'r2r_jitstressregs0x80') {
2515                                 runjitstressregsStr = '--jitstressregs=0x80'
2516                             }
2517                             else if (scenario == 'r2r_jitstressregs0x1000') {
2518                                 runjitstressregsStr = '--jitstressregs=0x1000'
2519                             }
2520                             else if (scenario == 'r2r_jitminopts') {
2521                                 runjitmioptsStr = '--jitminopts'
2522                             }
2523                             else if (scenario == 'r2r_jitforcerelocs') {
2524                                 runjitforcerelocsStr = '--jitforcerelocs'
2525                             }
2526                     }
2527                     if  (scenario == 'gcstress15_r2r')
2528                     {
2529                         gcstressStr = '--gcstresslevel=0xF'
2530                     }
2531
2532                     if (scenario == 'jitdiff')
2533                     {
2534                         runjitdisasmStr = '--jitdisasm --crossgen'
2535                     }
2536
2537                     if (scenario == 'illink')
2538                     {
2539                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2540                     }
2541
2542                     if (isLongGc(scenario)) {
2543                         // Long GC tests behave very poorly when they are not
2544                         // the only test running (many of them allocate until OOM).
2545                         sequentialString = '--sequential'
2546
2547                         // The Long GC playlist contains all of the tests that are
2548                         // going to be run. The GCSimulator playlist contains all of
2549                         // the GC simulator tests.
2550                         if (scenario == 'longgc') {
2551                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2552                         }
2553                         else if (scenario == 'gcsimulator') {
2554                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2555                         }
2556                     }
2557
2558                     if (isGcReliabilityFramework(scenario)) {
2559                         layoutOnlyStr = '--build-overlay-only'
2560                     }
2561
2562                     def folder = getJobFolder(scenario)
2563                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2564                         // Add parameters for the inputs
2565
2566                         parameters {
2567                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2568                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2569                         }
2570
2571                         steps {
2572                             // Set up the copies
2573
2574                             // Coreclr build containing the tests and mscorlib
2575
2576                             copyArtifacts(inputWindowTestsBuildName) {
2577                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2578                                 buildSelector {
2579                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2580                                 }
2581                             }
2582
2583                             // Coreclr build we are trying to test
2584
2585                             copyArtifacts(inputCoreCLRBuildName) {
2586                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2587                                 buildSelector {
2588                                     buildNumber('${CORECLR_BUILD}')
2589                                 }
2590                             }
2591
2592                             def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2593
2594                             // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2595                             def osJobName
2596                             if (os == 'Ubuntu') {
2597                                 osJobName = 'ubuntu14.04'
2598                             }
2599                             else {
2600                                 osJobName = os.toLowerCase()
2601                             }
2602                             copyArtifacts("${corefxFolder}/${osJobName}_release") {
2603                                 includePatterns('bin/build.tar.gz')
2604                                 buildSelector {
2605                                     latestSuccessful(true)
2606                                 }
2607                             }
2608
2609                             shell ("mkdir ./bin/CoreFxBinDir")
2610                             // Unpack the corefx binaries
2611                             shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2612
2613                             // HACK -- Arm64 does not have corefx jobs yet.
2614                             // Clone corefx and build the native packages overwriting the x64 packages.
2615                             if (architecture == 'arm64') {
2616                                 shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2617                                 shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2618                             }
2619
2620                             // Unzip the tests first.  Exit with 0
2621                             shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2622
2623                             // Execute the tests
2624                             // If we are running a stress mode, we'll set those variables first
2625                             def testEnvOpt = ""
2626                             if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2627                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2628                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2629                                 shell("${createScriptCmds}")
2630                                 testEnvOpt = "--test-env=" + scriptFileName
2631                             }
2632
2633                             if (isGCStressRelatedTesting(scenario)) {
2634                                 shell('./init-tools.sh')
2635                             }
2636
2637                             shell("""./tests/runtest.sh \\
2638             --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2639             --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2640             --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2641             --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2642             --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2643             --limitedDumpGeneration \\
2644             ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
2645             ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
2646             ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
2647
2648                             if (isGcReliabilityFramework(scenario)) {
2649                                 // runtest.sh doesn't actually execute the reliability framework - do it here.
2650                                 if (serverGCString != '') {
2651                                     shell("export COMPlus_gcServer=1")
2652                                 }
2653
2654                                 shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2655                             }
2656                         }
2657                     }
2658
2659                     if (scenario == 'jitdiff') {
2660                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2661                     }
2662
2663                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2664                     if (os in ['Ubuntu']) {
2665                         SummaryBuilder summaries = new SummaryBuilder()
2666                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2667                         summaries.emit(newJob)
2668                     }
2669
2670                     setMachineAffinity(newJob, os, architecture)
2671                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2672                     // Set timeouts to 240.
2673                     setTestJobTimeOut(newJob, scenario)
2674
2675                     if (architecture == 'arm64') {
2676                         Utilities.setJobTimeout(newJob, 240)
2677                     }
2678
2679                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2680
2681                     // Create a build flow to join together the build and tests required to run this
2682                     // test.
2683                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2684                     // Linux CoreCLR test
2685                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2686                     def fullTestJobName = projectFolder + '/' + newJob.name
2687                     // Add a reference to the input jobs for report purposes
2688                     JobReport.Report.addReference(inputCoreCLRBuildName)
2689                     JobReport.Report.addReference(inputWindowTestsBuildName)
2690                     JobReport.Report.addReference(fullTestJobName)
2691                     def newFlowJob;
2692
2693                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2694                         // Do not create the flow job for RHEL jobs.
2695                         return
2696                     }
2697
2698                     newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2699                     buildFlow("""
2700 // Build the input jobs in parallel
2701 parallel (
2702 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2703 { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2704 )
2705
2706 // And then build the test build
2707 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2708             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2709 """)
2710                     }
2711
2712                     setMachineAffinity(newFlowJob, os, architecture)
2713                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2714                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2715                 } // configuration
2716             } // os
2717         } // architecture
2718     } // isPR
2719 } // scenario
2720
2721 JobReport.Report.generateJobReport(out)
2722
2723 // Make the call to generate the help job
2724 Utilities.createHelperJob(this, project, branch,
2725     "Welcome to the ${project} Repository",  // This is prepended to the help message
2726     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2727
2728 Utilities.addCROSSCheck(this, project, branch)