netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs
  12 folder('jitstress')
  13
  14 // Create a folder for testing via illink
  15 folder('illink')
  16
  17 def static getOSGroup(def os) {
  18     def osGroupMap = ['Ubuntu':'Linux',
  19         'RHEL7.2': 'Linux',
  20         'Ubuntu16.04': 'Linux',
  21         'Ubuntu16.10': 'Linux',
  22         'Debian8.4':'Linux',
  23         'Fedora24':'Linux',
  24         'OSX10.12':'OSX',
  25         'Windows_NT':'Windows_NT',
  26         'FreeBSD':'FreeBSD',
  27         'CentOS7.1': 'Linux',
  28         'OpenSUSE42.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'FreeBSD',
  49                'CentOS7.1',
  50                'OpenSUSE42.1',
  51                'RHEL7.2',
  52                'Ubuntu16.04',
  53                'Ubuntu16.10',
  54                'Fedora24',
  55                'Tizen']
  56
  57     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  58
  59     // This is a set of JIT stress modes combined with the set of variables that
  60     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  61     // the values are the environment variables
  62     def static jitStressModeScenarios = [
  63                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  64                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  65                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  66                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  67                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  68                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  69                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  70                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  71                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  72                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  73                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  74                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  75                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  76                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  77                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  78                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  79                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  80                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  81                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  82                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  83                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  84                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  85                'corefx_baseline'                : [ : ], // corefx baseline
  86                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  87                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  88                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  89                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  90                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  91                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  92                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  93                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  94                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  95                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  96                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  97                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  98                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
  99                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 100                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 101                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 103                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 104                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 105                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 106                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 107                ]
 108
 109     // This is a set of r2r jit stress scenarios
 110     def static r2rJitStressScenarios = [
 111                'r2r_jitstress1',
 112                'r2r_jitstress2',
 113                'r2r_jitstressregs1',
 114                'r2r_jitstressregs2',
 115                'r2r_jitstressregs3',
 116                'r2r_jitstressregs4',
 117                'r2r_jitstressregs8',
 118                'r2r_jitstressregs0x10',
 119                'r2r_jitstressregs0x80',
 120                'r2r_jitstressregs0x1000',
 121                'r2r_jitminopts',
 122                'r2r_jitforcerelocs']
 123
 124     // This is the basic set of scenarios
 125     def static basicScenarios = [
 126                'default',
 127                'pri1',
 128                'ilrt',
 129                'r2r',
 130                'pri1r2r',
 131                'gcstress15_pri1r2r',
 132                'longgc',
 133                'coverage',
 134                'formatting',
 135                'gcsimulator',
 136                'jitdiff',
 137                'standalone_gc',
 138                'illink'] + r2rJitStressScenarios
 139
 140     def static configurationList = ['Debug', 'Checked', 'Release']
 141
 142     // This is the set of architectures
 143     def static architectureList = ['arm', 'arm64', 'x64', 'x86']
 144 }
 145
 146 def static setMachineAffinity(def job, def os, def architecture) {
 147     if (architecture == 'arm64' && os == 'Windows_NT') {
 148         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 149     } else if (architecture == 'arm64' && os == 'Ubuntu') {
 150         Utilities.setMachineAffinity(job, os, 'arm-cross-latest');
 151     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 152         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 153     } else {
 154         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 155     }
 156 }
 157
 158 def static isJITStressJob(def scenario) {
 159     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 160            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 161 }
 162
 163 def static isGCStressRelatedTesting(def scenario) {
 164     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 165     // Detect it and make it a GCStress related.
 166     if (scenario == 'gcstress15_pri1r2r')
 167     {
 168         return true;
 169     }
 170
 171     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 172     def scenarioName = scenario.toLowerCase()
 173     def isGCStressTesting = false
 174     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 175         if (k in gcStressTestEnvVars) {
 176             isGCStressTesting = true;
 177         }
 178     }
 179     return isGCStressTesting
 180 }
 181
 182 def static isCorefxTesting(def scenario) {
 183     def corefx_prefix = 'corefx_'
 184     if (scenario.length() < corefx_prefix.length()) {
 185         return false
 186     }
 187     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 188 }
 189
 190 def static isR2R(def scenario) {
 191     return (scenario == 'r2r' || scenario == 'pri1r2r')
 192 }
 193
 194 def static isCoverage(def scenario) {
 195     return (scenario == 'coverage')
 196 }
 197
 198 def static isLongGc(def scenario) {
 199     return (scenario == 'longgc' || scenario == 'gcsimulator')
 200 }
 201
 202 def static isJitDiff(def scenario) {
 203     return (scenario == 'jitdiff')
 204 }
 205
 206 def static setTestJobTimeOut(newJob, scenario) {
 207     if (isGCStressRelatedTesting(scenario)) {
 208         Utilities.setJobTimeout(newJob, 4320)
 209     }
 210     else if (isCorefxTesting(scenario)) {
 211         Utilities.setJobTimeout(newJob, 360)
 212     }
 213     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 214         Utilities.setJobTimeout(newJob, 240)
 215     }
 216     else if (isR2R(scenario)) {
 217         Utilities.setJobTimeout(newJob, 240)
 218     }
 219     else if (isCoverage(scenario)) {
 220         Utilities.setJobTimeout(newJob, 1440)
 221     }
 222     else if (isLongGc(scenario)) {
 223         Utilities.setJobTimeout(newJob, 1440)
 224     }
 225     else if (isJitDiff(scenario)) {
 226         Utilities.setJobTimeout(newJob, 240)
 227     }
 228     // Non-test jobs use the default timeout value.
 229 }
 230
 231 def static getJobFolder(def scenario) {
 232     if (isJITStressJob(scenario)) {
 233         return 'jitstress'
 234     }
 235     if (scenario == 'illink') {
 236         return 'illink'
 237     }
 238     return ''
 239 }
 240
 241 def static getStressModeDisplayName(def scenario) {
 242     def displayStr = ''
 243     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 244         def prefixLength = 'COMPlus_'.length()
 245         if (k.length() >= prefixLength) {
 246             def modeName = k.substring(prefixLength, k.length())
 247             displayStr += ' ' + modeName + '=' + v
 248         }
 249     }
 250     return displayStr
 251 }
 252
 253 def static getR2RStressModeDisplayName(def scenario) {
 254     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 255     // "r2r_" prefix.
 256     def displayStr = scenario
 257     def prefixLength = 'r2r_'.length()
 258     if (displayStr.length() >= prefixLength) {
 259         displayStr = displayStr.substring(prefixLength, displayStr.length())
 260     }
 261     return displayStr
 262 }
 263
 264 // Generates the string for creating a file that sets environment variables
 265 // that makes it possible to run stress modes.  Writes the script to the file
 266 // specified by the stepScriptLocation parameter.
 267 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 268     def stepScript = ''
 269     if (os == 'Windows_NT') {
 270         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 271         stepScript += "del ${stepScriptLocation}\r\n"
 272
 273         // Timeout in ms, default is 10 minutes. For stress
 274         // modes up this to 30 minutes
 275         def timeout = 1800000
 276
 277         // Set the Timeout
 278         stepScript += "set __TestTimeout=${timeout}\r\n"
 279         stepScript += "echo. > ${stepScriptLocation}\r\n"
 280         stressModeVars.each{ k, v ->
 281             // Write out what we are writing to the script file
 282             stepScript += "echo Setting ${k}=${v}\r\n"
 283             // Write out the set itself to the script file`
 284             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 285         }
 286     }
 287     else {
 288         stepScript += "echo Setting variables for ${stressModeName}\n"
 289         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 290         stressModeVars.each{ k, v ->
 291             // Write out what we are writing to the script file
 292             stepScript += "echo Setting ${k}=${v}\n"
 293             // Write out the set itself to the script file`
 294             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 295         }
 296         stepScript += "chmod +x ${stepScriptLocation}\n"
 297     }
 298     return stepScript
 299 }
 300
 301 // Calculates the name of the build job based on some typical parameters.
 302 //
 303 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 304     // If the architecture is x64, do not add that info into the build name.
 305     // Need to change around some systems and other builds to pick up the right builds
 306     // to do that.
 307
 308     def suffix = scenario != 'default' ? "_${scenario}" : '';
 309     if (isBuildOnly) {
 310         suffix += '_bld'
 311     }
 312     def baseName = ''
 313     switch (architecture) {
 314         case 'x64':
 315             if (scenario == 'default') {
 316                 // For now we leave x64 off of the name for compatibility with other jobs
 317                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 318             }
 319             else if (scenario == 'formatting') {
 320                 // we don't care about the configuration for the formatting job. It runs all configs
 321                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 322             }
 323             else {
 324                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 325             }
 326             break
 327         case 'arm64':
 328             // These are cross builds
 329             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 330             break
 331         case 'arm':
 332             // These are cross builds
 333             if (os == 'Tizen') {
 334                 // ABI: softfp
 335                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 336             }
 337             else {
 338                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 339             }
 340             break
 341         case 'x86':
 342             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 343             break
 344         default:
 345             println("Unknown architecture: ${architecture}");
 346             assert false
 347             break
 348     }
 349
 350     return baseName + suffix
 351 }
 352
 353 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 354     // Check scenario.
 355     switch (scenario) {
 356         case 'default':
 357             switch (architecture) {
 358                 case 'x64':
 359                 case 'x86':
 360                     if (architecture == 'x86' && os == 'Ubuntu') {
 361                         Utilities.addPeriodicTrigger(job, '@daily')
 362                     }
 363                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 364                         Utilities.addGithubPushTrigger(job)
 365                     }
 366                     break
 367                 case 'arm':
 368                     Utilities.addGithubPushTrigger(job)
 369                     break
 370                 case 'arm64':
 371                     if (os == 'Windows_NT') {
 372                         Utilities.addGithubPushTrigger(job)
 373                         // TODO: Add once external email sending is available again
 374                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 375                     }
 376                     break
 377                 default:
 378                     println("Unknown architecture: ${architecture}");
 379                     assert false
 380                     break
 381             }
 382             break
 383         case 'pri1':
 384             // Pri one gets a push trigger, and only for release
 385             if (architecture == 'x64') {
 386                 if (configuration == 'Release') {
 387                     // We expect release jobs to be Windows, or in the cross list
 388                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 389                     if (!os in bidailyCrossList) {
 390                         if (isFlowJob || os == 'Windows_NT') {
 391                             Utilities.addGithubPushTrigger(job)
 392                         }
 393                     }
 394                     else {
 395                         if (isFlowJob) {
 396                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 397                         }
 398                     }
 399                 }
 400             }
 401             break
 402         case 'r2r':
 403             //r2r jobs that aren't pri1 can only be triggered by phrase
 404             break
 405         case 'pri1r2r':
 406             assert !(os in bidailyCrossList)
 407             //pri1 r2r gets a push trigger for checked/release
 408             if (configuration == 'Checked' || configuration == 'Release') {
 409                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 410                 if (architecture == 'x64' && os != 'OSX10.12') {
 411                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 412                     if (isFlowJob || os == 'Windows_NT') {
 413                         Utilities.addGithubPushTrigger(job)
 414                     }
 415                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 416                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 417                     if (isFlowJob) {
 418                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 419                     }
 420                 }
 421                 // For x86, only add per-commit jobs for Windows
 422                 else if (architecture == 'x86') {
 423                     if (os == 'Windows_NT') {
 424                         Utilities.addGithubPushTrigger(job)
 425                     }
 426                 }
 427                 // arm64 pri1r2r jobs should only run every 12 hours.
 428                 else if (architecture == 'arm64') {
 429                     if (os == 'Windows_NT') {
 430                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 431                         // TODO: Add once external email sending is available again
 432                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 433                     }
 434                 }
 435             }
 436             break
 437         case 'r2r_jitstress1':
 438         case 'r2r_jitstress2':
 439         case 'r2r_jitstressregs1':
 440         case 'r2r_jitstressregs2':
 441         case 'r2r_jitstressregs3':
 442         case 'r2r_jitstressregs4':
 443         case 'r2r_jitstressregs8':
 444         case 'r2r_jitstressregs0x10':
 445         case 'r2r_jitstressregs0x80':
 446         case 'r2r_jitstressregs0x1000':
 447         case 'r2r_jitminopts':
 448         case 'r2r_jitforcerelocs':
 449         case 'gcstress15_pri1r2r':
 450             assert !(os in bidailyCrossList)
 451
 452             // GCStress=C is currently not supported on OS X
 453             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 454                 break
 455             }
 456
 457             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 458             if (configuration == 'Checked' || configuration == 'Release') {
 459                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 460                 if (architecture == 'x64') {
 461                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 462                     if (isFlowJob || os == 'Windows_NT') {
 463                         // Add a weekly periodic trigger
 464                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 465                     }
 466                 }
 467                 // For x86, only add per-commit jobs for Windows
 468                 else if (architecture == 'x86') {
 469                     if (os == 'Windows_NT') {
 470                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 471                     }
 472                 }
 473             }
 474             break
 475         case 'longgc':
 476             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 477             assert configuration == 'Release'
 478             assert architecture == 'x64'
 479             Utilities.addPeriodicTrigger(job, '@daily')
 480             // TODO: Add once external email sending is available again
 481             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 482             break
 483         case 'gcsimulator':
 484             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 485             assert configuration == 'Release'
 486             assert architecture == 'x64'
 487             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 488             // TODO: Add once external email sending is available again
 489             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 490             break
 491         case 'standalone_gc':
 492             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 493             assert (configuration == 'Release' || configuration == 'Checked')
 494             // TODO: Add once external email sending is available again
 495             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 496             Utilities.addPeriodicTrigger(job, '@weekly')
 497             break
 498         case 'ilrt':
 499             assert !(os in bidailyCrossList)
 500             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 501             if (architecture == 'x64' && configuration == 'Release') {
 502                 // We don't expect to see a job generated except in these scenarios
 503                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 504                 if (isFlowJob || os == 'Windows_NT') {
 505                     Utilities.addPeriodicTrigger(job, '@daily')
 506                 }
 507             }
 508             break
 509         case 'jitdiff':
 510             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 511             assert configuration == 'Checked'
 512             assert (architecture == 'x64' || architecture == 'x86')
 513             Utilities.addGithubPushTrigger(job)
 514             break
 515         case 'coverage':
 516             assert (os == 'Ubuntu' || os == 'Windows_NT')
 517             assert configuration == 'Release'
 518             assert architecture == 'x64'
 519             Utilities.addPeriodicTrigger(job, '@weekly')
 520             break
 521         case 'formatting':
 522             assert (os == 'Windows_NT' || os == "Ubuntu")
 523             assert architecture == 'x64'
 524             Utilities.addGithubPushTrigger(job)
 525             break
 526         case 'jitstressregs1':
 527         case 'jitstressregs2':
 528         case 'jitstressregs3':
 529         case 'jitstressregs4':
 530         case 'jitstressregs8':
 531         case 'jitstressregs0x10':
 532         case 'jitstressregs0x80':
 533         case 'jitstressregs0x1000':
 534         case 'minopts':
 535         case 'forcerelocs':
 536         case 'jitstress1':
 537         case 'jitstress2':
 538         case 'jitstress2_jitstressregs1':
 539         case 'jitstress2_jitstressregs2':
 540         case 'jitstress2_jitstressregs3':
 541         case 'jitstress2_jitstressregs4':
 542         case 'jitstress2_jitstressregs8':
 543         case 'jitstress2_jitstressregs0x10':
 544         case 'jitstress2_jitstressregs0x80':
 545         case 'jitstress2_jitstressregs0x1000':
 546         case 'tailcallstress':
 547         case 'jitsse2only':
 548         case 'corefx_baseline':
 549         case 'corefx_minopts':
 550         case 'corefx_jitstress1':
 551         case 'corefx_jitstress2':
 552         case 'corefx_jitstressregs1':
 553         case 'corefx_jitstressregs2':
 554         case 'corefx_jitstressregs3':
 555         case 'corefx_jitstressregs4':
 556         case 'corefx_jitstressregs8':
 557         case 'corefx_jitstressregs0x10':
 558         case 'corefx_jitstressregs0x80':
 559         case 'corefx_jitstressregs0x1000':
 560         case 'zapdisable':
 561             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 562             assert (os == 'Windows_NT') || (os in Constants.crossList)
 563             Utilities.addPeriodicTrigger(job, '@daily')
 564         }
 565         break
 566         case 'heapverify1':
 567         case 'gcstress0x3':
 568             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 569                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 570                 if (architecture == 'arm64') {
 571                     assert (os == 'Windows_NT')
 572                     Utilities.addPeriodicTrigger(job, '@daily')
 573                     // TODO: Add once external email sending is available again
 574                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 575                 }
 576                 else {
 577                     Utilities.addPeriodicTrigger(job, '@weekly')
 578                 }
 579             }
 580             break
 581         case 'gcstress0xc':
 582         case 'gcstress0xc_zapdisable':
 583         case 'gcstress0xc_zapdisable_jitstress2':
 584         case 'gcstress0xc_zapdisable_heapverify1':
 585         case 'gcstress0xc_jitstress1':
 586         case 'gcstress0xc_jitstress2':
 587         case 'gcstress0xc_minopts_heapverify1':
 588             // GCStress=C is currently not supported on OS X
 589             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 590                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 591                 if (architecture == 'arm64') {
 592                     assert (os == 'Windows_NT')
 593                     // TODO: Enable a periodic trigger after tests are updated.
 594                     // Utilities.addPeriodicTrigger(job, '@daily')
 595                     // TODO: Add once external email sending is available again
 596                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 597                 }
 598                 else {
 599                     Utilities.addPeriodicTrigger(job, '@weekly')
 600                 }
 601             }
 602             break
 603
 604         case 'illink':
 605             // Testing on other operating systems TBD
 606             assert (os == 'Windows_NT' || os == 'Ubuntu')
 607             if (architecture == 'x64' || architecture == 'x86') {
 608                 if (configuration == 'Checked') {
 609                     Utilities.addPeriodicTrigger(job, '@daily')
 610                 }
 611             }
 612             break
 613
 614         default:
 615             println("Unknown scenario: ${scenario}");
 616             assert false
 617             break
 618     }
 619     return
 620 }
 621
 622 // **************************
 623 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 624 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 625 // builds will do a couple extra steps.
 626 // **************************
 627
 628 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 629 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 630 // it's just used for internal builds
 631 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 632 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 633     if (isWindowsBuildOnlyJob) {
 634         return
 635     }
 636
 637     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 638     // Non pull request builds.
 639     if (!isPR) {
 640         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 641         return
 642     }
 643     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 644     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 645     def osGroup = getOSGroup(os)
 646     switch (architecture) {
 647         case 'x64': // editor brace matching: {
 648             if (scenario == 'coverage') {
 649                 assert configuration == 'Release'
 650                 if (os == 'Ubuntu') {
 651                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 652                 }
 653                 break
 654             }
 655
 656             if (scenario == 'formatting') {
 657                 assert configuration == 'Checked'
 658                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 659                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 660                 }
 661                 break
 662             }
 663
 664             switch (os) {
 665                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 666                 case 'Debian8.4':
 667                 case 'RHEL7.2':
 668                     if (scenario == 'default') {
 669                         assert !isFlowJob
 670                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 671                     }
 672                     else if (scenario == 'pri1' && isFlowJob) {
 673                         assert (configuration == 'Release')
 674                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 675                     }
 676                     break
 677                 case 'Ubuntu16.04':
 678                     assert !isFlowJob
 679                     assert scenario == 'default'
 680                     // Distinguish with the other architectures (arm and x86)
 681                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 682                     break
 683                 case 'Fedora24':
 684                 case 'Ubuntu16.10':
 685                 case 'OpenSUSE42.1':
 686                     assert !isFlowJob
 687                     assert scenario == 'default'
 688                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 689                     break
 690                 case 'Ubuntu':
 691                     if (scenario == 'illink') {
 692                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 693                         break
 694                     }
 695                     // fall through
 696                 case 'OSX10.12':
 697                     // Triggers on the non-flow jobs aren't necessary here
 698                     // Corefx testing uses non-flow jobs.
 699                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 700                         break
 701                     }
 702                     switch (scenario) {
 703                         case 'default':
 704                             // Ubuntu uses checked for default PR tests
 705                             if (configuration == 'Checked') {
 706                                 // Default trigger
 707                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 708                             }
 709                             break
 710                         case 'pri1':
 711                             if (configuration == 'Release') {
 712                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 713                             }
 714                             break
 715                         case 'jitdiff':
 716                             if (configuration == 'Checked') {
 717                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 718                             }
 719                             break
 720                         case 'ilrt':
 721                             if (configuration == 'Release') {
 722                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 723                             }
 724                             break
 725                         case 'r2r':
 726                             if (configuration == 'Release' || configuration == 'Checked') {
 727                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 728                             }
 729                             break
 730                         case 'pri1r2r':
 731                             if (configuration == 'Release' || configuration == 'Checked') {
 732                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 733                             }
 734                             break
 735                         case 'gcstress15_pri1r2r':
 736                             if (configuration == 'Release' || configuration == 'Checked') {
 737                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 738                             }
 739                             break
 740                         case 'r2r_jitstress1':
 741                         case 'r2r_jitstress2':
 742                         case 'r2r_jitstressregs1':
 743                         case 'r2r_jitstressregs2':
 744                         case 'r2r_jitstressregs3':
 745                         case 'r2r_jitstressregs4':
 746                         case 'r2r_jitstressregs8':
 747                         case 'r2r_jitstressregs0x10':
 748                         case 'r2r_jitstressregs0x80':
 749                         case 'r2r_jitstressregs0x1000':
 750                         case 'r2r_jitminopts':
 751                         case 'r2r_jitforcerelocs':
 752                             if (configuration == 'Release' || configuration == 'Checked') {
 753                                 def displayStr = getR2RStressModeDisplayName(scenario)
 754                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 755                             }
 756                             break
 757                         case 'longgc':
 758                             if (configuration == 'Release') {
 759                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 760                             }
 761                             break
 762                         case 'gcsimulator':
 763                             if (configuration == 'Release') {
 764                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 765                             }
 766                             break
 767                         case 'minopts':
 768                         case 'forcerelocs':
 769                         case 'jitstress1':
 770                         case 'jitstress2':
 771                         case 'jitstressregs1':
 772                         case 'jitstressregs2':
 773                         case 'jitstressregs3':
 774                         case 'jitstressregs4':
 775                         case 'jitstressregs8':
 776                         case 'jitstressregs0x10':
 777                         case 'jitstressregs0x80':
 778                         case 'jitstressregs0x1000':
 779                         case 'jitstress2_jitstressregs1':
 780                         case 'jitstress2_jitstressregs2':
 781                         case 'jitstress2_jitstressregs3':
 782                         case 'jitstress2_jitstressregs4':
 783                         case 'jitstress2_jitstressregs8':
 784                         case 'jitstress2_jitstressregs0x10':
 785                         case 'jitstress2_jitstressregs0x80':
 786                         case 'jitstress2_jitstressregs0x1000':
 787                         case 'tailcallstress':
 788                         case 'jitsse2only':
 789                         case 'gcstress0x3':
 790                         case 'gcstress0xc':
 791                         case 'zapdisable':
 792                         case 'heapverify1':
 793                         case 'gcstress0xc_zapdisable':
 794                         case 'gcstress0xc_zapdisable_jitstress2':
 795                         case 'gcstress0xc_zapdisable_heapverify1':
 796                         case 'gcstress0xc_jitstress1':
 797                         case 'gcstress0xc_jitstress2':
 798                         case 'gcstress0xc_minopts_heapverify1':
 799                             def displayStr = getStressModeDisplayName(scenario)
 800                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 801                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 802                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 803                             break
 804                         case 'corefx_baseline':
 805                         case 'corefx_minopts':
 806                         case 'corefx_jitstress1':
 807                         case 'corefx_jitstress2':
 808                         case 'corefx_jitstressregs1':
 809                         case 'corefx_jitstressregs2':
 810                         case 'corefx_jitstressregs3':
 811                         case 'corefx_jitstressregs4':
 812                         case 'corefx_jitstressregs8':
 813                         case 'corefx_jitstressregs0x10':
 814                         case 'corefx_jitstressregs0x80':
 815                         case 'corefx_jitstressregs0x1000':
 816                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 817                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 818                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 819                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 820                             break
 821                         default:
 822                             println("Unknown scenario: ${scenario}");
 823                             assert false
 824                             break
 825                     }
 826                     break
 827                 case 'CentOS7.1':
 828                     switch (scenario) {
 829                         case 'pri1':
 830                             if (configuration == 'Release') {
 831                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 832                             }
 833                             break
 834                         case 'r2r':
 835                             if (configuration == 'Checked' || configuration == 'Release') {
 836                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 837                             }
 838                             break
 839                         case 'pri1r2r':
 840                             if (configuration == 'Checked' || configuration == 'Release') {
 841                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 842                             }
 843                             break
 844                         case 'gcstress15_pri1r2r':
 845                             if (configuration == 'Release' || configuration == 'Checked') {
 846                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 847                             }
 848                             break
 849                         case 'r2r_jitstress1':
 850                         case 'r2r_jitstress2':
 851                         case 'r2r_jitstressregs1':
 852                         case 'r2r_jitstressregs2':
 853                         case 'r2r_jitstressregs3':
 854                         case 'r2r_jitstressregs4':
 855                         case 'r2r_jitstressregs8':
 856                         case 'r2r_jitstressregs0x10':
 857                         case 'r2r_jitstressregs0x80':
 858                         case 'r2r_jitstressregs0x1000':
 859                         case 'r2r_jitminopts':
 860                         case 'r2r_jitforcerelocs':
 861                             if (configuration == 'Release' || configuration == 'Checked') {
 862                                 def displayStr = getR2RStressModeDisplayName(scenario)
 863                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 864                             }
 865                             break
 866                         default:
 867                             break
 868                     }
 869                 case 'Windows_NT':
 870                     switch (scenario) {
 871                         case 'default':
 872                             // Default trigger
 873                             if (configuration == 'Debug') {
 874                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 875                             }
 876                             break
 877                         case 'pri1':
 878                             // Default trigger
 879                             if (configuration == 'Release') {
 880                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 881                             }
 882                             break
 883                         case 'jitdiff':
 884                             if (configuration == 'Checked') {
 885                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 886                             }
 887                             break
 888                         case 'ilrt':
 889                             if (configuration == 'Release') {
 890                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 891                             }
 892                             break
 893                         case 'r2r':
 894                             if (configuration == 'Checked' || configuration == 'Release') {
 895                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 896                             }
 897                             break
 898                         case 'pri1r2r':
 899                             if (configuration == 'Checked' || configuration == 'Release') {
 900                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 901                             }
 902                             break
 903                         case 'gcstress15_pri1r2r':
 904                             if (configuration == 'Release' || configuration == 'Checked') {
 905                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 906                             }
 907                             break
 908                         case 'r2r_jitstress1':
 909                         case 'r2r_jitstress2':
 910                         case 'r2r_jitstressregs1':
 911                         case 'r2r_jitstressregs2':
 912                         case 'r2r_jitstressregs3':
 913                         case 'r2r_jitstressregs4':
 914                         case 'r2r_jitstressregs8':
 915                         case 'r2r_jitstressregs0x10':
 916                         case 'r2r_jitstressregs0x80':
 917                         case 'r2r_jitstressregs0x1000':
 918                         case 'r2r_jitminopts':
 919                         case 'r2r_jitforcerelocs':
 920                             if (configuration == 'Release' || configuration == 'Checked') {
 921                                 def displayStr = getR2RStressModeDisplayName(scenario)
 922                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 923                             }
 924                             break
 925                         case 'longgc':
 926                             if (configuration == 'Release') {
 927                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 928                             }
 929                             break
 930                         case 'gcsimulator':
 931                             if (configuration == 'Release') {
 932                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 933                             }
 934                             break
 935                         case 'standalone_gc':
 936                             if (configuration == 'Release' || configuration == 'Checked') {
 937                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 938                             }
 939                             break
 940                         case 'minopts':
 941                         case 'forcerelocs':
 942                         case 'jitstress1':
 943                         case 'jitstress2':
 944                         case 'jitstressregs1':
 945                         case 'jitstressregs2':
 946                         case 'jitstressregs3':
 947                         case 'jitstressregs4':
 948                         case 'jitstressregs8':
 949                         case 'jitstressregs0x10':
 950                         case 'jitstressregs0x80':
 951                         case 'jitstressregs0x1000':
 952                         case 'jitstress2_jitstressregs1':
 953                         case 'jitstress2_jitstressregs2':
 954                         case 'jitstress2_jitstressregs3':
 955                         case 'jitstress2_jitstressregs4':
 956                         case 'jitstress2_jitstressregs8':
 957                         case 'jitstress2_jitstressregs0x10':
 958                         case 'jitstress2_jitstressregs0x80':
 959                         case 'jitstress2_jitstressregs0x1000':
 960                         case 'tailcallstress':
 961                         case 'jitsse2only':
 962                         case 'gcstress0x3':
 963                         case 'gcstress0xc':
 964                         case 'zapdisable':
 965                         case 'heapverify1':
 966                         case 'gcstress0xc_zapdisable':
 967                         case 'gcstress0xc_zapdisable_jitstress2':
 968                         case 'gcstress0xc_zapdisable_heapverify1':
 969                         case 'gcstress0xc_jitstress1':
 970                         case 'gcstress0xc_jitstress2':
 971                         case 'gcstress0xc_minopts_heapverify1':
 972                             def displayStr = getStressModeDisplayName(scenario)
 973                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 974                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 975                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 976                             break
 977                         case 'corefx_baseline':
 978                         case 'corefx_minopts':
 979                         case 'corefx_jitstress1':
 980                         case 'corefx_jitstress2':
 981                         case 'corefx_jitstressregs1':
 982                         case 'corefx_jitstressregs2':
 983                         case 'corefx_jitstressregs3':
 984                         case 'corefx_jitstressregs4':
 985                         case 'corefx_jitstressregs8':
 986                         case 'corefx_jitstressregs0x10':
 987                         case 'corefx_jitstressregs0x80':
 988                         case 'corefx_jitstressregs0x1000':
 989                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 990                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 991                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 992                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 993                             break
 994                         case 'illink':
 995                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 996                                     break
 997                         default:
 998                             println("Unknown scenario: ${scenario}");
 999                             assert false
1000                             break
1001                     }
1002                     break
1003                 case 'FreeBSD':
1004                     assert scenario == 'default'
1005                     if (configuration == 'Checked') {
1006                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1007                     }
1008                     break
1009                 default:
1010                     println("Unknown os: ${os}");
1011                     assert false
1012                     break
1013             }
1014             break
1015         // editor brace matching: }
1016         case 'arm': // editor brace matching: {
1017             assert scenario == 'default'
1018             switch (os) {
1019                 case 'Ubuntu':
1020                 case 'Ubuntu16.04':
1021                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1022                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1023                     }
1024                     else {
1025                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1026                     }
1027                     break;
1028                 case 'Tizen':
1029                     architecture='armel'
1030                     // Removing the regex will cause this to run on each PR.
1031                     if (configuration == 'Release' || configuration == 'Debug') {
1032                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1033                     }
1034                     else {
1035                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1036                     }
1037                     break;
1038                 case 'Windows_NT':
1039                     if (configuration == 'Debug' || configuration == 'Release')
1040                     {
1041                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1042                     }
1043                     break
1044                 default:
1045                     println("NYI os: ${os}");
1046                     assert false
1047                     break
1048             }
1049             break
1050         // editor brace matching: }
1051         case 'arm64': // editor brace matching: {
1052             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1053
1054             // Set up a private trigger
1055             def contextString = "${os} ${architecture} Cross ${configuration}"
1056             if (scenario != 'default')
1057                 contextString += " ${scenario}"
1058             contextString += " Build"
1059             // Debug builds only.
1060             if (configuration != 'Debug') {
1061                contextString += " and Test"
1062             }
1063
1064             def arm64Users = [
1065                 'adiaaida',
1066                 'AndyAyersMS',
1067                 'briansull',
1068                 'BruceForstall',
1069                 'CarolEidt',
1070                 'cmckinsey',
1071                 'erozenfeld',
1072                 'jashook',
1073                 'JosephTremoulet',
1074                 'pgavlin',
1075                 'pkukol',
1076                 'russellhadley',
1077                 'RussKeldorph',
1078                 'sandreenko',
1079                 'sivarv',
1080                 'swaroop-sridhar',
1081                 'gkhanna79',
1082                 'jkotas',
1083                 'markwilkie',
1084                 'rahku',
1085                 'ramarag',
1086                 'tzwlai',
1087                 'weshaggard'
1088             ]
1089
1090             switch (os) {
1091                 case 'Windows_NT':
1092                     switch (scenario) {
1093                         case 'default':
1094                             // For now only run Debug jobs on PR Trigger.
1095                             if (configuration != 'Debug') {
1096                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1097                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1098                             }
1099                             else {
1100                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1101                                 // are at higher risk of ARM64-breaking changes.
1102                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1103                             }
1104                             break
1105                         case 'pri1r2r':
1106                         case 'gcstress0x3':
1107                         case 'gcstress0xc':
1108                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1109                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1110                             break
1111                     }
1112                     break
1113                 default:
1114                     println("NYI os: ${os}");
1115                     assert false
1116                     break
1117             }
1118             break
1119         // editor brace matching: }
1120         case 'x86': // editor brace matching: {
1121             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1122             if (os == 'Ubuntu') {
1123                 // on-demand only for ubuntu x86
1124                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1125                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1126                 break
1127             }
1128             switch (scenario) {
1129                 case 'default':
1130                     if (configuration == 'Checked') {
1131                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1132                     }
1133                     else if (configuration == 'Release') {
1134                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1135                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1136                     }
1137                     break
1138                 case 'pri1':
1139                     if (configuration == 'Release') {
1140                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1141                     }
1142                     break
1143                 case 'ilrt':
1144                     if (configuration == 'Release') {
1145                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1146                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1147                     }
1148                     break
1149                 case 'r2r':
1150                     if (configuration == 'Checked' || configuration == 'Release') {
1151                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1152                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1153                     }
1154                     break
1155                 case 'pri1r2r':
1156                     if (configuration == 'Checked' || configuration == 'Release') {
1157                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1158                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1159                     }
1160                     break
1161                 case 'gcstress15_pri1r2r':
1162                     if (configuration == 'Release' || configuration == 'Checked') {
1163                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1164                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1165                     }
1166                     break
1167                 case 'r2r_jitstress1':
1168                 case 'r2r_jitstress2':
1169                 case 'r2r_jitstressregs1':
1170                 case 'r2r_jitstressregs2':
1171                 case 'r2r_jitstressregs3':
1172                 case 'r2r_jitstressregs4':
1173                 case 'r2r_jitstressregs8':
1174                 case 'r2r_jitstressregs0x10':
1175                 case 'r2r_jitstressregs0x80':
1176                 case 'r2r_jitstressregs0x1000':
1177                 case 'r2r_jitminopts':
1178                 case 'r2r_jitforcerelocs':
1179                     if (configuration == 'Release' || configuration == 'Checked') {
1180                         def displayStr = getR2RStressModeDisplayName(scenario)
1181                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1182                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1183                     }
1184                     break
1185                 case 'longgc':
1186                     if (configuration == 'Release') {
1187                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1188                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1189                     }
1190                     break
1191                 case 'gcsimulator':
1192                     if (configuration == 'Release') {
1193                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1194                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1195                     }
1196                     break
1197                 case 'standalone_gc':
1198                     if (configuration == 'Release' || configuration == 'Checked') {
1199                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1200                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1201                     }
1202                     break
1203                 case 'minopts':
1204                 case 'forcerelocs':
1205                 case 'jitstress1':
1206                 case 'jitstress2':
1207                 case 'jitstressregs1':
1208                 case 'jitstressregs2':
1209                 case 'jitstressregs3':
1210                 case 'jitstressregs4':
1211                 case 'jitstressregs8':
1212                 case 'jitstressregs0x10':
1213                 case 'jitstressregs0x80':
1214                 case 'jitstressregs0x1000':
1215                 case 'jitstress2_jitstressregs1':
1216                 case 'jitstress2_jitstressregs2':
1217                 case 'jitstress2_jitstressregs3':
1218                 case 'jitstress2_jitstressregs4':
1219                 case 'jitstress2_jitstressregs8':
1220                 case 'jitstress2_jitstressregs0x10':
1221                 case 'jitstress2_jitstressregs0x80':
1222                 case 'jitstress2_jitstressregs0x1000':
1223                 case 'tailcallstress':
1224                 case 'jitsse2only':
1225                 case 'gcstress0x3':
1226                 case 'gcstress0xc':
1227                 case 'zapdisable':
1228                 case 'heapverify1':
1229                 case 'gcstress0xc_zapdisable':
1230                 case 'gcstress0xc_zapdisable_jitstress2':
1231                 case 'gcstress0xc_zapdisable_heapverify1':
1232                 case 'gcstress0xc_jitstress1':
1233                 case 'gcstress0xc_jitstress2':
1234                 case 'gcstress0xc_minopts_heapverify1':
1235                     def displayStr = getStressModeDisplayName(scenario)
1236                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1237                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1238                     break
1239                 case 'corefx_baseline':
1240                 case 'corefx_minopts':
1241                 case 'corefx_jitstress1':
1242                 case 'corefx_jitstress2':
1243                 case 'corefx_jitstressregs1':
1244                 case 'corefx_jitstressregs2':
1245                 case 'corefx_jitstressregs3':
1246                 case 'corefx_jitstressregs4':
1247                 case 'corefx_jitstressregs8':
1248                 case 'corefx_jitstressregs0x10':
1249                 case 'corefx_jitstressregs0x80':
1250                 case 'corefx_jitstressregs0x1000':
1251                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1252                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1253                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1254                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1255                     break
1256                 case 'illink':
1257                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1258                     break
1259                 default:
1260                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1261                     assert false
1262                     break
1263             }
1264             break
1265         // editor brace matching: }
1266         default:
1267             println("Unknown architecture: ${architecture}");
1268             assert false
1269             break
1270     }
1271 }
1272
1273 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1274     def buildCommands = [];
1275     def osGroup = getOSGroup(os)
1276     def lowerConfiguration = configuration.toLowerCase()
1277
1278     // Calculate the build steps, archival, and xunit results
1279     switch (os) {
1280         case 'Windows_NT': // editor brace matching: {
1281             switch (architecture) {
1282                 case 'x64':
1283                 case 'x86':
1284                     def arch = architecture
1285                     def buildOpts = ''
1286
1287                     if (scenario == 'illink') {
1288                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1289                     }
1290
1291                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1292                             scenario == 'default' ||
1293                             scenario == 'r2r' ||
1294                             scenario == 'jitdiff' ||
1295                             scenario == 'illink' ||
1296                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1297                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1298                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1299                     }
1300
1301                     // For Pri 1 tests, we must shorten the output test binary path names.
1302                     // if __TestIntermediateDir is already set, build-test.cmd will
1303                     // output test binaries to that directory. If it is not set, the
1304                     // binaries are sent to a default directory whose name is about
1305                     // 35 characters long.
1306
1307                     else if (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage') {
1308                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1309                     }
1310                     else if (scenario == 'ilrt') {
1311                         // First do the build with skiptests and then build the tests with ilasm roundtrip
1312                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1313                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch} -ilasmroundtrip"
1314                     }
1315                     else if (isLongGc(scenario)) {
1316                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1317                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1318                     }
1319                     else if (scenario == 'standalone_gc') {
1320                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1321                     }
1322                     else if (scenario == 'formatting') {
1323                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1324                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1325                         break
1326                     }
1327                     else {
1328                         println("Unknown scenario: ${scenario}")
1329                         assert false
1330                     }
1331
1332                     // If we are running a stress mode, we should write out the set of key
1333                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1334
1335                     if (!isBuildOnly) {
1336                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1337                         def crossgenStr = ''
1338                         def runcrossgentestsStr = ''
1339                         def runjitstressStr = ''
1340                         def runjitstressregsStr = ''
1341                         def runjitmioptsStr = ''
1342                         def runjitforcerelocsStr = ''
1343                         def runjitdisasmStr = ''
1344                         def gcstressStr = ''
1345                         def runtestArguments = ''
1346                         def gcTestArguments = ''
1347                         def illinkArguments = ''
1348
1349                         if (scenario == 'r2r' ||
1350                             scenario == 'pri1r2r' ||
1351                             scenario == 'gcstress15_pri1r2r' ||
1352                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1353                                 crossgenStr = 'crossgen'
1354                                 runcrossgentestsStr = 'runcrossgentests'
1355
1356                                 if (scenario == 'r2r_jitstress1'){
1357                                     runjitstressStr = 'jitstress 1'
1358                                 }
1359                                 else if (scenario == 'r2r_jitstress2') {
1360                                     runjitstressStr = 'jitstress 2'
1361                                 }
1362                                 else if (scenario == 'r2r_jitstressregs1'){
1363                                     runjitstressregsStr = 'jitstressregs 1'
1364                                 }
1365                                 else if (scenario == 'r2r_jitstressregs2') {
1366                                     runjitstressregsStr = 'jitstressregs 2'
1367                                 }
1368                                 else if (scenario == 'r2r_jitstressregs3') {
1369                                     runjitstressregsStr = 'jitstressregs 3'
1370                                 }
1371                                 else if (scenario == 'r2r_jitstressregs4') {
1372                                     runjitstressregsStr = 'jitstressregs 4'
1373                                 }
1374                                 else if (scenario == 'r2r_jitstressregs8') {
1375                                     runjitstressregsStr = 'jitstressregs 8'
1376                                 }
1377                                 else if (scenario == 'r2r_jitstressregs0x10') {
1378                                     runjitstressregsStr = 'jitstressregs 0x10'
1379                                 }
1380                                 else if (scenario == 'r2r_jitstressregs0x80') {
1381                                     runjitstressregsStr = 'jitstressregs 0x80'
1382                                 }
1383                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1384                                     runjitstressregsStr = 'jitstressregs 0x1000'
1385                                 }
1386                                 else if (scenario == 'r2r_jitminopts') {
1387                                     runjitmioptsStr = 'jitminopts'
1388                                 }
1389                                 else if (scenario == 'r2r_jitforcerelocs') {
1390                                     runjitforcerelocsStr = 'jitforcerelocs'
1391                                 }
1392                         }
1393                         if (scenario == 'gcstress15_pri1r2r')
1394                         {
1395                             gcstressStr = 'gcstresslevel 0xF'
1396                         }
1397
1398                         if (scenario == 'jitdiff')
1399                         {
1400                             runjitdisasmStr = 'jitdisasm crossgen'
1401                         }
1402
1403                         if (isLongGc(scenario)) {
1404                             gcTestArguments = "${scenario} sequential"
1405                         }
1406
1407                         if (scenario == 'illink')
1408                         {
1409                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1410                         }
1411
1412                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1413
1414                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1415                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1416                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1417
1418                             if (enableCorefxTesting) {
1419                                 def workspaceRelativeFxRoot = "_/fx"
1420                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1421
1422                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1423
1424                                 setTestJobTimeOut(newJob, scenario)
1425
1426                                 // Archive and process (only) the test results
1427                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1428                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1429                             }
1430                             else {
1431                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1432                             }
1433                         }
1434                         else if (architecture == 'x64' || architecture == 'x86') {
1435                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1436                         }
1437                     }
1438
1439                     if (!enableCorefxTesting) {
1440                         // Run the rest of the build
1441                         // Build the mscorlib for the other OS's
1442                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1443                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1444                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1445
1446                         // Zip up the tests directory so that we don't use so much space/time copying
1447                         // 10s of thousands of files around.
1448                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1449
1450                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1451                             // For windows, pull full test results and test drops for x86/x64.
1452                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1453                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1454                         }
1455
1456                         if (scenario == 'jitdiff') {
1457                             // retrive jit-dasm output for base commit, and run jit-diff
1458                             if (!isBuildOnly) {
1459                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1460                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1461                             }
1462                         }
1463
1464                         if (!isBuildOnly) {
1465                             if (architecture == 'x64' || !isPR) {
1466                                 Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml')
1467                             }
1468                             setTestJobTimeOut(newJob, scenario)
1469                         }
1470                     }
1471                     break
1472                 case 'arm':
1473                     assert (scenario == 'default')
1474
1475                     // Set time out
1476                     setTestJobTimeOut(newJob, scenario)
1477
1478                     if ( lowerConfiguration == "debug" ) {
1479                         // For Debug builds, we will do a P1 test build
1480                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1481                     }
1482                     else {
1483                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1484                     }
1485                     // Add archival.
1486                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1487                     break
1488                 case 'arm64':
1489                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1490
1491                     // Set time out
1492                     setTestJobTimeOut(newJob, scenario)
1493
1494                     // Debug runs take too long to run. So build job only.
1495                     if (lowerConfiguration == "debug") {
1496                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1497                     }
1498                     else {
1499                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1500                        {
1501                            // Up the timeout for arm64 checked testing only.
1502                            // Keep the longer timeout for gcstress.
1503                            Utilities.setJobTimeout(newJob, 240)
1504                        }
1505
1506                        buildCommands += "set __TestIntermediateDir=int&&build.cmd skiptests ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1507                        // Test build and run are launched together.
1508                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1509                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1510                     }
1511
1512                     // Add archival.
1513                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1514                     break
1515                 default:
1516                     println("Unknown architecture: ${architecture}");
1517                     assert false
1518                     break
1519             }
1520             break
1521         // editor brace matching: }
1522         case 'Ubuntu':
1523         case 'Ubuntu16.04':
1524         case 'Ubuntu16.10':
1525         case 'Debian8.4':
1526         case 'OSX10.12':
1527         case 'FreeBSD':
1528         case 'CentOS7.1':
1529         case 'RHEL7.2':
1530         case 'OpenSUSE42.1':
1531         case 'Tizen':
1532         case 'Fedora24': // editor brace matching: {
1533             switch (architecture) {
1534                 case 'x64':
1535                 case 'x86':
1536                     if (architecture == 'x86' && os == 'Ubuntu') {
1537                         // build only, not test yet
1538                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1539                         break;
1540                     }
1541
1542                     if (scenario == 'formatting') {
1543                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1544                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1545                         break
1546                     }
1547
1548                     if (scenario == 'illink') {
1549                         assert(os == 'Ubuntu')
1550                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1551                     }
1552
1553                     def standaloneGc = ''
1554                     if (scenario == 'standalone_gc') {
1555                         standaloneGc = 'buildstandalonegc'
1556                     }
1557
1558                     if (!enableCorefxTesting) {
1559                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1560                         // only on supported OS platforms.
1561                         if (os == 'FreeBSD')
1562                         {
1563                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1564                         }
1565                         else
1566                         {
1567                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1568                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1569                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1570                         }
1571                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1572
1573                         // Set time out
1574                         setTestJobTimeOut(newJob, scenario)
1575                         // Basic archiving of the build
1576                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1577                         // And pal tests
1578                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1579                     }
1580                     else {
1581                         // Corefx stress testing
1582                         assert os == 'Ubuntu'
1583                         assert architecture == 'x64'
1584                         assert lowerConfiguration == 'checked'
1585                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1586
1587                         // Build coreclr
1588                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1589
1590                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1591                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1592
1593                         // Build and text corefx
1594                         def workspaceRelativeFxRoot = "_/fx"
1595                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1596
1597                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1598
1599                         setTestJobTimeOut(newJob, scenario)
1600
1601                         // Archive and process (only) the test results
1602                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1603                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1604                     }
1605                     break
1606                 case 'arm64':
1607                     // We don't run the cross build except on Ubuntu
1608                     assert os == 'Ubuntu'
1609
1610                     buildCommands += """echo \"Using rootfs in /opt/aarch64-linux-gnu-root\"
1611                         ROOTFS_DIR=/opt/aarch64-linux-gnu-root ./build.sh skipmscorlib arm64 cross verbose ${lowerConfiguration}"""
1612
1613                     // Basic archiving of the build, no pal tests
1614                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1615                     break
1616                 case 'arm':
1617                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1618                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1619
1620                     // default values for Ubuntu
1621                     def arm_abi="arm"
1622                     def linuxCodeName="trusty"
1623                     if (os == 'Ubuntu16.04') {
1624                         linuxCodeName="xenial"
1625                     }
1626                     else if (os == 'Tizen') {
1627                         arm_abi="armel"
1628                         linuxCodeName="tizen"
1629                     }
1630
1631                     // Unzip the Windows test binaries first. Exit with 0
1632                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1633
1634                     // Unpack the corefx binaries
1635                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1636                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1637
1638                     // Call the ARM CI script to cross build and test using docker
1639                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1640                     --mode=docker \\
1641                     --${arm_abi} \\
1642                     --linuxCodeName=${linuxCodeName} \\
1643                     --buildConfig=${lowerConfiguration} \\
1644                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1645                     --coreFxBinDir=./bin/CoreFxBinDir \\
1646                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1647
1648                     // Basic archiving of the build, no pal tests
1649                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1650                     break
1651                 default:
1652                     println("Unknown architecture: ${architecture}");
1653                     assert false
1654                     break
1655             }
1656             break
1657         // editor brace matching: }
1658         default:
1659             println("Unknown os: ${os}");
1660             assert false
1661             break
1662     } // os
1663
1664     return buildCommands
1665 }
1666
1667 // Additional scenario which can alter behavior
1668
1669 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1670 combinedScenarios.each { scenario ->
1671     [true, false].each { isPR ->
1672         Constants.architectureList.each { architecture ->
1673             Constants.configurationList.each { configuration ->
1674                 Constants.osList.each { os ->
1675                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1676                     // and reset the os to Windows_NT
1677                     def isBuildOnly = false
1678                     if (os == 'Windows_NT_BuildOnly') {
1679                         isBuildOnly = true
1680                         os = 'Windows_NT'
1681                     }
1682
1683                     // WinArm32 is only built for Debug and Release
1684                     if (os == 'Windows_NT' && architecture == 'arm')
1685                     {
1686                         if (configuration == 'Checked')
1687                         {
1688                             return
1689                         }
1690                     }
1691                     // Tizen is only supported for arm architecture
1692                     if (os == 'Tizen' && architecture != 'arm') {
1693                         return
1694                     }
1695
1696                     // Skip totally unimplemented (in CI) configurations.
1697                     switch (architecture) {
1698                         case 'arm64':
1699                             // Windows only
1700                             if (os != 'Windows_NT' || isBuildOnly) {
1701                                 return
1702                             }
1703                             break
1704                         case 'arm':
1705                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1706                                 return
1707                             }
1708                             break
1709                         case 'x86':
1710                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1711                                 return
1712                             }
1713                             break
1714                         case 'x64':
1715                             // Everything implemented
1716                             break
1717                         default:
1718                             println("Unknown architecture: ${architecture}")
1719                             assert false
1720                             break
1721                     }
1722
1723                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1724                     // with checked builds
1725                     def enableCorefxTesting = false
1726                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1727                         if (configuration != 'Checked') {
1728                             return
1729                         }
1730
1731                         enableCorefxTesting = isCorefxTesting(scenario)
1732
1733                         // Since these are just execution time differences,
1734                         // skip platforms that don't execute the tests here (Windows_NT only)
1735                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1736                         if (!isEnabledOS || isBuildOnly) {
1737                             return
1738                         }
1739
1740                         switch (architecture) {
1741                             case 'arm64':
1742                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1743                                     return
1744                                 }
1745                                 break
1746                             case 'x64':
1747                             case 'x86':
1748                                 // x86 ubuntu: default only
1749                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1750                                     return
1751                                 }
1752                                 // Windows: Everything implemented
1753                                 break
1754                             default:
1755                                 return
1756                         }
1757                     }
1758                     else {
1759                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
1760                         // and configuration is not Checked, bail out.
1761                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1762                             return;
1763                         }
1764
1765                         // Skip scenarios
1766                         switch (scenario) {
1767                             case 'pri1':
1768                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1769                                 // the default scenario build
1770                                 if (os != 'Windows_NT') {
1771                                     return
1772                                 }
1773                                 // Only x64 for now
1774                                 if (architecture != 'x64') {
1775                                     return
1776                                 }
1777                                 break
1778                             case 'ilrt':
1779                                 // The ilrt build isn't necessary except for Windows_NT.  Non-Windows NT uses
1780                                 // the default scenario build
1781                                 if (os != 'Windows_NT') {
1782                                     return
1783                                 }
1784                                 // Only x64 for now
1785                                 if (architecture != 'x64') {
1786                                     return
1787                                 }
1788                                 // Release only
1789                                 if (configuration != 'Release') {
1790                                     return
1791                                 }
1792                                 break
1793                             case 'jitdiff':
1794                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1795                                     return
1796                                 }
1797                                 if (architecture != 'x64') {
1798                                     return
1799                                 }
1800                                 if (configuration != 'Checked') {
1801                                     return
1802                                 }
1803                                 break
1804                             case 'r2r':
1805                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1806                                 // the default scenario build
1807                                 if (os != 'Windows_NT') {
1808                                     return
1809                                 }
1810                                 if (architecture != 'x64') {
1811                                     return
1812                                 }
1813                                 break
1814                             case 'pri1r2r':
1815                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1816                                 // the default scenario build
1817                                 if (os != 'Windows_NT') {
1818                                     return
1819                                 }
1820                                 if (architecture != 'x64') {
1821                                     if (architecture != 'arm64' || configuration == 'Debug') {
1822                                         return
1823                                     }
1824                                 }
1825                                 break
1826                             case 'gcstress15_pri1r2r':
1827                             case 'r2r_jitstress1':
1828                             case 'r2r_jitstress2':
1829                             case 'r2r_jitstressregs1':
1830                             case 'r2r_jitstressregs2':
1831                             case 'r2r_jitstressregs3':
1832                             case 'r2r_jitstressregs4':
1833                             case 'r2r_jitstressregs8':
1834                             case 'r2r_jitstressregs0x10':
1835                             case 'r2r_jitstressregs0x80':
1836                             case 'r2r_jitstressregs0x1000':
1837                             case 'r2r_jitminopts':
1838                             case 'r2r_jitforcerelocs':
1839                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
1840                                 // the default scenario build
1841                                 if (os != 'Windows_NT') {
1842                                     return
1843                                 }
1844                                 if (architecture != 'x64') {
1845                                     return
1846                                 }
1847                                 break
1848                             case 'longgc':
1849                             case 'gcsimulator':
1850                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1851                                     return
1852                                 }
1853                                 if (architecture != 'x64') {
1854                                     return
1855                                 }
1856                                 if (configuration != 'Release') {
1857                                     return
1858                                 }
1859                                 break
1860                             case 'standalone_gc':
1861                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1862                                     return
1863                                 }
1864
1865                                 if (architecture != 'x64') {
1866                                     return
1867                                 }
1868
1869                                 if (configuration != 'Release' && configuration != 'Checked') {
1870                                     return
1871                                 }
1872                                 break
1873                             // We need Windows x64 Release bits for the code coverage build
1874                             case 'coverage':
1875                                 if (os != 'Windows_NT') {
1876                                     return
1877                                 }
1878                                 if (architecture != 'x64') {
1879                                     return
1880                                 }
1881                                 if (configuration != 'Release') {
1882                                     return
1883                                 }
1884                                 break
1885                             // We only run Windows and Ubuntu x64 Checked for formatting right now
1886                             case 'formatting':
1887                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
1888                                     return
1889                                 }
1890                                 if (architecture != 'x64') {
1891                                     return
1892                                 }
1893                                 if (configuration != 'Checked') {
1894                                     return
1895                                 }
1896                                 if (isBuildOnly) {
1897                                     return
1898                                 }
1899                                 break
1900                             case 'illink':
1901                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
1902                                     return
1903                                 }
1904                                 if (architecture != 'x64' && architecture != 'x86') {
1905                                     return
1906                                 }
1907                                 if (isBuildOnly) {
1908                                     return
1909                                 }
1910                                 break
1911                             case 'default':
1912                                 // Nothing skipped
1913                                 break
1914                             default:
1915                                 println("Unknown scenario: ${scenario}")
1916                                 assert false
1917                                 break
1918                         }
1919                     }
1920
1921                     // Calculate names
1922                     def lowerConfiguration = configuration.toLowerCase()
1923                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
1924                     def folderName = getJobFolder(scenario)
1925
1926                     // Create the new job
1927                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
1928
1929                     setMachineAffinity(newJob, os, architecture)
1930
1931                     // Add all the standard options
1932                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
1933                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
1934
1935                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
1936                     def osGroup = getOSGroup(os)
1937
1938                     newJob.with {
1939                         steps {
1940                             if (os == 'Windows_NT') {
1941                                 buildCommands.each { buildCommand ->
1942                                     batchFile(buildCommand)
1943                                 }
1944                             }
1945                             else {
1946                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
1947                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
1948                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
1949                                     // Define the Windows Tests and Corefx build job names
1950                                     def WindowTestsName = projectFolder + '/' +
1951                                                           Utilities.getFullJobName(project,
1952                                                                                    getJobName(lowerConfiguration,
1953                                                                                               'x64' ,
1954                                                                                               'windows_nt',
1955                                                                                               'default',
1956                                                                                               true),
1957                                                                                    false)
1958                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
1959                                                        Utilities.getFolderName(branch)
1960
1961                                     // Copy the Windows test binaries and the Corefx build binaries
1962                                     copyArtifacts(WindowTestsName) {
1963                                         excludePatterns('**/testResults.xml', '**/*.ni.dll')
1964                                         buildSelector {
1965                                             latestSuccessful(true)
1966                                         }
1967                                     }
1968
1969                                     // Defaults for Ubuntu
1970                                     def arm_abi = 'arm'
1971                                     def corefx_os = 'ubuntu14.04'
1972                                     if (os == 'Ubuntu16.04') {
1973                                         arm_abi = 'arm'
1974                                         corefx_os = 'ubuntu16.04'
1975                                     }
1976                                     else if (os == 'Tizen') {
1977                                         arm_abi = 'armel'
1978                                         corefx_os = 'tizen'
1979                                     }
1980
1981                                     // Let's use release CoreFX to test checked CoreCLR,
1982                                     // because we do not generate checked CoreFX in CoreFX CI yet.
1983                                     def corefx_lowerConfiguration = lowerConfiguration
1984                                     if ( lowerConfiguration == 'checked' ) {
1985                                         corefx_lowerConfiguration='release'
1986                                     }
1987                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
1988                                         includePatterns('bin/build.tar.gz')
1989                                         buildSelector {
1990                                             latestSuccessful(true)
1991                                         }
1992                                     }
1993                                 }
1994
1995                                 buildCommands.each { buildCommand ->
1996                                     shell(buildCommand)
1997                                 }
1998                             }
1999                         }
2000                     } // newJob.with
2001
2002                 } // os
2003             } // configuration
2004         } // architecture
2005     } // isPR
2006 } // scenario
2007
2008
2009 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2010 combinedScenarios.each { scenario ->
2011     [true, false].each { isPR ->
2012         // Architectures.  x64 only at this point
2013         ['x64'].each { architecture ->
2014             // Put the OS's supported for coreclr cross testing here
2015             Constants.crossList.each { os ->
2016                 Constants.configurationList.each { configuration ->
2017
2018                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2019                         if (configuration != 'Checked') {
2020                             return
2021                         }
2022                         if (isCorefxTesting(scenario)) {
2023                             return
2024                         }
2025                         //Skip stress modes for these scenarios
2026                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2027                             return
2028                         }
2029                     }
2030                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2031                     // and configuration is not Checked, bail out.
2032                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2033                         return;
2034                     }
2035                     // For CentOS, we only want Checked/Release pri1 builds.
2036                     else if (os == 'CentOS7.1') {
2037                         if (scenario != 'pri1' &&
2038                             scenario != 'r2r' &&
2039                             scenario != 'pri1r2r' &&
2040                             scenario != 'gcstress15_pri1r2r' &&
2041                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2042                             return
2043                         }
2044                         if (configuration != 'Checked' && configuration != 'Release') {
2045                             return
2046                         }
2047                     }
2048                     // For RedHat and Debian, we only do Release pri1 builds.
2049                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2050                         if (scenario != 'pri1') {
2051                             return
2052                         }
2053                         if (configuration != 'Release') {
2054                             return
2055                         }
2056                     }
2057                     else {
2058                         // Skip scenarios
2059                         switch (scenario) {
2060                             case 'pri1':
2061                                 // Nothing skipped
2062                                 break
2063                             case 'ilrt':
2064                                 // Release only
2065                                 if (configuration != 'Release') {
2066                                     return
2067                                 }
2068                                 break
2069                             case 'jitdiff':
2070                                 if (configuration != 'Checked') {
2071                                     return;
2072                                 }
2073                             case 'r2r':
2074                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2075                                 if (configuration != 'Checked' && configuration != 'Release') {
2076                                     return
2077                                 }
2078                                 break
2079                             case 'pri1r2r':
2080                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2081                                 if (configuration != 'Checked' && configuration != 'Release') {
2082                                     return
2083                                 }
2084                                 break
2085                             case 'gcstress15_pri1r2r':
2086                             case 'r2r_jitstress1':
2087                             case 'r2r_jitstress2':
2088                             case 'r2r_jitstressregs1':
2089                             case 'r2r_jitstressregs2':
2090                             case 'r2r_jitstressregs3':
2091                             case 'r2r_jitstressregs4':
2092                             case 'r2r_jitstressregs8':
2093                             case 'r2r_jitstressregs0x10':
2094                             case 'r2r_jitstressregs0x80':
2095                             case 'r2r_jitstressregs0x1000':
2096                             case 'r2r_jitminopts':
2097                             case 'r2r_jitforcerelocs':
2098                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2099                                 if (configuration != 'Checked' && configuration != 'Release') {
2100                                     return
2101                                 }
2102                                 break
2103                             case 'longgc':
2104                             case 'gcsimulator':
2105                                 // Long GC tests take a long time on non-Release builds
2106                                 if (configuration != 'Release') {
2107                                     return
2108                                 }
2109                                 break
2110                             case 'standalone_gc':
2111                                 if (configuration != 'Release' && configuration != 'Checked') {
2112                                     return
2113                                 }
2114                             case 'coverage':
2115                                 //We only want Ubuntu Release for coverage
2116                                 if (os != 'Ubuntu') {
2117                                     return
2118                                 }
2119                                 if (configuration != 'Release') {
2120                                     return
2121                                 }
2122                             case 'formatting':
2123                                 return
2124                             case 'illink':
2125                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2126                                     return
2127                                 }
2128                                 break
2129                             case 'default':
2130                                 // Nothing skipped
2131                                 break
2132                             default:
2133                                 println("Unknown scenario: ${scenario}")
2134                                 assert false
2135                                 break
2136                         }
2137                     }
2138
2139                     def lowerConfiguration = configuration.toLowerCase()
2140                     def osGroup = getOSGroup(os)
2141                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2142
2143                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2144                     def inputScenario = 'default'
2145                     if (scenario == 'coverage') {
2146                         inputScenario = 'coverage'
2147                     }
2148                     def inputCoreCLRBuildName = projectFolder + '/' +
2149                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2150                     // If this is a stress scenario, there isn't any difference in the build job
2151                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2152                     // from the default scenario
2153                     def testBuildScenario = scenario
2154                     if (testBuildScenario == 'coverage' || testBuildScenario == 'pri1r2r'|| testBuildScenario == 'gcstress15_pri1r2r') {
2155                         testBuildScenario = 'pri1'
2156                     }
2157                     else if ( testBuildScenario == 'r2r' || isLongGc(testBuildScenario)) {
2158                         testBuildScenario = 'default'
2159                     }
2160                     def inputWindowTestsBuildName = ''
2161                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2162                         inputWindowTestsBuildName = projectFolder + '/' +
2163                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', 'default', true), isPR)
2164                     }
2165                     else {
2166                         inputWindowTestsBuildName = projectFolder + '/' +
2167                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', testBuildScenario, true), isPR)
2168                     }
2169                     // Enable Server GC for Ubuntu PR builds
2170                     def serverGCString = ''
2171
2172                     // Whether or not this test run should be run sequentially instead
2173                     // of in parallel. Only used for long GC tests.
2174                     def sequentialString = ''
2175
2176                     // Whether or not this test run should run a specific playlist.
2177                     // Only used for long GC tests.
2178
2179                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2180                     // for running long GC and GCSimulator tests, respectively. We don't use them
2181                     // here because using a playlist file produces much more readable output on the CI machines
2182                     // and reduces running time.
2183                     def playlistString = ''
2184
2185                     if (os == 'Ubuntu' && isPR){
2186                         serverGCString = '--useServerGC'
2187                     }
2188
2189                     // pass --crossgen to runtest.sh for crossgen builds
2190                     def crossgenStr = ''
2191                     def runcrossgentestsStr = ''
2192                     def runjitstressStr = ''
2193                     def runjitstressregsStr = ''
2194                     def runjitmioptsStr = ''
2195                     def runjitforcerelocsStr = ''
2196                     def runjitdisasmStr = ''
2197                     def gcstressStr = ''
2198                     def illinkStr = ''
2199
2200                     if (scenario == 'r2r' ||
2201                         scenario == 'pri1r2r' ||
2202                         scenario == 'gcstress15_pri1r2r' ||
2203                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2204                             crossgenStr = '--crossgen'
2205                             runcrossgentestsStr = '--runcrossgentests'
2206
2207                             if (scenario == 'r2r_jitstress1'){
2208                                 runjitstressStr = '--jitstress=1'
2209                             }
2210                             else if (scenario == 'r2r_jitstress2') {
2211                                 runjitstressStr = '--jitstress=2'
2212                             }
2213                             else if (scenario == 'r2r_jitstressregs1'){
2214                                 runjitstressregsStr = '--jitstressregs=1'
2215                             }
2216                             else if (scenario == 'r2r_jitstressregs2') {
2217                                 runjitstressregsStr = '--jitstressregs=2'
2218                             }
2219                             else if (scenario == 'r2r_jitstressregs3') {
2220                                 runjitstressregsStr = '--jitstressregs=3'
2221                             }
2222                             else if (scenario == 'r2r_jitstressregs4') {
2223                                 runjitstressregsStr = '--jitstressregs=4'
2224                             }
2225                             else if (scenario == 'r2r_jitstressregs8') {
2226                                 runjitstressregsStr = '--jitstressregs=8'
2227                             }
2228                             else if (scenario == 'r2r_jitstressregs0x10') {
2229                                 runjitstressregsStr = '--jitstressregs=0x10'
2230                             }
2231                             else if (scenario == 'r2r_jitstressregs0x80') {
2232                                 runjitstressregsStr = '--jitstressregs=0x80'
2233                             }
2234                             else if (scenario == 'r2r_jitstressregs0x1000') {
2235                                 runjitstressregsStr = '--jitstressregs=0x1000'
2236                             }
2237                             else if (scenario == 'r2r_jitminopts') {
2238                                 runjitmioptsStr = '--jitminopts'
2239                             }
2240                             else if (scenario == 'r2r_jitforcerelocs') {
2241                                 runjitforcerelocsStr = '--jitforcerelocs'
2242                             }
2243                     }
2244                     if  (scenario == 'gcstress15_pri1r2r')
2245                     {
2246                         gcstressStr = '--gcstresslevel=0xF'
2247                     }
2248
2249                     if (scenario == 'jitdiff')
2250                     {
2251                         runjitdisasmStr = '--jitdisasm --crossgen'
2252                     }
2253
2254                     if (scenario == 'illink')
2255                     {
2256                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2257                     }
2258
2259                     if (isLongGc(scenario)) {
2260                         // Long GC tests behave very poorly when they are not
2261                         // the only test running (many of them allocate until OOM).
2262                         sequentialString = '--sequential'
2263
2264                         // The Long GC playlist contains all of the tests that are
2265                         // going to be run. The GCSimulator playlist contains all of
2266                         // the GC simulator tests.
2267                         if (scenario == 'longgc') {
2268                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2269                         }
2270                         else if (scenario == 'gcsimulator') {
2271                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2272                         }
2273                     }
2274
2275                     def folder = getJobFolder(scenario)
2276                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2277                         // Add parameters for the inputs
2278
2279                         parameters {
2280                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2281                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2282                         }
2283
2284                         steps {
2285                             // Set up the copies
2286
2287                             // Coreclr build containing the tests and mscorlib
2288
2289                             copyArtifacts(inputWindowTestsBuildName) {
2290                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2291                                 buildSelector {
2292                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2293                                 }
2294                             }
2295
2296                             if (scenario == 'coverage') {
2297
2298                                 // Move coreclr to clr directory
2299                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2300
2301                                 // Build coreclr
2302                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2303
2304                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2305                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2306                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2307                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2308                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2309                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2310                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2311                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2312
2313                                 // Run PAL tests
2314                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2315
2316                                 // Remove obj files for PAL tests so they're not included in coverage results
2317                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2318
2319                                 // Unzip the tests first.  Exit with 0
2320                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2321
2322                                 // Get corefx
2323                                 shell("git clone https://github.com/dotnet/corefx fx")
2324
2325                                 // Build Linux corefx
2326                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2327                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2328
2329                                 def testEnvOpt = ""
2330                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2331                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2332                                 shell("${createScriptCmds}")
2333                                 testEnvOpt = "--test-env=" + scriptFileName
2334
2335                                 // Run corefx tests
2336                                 shell("""./fx/run-test.sh \\
2337                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2338                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2339                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2340                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2341                 --configurationGroup Release""")
2342
2343
2344                                 // Run coreclr tests w/ workstation GC
2345                                 shell("""./clr/tests/runtest.sh \\
2346                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2347                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2348                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2349                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2350                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2351                 --crossgen --runcrossgentests""")
2352
2353                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2354                                 shell("""./clr/tests/runtest.sh \\
2355                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2356                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2357                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2358                 --useServerGC ${testEnvOpt}""")
2359
2360                                  // Run long-running coreclr GC tests & produce coverage reports
2361                                 shell("""./clr/tests/runtest.sh \\
2362                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2363                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2364                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2365                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2366                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2367                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2368                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2369
2370                             }
2371                             else {
2372
2373                                 // Coreclr build we are trying to test
2374
2375                                 copyArtifacts(inputCoreCLRBuildName) {
2376                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2377                                     buildSelector {
2378                                         buildNumber('${CORECLR_BUILD}')
2379                                     }
2380                                 }
2381
2382                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2383
2384                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2385                                 def osJobName
2386                                 if (os == 'Ubuntu') {
2387                                     osJobName = 'ubuntu14.04'
2388                                 }
2389                                 else {
2390                                     osJobName = os.toLowerCase()
2391                                 }
2392                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2393                                     includePatterns('bin/build.tar.gz')
2394                                     buildSelector {
2395                                         latestSuccessful(true)
2396                                     }
2397                                 }
2398
2399                                 shell ("mkdir ./bin/CoreFxBinDir")
2400                                 // Unpack the corefx binaries
2401                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2402
2403                                 // Unzip the tests first.  Exit with 0
2404                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2405
2406                                 // Execute the tests
2407                                 // If we are running a stress mode, we'll set those variables first
2408                                 def testEnvOpt = ""
2409                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2410                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2411                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2412                                     shell("${createScriptCmds}")
2413                                     testEnvOpt = "--test-env=" + scriptFileName
2414                                 }
2415
2416                                 if (isGCStressRelatedTesting(scenario)) {
2417                                     shell('./init-tools.sh')
2418                                 }
2419
2420                                 shell("""./tests/runtest.sh \\
2421                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2422                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2423                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2424                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2425                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2426                 --limitedDumpGeneration \\
2427                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${illinkStr} ${sequentialString} ${playlistString}""")
2428                             }
2429                         }
2430                     }
2431
2432                     if (scenario == 'coverage') {
2433                         // Publish coverage reports
2434                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2435                         // TODO: Add once external email sending is available again
2436                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2437                     }
2438
2439                     if (scenario == 'jitdiff') {
2440                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2441                     }
2442
2443                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2444                     if (os in ['Ubuntu']) {
2445                         SummaryBuilder summaries = new SummaryBuilder()
2446                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2447                         summaries.emit(newJob)
2448                     }
2449
2450                     setMachineAffinity(newJob, os, architecture)
2451                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2452                     // Set timeouts to 240.
2453                     setTestJobTimeOut(newJob, scenario)
2454                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2455
2456                     // Create a build flow to join together the build and tests required to run this
2457                     // test.
2458                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2459                     // Linux CoreCLR test
2460                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2461                     def fullTestJobName = projectFolder + '/' + newJob.name
2462                     // Add a reference to the input jobs for report purposes
2463                     JobReport.Report.addReference(inputCoreCLRBuildName)
2464                     JobReport.Report.addReference(inputWindowTestsBuildName)
2465                     JobReport.Report.addReference(fullTestJobName)
2466                     def newFlowJob;
2467
2468                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2469                     // so that coverage data can be preserved.
2470                     if (scenario == 'coverage') {
2471                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2472                         buildFlow("""
2473 // Build the input Windows job
2474 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2475
2476 // And then build the test build
2477 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2478 """)
2479                         }
2480                     // Normal jobs copy a Windows build & a non-Windows build
2481                     } else {
2482                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2483                         buildFlow("""
2484 // Build the input jobs in parallel
2485 parallel (
2486     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2487     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2488 )
2489
2490 // And then build the test build
2491 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2492                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2493 """)
2494                         }
2495                     }
2496
2497                     setMachineAffinity(newFlowJob, os, architecture)
2498                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2499                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2500                 } // configuration
2501             } // os
2502         } // architecture
2503     } // isPR
2504 } // scenario
2505
2506 JobReport.Report.generateJobReport(out)
2507
2508 // Make the call to generate the help job
2509 Utilities.createHelperJob(this, project, branch,
2510     "Welcome to the ${project} Repository",  // This is prepended to the help message
2511     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2512
2513 Utilities.addCROSSCheck(this, project, branch)