netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs
  12 folder('jitstress')
  13
  14 // Create a folder for testing via illink
  15 folder('illink')
  16
  17 def static getOSGroup(def os) {
  18     def osGroupMap = ['Ubuntu':'Linux',
  19         'RHEL7.2': 'Linux',
  20         'Ubuntu16.04': 'Linux',
  21         'Ubuntu16.10': 'Linux',
  22         'Debian8.4':'Linux',
  23         'Fedora24':'Linux',
  24         'OSX10.12':'OSX',
  25         'Windows_NT':'Windows_NT',
  26         'FreeBSD':'FreeBSD',
  27         'CentOS7.1': 'Linux',
  28         'OpenSUSE42.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'FreeBSD',
  49                'CentOS7.1',
  50                'OpenSUSE42.1',
  51                'RHEL7.2',
  52                'Ubuntu16.04',
  53                'Ubuntu16.10',
  54                'Fedora24',
  55                'Tizen']
  56
  57     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  58
  59     // This is a set of JIT stress modes combined with the set of variables that
  60     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  61     // the values are the environment variables
  62     def static jitStressModeScenarios = [
  63                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  64                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  65                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  66                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  67                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  68                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  69                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  70                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  71                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  72                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  73                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  74                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  75                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  76                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  77                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  78                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  79                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  80                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  81                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  82                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  83                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  84                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  85                'corefx_baseline'                : [ : ], // corefx baseline
  86                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  87                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  88                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  89                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  90                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  91                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  92                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  93                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  94                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  95                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  96                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  97                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  98                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
  99                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 100                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 101                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 103                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 104                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 105                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 106                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 107                ]
 108
 109     // This is a set of r2r jit stress scenarios
 110     def static r2rJitStressScenarios = [
 111                'r2r_jitstress1',
 112                'r2r_jitstress2',
 113                'r2r_jitstressregs1',
 114                'r2r_jitstressregs2',
 115                'r2r_jitstressregs3',
 116                'r2r_jitstressregs4',
 117                'r2r_jitstressregs8',
 118                'r2r_jitstressregs0x10',
 119                'r2r_jitstressregs0x80',
 120                'r2r_jitstressregs0x1000',
 121                'r2r_jitminopts',
 122                'r2r_jitforcerelocs']
 123
 124     // This is the basic set of scenarios
 125     def static basicScenarios = [
 126                'default',
 127                'pri1',
 128                'ilrt',
 129                'r2r',
 130                'pri1r2r',
 131                'gcstress15_pri1r2r',
 132                'longgc',
 133                'coverage',
 134                'formatting',
 135                'gcsimulator',
 136                'jitdiff',
 137                'standalone_gc',
 138                'illink'] + r2rJitStressScenarios
 139
 140     def static configurationList = ['Debug', 'Checked', 'Release']
 141
 142     // This is the set of architectures
 143     def static architectureList = ['arm', 'arm64', 'x64', 'x86']
 144 }
 145
 146 def static setMachineAffinity(def job, def os, def architecture) {
 147     if (architecture == 'arm64' && os == 'Windows_NT') {
 148         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 149     } else if (architecture == 'arm64' && os == 'Ubuntu') {
 150         Utilities.setMachineAffinity(job, os, 'arm-cross-latest');
 151     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 152         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 153     } else {
 154         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 155     }
 156 }
 157
 158 def static isJITStressJob(def scenario) {
 159     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 160            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 161 }
 162
 163 def static isGCStressRelatedTesting(def scenario) {
 164     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 165     // Detect it and make it a GCStress related.
 166     if (scenario == 'gcstress15_pri1r2r')
 167     {
 168         return true;
 169     }
 170
 171     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 172     def scenarioName = scenario.toLowerCase()
 173     def isGCStressTesting = false
 174     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 175         if (k in gcStressTestEnvVars) {
 176             isGCStressTesting = true;
 177         }
 178     }
 179     return isGCStressTesting
 180 }
 181
 182 def static isCorefxTesting(def scenario) {
 183     def corefx_prefix = 'corefx_'
 184     if (scenario.length() < corefx_prefix.length()) {
 185         return false
 186     }
 187     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 188 }
 189
 190 def static isR2R(def scenario) {
 191     return (scenario == 'r2r' || scenario == 'pri1r2r')
 192 }
 193
 194 def static isCoverage(def scenario) {
 195     return (scenario == 'coverage')
 196 }
 197
 198 def static isLongGc(def scenario) {
 199     return (scenario == 'longgc' || scenario == 'gcsimulator')
 200 }
 201
 202 def static isJitDiff(def scenario) {
 203     return (scenario == 'jitdiff')
 204 }
 205
 206 def static setTestJobTimeOut(newJob, scenario) {
 207     if (isGCStressRelatedTesting(scenario)) {
 208         Utilities.setJobTimeout(newJob, 4320)
 209     }
 210     else if (isCorefxTesting(scenario)) {
 211         Utilities.setJobTimeout(newJob, 360)
 212     }
 213     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 214         Utilities.setJobTimeout(newJob, 240)
 215     }
 216     else if (isR2R(scenario)) {
 217         Utilities.setJobTimeout(newJob, 240)
 218     }
 219     else if (isCoverage(scenario)) {
 220         Utilities.setJobTimeout(newJob, 1440)
 221     }
 222     else if (isLongGc(scenario)) {
 223         Utilities.setJobTimeout(newJob, 1440)
 224     }
 225     else if (isJitDiff(scenario)) {
 226         Utilities.setJobTimeout(newJob, 240)
 227     }
 228     // Non-test jobs use the default timeout value.
 229 }
 230
 231 def static getJobFolder(def scenario) {
 232     if (isJITStressJob(scenario)) {
 233         return 'jitstress'
 234     }
 235     if (scenario == 'illink') {
 236         return 'illink'
 237     }
 238     return ''
 239 }
 240
 241 def static getStressModeDisplayName(def scenario) {
 242     def displayStr = ''
 243     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 244         def prefixLength = 'COMPlus_'.length()
 245         if (k.length() >= prefixLength) {
 246             def modeName = k.substring(prefixLength, k.length())
 247             displayStr += ' ' + modeName + '=' + v
 248         }
 249     }
 250     return displayStr
 251 }
 252
 253 def static getR2RStressModeDisplayName(def scenario) {
 254     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 255     // "r2r_" prefix.
 256     def displayStr = scenario
 257     def prefixLength = 'r2r_'.length()
 258     if (displayStr.length() >= prefixLength) {
 259         displayStr = displayStr.substring(prefixLength, displayStr.length())
 260     }
 261     return displayStr
 262 }
 263
 264 // Generates the string for creating a file that sets environment variables
 265 // that makes it possible to run stress modes.  Writes the script to the file
 266 // specified by the stepScriptLocation parameter.
 267 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 268     def stepScript = ''
 269     if (os == 'Windows_NT') {
 270         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 271         stepScript += "del ${stepScriptLocation}\r\n"
 272
 273         // Timeout in ms, default is 10 minutes. For stress
 274         // modes up this to 30 minutes
 275         def timeout = 1800000
 276
 277         // Set the Timeout
 278         stepScript += "set __TestTimeout=${timeout}\r\n"
 279         stepScript += "echo. > ${stepScriptLocation}\r\n"
 280         stressModeVars.each{ k, v ->
 281             // Write out what we are writing to the script file
 282             stepScript += "echo Setting ${k}=${v}\r\n"
 283             // Write out the set itself to the script file`
 284             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 285         }
 286     }
 287     else {
 288         stepScript += "echo Setting variables for ${stressModeName}\n"
 289         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 290         stressModeVars.each{ k, v ->
 291             // Write out what we are writing to the script file
 292             stepScript += "echo Setting ${k}=${v}\n"
 293             // Write out the set itself to the script file`
 294             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 295         }
 296         stepScript += "chmod +x ${stepScriptLocation}\n"
 297     }
 298     return stepScript
 299 }
 300
 301 // Calculates the name of the build job based on some typical parameters.
 302 //
 303 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 304     // If the architecture is x64, do not add that info into the build name.
 305     // Need to change around some systems and other builds to pick up the right builds
 306     // to do that.
 307
 308     def suffix = scenario != 'default' ? "_${scenario}" : '';
 309     if (isBuildOnly) {
 310         suffix += '_bld'
 311     }
 312     def baseName = ''
 313     switch (architecture) {
 314         case 'x64':
 315             if (scenario == 'default') {
 316                 // For now we leave x64 off of the name for compatibility with other jobs
 317                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 318             }
 319             else if (scenario == 'formatting') {
 320                 // we don't care about the configuration for the formatting job. It runs all configs
 321                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 322             }
 323             else {
 324                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 325             }
 326             break
 327         case 'arm64':
 328             // These are cross builds
 329             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 330             break
 331         case 'arm':
 332             // These are cross builds
 333             if (os == 'Tizen') {
 334                 // ABI: softfp
 335                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 336             }
 337             else {
 338                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 339             }
 340             break
 341         case 'x86':
 342             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 343             break
 344         default:
 345             println("Unknown architecture: ${architecture}");
 346             assert false
 347             break
 348     }
 349
 350     return baseName + suffix
 351 }
 352
 353 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 354     // Check scenario.
 355     switch (scenario) {
 356         case 'default':
 357             switch (architecture) {
 358                 case 'x64':
 359                 case 'x86':
 360                     if (architecture == 'x86' && os == 'Ubuntu') {
 361                         Utilities.addPeriodicTrigger(job, '@daily')
 362                     }
 363                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 364                         Utilities.addGithubPushTrigger(job)
 365                     }
 366                     break
 367                 case 'arm':
 368                     Utilities.addGithubPushTrigger(job)
 369                     break
 370                 case 'arm64':
 371                     if (os == 'Windows_NT') {
 372                         Utilities.addGithubPushTrigger(job)
 373                         // TODO: Add once external email sending is available again
 374                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 375                     }
 376                     break
 377                 default:
 378                     println("Unknown architecture: ${architecture}");
 379                     assert false
 380                     break
 381             }
 382             break
 383         case 'pri1':
 384             // Pri one gets a push trigger, and only for release
 385             if (architecture == 'x64') {
 386                 if (configuration == 'Release') {
 387                     // We expect release jobs to be Windows, or in the cross list
 388                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 389                     if (!os in bidailyCrossList) {
 390                         if (isFlowJob || os == 'Windows_NT') {
 391                             Utilities.addGithubPushTrigger(job)
 392                         }
 393                     }
 394                     else {
 395                         if (isFlowJob) {
 396                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 397                         }
 398                     }
 399                 }
 400             }
 401             break
 402         case 'r2r':
 403             //r2r jobs that aren't pri1 can only be triggered by phrase
 404             break
 405         case 'pri1r2r':
 406             assert !(os in bidailyCrossList)
 407             //pri1 r2r gets a push trigger for checked/release
 408             if (configuration == 'Checked' || configuration == 'Release') {
 409                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 410                 if (architecture == 'x64' && os != 'OSX10.12') {
 411                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 412                     if (isFlowJob || os == 'Windows_NT') {
 413                         Utilities.addGithubPushTrigger(job)
 414                     }
 415                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 416                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 417                     if (isFlowJob) {
 418                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 419                     }
 420                 }
 421                 // For x86, only add per-commit jobs for Windows
 422                 else if (architecture == 'x86') {
 423                     if (os == 'Windows_NT') {
 424                         Utilities.addGithubPushTrigger(job)
 425                     }
 426                 }
 427                 // arm64 pri1r2r jobs should only run every 12 hours.
 428                 else if (architecture == 'arm64') {
 429                     if (os == 'Windows_NT') {
 430                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 431                         // TODO: Add once external email sending is available again
 432                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 433                     }
 434                 }
 435             }
 436             break
 437         case 'r2r_jitstress1':
 438         case 'r2r_jitstress2':
 439         case 'r2r_jitstressregs1':
 440         case 'r2r_jitstressregs2':
 441         case 'r2r_jitstressregs3':
 442         case 'r2r_jitstressregs4':
 443         case 'r2r_jitstressregs8':
 444         case 'r2r_jitstressregs0x10':
 445         case 'r2r_jitstressregs0x80':
 446         case 'r2r_jitstressregs0x1000':
 447         case 'r2r_jitminopts':
 448         case 'r2r_jitforcerelocs':
 449         case 'gcstress15_pri1r2r':
 450             assert !(os in bidailyCrossList)
 451
 452             // GCStress=C is currently not supported on OS X
 453             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 454                 break
 455             }
 456
 457             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 458             if (configuration == 'Checked' || configuration == 'Release') {
 459                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 460                 if (architecture == 'x64') {
 461                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 462                     if (isFlowJob || os == 'Windows_NT') {
 463                         // Add a weekly periodic trigger
 464                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 465                     }
 466                 }
 467                 // For x86, only add per-commit jobs for Windows
 468                 else if (architecture == 'x86') {
 469                     if (os == 'Windows_NT') {
 470                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 471                     }
 472                 }
 473             }
 474             break
 475         case 'longgc':
 476             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 477             assert configuration == 'Release'
 478             assert architecture == 'x64'
 479             Utilities.addPeriodicTrigger(job, '@daily')
 480             // TODO: Add once external email sending is available again
 481             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 482             break
 483         case 'gcsimulator':
 484             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 485             assert configuration == 'Release'
 486             assert architecture == 'x64'
 487             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 488             // TODO: Add once external email sending is available again
 489             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 490             break
 491         case 'standalone_gc':
 492             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 493             assert (configuration == 'Release' || configuration == 'Checked')
 494             // TODO: Add once external email sending is available again
 495             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 496             Utilities.addPeriodicTrigger(job, '@weekly')
 497             break
 498         case 'ilrt':
 499             assert !(os in bidailyCrossList)
 500             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 501             if (architecture == 'x64' && configuration == 'Release') {
 502                 // We don't expect to see a job generated except in these scenarios
 503                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 504                 if (isFlowJob || os == 'Windows_NT') {
 505                     Utilities.addPeriodicTrigger(job, '@daily')
 506                 }
 507             }
 508             break
 509         case 'jitdiff':
 510             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 511             assert configuration == 'Checked'
 512             assert (architecture == 'x64' || architecture == 'x86')
 513             Utilities.addGithubPushTrigger(job)
 514             break
 515         case 'coverage':
 516             assert (os == 'Ubuntu' || os == 'Windows_NT')
 517             assert configuration == 'Release'
 518             assert architecture == 'x64'
 519             Utilities.addPeriodicTrigger(job, '@weekly')
 520             break
 521         case 'formatting':
 522             assert (os == 'Windows_NT' || os == "Ubuntu")
 523             assert architecture == 'x64'
 524             Utilities.addGithubPushTrigger(job)
 525             break
 526         case 'jitstressregs1':
 527         case 'jitstressregs2':
 528         case 'jitstressregs3':
 529         case 'jitstressregs4':
 530         case 'jitstressregs8':
 531         case 'jitstressregs0x10':
 532         case 'jitstressregs0x80':
 533         case 'jitstressregs0x1000':
 534         case 'minopts':
 535         case 'forcerelocs':
 536         case 'jitstress1':
 537         case 'jitstress2':
 538         case 'jitstress2_jitstressregs1':
 539         case 'jitstress2_jitstressregs2':
 540         case 'jitstress2_jitstressregs3':
 541         case 'jitstress2_jitstressregs4':
 542         case 'jitstress2_jitstressregs8':
 543         case 'jitstress2_jitstressregs0x10':
 544         case 'jitstress2_jitstressregs0x80':
 545         case 'jitstress2_jitstressregs0x1000':
 546         case 'tailcallstress':
 547         case 'jitsse2only':
 548         case 'corefx_baseline':
 549         case 'corefx_minopts':
 550         case 'corefx_jitstress1':
 551         case 'corefx_jitstress2':
 552         case 'corefx_jitstressregs1':
 553         case 'corefx_jitstressregs2':
 554         case 'corefx_jitstressregs3':
 555         case 'corefx_jitstressregs4':
 556         case 'corefx_jitstressregs8':
 557         case 'corefx_jitstressregs0x10':
 558         case 'corefx_jitstressregs0x80':
 559         case 'corefx_jitstressregs0x1000':
 560         case 'zapdisable':
 561             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 562             assert (os == 'Windows_NT') || (os in Constants.crossList)
 563             Utilities.addPeriodicTrigger(job, '@daily')
 564         }
 565         break
 566         case 'heapverify1':
 567         case 'gcstress0x3':
 568             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 569                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 570                 if (architecture == 'arm64') {
 571                     assert (os == 'Windows_NT')
 572                     Utilities.addPeriodicTrigger(job, '@daily')
 573                     // TODO: Add once external email sending is available again
 574                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 575                 }
 576                 else {
 577                     Utilities.addPeriodicTrigger(job, '@weekly')
 578                 }
 579             }
 580             break
 581         case 'gcstress0xc':
 582         case 'gcstress0xc_zapdisable':
 583         case 'gcstress0xc_zapdisable_jitstress2':
 584         case 'gcstress0xc_zapdisable_heapverify1':
 585         case 'gcstress0xc_jitstress1':
 586         case 'gcstress0xc_jitstress2':
 587         case 'gcstress0xc_minopts_heapverify1':
 588             // GCStress=C is currently not supported on OS X
 589             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 590                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 591                 if (architecture == 'arm64') {
 592                     assert (os == 'Windows_NT')
 593                     // TODO: Enable a periodic trigger after tests are updated.
 594                     // Utilities.addPeriodicTrigger(job, '@daily')
 595                     // TODO: Add once external email sending is available again
 596                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 597                 }
 598                 else {
 599                     Utilities.addPeriodicTrigger(job, '@weekly')
 600                 }
 601             }
 602             break
 603
 604         case 'illink':
 605             // Testing on other operating systems TBD
 606             assert (os == 'Windows_NT' || os == 'Ubuntu')
 607             if (architecture == 'x64' || architecture == 'x86') {
 608                 if (configuration == 'Checked') {
 609                     Utilities.addPeriodicTrigger(job, '@daily')
 610                 }
 611             }
 612             break
 613
 614         default:
 615             println("Unknown scenario: ${scenario}");
 616             assert false
 617             break
 618     }
 619     return
 620 }
 621
 622 // **************************
 623 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 624 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 625 // builds will do a couple extra steps.
 626 // **************************
 627
 628 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 629 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 630 // it's just used for internal builds
 631 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 632 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 633     if (isWindowsBuildOnlyJob) {
 634         return
 635     }
 636
 637     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 638     // Non pull request builds.
 639     if (!isPR) {
 640         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 641         return
 642     }
 643     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 644     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 645     def osGroup = getOSGroup(os)
 646     switch (architecture) {
 647         case 'x64': // editor brace matching: {
 648             if (scenario == 'coverage') {
 649                 assert configuration == 'Release'
 650                 if (os == 'Ubuntu') {
 651                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 652                 }
 653                 break
 654             }
 655
 656             if (scenario == 'formatting') {
 657                 assert configuration == 'Checked'
 658                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 659                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 660                 }
 661                 break
 662             }
 663
 664             switch (os) {
 665                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 666                 case 'Debian8.4':
 667                 case 'RHEL7.2':
 668                     if (scenario == 'default') {
 669                         assert !isFlowJob
 670                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 671                     }
 672                     else if (scenario == 'pri1' && isFlowJob) {
 673                         assert (configuration == 'Release')
 674                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 675                     }
 676                     break
 677                 case 'Ubuntu16.04':
 678                     assert !isFlowJob
 679                     assert scenario == 'default'
 680                     // Distinguish with the other architectures (arm and x86)
 681                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 682                     break
 683                 case 'Fedora24':
 684                 case 'Ubuntu16.10':
 685                 case 'OpenSUSE42.1':
 686                     assert !isFlowJob
 687                     assert scenario == 'default'
 688                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 689                     break
 690                 case 'Ubuntu':
 691                     if (scenario == 'illink') {
 692                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 693                         break
 694                     }
 695                     // fall through
 696                 case 'OSX10.12':
 697                     // Triggers on the non-flow jobs aren't necessary here
 698                     // Corefx testing uses non-flow jobs.
 699                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 700                         break
 701                     }
 702                     switch (scenario) {
 703                         case 'default':
 704                             // Ubuntu uses checked for default PR tests
 705                             if (configuration == 'Checked') {
 706                                 // Default trigger
 707                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 708                             }
 709                             break
 710                         case 'pri1':
 711                             if (configuration == 'Release') {
 712                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 713                             }
 714                             break
 715                         case 'jitdiff':
 716                             if (configuration == 'Checked') {
 717                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 718                             }
 719                             break
 720                         case 'ilrt':
 721                             if (configuration == 'Release') {
 722                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 723                             }
 724                             break
 725                         case 'r2r':
 726                             if (configuration == 'Release' || configuration == 'Checked') {
 727                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 728                             }
 729                             break
 730                         case 'pri1r2r':
 731                             if (configuration == 'Release' || configuration == 'Checked') {
 732                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 733                             }
 734                             break
 735                         case 'gcstress15_pri1r2r':
 736                             if (configuration == 'Release' || configuration == 'Checked') {
 737                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 738                             }
 739                             break
 740                         case 'r2r_jitstress1':
 741                         case 'r2r_jitstress2':
 742                         case 'r2r_jitstressregs1':
 743                         case 'r2r_jitstressregs2':
 744                         case 'r2r_jitstressregs3':
 745                         case 'r2r_jitstressregs4':
 746                         case 'r2r_jitstressregs8':
 747                         case 'r2r_jitstressregs0x10':
 748                         case 'r2r_jitstressregs0x80':
 749                         case 'r2r_jitstressregs0x1000':
 750                         case 'r2r_jitminopts':
 751                         case 'r2r_jitforcerelocs':
 752                             if (configuration == 'Release' || configuration == 'Checked') {
 753                                 def displayStr = getR2RStressModeDisplayName(scenario)
 754                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 755                             }
 756                             break
 757                         case 'longgc':
 758                             if (configuration == 'Release') {
 759                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 760                             }
 761                             break
 762                         case 'gcsimulator':
 763                             if (configuration == 'Release') {
 764                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 765                             }
 766                             break
 767                         case 'minopts':
 768                         case 'forcerelocs':
 769                         case 'jitstress1':
 770                         case 'jitstress2':
 771                         case 'jitstressregs1':
 772                         case 'jitstressregs2':
 773                         case 'jitstressregs3':
 774                         case 'jitstressregs4':
 775                         case 'jitstressregs8':
 776                         case 'jitstressregs0x10':
 777                         case 'jitstressregs0x80':
 778                         case 'jitstressregs0x1000':
 779                         case 'jitstress2_jitstressregs1':
 780                         case 'jitstress2_jitstressregs2':
 781                         case 'jitstress2_jitstressregs3':
 782                         case 'jitstress2_jitstressregs4':
 783                         case 'jitstress2_jitstressregs8':
 784                         case 'jitstress2_jitstressregs0x10':
 785                         case 'jitstress2_jitstressregs0x80':
 786                         case 'jitstress2_jitstressregs0x1000':
 787                         case 'tailcallstress':
 788                         case 'jitsse2only':
 789                         case 'gcstress0x3':
 790                         case 'gcstress0xc':
 791                         case 'zapdisable':
 792                         case 'heapverify1':
 793                         case 'gcstress0xc_zapdisable':
 794                         case 'gcstress0xc_zapdisable_jitstress2':
 795                         case 'gcstress0xc_zapdisable_heapverify1':
 796                         case 'gcstress0xc_jitstress1':
 797                         case 'gcstress0xc_jitstress2':
 798                         case 'gcstress0xc_minopts_heapverify1':
 799                             def displayStr = getStressModeDisplayName(scenario)
 800                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 801                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 802                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 803                             break
 804                         case 'corefx_baseline':
 805                         case 'corefx_minopts':
 806                         case 'corefx_jitstress1':
 807                         case 'corefx_jitstress2':
 808                         case 'corefx_jitstressregs1':
 809                         case 'corefx_jitstressregs2':
 810                         case 'corefx_jitstressregs3':
 811                         case 'corefx_jitstressregs4':
 812                         case 'corefx_jitstressregs8':
 813                         case 'corefx_jitstressregs0x10':
 814                         case 'corefx_jitstressregs0x80':
 815                         case 'corefx_jitstressregs0x1000':
 816                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 817                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 818                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 819                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 820                             break
 821                         default:
 822                             println("Unknown scenario: ${scenario}");
 823                             assert false
 824                             break
 825                     }
 826                     break
 827                 case 'CentOS7.1':
 828                     switch (scenario) {
 829                         case 'pri1':
 830                             if (configuration == 'Release') {
 831                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 832                             }
 833                             break
 834                         case 'r2r':
 835                             if (configuration == 'Checked' || configuration == 'Release') {
 836                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 837                             }
 838                             break
 839                         case 'pri1r2r':
 840                             if (configuration == 'Checked' || configuration == 'Release') {
 841                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 842                             }
 843                             break
 844                         case 'gcstress15_pri1r2r':
 845                             if (configuration == 'Release' || configuration == 'Checked') {
 846                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 847                             }
 848                             break
 849                         case 'r2r_jitstress1':
 850                         case 'r2r_jitstress2':
 851                         case 'r2r_jitstressregs1':
 852                         case 'r2r_jitstressregs2':
 853                         case 'r2r_jitstressregs3':
 854                         case 'r2r_jitstressregs4':
 855                         case 'r2r_jitstressregs8':
 856                         case 'r2r_jitstressregs0x10':
 857                         case 'r2r_jitstressregs0x80':
 858                         case 'r2r_jitstressregs0x1000':
 859                         case 'r2r_jitminopts':
 860                         case 'r2r_jitforcerelocs':
 861                             if (configuration == 'Release' || configuration == 'Checked') {
 862                                 def displayStr = getR2RStressModeDisplayName(scenario)
 863                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 864                             }
 865                             break
 866                         default:
 867                             break
 868                     }
 869                 case 'Windows_NT':
 870                     switch (scenario) {
 871                         case 'default':
 872                             // Default trigger
 873                             if (configuration == 'Debug') {
 874                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 875                             }
 876                             break
 877                         case 'pri1':
 878                             // Default trigger
 879                             if (configuration == 'Release') {
 880                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 881                             }
 882                             break
 883                         case 'jitdiff':
 884                             if (configuration == 'Checked') {
 885                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 886                             }
 887                             break
 888                         case 'ilrt':
 889                             if (configuration == 'Release') {
 890                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 891                             }
 892                             break
 893                         case 'r2r':
 894                             if (configuration == 'Checked' || configuration == 'Release') {
 895                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 896                             }
 897                             break
 898                         case 'pri1r2r':
 899                             if (configuration == 'Checked' || configuration == 'Release') {
 900                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 901                             }
 902                             break
 903                         case 'gcstress15_pri1r2r':
 904                             if (configuration == 'Release' || configuration == 'Checked') {
 905                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 906                             }
 907                             break
 908                         case 'r2r_jitstress1':
 909                         case 'r2r_jitstress2':
 910                         case 'r2r_jitstressregs1':
 911                         case 'r2r_jitstressregs2':
 912                         case 'r2r_jitstressregs3':
 913                         case 'r2r_jitstressregs4':
 914                         case 'r2r_jitstressregs8':
 915                         case 'r2r_jitstressregs0x10':
 916                         case 'r2r_jitstressregs0x80':
 917                         case 'r2r_jitstressregs0x1000':
 918                         case 'r2r_jitminopts':
 919                         case 'r2r_jitforcerelocs':
 920                             if (configuration == 'Release' || configuration == 'Checked') {
 921                                 def displayStr = getR2RStressModeDisplayName(scenario)
 922                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 923                             }
 924                             break
 925                         case 'longgc':
 926                             if (configuration == 'Release') {
 927                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 928                             }
 929                             break
 930                         case 'gcsimulator':
 931                             if (configuration == 'Release') {
 932                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 933                             }
 934                             break
 935                         case 'standalone_gc':
 936                             if (configuration == 'Release' || configuration == 'Checked') {
 937                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 938                             }
 939                             break
 940                         case 'minopts':
 941                         case 'forcerelocs':
 942                         case 'jitstress1':
 943                         case 'jitstress2':
 944                         case 'jitstressregs1':
 945                         case 'jitstressregs2':
 946                         case 'jitstressregs3':
 947                         case 'jitstressregs4':
 948                         case 'jitstressregs8':
 949                         case 'jitstressregs0x10':
 950                         case 'jitstressregs0x80':
 951                         case 'jitstressregs0x1000':
 952                         case 'jitstress2_jitstressregs1':
 953                         case 'jitstress2_jitstressregs2':
 954                         case 'jitstress2_jitstressregs3':
 955                         case 'jitstress2_jitstressregs4':
 956                         case 'jitstress2_jitstressregs8':
 957                         case 'jitstress2_jitstressregs0x10':
 958                         case 'jitstress2_jitstressregs0x80':
 959                         case 'jitstress2_jitstressregs0x1000':
 960                         case 'tailcallstress':
 961                         case 'jitsse2only':
 962                         case 'gcstress0x3':
 963                         case 'gcstress0xc':
 964                         case 'zapdisable':
 965                         case 'heapverify1':
 966                         case 'gcstress0xc_zapdisable':
 967                         case 'gcstress0xc_zapdisable_jitstress2':
 968                         case 'gcstress0xc_zapdisable_heapverify1':
 969                         case 'gcstress0xc_jitstress1':
 970                         case 'gcstress0xc_jitstress2':
 971                         case 'gcstress0xc_minopts_heapverify1':
 972                             def displayStr = getStressModeDisplayName(scenario)
 973                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 974                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 975                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 976                             break
 977                         case 'corefx_baseline':
 978                         case 'corefx_minopts':
 979                         case 'corefx_jitstress1':
 980                         case 'corefx_jitstress2':
 981                         case 'corefx_jitstressregs1':
 982                         case 'corefx_jitstressregs2':
 983                         case 'corefx_jitstressregs3':
 984                         case 'corefx_jitstressregs4':
 985                         case 'corefx_jitstressregs8':
 986                         case 'corefx_jitstressregs0x10':
 987                         case 'corefx_jitstressregs0x80':
 988                         case 'corefx_jitstressregs0x1000':
 989                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 990                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 991                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 992                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 993                             break
 994                         case 'illink':
 995                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 996                                     break
 997                         default:
 998                             println("Unknown scenario: ${scenario}");
 999                             assert false
1000                             break
1001                     }
1002                     break
1003                 case 'FreeBSD':
1004                     assert scenario == 'default'
1005                     if (configuration == 'Checked') {
1006                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1007                     }
1008                     break
1009                 default:
1010                     println("Unknown os: ${os}");
1011                     assert false
1012                     break
1013             }
1014             break
1015         // editor brace matching: }
1016         case 'arm': // editor brace matching: {
1017             assert scenario == 'default'
1018             switch (os) {
1019                 case 'Ubuntu':
1020                 case 'Ubuntu16.04':
1021                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1022                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1023                     }
1024                     else {
1025                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1026                     }
1027                     break;
1028                 case 'Tizen':
1029                     architecture='armel'
1030                     // Removing the regex will cause this to run on each PR.
1031                     if (configuration == 'Release' || configuration == 'Debug') {
1032                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1033                     }
1034                     else {
1035                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1036                     }
1037                     break;
1038                 case 'Windows_NT':
1039                     if (configuration == 'Debug' || configuration == 'Release')
1040                     {
1041                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1042                     }
1043                     break
1044                 default:
1045                     println("NYI os: ${os}");
1046                     assert false
1047                     break
1048             }
1049             break
1050         // editor brace matching: }
1051         case 'arm64': // editor brace matching: {
1052             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1053
1054             // Set up a private trigger
1055             def contextString = "${os} ${architecture} Cross ${configuration}"
1056             if (scenario != 'default')
1057                 contextString += " ${scenario}"
1058             contextString += " Build"
1059             // Debug builds only.
1060             if (configuration != 'Debug') {
1061                contextString += " and Test"
1062             }
1063
1064             def arm64Users = [
1065                 'adiaaida',
1066                 'AndyAyersMS',
1067                 'briansull',
1068                 'BruceForstall',
1069                 'CarolEidt',
1070                 'cmckinsey',
1071                 'erozenfeld',
1072                 'jashook',
1073                 'JosephTremoulet',
1074                 'pgavlin',
1075                 'pkukol',
1076                 'russellhadley',
1077                 'RussKeldorph',
1078                 'sandreenko',
1079                 'sivarv',
1080                 'swaroop-sridhar',
1081                 'gkhanna79',
1082                 'jkotas',
1083                 'markwilkie',
1084                 'rahku',
1085                 'ramarag',
1086                 'tzwlai',
1087                 'weshaggard'
1088             ]
1089
1090             switch (os) {
1091                 case 'Windows_NT':
1092                     switch (scenario) {
1093                         case 'default':
1094                             // For now only run Debug jobs on PR Trigger.
1095                             if (configuration != 'Debug') {
1096                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1097                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1098                             }
1099                             else {
1100                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1101                                 // are at higher risk of ARM64-breaking changes.
1102                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1103                             }
1104                             break
1105                         case 'pri1r2r':
1106                         case 'gcstress0x3':
1107                         case 'gcstress0xc':
1108                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1109                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1110                             break
1111                     }
1112                     break
1113                 default:
1114                     println("NYI os: ${os}");
1115                     assert false
1116                     break
1117             }
1118             break
1119         // editor brace matching: }
1120         case 'x86': // editor brace matching: {
1121             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1122             if (os == 'Ubuntu') {
1123                 // on-demand only for ubuntu x86
1124                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1125                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1126                 break
1127             }
1128             switch (scenario) {
1129                 case 'default':
1130                     if (configuration == 'Checked') {
1131                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1132                     }
1133                     else if (configuration == 'Release') {
1134                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1135                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1136                     }
1137                     break
1138                 case 'pri1':
1139                     if (configuration == 'Release') {
1140                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1141                     }
1142                     break
1143                 case 'ilrt':
1144                     if (configuration == 'Release') {
1145                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1146                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1147                     }
1148                     break
1149                 case 'r2r':
1150                     if (configuration == 'Checked' || configuration == 'Release') {
1151                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1152                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1153                     }
1154                     break
1155                 case 'pri1r2r':
1156                     if (configuration == 'Checked' || configuration == 'Release') {
1157                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1158                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1159                     }
1160                     break
1161                 case 'gcstress15_pri1r2r':
1162                     if (configuration == 'Release' || configuration == 'Checked') {
1163                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1164                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1165                     }
1166                     break
1167                 case 'r2r_jitstress1':
1168                 case 'r2r_jitstress2':
1169                 case 'r2r_jitstressregs1':
1170                 case 'r2r_jitstressregs2':
1171                 case 'r2r_jitstressregs3':
1172                 case 'r2r_jitstressregs4':
1173                 case 'r2r_jitstressregs8':
1174                 case 'r2r_jitstressregs0x10':
1175                 case 'r2r_jitstressregs0x80':
1176                 case 'r2r_jitstressregs0x1000':
1177                 case 'r2r_jitminopts':
1178                 case 'r2r_jitforcerelocs':
1179                     if (configuration == 'Release' || configuration == 'Checked') {
1180                         def displayStr = getR2RStressModeDisplayName(scenario)
1181                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1182                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1183                     }
1184                     break
1185                 case 'longgc':
1186                     if (configuration == 'Release') {
1187                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1188                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1189                     }
1190                     break
1191                 case 'gcsimulator':
1192                     if (configuration == 'Release') {
1193                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1194                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1195                     }
1196                     break
1197                 case 'standalone_gc':
1198                     if (configuration == 'Release' || configuration == 'Checked') {
1199                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1200                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1201                     }
1202                     break
1203                 case 'minopts':
1204                 case 'forcerelocs':
1205                 case 'jitstress1':
1206                 case 'jitstress2':
1207                 case 'jitstressregs1':
1208                 case 'jitstressregs2':
1209                 case 'jitstressregs3':
1210                 case 'jitstressregs4':
1211                 case 'jitstressregs8':
1212                 case 'jitstressregs0x10':
1213                 case 'jitstressregs0x80':
1214                 case 'jitstressregs0x1000':
1215                 case 'jitstress2_jitstressregs1':
1216                 case 'jitstress2_jitstressregs2':
1217                 case 'jitstress2_jitstressregs3':
1218                 case 'jitstress2_jitstressregs4':
1219                 case 'jitstress2_jitstressregs8':
1220                 case 'jitstress2_jitstressregs0x10':
1221                 case 'jitstress2_jitstressregs0x80':
1222                 case 'jitstress2_jitstressregs0x1000':
1223                 case 'tailcallstress':
1224                 case 'jitsse2only':
1225                 case 'gcstress0x3':
1226                 case 'gcstress0xc':
1227                 case 'zapdisable':
1228                 case 'heapverify1':
1229                 case 'gcstress0xc_zapdisable':
1230                 case 'gcstress0xc_zapdisable_jitstress2':
1231                 case 'gcstress0xc_zapdisable_heapverify1':
1232                 case 'gcstress0xc_jitstress1':
1233                 case 'gcstress0xc_jitstress2':
1234                 case 'gcstress0xc_minopts_heapverify1':
1235                     def displayStr = getStressModeDisplayName(scenario)
1236                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1237                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1238                     break
1239                 case 'corefx_baseline':
1240                 case 'corefx_minopts':
1241                 case 'corefx_jitstress1':
1242                 case 'corefx_jitstress2':
1243                 case 'corefx_jitstressregs1':
1244                 case 'corefx_jitstressregs2':
1245                 case 'corefx_jitstressregs3':
1246                 case 'corefx_jitstressregs4':
1247                 case 'corefx_jitstressregs8':
1248                 case 'corefx_jitstressregs0x10':
1249                 case 'corefx_jitstressregs0x80':
1250                 case 'corefx_jitstressregs0x1000':
1251                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1252                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1253                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1254                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1255                     break
1256                 case 'illink':
1257                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1258                     break
1259                 default:
1260                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1261                     assert false
1262                     break
1263             }
1264             break
1265         // editor brace matching: }
1266         default:
1267             println("Unknown architecture: ${architecture}");
1268             assert false
1269             break
1270     }
1271 }
1272
1273 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1274     def buildCommands = [];
1275     def osGroup = getOSGroup(os)
1276     def lowerConfiguration = configuration.toLowerCase()
1277
1278     // Calculate the build steps, archival, and xunit results
1279     switch (os) {
1280         case 'Windows_NT': // editor brace matching: {
1281             switch (architecture) {
1282                 case 'x64':
1283                 case 'x86':
1284                     def arch = architecture
1285                     def buildOpts = ''
1286
1287                     if (scenario == 'illink') {
1288                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1289                     }
1290
1291                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1292                             scenario == 'default' ||
1293                             scenario == 'r2r' ||
1294                             scenario == 'jitdiff' ||
1295                             scenario == 'illink' ||
1296                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1297                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1298                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1299                     }
1300
1301                     // For Pri 1 tests, we must shorten the output test binary path names.
1302                     // if __TestIntermediateDir is already set, build-test.cmd will
1303                     // output test binaries to that directory. If it is not set, the
1304                     // binaries are sent to a default directory whose name is about
1305                     // 35 characters long.
1306
1307                     else if (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage') {
1308                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1309                     }
1310                     else if (scenario == 'ilrt') {
1311                         // First do the build with skiptests and then build the tests with ilasm roundtrip
1312                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1313                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch} -ilasmroundtrip"
1314                     }
1315                     else if (isLongGc(scenario)) {
1316                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1317                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1318                     }
1319                     else if (scenario == 'standalone_gc') {
1320                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1321                     }
1322                     else if (scenario == 'formatting') {
1323                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1324                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1325                         break
1326                     }
1327                     else {
1328                         println("Unknown scenario: ${scenario}")
1329                         assert false
1330                     }
1331
1332                     // If we are running a stress mode, we should write out the set of key
1333                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1334
1335                     if (!isBuildOnly) {
1336                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1337                         def crossgenStr = ''
1338                         def runcrossgentestsStr = ''
1339                         def runjitstressStr = ''
1340                         def runjitstressregsStr = ''
1341                         def runjitmioptsStr = ''
1342                         def runjitforcerelocsStr = ''
1343                         def runjitdisasmStr = ''
1344                         def gcstressStr = ''
1345                         def runtestArguments = ''
1346                         def gcTestArguments = ''
1347                         def illinkArguments = ''
1348
1349                         if (scenario == 'r2r' ||
1350                             scenario == 'pri1r2r' ||
1351                             scenario == 'gcstress15_pri1r2r' ||
1352                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1353                                 crossgenStr = 'crossgen'
1354                                 runcrossgentestsStr = 'runcrossgentests'
1355
1356                                 if (scenario == 'r2r_jitstress1'){
1357                                     runjitstressStr = 'jitstress 1'
1358                                 }
1359                                 else if (scenario == 'r2r_jitstress2') {
1360                                     runjitstressStr = 'jitstress 2'
1361                                 }
1362                                 else if (scenario == 'r2r_jitstressregs1'){
1363                                     runjitstressregsStr = 'jitstressregs 1'
1364                                 }
1365                                 else if (scenario == 'r2r_jitstressregs2') {
1366                                     runjitstressregsStr = 'jitstressregs 2'
1367                                 }
1368                                 else if (scenario == 'r2r_jitstressregs3') {
1369                                     runjitstressregsStr = 'jitstressregs 3'
1370                                 }
1371                                 else if (scenario == 'r2r_jitstressregs4') {
1372                                     runjitstressregsStr = 'jitstressregs 4'
1373                                 }
1374                                 else if (scenario == 'r2r_jitstressregs8') {
1375                                     runjitstressregsStr = 'jitstressregs 8'
1376                                 }
1377                                 else if (scenario == 'r2r_jitstressregs0x10') {
1378                                     runjitstressregsStr = 'jitstressregs 0x10'
1379                                 }
1380                                 else if (scenario == 'r2r_jitstressregs0x80') {
1381                                     runjitstressregsStr = 'jitstressregs 0x80'
1382                                 }
1383                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1384                                     runjitstressregsStr = 'jitstressregs 0x1000'
1385                                 }
1386                                 else if (scenario == 'r2r_jitminopts') {
1387                                     runjitmioptsStr = 'jitminopts'
1388                                 }
1389                                 else if (scenario == 'r2r_jitforcerelocs') {
1390                                     runjitforcerelocsStr = 'jitforcerelocs'
1391                                 }
1392                         }
1393                         if (scenario == 'gcstress15_pri1r2r')
1394                         {
1395                             gcstressStr = 'gcstresslevel 0xF'
1396                         }
1397
1398                         if (scenario == 'jitdiff')
1399                         {
1400                             runjitdisasmStr = 'jitdisasm crossgen'
1401                         }
1402
1403                         if (isLongGc(scenario)) {
1404                             gcTestArguments = "${scenario} sequential"
1405                         }
1406
1407                         if (scenario == 'illink')
1408                         {
1409                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1410                         }
1411
1412                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1413
1414                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1415                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1416                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1417
1418                             if (enableCorefxTesting) {
1419                                 def workspaceRelativeFxRoot = "_/fx"
1420                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1421
1422                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1423
1424                                 setTestJobTimeOut(newJob, scenario)
1425
1426                                 // Archive and process (only) the test results
1427                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1428                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1429                             }
1430                             else {
1431                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1432                             }
1433                         }
1434                         else if (architecture == 'x64' || architecture == 'x86') {
1435                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1436                         }
1437                     }
1438
1439                     if (!enableCorefxTesting) {
1440                         // Run the rest of the build
1441                         // Build the mscorlib for the other OS's
1442                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1443                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1444                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1445
1446                         // Zip up the tests directory so that we don't use so much space/time copying
1447                         // 10s of thousands of files around.
1448                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1449
1450                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1451                             // For windows, pull full test results and test drops for x86/x64.
1452                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1453                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1454                         }
1455
1456                         if (scenario == 'jitdiff') {
1457                             // retrive jit-dasm output for base commit, and run jit-diff
1458                             if (!isBuildOnly) {
1459                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1460                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1461                             }
1462                         }
1463
1464                         if (!isBuildOnly) {
1465                             if (architecture == 'x64' || !isPR) {
1466                                 Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml')
1467                             }
1468                             setTestJobTimeOut(newJob, scenario)
1469                         }
1470                     }
1471                     break
1472                 case 'arm':
1473                     assert (scenario == 'default')
1474
1475                     // Set time out
1476                     setTestJobTimeOut(newJob, scenario)
1477
1478                     if ( lowerConfiguration == "debug" ) {
1479                         // For Debug builds, we will do a P1 test build
1480                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1481                     }
1482                     else {
1483                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1484                     }
1485                     // Add archival.
1486                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1487                     break
1488                 case 'arm64':
1489                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1490
1491                     // Set time out
1492                     setTestJobTimeOut(newJob, scenario)
1493
1494                     // Debug runs take too long to run. So build job only.
1495                     if (lowerConfiguration == "debug") {
1496                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1497                     }
1498                     else {
1499                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1500                        {
1501                            // Up the timeout for arm64 checked testing only.
1502                            // Keep the longer timeout for gcstress.
1503                            Utilities.setJobTimeout(newJob, 240)
1504                        }
1505
1506                        buildCommands += "set __TestIntermediateDir=int&&build.cmd skiptests ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1507                        // Test build and run are launched together.
1508                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1509                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1510                     }
1511
1512                     // Add archival.
1513                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1514                     break
1515                 default:
1516                     println("Unknown architecture: ${architecture}");
1517                     assert false
1518                     break
1519             }
1520             break
1521         // editor brace matching: }
1522         case 'Ubuntu':
1523         case 'Ubuntu16.04':
1524         case 'Ubuntu16.10':
1525         case 'Debian8.4':
1526         case 'OSX10.12':
1527         case 'FreeBSD':
1528         case 'CentOS7.1':
1529         case 'RHEL7.2':
1530         case 'OpenSUSE42.1':
1531         case 'Tizen':
1532         case 'Fedora24': // editor brace matching: {
1533             switch (architecture) {
1534                 case 'x64':
1535                 case 'x86':
1536                     if (architecture == 'x86' && os == 'Ubuntu') {
1537                         // build and PAL test
1538                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1539                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1540                         break;
1541                     }
1542
1543                     if (scenario == 'formatting') {
1544                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1545                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1546                         break
1547                     }
1548
1549                     if (scenario == 'illink') {
1550                         assert(os == 'Ubuntu')
1551                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1552                     }
1553
1554                     def standaloneGc = ''
1555                     if (scenario == 'standalone_gc') {
1556                         standaloneGc = 'buildstandalonegc'
1557                     }
1558
1559                     if (!enableCorefxTesting) {
1560                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1561                         // only on supported OS platforms.
1562                         if (os == 'FreeBSD')
1563                         {
1564                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1565                         }
1566                         else
1567                         {
1568                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1569                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1570                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1571                         }
1572                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1573
1574                         // Set time out
1575                         setTestJobTimeOut(newJob, scenario)
1576                         // Basic archiving of the build
1577                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1578                         // And pal tests
1579                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1580                     }
1581                     else {
1582                         // Corefx stress testing
1583                         assert os == 'Ubuntu'
1584                         assert architecture == 'x64'
1585                         assert lowerConfiguration == 'checked'
1586                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1587
1588                         // Build coreclr
1589                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1590
1591                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1592                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1593
1594                         // Build and text corefx
1595                         def workspaceRelativeFxRoot = "_/fx"
1596                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1597
1598                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1599
1600                         setTestJobTimeOut(newJob, scenario)
1601
1602                         // Archive and process (only) the test results
1603                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1604                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1605                     }
1606                     break
1607                 case 'arm64':
1608                     // We don't run the cross build except on Ubuntu
1609                     assert os == 'Ubuntu'
1610
1611                     buildCommands += """echo \"Using rootfs in /opt/aarch64-linux-gnu-root\"
1612                         ROOTFS_DIR=/opt/aarch64-linux-gnu-root ./build.sh skipmscorlib arm64 cross verbose ${lowerConfiguration}"""
1613
1614                     // Basic archiving of the build, no pal tests
1615                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1616                     break
1617                 case 'arm':
1618                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1619                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1620
1621                     // default values for Ubuntu
1622                     def arm_abi="arm"
1623                     def linuxCodeName="trusty"
1624                     if (os == 'Ubuntu16.04') {
1625                         linuxCodeName="xenial"
1626                     }
1627                     else if (os == 'Tizen') {
1628                         arm_abi="armel"
1629                         linuxCodeName="tizen"
1630                     }
1631
1632                     // Unzip the Windows test binaries first. Exit with 0
1633                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1634
1635                     // Unpack the corefx binaries
1636                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1637                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1638
1639                     // Call the ARM CI script to cross build and test using docker
1640                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1641                     --mode=docker \\
1642                     --${arm_abi} \\
1643                     --linuxCodeName=${linuxCodeName} \\
1644                     --buildConfig=${lowerConfiguration} \\
1645                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1646                     --coreFxBinDir=./bin/CoreFxBinDir \\
1647                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1648
1649                     // Basic archiving of the build, no pal tests
1650                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1651                     break
1652                 default:
1653                     println("Unknown architecture: ${architecture}");
1654                     assert false
1655                     break
1656             }
1657             break
1658         // editor brace matching: }
1659         default:
1660             println("Unknown os: ${os}");
1661             assert false
1662             break
1663     } // os
1664
1665     return buildCommands
1666 }
1667
1668 // Additional scenario which can alter behavior
1669
1670 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1671 combinedScenarios.each { scenario ->
1672     [true, false].each { isPR ->
1673         Constants.architectureList.each { architecture ->
1674             Constants.configurationList.each { configuration ->
1675                 Constants.osList.each { os ->
1676                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1677                     // and reset the os to Windows_NT
1678                     def isBuildOnly = false
1679                     if (os == 'Windows_NT_BuildOnly') {
1680                         isBuildOnly = true
1681                         os = 'Windows_NT'
1682                     }
1683
1684                     // WinArm32 is only built for Debug and Release
1685                     if (os == 'Windows_NT' && architecture == 'arm')
1686                     {
1687                         if (configuration == 'Checked')
1688                         {
1689                             return
1690                         }
1691                     }
1692                     // Tizen is only supported for arm architecture
1693                     if (os == 'Tizen' && architecture != 'arm') {
1694                         return
1695                     }
1696
1697                     // Skip totally unimplemented (in CI) configurations.
1698                     switch (architecture) {
1699                         case 'arm64':
1700                             // Windows only
1701                             if (os != 'Windows_NT' || isBuildOnly) {
1702                                 return
1703                             }
1704                             break
1705                         case 'arm':
1706                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1707                                 return
1708                             }
1709                             break
1710                         case 'x86':
1711                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1712                                 return
1713                             }
1714                             break
1715                         case 'x64':
1716                             // Everything implemented
1717                             break
1718                         default:
1719                             println("Unknown architecture: ${architecture}")
1720                             assert false
1721                             break
1722                     }
1723
1724                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1725                     // with checked builds
1726                     def enableCorefxTesting = false
1727                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1728                         if (configuration != 'Checked') {
1729                             return
1730                         }
1731
1732                         enableCorefxTesting = isCorefxTesting(scenario)
1733
1734                         // Since these are just execution time differences,
1735                         // skip platforms that don't execute the tests here (Windows_NT only)
1736                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1737                         if (!isEnabledOS || isBuildOnly) {
1738                             return
1739                         }
1740
1741                         switch (architecture) {
1742                             case 'arm64':
1743                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1744                                     return
1745                                 }
1746                                 break
1747                             case 'x64':
1748                             case 'x86':
1749                                 // x86 ubuntu: default only
1750                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1751                                     return
1752                                 }
1753                                 // Windows: Everything implemented
1754                                 break
1755                             default:
1756                                 return
1757                         }
1758                     }
1759                     else {
1760                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
1761                         // and configuration is not Checked, bail out.
1762                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1763                             return;
1764                         }
1765
1766                         // Skip scenarios
1767                         switch (scenario) {
1768                             case 'pri1':
1769                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1770                                 // the default scenario build
1771                                 if (os != 'Windows_NT') {
1772                                     return
1773                                 }
1774                                 // Only x64 for now
1775                                 if (architecture != 'x64') {
1776                                     return
1777                                 }
1778                                 break
1779                             case 'ilrt':
1780                                 // The ilrt build isn't necessary except for Windows_NT.  Non-Windows NT uses
1781                                 // the default scenario build
1782                                 if (os != 'Windows_NT') {
1783                                     return
1784                                 }
1785                                 // Only x64 for now
1786                                 if (architecture != 'x64') {
1787                                     return
1788                                 }
1789                                 // Release only
1790                                 if (configuration != 'Release') {
1791                                     return
1792                                 }
1793                                 break
1794                             case 'jitdiff':
1795                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1796                                     return
1797                                 }
1798                                 if (architecture != 'x64') {
1799                                     return
1800                                 }
1801                                 if (configuration != 'Checked') {
1802                                     return
1803                                 }
1804                                 break
1805                             case 'r2r':
1806                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1807                                 // the default scenario build
1808                                 if (os != 'Windows_NT') {
1809                                     return
1810                                 }
1811                                 if (architecture != 'x64') {
1812                                     return
1813                                 }
1814                                 break
1815                             case 'pri1r2r':
1816                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1817                                 // the default scenario build
1818                                 if (os != 'Windows_NT') {
1819                                     return
1820                                 }
1821                                 if (architecture != 'x64') {
1822                                     if (architecture != 'arm64' || configuration == 'Debug') {
1823                                         return
1824                                     }
1825                                 }
1826                                 break
1827                             case 'gcstress15_pri1r2r':
1828                             case 'r2r_jitstress1':
1829                             case 'r2r_jitstress2':
1830                             case 'r2r_jitstressregs1':
1831                             case 'r2r_jitstressregs2':
1832                             case 'r2r_jitstressregs3':
1833                             case 'r2r_jitstressregs4':
1834                             case 'r2r_jitstressregs8':
1835                             case 'r2r_jitstressregs0x10':
1836                             case 'r2r_jitstressregs0x80':
1837                             case 'r2r_jitstressregs0x1000':
1838                             case 'r2r_jitminopts':
1839                             case 'r2r_jitforcerelocs':
1840                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
1841                                 // the default scenario build
1842                                 if (os != 'Windows_NT') {
1843                                     return
1844                                 }
1845                                 if (architecture != 'x64') {
1846                                     return
1847                                 }
1848                                 break
1849                             case 'longgc':
1850                             case 'gcsimulator':
1851                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1852                                     return
1853                                 }
1854                                 if (architecture != 'x64') {
1855                                     return
1856                                 }
1857                                 if (configuration != 'Release') {
1858                                     return
1859                                 }
1860                                 break
1861                             case 'standalone_gc':
1862                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1863                                     return
1864                                 }
1865
1866                                 if (architecture != 'x64') {
1867                                     return
1868                                 }
1869
1870                                 if (configuration != 'Release' && configuration != 'Checked') {
1871                                     return
1872                                 }
1873                                 break
1874                             // We need Windows x64 Release bits for the code coverage build
1875                             case 'coverage':
1876                                 if (os != 'Windows_NT') {
1877                                     return
1878                                 }
1879                                 if (architecture != 'x64') {
1880                                     return
1881                                 }
1882                                 if (configuration != 'Release') {
1883                                     return
1884                                 }
1885                                 break
1886                             // We only run Windows and Ubuntu x64 Checked for formatting right now
1887                             case 'formatting':
1888                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
1889                                     return
1890                                 }
1891                                 if (architecture != 'x64') {
1892                                     return
1893                                 }
1894                                 if (configuration != 'Checked') {
1895                                     return
1896                                 }
1897                                 if (isBuildOnly) {
1898                                     return
1899                                 }
1900                                 break
1901                             case 'illink':
1902                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
1903                                     return
1904                                 }
1905                                 if (architecture != 'x64' && architecture != 'x86') {
1906                                     return
1907                                 }
1908                                 if (isBuildOnly) {
1909                                     return
1910                                 }
1911                                 break
1912                             case 'default':
1913                                 // Nothing skipped
1914                                 break
1915                             default:
1916                                 println("Unknown scenario: ${scenario}")
1917                                 assert false
1918                                 break
1919                         }
1920                     }
1921
1922                     // Calculate names
1923                     def lowerConfiguration = configuration.toLowerCase()
1924                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
1925                     def folderName = getJobFolder(scenario)
1926
1927                     // Create the new job
1928                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
1929
1930                     setMachineAffinity(newJob, os, architecture)
1931
1932                     // Add all the standard options
1933                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
1934                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
1935
1936                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
1937                     def osGroup = getOSGroup(os)
1938
1939                     newJob.with {
1940                         steps {
1941                             if (os == 'Windows_NT') {
1942                                 buildCommands.each { buildCommand ->
1943                                     batchFile(buildCommand)
1944                                 }
1945                             }
1946                             else {
1947                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
1948                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
1949                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
1950                                     // Define the Windows Tests and Corefx build job names
1951                                     def WindowTestsName = projectFolder + '/' +
1952                                                           Utilities.getFullJobName(project,
1953                                                                                    getJobName(lowerConfiguration,
1954                                                                                               'x64' ,
1955                                                                                               'windows_nt',
1956                                                                                               'default',
1957                                                                                               true),
1958                                                                                    false)
1959                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
1960                                                        Utilities.getFolderName(branch)
1961
1962                                     // Copy the Windows test binaries and the Corefx build binaries
1963                                     copyArtifacts(WindowTestsName) {
1964                                         excludePatterns('**/testResults.xml', '**/*.ni.dll')
1965                                         buildSelector {
1966                                             latestSuccessful(true)
1967                                         }
1968                                     }
1969
1970                                     // Defaults for Ubuntu
1971                                     def arm_abi = 'arm'
1972                                     def corefx_os = 'ubuntu14.04'
1973                                     if (os == 'Ubuntu16.04') {
1974                                         arm_abi = 'arm'
1975                                         corefx_os = 'ubuntu16.04'
1976                                     }
1977                                     else if (os == 'Tizen') {
1978                                         arm_abi = 'armel'
1979                                         corefx_os = 'tizen'
1980                                     }
1981
1982                                     // Let's use release CoreFX to test checked CoreCLR,
1983                                     // because we do not generate checked CoreFX in CoreFX CI yet.
1984                                     def corefx_lowerConfiguration = lowerConfiguration
1985                                     if ( lowerConfiguration == 'checked' ) {
1986                                         corefx_lowerConfiguration='release'
1987                                     }
1988                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
1989                                         includePatterns('bin/build.tar.gz')
1990                                         buildSelector {
1991                                             latestSuccessful(true)
1992                                         }
1993                                     }
1994                                 }
1995
1996                                 buildCommands.each { buildCommand ->
1997                                     shell(buildCommand)
1998                                 }
1999                             }
2000                         }
2001                     } // newJob.with
2002
2003                 } // os
2004             } // configuration
2005         } // architecture
2006     } // isPR
2007 } // scenario
2008
2009
2010 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2011 combinedScenarios.each { scenario ->
2012     [true, false].each { isPR ->
2013         // Architectures.  x64 only at this point
2014         ['x64'].each { architecture ->
2015             // Put the OS's supported for coreclr cross testing here
2016             Constants.crossList.each { os ->
2017                 Constants.configurationList.each { configuration ->
2018
2019                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2020                         if (configuration != 'Checked') {
2021                             return
2022                         }
2023                         if (isCorefxTesting(scenario)) {
2024                             return
2025                         }
2026                         //Skip stress modes for these scenarios
2027                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2028                             return
2029                         }
2030                     }
2031                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2032                     // and configuration is not Checked, bail out.
2033                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2034                         return;
2035                     }
2036                     // For CentOS, we only want Checked/Release pri1 builds.
2037                     else if (os == 'CentOS7.1') {
2038                         if (scenario != 'pri1' &&
2039                             scenario != 'r2r' &&
2040                             scenario != 'pri1r2r' &&
2041                             scenario != 'gcstress15_pri1r2r' &&
2042                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2043                             return
2044                         }
2045                         if (configuration != 'Checked' && configuration != 'Release') {
2046                             return
2047                         }
2048                     }
2049                     // For RedHat and Debian, we only do Release pri1 builds.
2050                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2051                         if (scenario != 'pri1') {
2052                             return
2053                         }
2054                         if (configuration != 'Release') {
2055                             return
2056                         }
2057                     }
2058                     else {
2059                         // Skip scenarios
2060                         switch (scenario) {
2061                             case 'pri1':
2062                                 // Nothing skipped
2063                                 break
2064                             case 'ilrt':
2065                                 // Release only
2066                                 if (configuration != 'Release') {
2067                                     return
2068                                 }
2069                                 break
2070                             case 'jitdiff':
2071                                 if (configuration != 'Checked') {
2072                                     return;
2073                                 }
2074                             case 'r2r':
2075                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2076                                 if (configuration != 'Checked' && configuration != 'Release') {
2077                                     return
2078                                 }
2079                                 break
2080                             case 'pri1r2r':
2081                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2082                                 if (configuration != 'Checked' && configuration != 'Release') {
2083                                     return
2084                                 }
2085                                 break
2086                             case 'gcstress15_pri1r2r':
2087                             case 'r2r_jitstress1':
2088                             case 'r2r_jitstress2':
2089                             case 'r2r_jitstressregs1':
2090                             case 'r2r_jitstressregs2':
2091                             case 'r2r_jitstressregs3':
2092                             case 'r2r_jitstressregs4':
2093                             case 'r2r_jitstressregs8':
2094                             case 'r2r_jitstressregs0x10':
2095                             case 'r2r_jitstressregs0x80':
2096                             case 'r2r_jitstressregs0x1000':
2097                             case 'r2r_jitminopts':
2098                             case 'r2r_jitforcerelocs':
2099                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2100                                 if (configuration != 'Checked' && configuration != 'Release') {
2101                                     return
2102                                 }
2103                                 break
2104                             case 'longgc':
2105                             case 'gcsimulator':
2106                                 // Long GC tests take a long time on non-Release builds
2107                                 if (configuration != 'Release') {
2108                                     return
2109                                 }
2110                                 break
2111                             case 'standalone_gc':
2112                                 if (configuration != 'Release' && configuration != 'Checked') {
2113                                     return
2114                                 }
2115                             case 'coverage':
2116                                 //We only want Ubuntu Release for coverage
2117                                 if (os != 'Ubuntu') {
2118                                     return
2119                                 }
2120                                 if (configuration != 'Release') {
2121                                     return
2122                                 }
2123                             case 'formatting':
2124                                 return
2125                             case 'illink':
2126                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2127                                     return
2128                                 }
2129                                 break
2130                             case 'default':
2131                                 // Nothing skipped
2132                                 break
2133                             default:
2134                                 println("Unknown scenario: ${scenario}")
2135                                 assert false
2136                                 break
2137                         }
2138                     }
2139
2140                     def lowerConfiguration = configuration.toLowerCase()
2141                     def osGroup = getOSGroup(os)
2142                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2143
2144                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2145                     def inputScenario = 'default'
2146                     if (scenario == 'coverage') {
2147                         inputScenario = 'coverage'
2148                     }
2149                     def inputCoreCLRBuildName = projectFolder + '/' +
2150                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2151                     // If this is a stress scenario, there isn't any difference in the build job
2152                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2153                     // from the default scenario
2154                     def testBuildScenario = scenario
2155                     if (testBuildScenario == 'coverage' || testBuildScenario == 'pri1r2r'|| testBuildScenario == 'gcstress15_pri1r2r') {
2156                         testBuildScenario = 'pri1'
2157                     }
2158                     else if ( testBuildScenario == 'r2r' || isLongGc(testBuildScenario)) {
2159                         testBuildScenario = 'default'
2160                     }
2161                     def inputWindowTestsBuildName = ''
2162                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2163                         inputWindowTestsBuildName = projectFolder + '/' +
2164                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', 'default', true), isPR)
2165                     }
2166                     else {
2167                         inputWindowTestsBuildName = projectFolder + '/' +
2168                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', testBuildScenario, true), isPR)
2169                     }
2170                     // Enable Server GC for Ubuntu PR builds
2171                     def serverGCString = ''
2172
2173                     // Whether or not this test run should be run sequentially instead
2174                     // of in parallel. Only used for long GC tests.
2175                     def sequentialString = ''
2176
2177                     // Whether or not this test run should run a specific playlist.
2178                     // Only used for long GC tests.
2179
2180                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2181                     // for running long GC and GCSimulator tests, respectively. We don't use them
2182                     // here because using a playlist file produces much more readable output on the CI machines
2183                     // and reduces running time.
2184                     def playlistString = ''
2185
2186                     if (os == 'Ubuntu' && isPR){
2187                         serverGCString = '--useServerGC'
2188                     }
2189
2190                     // pass --crossgen to runtest.sh for crossgen builds
2191                     def crossgenStr = ''
2192                     def runcrossgentestsStr = ''
2193                     def runjitstressStr = ''
2194                     def runjitstressregsStr = ''
2195                     def runjitmioptsStr = ''
2196                     def runjitforcerelocsStr = ''
2197                     def runjitdisasmStr = ''
2198                     def gcstressStr = ''
2199                     def illinkStr = ''
2200
2201                     if (scenario == 'r2r' ||
2202                         scenario == 'pri1r2r' ||
2203                         scenario == 'gcstress15_pri1r2r' ||
2204                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2205                             crossgenStr = '--crossgen'
2206                             runcrossgentestsStr = '--runcrossgentests'
2207
2208                             if (scenario == 'r2r_jitstress1'){
2209                                 runjitstressStr = '--jitstress=1'
2210                             }
2211                             else if (scenario == 'r2r_jitstress2') {
2212                                 runjitstressStr = '--jitstress=2'
2213                             }
2214                             else if (scenario == 'r2r_jitstressregs1'){
2215                                 runjitstressregsStr = '--jitstressregs=1'
2216                             }
2217                             else if (scenario == 'r2r_jitstressregs2') {
2218                                 runjitstressregsStr = '--jitstressregs=2'
2219                             }
2220                             else if (scenario == 'r2r_jitstressregs3') {
2221                                 runjitstressregsStr = '--jitstressregs=3'
2222                             }
2223                             else if (scenario == 'r2r_jitstressregs4') {
2224                                 runjitstressregsStr = '--jitstressregs=4'
2225                             }
2226                             else if (scenario == 'r2r_jitstressregs8') {
2227                                 runjitstressregsStr = '--jitstressregs=8'
2228                             }
2229                             else if (scenario == 'r2r_jitstressregs0x10') {
2230                                 runjitstressregsStr = '--jitstressregs=0x10'
2231                             }
2232                             else if (scenario == 'r2r_jitstressregs0x80') {
2233                                 runjitstressregsStr = '--jitstressregs=0x80'
2234                             }
2235                             else if (scenario == 'r2r_jitstressregs0x1000') {
2236                                 runjitstressregsStr = '--jitstressregs=0x1000'
2237                             }
2238                             else if (scenario == 'r2r_jitminopts') {
2239                                 runjitmioptsStr = '--jitminopts'
2240                             }
2241                             else if (scenario == 'r2r_jitforcerelocs') {
2242                                 runjitforcerelocsStr = '--jitforcerelocs'
2243                             }
2244                     }
2245                     if  (scenario == 'gcstress15_pri1r2r')
2246                     {
2247                         gcstressStr = '--gcstresslevel=0xF'
2248                     }
2249
2250                     if (scenario == 'jitdiff')
2251                     {
2252                         runjitdisasmStr = '--jitdisasm --crossgen'
2253                     }
2254
2255                     if (scenario == 'illink')
2256                     {
2257                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2258                     }
2259
2260                     if (isLongGc(scenario)) {
2261                         // Long GC tests behave very poorly when they are not
2262                         // the only test running (many of them allocate until OOM).
2263                         sequentialString = '--sequential'
2264
2265                         // The Long GC playlist contains all of the tests that are
2266                         // going to be run. The GCSimulator playlist contains all of
2267                         // the GC simulator tests.
2268                         if (scenario == 'longgc') {
2269                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2270                         }
2271                         else if (scenario == 'gcsimulator') {
2272                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2273                         }
2274                     }
2275
2276                     def folder = getJobFolder(scenario)
2277                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2278                         // Add parameters for the inputs
2279
2280                         parameters {
2281                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2282                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2283                         }
2284
2285                         steps {
2286                             // Set up the copies
2287
2288                             // Coreclr build containing the tests and mscorlib
2289
2290                             copyArtifacts(inputWindowTestsBuildName) {
2291                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2292                                 buildSelector {
2293                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2294                                 }
2295                             }
2296
2297                             if (scenario == 'coverage') {
2298
2299                                 // Move coreclr to clr directory
2300                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2301
2302                                 // Build coreclr
2303                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2304
2305                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2306                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2307                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2308                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2309                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2310                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2311                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2312                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2313
2314                                 // Run PAL tests
2315                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2316
2317                                 // Remove obj files for PAL tests so they're not included in coverage results
2318                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2319
2320                                 // Unzip the tests first.  Exit with 0
2321                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2322
2323                                 // Get corefx
2324                                 shell("git clone https://github.com/dotnet/corefx fx")
2325
2326                                 // Build Linux corefx
2327                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2328                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2329
2330                                 def testEnvOpt = ""
2331                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2332                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2333                                 shell("${createScriptCmds}")
2334                                 testEnvOpt = "--test-env=" + scriptFileName
2335
2336                                 // Run corefx tests
2337                                 shell("""./fx/run-test.sh \\
2338                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2339                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2340                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2341                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2342                 --configurationGroup Release""")
2343
2344
2345                                 // Run coreclr tests w/ workstation GC
2346                                 shell("""./clr/tests/runtest.sh \\
2347                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2348                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2349                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2350                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2351                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2352                 --crossgen --runcrossgentests""")
2353
2354                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2355                                 shell("""./clr/tests/runtest.sh \\
2356                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2357                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2358                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2359                 --useServerGC ${testEnvOpt}""")
2360
2361                                  // Run long-running coreclr GC tests & produce coverage reports
2362                                 shell("""./clr/tests/runtest.sh \\
2363                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2364                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2365                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2366                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2367                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2368                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2369                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2370
2371                             }
2372                             else {
2373
2374                                 // Coreclr build we are trying to test
2375
2376                                 copyArtifacts(inputCoreCLRBuildName) {
2377                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2378                                     buildSelector {
2379                                         buildNumber('${CORECLR_BUILD}')
2380                                     }
2381                                 }
2382
2383                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2384
2385                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2386                                 def osJobName
2387                                 if (os == 'Ubuntu') {
2388                                     osJobName = 'ubuntu14.04'
2389                                 }
2390                                 else {
2391                                     osJobName = os.toLowerCase()
2392                                 }
2393                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2394                                     includePatterns('bin/build.tar.gz')
2395                                     buildSelector {
2396                                         latestSuccessful(true)
2397                                     }
2398                                 }
2399
2400                                 shell ("mkdir ./bin/CoreFxBinDir")
2401                                 // Unpack the corefx binaries
2402                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2403
2404                                 // Unzip the tests first.  Exit with 0
2405                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2406
2407                                 // Execute the tests
2408                                 // If we are running a stress mode, we'll set those variables first
2409                                 def testEnvOpt = ""
2410                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2411                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2412                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2413                                     shell("${createScriptCmds}")
2414                                     testEnvOpt = "--test-env=" + scriptFileName
2415                                 }
2416
2417                                 if (isGCStressRelatedTesting(scenario)) {
2418                                     shell('./init-tools.sh')
2419                                 }
2420
2421                                 shell("""./tests/runtest.sh \\
2422                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2423                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2424                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2425                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2426                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2427                 --limitedDumpGeneration \\
2428                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${illinkStr} ${sequentialString} ${playlistString}""")
2429                             }
2430                         }
2431                     }
2432
2433                     if (scenario == 'coverage') {
2434                         // Publish coverage reports
2435                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2436                         // TODO: Add once external email sending is available again
2437                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2438                     }
2439
2440                     if (scenario == 'jitdiff') {
2441                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2442                     }
2443
2444                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2445                     if (os in ['Ubuntu']) {
2446                         SummaryBuilder summaries = new SummaryBuilder()
2447                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2448                         summaries.emit(newJob)
2449                     }
2450
2451                     setMachineAffinity(newJob, os, architecture)
2452                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2453                     // Set timeouts to 240.
2454                     setTestJobTimeOut(newJob, scenario)
2455                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2456
2457                     // Create a build flow to join together the build and tests required to run this
2458                     // test.
2459                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2460                     // Linux CoreCLR test
2461                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2462                     def fullTestJobName = projectFolder + '/' + newJob.name
2463                     // Add a reference to the input jobs for report purposes
2464                     JobReport.Report.addReference(inputCoreCLRBuildName)
2465                     JobReport.Report.addReference(inputWindowTestsBuildName)
2466                     JobReport.Report.addReference(fullTestJobName)
2467                     def newFlowJob;
2468
2469                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2470                     // so that coverage data can be preserved.
2471                     if (scenario == 'coverage') {
2472                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2473                         buildFlow("""
2474 // Build the input Windows job
2475 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2476
2477 // And then build the test build
2478 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2479 """)
2480                         }
2481                     // Normal jobs copy a Windows build & a non-Windows build
2482                     } else {
2483                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2484                         buildFlow("""
2485 // Build the input jobs in parallel
2486 parallel (
2487     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2488     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2489 )
2490
2491 // And then build the test build
2492 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2493                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2494 """)
2495                         }
2496                     }
2497
2498                     setMachineAffinity(newFlowJob, os, architecture)
2499                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2500                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2501                 } // configuration
2502             } // os
2503         } // architecture
2504     } // isPR
2505 } // scenario
2506
2507 JobReport.Report.generateJobReport(out)
2508
2509 // Make the call to generate the help job
2510 Utilities.createHelperJob(this, project, branch,
2511     "Welcome to the ${project} Repository",  // This is prepended to the help message
2512     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2513
2514 Utilities.addCROSSCheck(this, project, branch)