netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs
  12 folder('jitstress')
  13
  14 // Create a folder for testing via illink
  15 folder('illink')
  16
  17 def static getOSGroup(def os) {
  18     def osGroupMap = ['Ubuntu':'Linux',
  19         'RHEL7.2': 'Linux',
  20         'Ubuntu16.04': 'Linux',
  21         'Ubuntu16.10': 'Linux',
  22         'Debian8.4':'Linux',
  23         'Fedora23':'Linux',
  24         'OSX10.12':'OSX',
  25         'Windows_NT':'Windows_NT',
  26         'FreeBSD':'FreeBSD',
  27         'CentOS7.1': 'Linux',
  28         'OpenSUSE42.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'FreeBSD',
  49                'CentOS7.1',
  50                'OpenSUSE42.1',
  51                'RHEL7.2',
  52                'Ubuntu16.04',
  53                'Ubuntu16.10',
  54                'Tizen',
  55                'Fedora23']
  56
  57     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  58
  59     // This is a set of JIT stress modes combined with the set of variables that
  60     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  61     // the values are the environment variables
  62     def static jitStressModeScenarios = [
  63                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  64                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  65                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  66                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  67                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  68                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  69                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  70                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  71                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  72                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  73                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  74                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  75                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  76                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  77                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  78                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  79                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  80                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  81                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  82                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  83                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  84                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  85                'corefx_baseline'                : [ : ], // corefx baseline
  86                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  87                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  88                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  89                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  90                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  91                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  92                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  93                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  94                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  95                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  96                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  97                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  98                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
  99                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 100                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 101                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 103                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 104                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 105                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 106                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 107                ]
 108
 109     // This is a set of r2r jit stress scenarios
 110     def static r2rJitStressScenarios = [
 111                'r2r_jitstress1',
 112                'r2r_jitstress2',
 113                'r2r_jitstressregs1',
 114                'r2r_jitstressregs2',
 115                'r2r_jitstressregs3',
 116                'r2r_jitstressregs4',
 117                'r2r_jitstressregs8',
 118                'r2r_jitstressregs0x10',
 119                'r2r_jitstressregs0x80',
 120                'r2r_jitstressregs0x1000',
 121                'r2r_jitminopts',
 122                'r2r_jitforcerelocs']
 123
 124     // This is the basic set of scenarios
 125     def static basicScenarios = [
 126                'default',
 127                'pri1',
 128                'ilrt',
 129                'r2r',
 130                'pri1r2r',
 131                'gcstress15_pri1r2r',
 132                'longgc',
 133                'coverage',
 134                'formatting',
 135                'gcsimulator',
 136                'jitdiff',
 137                'standalone_gc',
 138                'illink'] + r2rJitStressScenarios
 139
 140     def static configurationList = ['Debug', 'Checked', 'Release']
 141
 142     // This is the set of architectures
 143     def static architectureList = ['arm', 'arm64', 'x64', 'x86', 'x86compatjit']
 144 }
 145
 146 def static setMachineAffinity(def job, def os, def architecture) {
 147     if (architecture == 'arm64' && os == 'Windows_NT') {
 148         // For cross compilation
 149         job.with {
 150             label('arm64')
 151         }
 152     } else if (architecture == 'arm64' && os == 'Ubuntu') {
 153         Utilities.setMachineAffinity(job, os, 'arm-cross-latest');
 154     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 155         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 156     } else {
 157         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 158     }
 159 }
 160
 161 def static isJITStressJob(def scenario) {
 162     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 163            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 164 }
 165
 166 def static isGCStressRelatedTesting(def scenario) {
 167     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 168     // Detect it and make it a GCStress related.
 169     if (scenario == 'gcstress15_pri1r2r')
 170     {
 171         return true;
 172     }
 173
 174     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 175     def scenarioName = scenario.toLowerCase()
 176     def isGCStressTesting = false
 177     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 178         if (k in gcStressTestEnvVars) {
 179             isGCStressTesting = true;
 180         }
 181     }
 182     return isGCStressTesting
 183 }
 184
 185 def static isCorefxTesting(def scenario) {
 186     def corefx_prefix = 'corefx_'
 187     if (scenario.length() < corefx_prefix.length()) {
 188         return false
 189     }
 190     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 191 }
 192
 193 def static isR2R(def scenario) {
 194     return (scenario == 'r2r' || scenario == 'pri1r2r')
 195 }
 196
 197 def static isCoverage(def scenario) {
 198     return (scenario == 'coverage')
 199 }
 200
 201 def static isLongGc(def scenario) {
 202     return (scenario == 'longgc' || scenario == 'gcsimulator')
 203 }
 204
 205 def static isJitDiff(def scenario) {
 206     return (scenario == 'jitdiff')
 207 }
 208
 209 def static setTestJobTimeOut(newJob, scenario) {
 210     if (isGCStressRelatedTesting(scenario)) {
 211         Utilities.setJobTimeout(newJob, 4320)
 212     }
 213     else if (isCorefxTesting(scenario)) {
 214         Utilities.setJobTimeout(newJob, 360)
 215     }
 216     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 217         Utilities.setJobTimeout(newJob, 240)
 218     }
 219     else if (isR2R(scenario)) {
 220         Utilities.setJobTimeout(newJob, 240)
 221     }
 222     else if (isCoverage(scenario)) {
 223         Utilities.setJobTimeout(newJob, 1440)
 224     }
 225     else if (isLongGc(scenario)) {
 226         Utilities.setJobTimeout(newJob, 1440)
 227     }
 228     else if (isJitDiff(scenario)) {
 229         Utilities.setJobTimeout(newJob, 240)
 230     }
 231     // Non-test jobs use the default timeout value.
 232 }
 233
 234 def static getJobFolder(def scenario) {
 235     if (isJITStressJob(scenario)) {
 236         return 'jitstress'
 237     }
 238     if (scenario == 'illink') {
 239         return 'illink'
 240     }
 241     return ''
 242 }
 243
 244 def static getStressModeDisplayName(def scenario) {
 245     def displayStr = ''
 246     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 247         def prefixLength = 'COMPlus_'.length()
 248         if (k.length() >= prefixLength) {
 249             def modeName = k.substring(prefixLength, k.length())
 250             displayStr += ' ' + modeName + '=' + v
 251         }
 252     }
 253     return displayStr
 254 }
 255
 256 def static getR2RStressModeDisplayName(def scenario) {
 257     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 258     // "r2r_" prefix.
 259     def displayStr = scenario
 260     def prefixLength = 'r2r_'.length()
 261     if (displayStr.length() >= prefixLength) {
 262         displayStr = displayStr.substring(prefixLength, displayStr.length())
 263     }
 264     return displayStr
 265 }
 266
 267 // Generates the string for creating a file that sets environment variables
 268 // that makes it possible to run stress modes.  Writes the script to the file
 269 // specified by the stepScriptLocation parameter.
 270 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 271     def stepScript = ''
 272     if (os == 'Windows_NT') {
 273         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 274         stepScript += "del ${stepScriptLocation}\r\n"
 275
 276         // Timeout in ms, default is 10 minutes. For stress
 277         // modes up this to 30 minutes
 278         def timeout = 1800000
 279
 280         // Set the Timeout
 281         stepScript += "set __TestTimeout=${timeout}\r\n"
 282         stepScript += "echo. > ${stepScriptLocation}\r\n"
 283         stressModeVars.each{ k, v ->
 284             // Write out what we are writing to the script file
 285             stepScript += "echo Setting ${k}=${v}\r\n"
 286             // Write out the set itself to the script file`
 287             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 288         }
 289     }
 290     else {
 291         stepScript += "echo Setting variables for ${stressModeName}\n"
 292         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 293         stressModeVars.each{ k, v ->
 294             // Write out what we are writing to the script file
 295             stepScript += "echo Setting ${k}=${v}\n"
 296             // Write out the set itself to the script file`
 297             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 298         }
 299         stepScript += "chmod +x ${stepScriptLocation}\n"
 300     }
 301     return stepScript
 302 }
 303
 304 // Calculates the name of the build job based on some typical parameters.
 305 //
 306 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 307     // If the architecture is x64, do not add that info into the build name.
 308     // Need to change around some systems and other builds to pick up the right builds
 309     // to do that.
 310
 311     def suffix = scenario != 'default' ? "_${scenario}" : '';
 312     if (isBuildOnly) {
 313         suffix += '_bld'
 314     }
 315     def baseName = ''
 316     switch (architecture) {
 317         case 'x64':
 318             if (scenario == 'default') {
 319                 // For now we leave x64 off of the name for compatibility with other jobs
 320                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 321             }
 322             else if (scenario == 'formatting') {
 323                 // we don't care about the configuration for the formatting job. It runs all configs
 324                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 325             }
 326             else {
 327                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 328             }
 329             break
 330         case 'arm64':
 331             // These are cross builds
 332             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 333             break
 334         case 'arm':
 335             // These are cross builds
 336             if (os == 'Tizen') {
 337                 // ABI: softfp
 338                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 339             }
 340             else {
 341                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 342             }
 343             break
 344         case 'x86':
 345             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 346             break
 347         case 'x86compatjit':
 348             baseName = 'x86_compatjit_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 349             break
 350         default:
 351             println("Unknown architecture: ${architecture}");
 352             assert false
 353             break
 354     }
 355
 356     return baseName + suffix
 357 }
 358
 359 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 360     // Check scenario.
 361     switch (scenario) {
 362         case 'default':
 363             switch (architecture) {
 364                 case 'x64':
 365                 case 'x86':
 366                 case 'x86compatjit':
 367                     if (architecture == 'x86' && os == 'Ubuntu') {
 368                         Utilities.addPeriodicTrigger(job, '@daily')
 369                     }
 370                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 371                         Utilities.addGithubPushTrigger(job)
 372                     }
 373                     break
 374                 case 'arm':
 375                     Utilities.addGithubPushTrigger(job)
 376                     break
 377                 case 'arm64':
 378                     if (os == 'Windows_NT') {
 379                         Utilities.addGithubPushTrigger(job)
 380                         // TODO: Add once external email sending is available again
 381                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 382                     }
 383                     break
 384                 default:
 385                     println("Unknown architecture: ${architecture}");
 386                     assert false
 387                     break
 388             }
 389             break
 390         case 'pri1':
 391             // Pri one gets a push trigger, and only for release
 392             if (architecture == 'x64') {
 393                 if (configuration == 'Release') {
 394                     // We expect release jobs to be Windows, or in the cross list
 395                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 396                     if (!os in bidailyCrossList) {
 397                         if (isFlowJob || os == 'Windows_NT') {
 398                             Utilities.addGithubPushTrigger(job)
 399                         }
 400                     }
 401                     else {
 402                         if (isFlowJob) {
 403                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 404                         }
 405                     }
 406                 }
 407             }
 408             break
 409         case 'r2r':
 410             //r2r jobs that aren't pri1 can only be triggered by phrase
 411             break
 412         case 'pri1r2r':
 413             assert !(os in bidailyCrossList)
 414             //pri1 r2r gets a push trigger for checked/release
 415             if (configuration == 'Checked' || configuration == 'Release') {
 416                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 417                 if (architecture == 'x64' && os != 'OSX10.12') {
 418                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 419                     if (isFlowJob || os == 'Windows_NT') {
 420                         Utilities.addGithubPushTrigger(job)
 421                     }
 422                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 423                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 424                     if (isFlowJob) {
 425                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 426                     }
 427                 }
 428                 // For x86, only add per-commit jobs for Windows
 429                 else if (architecture == 'x86' || architecture == 'x86compatjit') {
 430                     if (os == 'Windows_NT') {
 431                         Utilities.addGithubPushTrigger(job)
 432                     }
 433                 }
 434                 // arm64 pri1r2r jobs should only run every 12 hours.
 435                 else if (architecture == 'arm64') {
 436                     if (os == 'Windows_NT') {
 437                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 438                         // TODO: Add once external email sending is available again
 439                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 440                     }
 441                 }
 442             }
 443             break
 444         case 'r2r_jitstress1':
 445         case 'r2r_jitstress2':
 446         case 'r2r_jitstressregs1':
 447         case 'r2r_jitstressregs2':
 448         case 'r2r_jitstressregs3':
 449         case 'r2r_jitstressregs4':
 450         case 'r2r_jitstressregs8':
 451         case 'r2r_jitstressregs0x10':
 452         case 'r2r_jitstressregs0x80':
 453         case 'r2r_jitstressregs0x1000':
 454         case 'r2r_jitminopts':
 455         case 'r2r_jitforcerelocs':
 456         case 'gcstress15_pri1r2r':
 457             assert !(os in bidailyCrossList)
 458
 459             // GCStress=C is currently not supported on OS X
 460             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 461                 break
 462             }
 463
 464             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 465             if (configuration == 'Checked' || configuration == 'Release') {
 466                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 467                 if (architecture == 'x64') {
 468                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 469                     if (isFlowJob || os == 'Windows_NT') {
 470                         // Add a weekly periodic trigger
 471                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 472                     }
 473                 }
 474                 // For x86, only add per-commit jobs for Windows
 475                 else if (architecture == 'x86' || architecture == 'x86compatjit') {
 476                     if (os == 'Windows_NT') {
 477                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 478                     }
 479                 }
 480             }
 481             break
 482         case 'longgc':
 483             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 484             assert configuration == 'Release'
 485             assert architecture == 'x64'
 486             Utilities.addPeriodicTrigger(job, '@daily')
 487             // TODO: Add once external email sending is available again
 488             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 489             break
 490         case 'gcsimulator':
 491             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 492             assert configuration == 'Release'
 493             assert architecture == 'x64'
 494             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 495             // TODO: Add once external email sending is available again
 496             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 497             break
 498         case 'standalone_gc':
 499             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 500             assert (configuration == 'Release' || configuration == 'Checked')
 501             // TODO: Add once external email sending is available again
 502             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 503             Utilities.addPeriodicTrigger(job, '@weekly')
 504             break
 505         case 'ilrt':
 506             assert !(os in bidailyCrossList)
 507             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 508             if (architecture == 'x64' && configuration == 'Release') {
 509                 // We don't expect to see a job generated except in these scenarios
 510                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 511                 if (isFlowJob || os == 'Windows_NT') {
 512                     Utilities.addPeriodicTrigger(job, '@daily')
 513                 }
 514             }
 515             break
 516         case 'jitdiff':
 517             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 518             assert configuration == 'Checked'
 519             assert (architecture == 'x64' || architecture == 'x86')
 520             Utilities.addGithubPushTrigger(job)
 521             break
 522         case 'coverage':
 523             assert (os == 'Ubuntu' || os == 'Windows_NT')
 524             assert configuration == 'Release'
 525             assert architecture == 'x64'
 526             Utilities.addPeriodicTrigger(job, '@weekly')
 527             break
 528         case 'formatting':
 529             assert (os == 'Windows_NT' || os == "Ubuntu")
 530             assert architecture == 'x64'
 531             Utilities.addGithubPushTrigger(job)
 532             break
 533         case 'jitstressregs1':
 534         case 'jitstressregs2':
 535         case 'jitstressregs3':
 536         case 'jitstressregs4':
 537         case 'jitstressregs8':
 538         case 'jitstressregs0x10':
 539         case 'jitstressregs0x80':
 540         case 'jitstressregs0x1000':
 541         case 'minopts':
 542         case 'forcerelocs':
 543         case 'jitstress1':
 544         case 'jitstress2':
 545         case 'jitstress2_jitstressregs1':
 546         case 'jitstress2_jitstressregs2':
 547         case 'jitstress2_jitstressregs3':
 548         case 'jitstress2_jitstressregs4':
 549         case 'jitstress2_jitstressregs8':
 550         case 'jitstress2_jitstressregs0x10':
 551         case 'jitstress2_jitstressregs0x80':
 552         case 'jitstress2_jitstressregs0x1000':
 553         case 'tailcallstress':
 554         case 'jitsse2only':
 555         case 'corefx_baseline':
 556         case 'corefx_minopts':
 557         case 'corefx_jitstress1':
 558         case 'corefx_jitstress2':
 559         case 'corefx_jitstressregs1':
 560         case 'corefx_jitstressregs2':
 561         case 'corefx_jitstressregs3':
 562         case 'corefx_jitstressregs4':
 563         case 'corefx_jitstressregs8':
 564         case 'corefx_jitstressregs0x10':
 565         case 'corefx_jitstressregs0x80':
 566         case 'corefx_jitstressregs0x1000':
 567         case 'zapdisable':
 568             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 569             assert (os == 'Windows_NT') || (os in Constants.crossList)
 570             Utilities.addPeriodicTrigger(job, '@daily')
 571         }
 572         break
 573         case 'heapverify1':
 574         case 'gcstress0x3':
 575             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 576                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 577                 if (architecture == 'arm64') {
 578                     assert (os == 'Windows_NT')
 579                     Utilities.addPeriodicTrigger(job, '@daily')
 580                     // TODO: Add once external email sending is available again
 581                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 582                 }
 583                 else {
 584                     Utilities.addPeriodicTrigger(job, '@weekly')
 585                 }
 586             }
 587             break
 588         case 'gcstress0xc':
 589         case 'gcstress0xc_zapdisable':
 590         case 'gcstress0xc_zapdisable_jitstress2':
 591         case 'gcstress0xc_zapdisable_heapverify1':
 592         case 'gcstress0xc_jitstress1':
 593         case 'gcstress0xc_jitstress2':
 594         case 'gcstress0xc_minopts_heapverify1':
 595             // GCStress=C is currently not supported on OS X
 596             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 597                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 598                 if (architecture == 'arm64') {
 599                     assert (os == 'Windows_NT')
 600                     // TODO: Enable a periodic trigger after tests are updated.
 601                     // Utilities.addPeriodicTrigger(job, '@daily')
 602                     // TODO: Add once external email sending is available again
 603                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 604                 }
 605                 else {
 606                     Utilities.addPeriodicTrigger(job, '@weekly')
 607                 }
 608             }
 609             break
 610
 611         case 'illink':
 612             // Testing on other operating systems TBD
 613             assert (os == 'Windows_NT')
 614             if (architecture == 'x64' || architecture == 'x86') {
 615                 Utilities.addPeriodicTrigger(job, '@daily')
 616             }
 617             break
 618
 619         default:
 620             println("Unknown scenario: ${scenario}");
 621             assert false
 622             break
 623     }
 624     return
 625 }
 626
 627 // **************************
 628 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 629 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 630 // builds will do a couple extra steps.
 631 // **************************
 632
 633 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 634 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 635 // it's just used for internal builds
 636 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 637 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 638     if (isWindowsBuildOnlyJob) {
 639         return
 640     }
 641
 642     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 643     // Non pull request builds.
 644     if (!isPR) {
 645         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 646         return
 647     }
 648     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 649     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 650     def osGroup = getOSGroup(os)
 651     switch (architecture) {
 652         case 'x64': // editor brace matching: {
 653             if (scenario == 'coverage') {
 654                 assert configuration == 'Release'
 655                 if (os == 'Ubuntu') {
 656                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 657                 }
 658                 break
 659             }
 660
 661             if (scenario == 'formatting') {
 662                 assert configuration == 'Checked'
 663                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 664                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 665                 }
 666                 break
 667             }
 668
 669             switch (os) {
 670                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 671                 case 'Debian8.4':
 672                 case 'RHEL7.2':
 673                     if (scenario == 'default') {
 674                         assert !isFlowJob
 675                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 676                     }
 677                     else if (scenario == 'pri1' && isFlowJob) {
 678                         assert (configuration == 'Release')
 679                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 680                     }
 681                     break
 682                 case 'Ubuntu16.04':
 683                     assert !isFlowJob
 684                     assert scenario == 'default'
 685                     // Distinguish with the other architectures (arm and x86)
 686                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 687                     break
 688                 case 'Fedora23':
 689                 case 'Ubuntu16.10':
 690                 case 'OpenSUSE42.1':
 691                     assert !isFlowJob
 692                     assert scenario == 'default'
 693                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 694                     break
 695                 case 'Ubuntu':
 696                 case 'OSX10.12':
 697                     // Triggers on the non-flow jobs aren't necessary here
 698                     // Corefx testing uses non-flow jobs.
 699                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 700                         break
 701                     }
 702                     switch (scenario) {
 703                         case 'default':
 704                             // Ubuntu uses checked for default PR tests
 705                             if (configuration == 'Checked') {
 706                                 // Default trigger
 707                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 708                             }
 709                             break
 710                         case 'pri1':
 711                             if (configuration == 'Release') {
 712                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 713                             }
 714                             break
 715                         case 'jitdiff':
 716                             if (configuration == 'Checked') {
 717                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 718                             }
 719                             break
 720                         case 'ilrt':
 721                             if (configuration == 'Release') {
 722                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 723                             }
 724                             break
 725                         case 'r2r':
 726                             if (configuration == 'Release' || configuration == 'Checked') {
 727                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 728                             }
 729                             break
 730                         case 'pri1r2r':
 731                             if (configuration == 'Release' || configuration == 'Checked') {
 732                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 733                             }
 734                             break
 735                         case 'gcstress15_pri1r2r':
 736                             if (configuration == 'Release' || configuration == 'Checked') {
 737                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 738                             }
 739                             break
 740                         case 'r2r_jitstress1':
 741                         case 'r2r_jitstress2':
 742                         case 'r2r_jitstressregs1':
 743                         case 'r2r_jitstressregs2':
 744                         case 'r2r_jitstressregs3':
 745                         case 'r2r_jitstressregs4':
 746                         case 'r2r_jitstressregs8':
 747                         case 'r2r_jitstressregs0x10':
 748                         case 'r2r_jitstressregs0x80':
 749                         case 'r2r_jitstressregs0x1000':
 750                         case 'r2r_jitminopts':
 751                         case 'r2r_jitforcerelocs':
 752                             if (configuration == 'Release' || configuration == 'Checked') {
 753                                 def displayStr = getR2RStressModeDisplayName(scenario)
 754                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 755                             }
 756                             break
 757                         case 'longgc':
 758                             if (configuration == 'Release') {
 759                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 760                             }
 761                             break
 762                         case 'gcsimulator':
 763                             if (configuration == 'Release') {
 764                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 765                             }
 766                             break
 767                         case 'minopts':
 768                         case 'forcerelocs':
 769                         case 'jitstress1':
 770                         case 'jitstress2':
 771                         case 'jitstressregs1':
 772                         case 'jitstressregs2':
 773                         case 'jitstressregs3':
 774                         case 'jitstressregs4':
 775                         case 'jitstressregs8':
 776                         case 'jitstressregs0x10':
 777                         case 'jitstressregs0x80':
 778                         case 'jitstressregs0x1000':
 779                         case 'jitstress2_jitstressregs1':
 780                         case 'jitstress2_jitstressregs2':
 781                         case 'jitstress2_jitstressregs3':
 782                         case 'jitstress2_jitstressregs4':
 783                         case 'jitstress2_jitstressregs8':
 784                         case 'jitstress2_jitstressregs0x10':
 785                         case 'jitstress2_jitstressregs0x80':
 786                         case 'jitstress2_jitstressregs0x1000':
 787                         case 'tailcallstress':
 788                         case 'jitsse2only':
 789                         case 'gcstress0x3':
 790                         case 'gcstress0xc':
 791                         case 'zapdisable':
 792                         case 'heapverify1':
 793                         case 'gcstress0xc_zapdisable':
 794                         case 'gcstress0xc_zapdisable_jitstress2':
 795                         case 'gcstress0xc_zapdisable_heapverify1':
 796                         case 'gcstress0xc_jitstress1':
 797                         case 'gcstress0xc_jitstress2':
 798                         case 'gcstress0xc_minopts_heapverify1':
 799                             def displayStr = getStressModeDisplayName(scenario)
 800                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 801                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 802                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 803                             break
 804                         case 'corefx_baseline':
 805                         case 'corefx_minopts':
 806                         case 'corefx_jitstress1':
 807                         case 'corefx_jitstress2':
 808                         case 'corefx_jitstressregs1':
 809                         case 'corefx_jitstressregs2':
 810                         case 'corefx_jitstressregs3':
 811                         case 'corefx_jitstressregs4':
 812                         case 'corefx_jitstressregs8':
 813                         case 'corefx_jitstressregs0x10':
 814                         case 'corefx_jitstressregs0x80':
 815                         case 'corefx_jitstressregs0x1000':
 816                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 817                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 818                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 819                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 820                             break
 821                         default:
 822                             println("Unknown scenario: ${scenario}");
 823                             assert false
 824                             break
 825                     }
 826                     break
 827                 case 'CentOS7.1':
 828                     switch (scenario) {
 829                         case 'pri1':
 830                             if (configuration == 'Release') {
 831                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 832                             }
 833                             break
 834                         case 'r2r':
 835                             if (configuration == 'Checked' || configuration == 'Release') {
 836                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 837                             }
 838                             break
 839                         case 'pri1r2r':
 840                             if (configuration == 'Checked' || configuration == 'Release') {
 841                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 842                             }
 843                             break
 844                         case 'gcstress15_pri1r2r':
 845                             if (configuration == 'Release' || configuration == 'Checked') {
 846                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 847                             }
 848                             break
 849                         case 'r2r_jitstress1':
 850                         case 'r2r_jitstress2':
 851                         case 'r2r_jitstressregs1':
 852                         case 'r2r_jitstressregs2':
 853                         case 'r2r_jitstressregs3':
 854                         case 'r2r_jitstressregs4':
 855                         case 'r2r_jitstressregs8':
 856                         case 'r2r_jitstressregs0x10':
 857                         case 'r2r_jitstressregs0x80':
 858                         case 'r2r_jitstressregs0x1000':
 859                         case 'r2r_jitminopts':
 860                         case 'r2r_jitforcerelocs':
 861                             if (configuration == 'Release' || configuration == 'Checked') {
 862                                 def displayStr = getR2RStressModeDisplayName(scenario)
 863                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 864                             }
 865                             break
 866                         default:
 867                             break
 868                     }
 869                 case 'Windows_NT':
 870                     switch (scenario) {
 871                         case 'default':
 872                             // Default trigger
 873                             if (configuration == 'Debug') {
 874                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 875                             }
 876                             break
 877                         case 'pri1':
 878                             // Default trigger
 879                             if (configuration == 'Release') {
 880                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 881                             }
 882                             break
 883                         case 'jitdiff':
 884                             if (configuration == 'Checked') {
 885                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 886                             }
 887                             break
 888                         case 'ilrt':
 889                             if (configuration == 'Release') {
 890                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 891                             }
 892                             break
 893                         case 'r2r':
 894                             if (configuration == 'Checked' || configuration == 'Release') {
 895                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 896                             }
 897                             break
 898                         case 'pri1r2r':
 899                             if (configuration == 'Checked' || configuration == 'Release') {
 900                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 901                             }
 902                             break
 903                         case 'gcstress15_pri1r2r':
 904                             if (configuration == 'Release' || configuration == 'Checked') {
 905                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 906                             }
 907                             break
 908                         case 'r2r_jitstress1':
 909                         case 'r2r_jitstress2':
 910                         case 'r2r_jitstressregs1':
 911                         case 'r2r_jitstressregs2':
 912                         case 'r2r_jitstressregs3':
 913                         case 'r2r_jitstressregs4':
 914                         case 'r2r_jitstressregs8':
 915                         case 'r2r_jitstressregs0x10':
 916                         case 'r2r_jitstressregs0x80':
 917                         case 'r2r_jitstressregs0x1000':
 918                         case 'r2r_jitminopts':
 919                         case 'r2r_jitforcerelocs':
 920                             if (configuration == 'Release' || configuration == 'Checked') {
 921                                 def displayStr = getR2RStressModeDisplayName(scenario)
 922                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 923                             }
 924                             break
 925                         case 'longgc':
 926                             if (configuration == 'Release') {
 927                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 928                             }
 929                             break
 930                         case 'gcsimulator':
 931                             if (configuration == 'Release') {
 932                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 933                             }
 934                             break
 935                         case 'standalone_gc':
 936                             if (configuration == 'Release' || configuration == 'Checked') {
 937                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 938                             }
 939                             break
 940                         case 'minopts':
 941                         case 'forcerelocs':
 942                         case 'jitstress1':
 943                         case 'jitstress2':
 944                         case 'jitstressregs1':
 945                         case 'jitstressregs2':
 946                         case 'jitstressregs3':
 947                         case 'jitstressregs4':
 948                         case 'jitstressregs8':
 949                         case 'jitstressregs0x10':
 950                         case 'jitstressregs0x80':
 951                         case 'jitstressregs0x1000':
 952                         case 'jitstress2_jitstressregs1':
 953                         case 'jitstress2_jitstressregs2':
 954                         case 'jitstress2_jitstressregs3':
 955                         case 'jitstress2_jitstressregs4':
 956                         case 'jitstress2_jitstressregs8':
 957                         case 'jitstress2_jitstressregs0x10':
 958                         case 'jitstress2_jitstressregs0x80':
 959                         case 'jitstress2_jitstressregs0x1000':
 960                         case 'tailcallstress':
 961                         case 'jitsse2only':
 962                         case 'gcstress0x3':
 963                         case 'gcstress0xc':
 964                         case 'zapdisable':
 965                         case 'heapverify1':
 966                         case 'gcstress0xc_zapdisable':
 967                         case 'gcstress0xc_zapdisable_jitstress2':
 968                         case 'gcstress0xc_zapdisable_heapverify1':
 969                         case 'gcstress0xc_jitstress1':
 970                         case 'gcstress0xc_jitstress2':
 971                         case 'gcstress0xc_minopts_heapverify1':
 972                             def displayStr = getStressModeDisplayName(scenario)
 973                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 974                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 975                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 976                             break
 977                         case 'corefx_baseline':
 978                         case 'corefx_minopts':
 979                         case 'corefx_jitstress1':
 980                         case 'corefx_jitstress2':
 981                         case 'corefx_jitstressregs1':
 982                         case 'corefx_jitstressregs2':
 983                         case 'corefx_jitstressregs3':
 984                         case 'corefx_jitstressregs4':
 985                         case 'corefx_jitstressregs8':
 986                         case 'corefx_jitstressregs0x10':
 987                         case 'corefx_jitstressregs0x80':
 988                         case 'corefx_jitstressregs0x1000':
 989                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 990                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 991                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 992                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 993                             break
 994                         case 'illink':
 995                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 996                                     break
 997                         default:
 998                             println("Unknown scenario: ${scenario}");
 999                             assert false
1000                             break
1001                     }
1002                     break
1003                 case 'FreeBSD':
1004                     assert scenario == 'default'
1005                     if (configuration == 'Checked') {
1006                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1007                     }
1008                     break
1009                 default:
1010                     println("Unknown os: ${os}");
1011                     assert false
1012                     break
1013             }
1014             break
1015         // editor brace matching: }
1016         case 'arm': // editor brace matching: {
1017             assert scenario == 'default'
1018             switch (os) {
1019                 case 'Ubuntu':
1020                 case 'Ubuntu16.04':
1021                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1022                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1023                     }
1024                     else {
1025                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1026                     }
1027                     break;
1028                 case 'Tizen':
1029                     architecture='armel'
1030                     // Removing the regex will cause this to run on each PR.
1031                     if (configuration == 'Release' || configuration == 'Debug') {
1032                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1033                     }
1034                     else {
1035                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1036                     }
1037                     break;
1038                 case 'Windows_NT':
1039                     if (configuration == 'Debug' || configuration == 'Release')
1040                     {
1041                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1042                     }
1043                     break
1044                 default:
1045                     println("NYI os: ${os}");
1046                     assert false
1047                     break
1048             }
1049             break
1050         // editor brace matching: }
1051         case 'arm64': // editor brace matching: {
1052             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1053
1054             // Set up a private trigger
1055             def contextString = "${os} ${architecture} Cross ${configuration}"
1056             if (scenario != 'default')
1057                 contextString += " ${scenario}"
1058             contextString += " Build"
1059             // Debug builds only.
1060             if (configuration != 'Debug') {
1061                contextString += " and Test"
1062             }
1063
1064             def arm64Users = [
1065                 'adiaaida',
1066                 'AndyAyersMS',
1067                 'briansull',
1068                 'BruceForstall',
1069                 'CarolEidt',
1070                 'cmckinsey',
1071                 'erozenfeld',
1072                 'jashook',
1073                 'JosephTremoulet',
1074                 'pgavlin',
1075                 'pkukol',
1076                 'russellhadley',
1077                 'RussKeldorph',
1078                 'sandreenko',
1079                 'sivarv',
1080                 'swaroop-sridhar',
1081                 'gkhanna79',
1082                 'jkotas',
1083                 'markwilkie',
1084                 'rahku',
1085                 'ramarag',
1086                 'tzwlai',
1087                 'weshaggard'
1088             ]
1089
1090             switch (os) {
1091                 case 'Windows_NT':
1092                     switch (scenario) {
1093                         case 'default':
1094                             // For now only run Debug jobs on PR Trigger.
1095                             if (configuration != 'Debug') {
1096                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1097                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1098                             }
1099                             else {
1100                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1101                                 // are at higher risk of ARM64-breaking changes.
1102                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1103                             }
1104                             break
1105                         case 'pri1r2r':
1106                         case 'gcstress0x3':
1107                         case 'gcstress0xc':
1108                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1109                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1110                             break
1111                     }
1112                     break
1113                 default:
1114                     println("NYI os: ${os}");
1115                     assert false
1116                     break
1117             }
1118             break
1119         // editor brace matching: }
1120         case 'x86': // editor brace matching: {
1121             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1122             if (os == 'Ubuntu') {
1123                 // on-demand only for ubuntu x86
1124                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1125                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1126                 break
1127             }
1128             switch (scenario) {
1129                 case 'default':
1130                     if (configuration == 'Checked') {
1131                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1132                     }
1133                     else if (configuration == 'Release') {
1134                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1135                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1136                     }
1137                     break
1138                 case 'pri1':
1139                     if (configuration == 'Release') {
1140                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1141                     }
1142                     break
1143                 case 'ilrt':
1144                     if (configuration == 'Release') {
1145                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1146                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1147                     }
1148                     break
1149                 case 'r2r':
1150                     if (configuration == 'Checked' || configuration == 'Release') {
1151                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1152                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1153                     }
1154                     break
1155                 case 'pri1r2r':
1156                     if (configuration == 'Checked' || configuration == 'Release') {
1157                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1158                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1159                     }
1160                     break
1161                 case 'gcstress15_pri1r2r':
1162                     if (configuration == 'Release' || configuration == 'Checked') {
1163                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1164                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1165                     }
1166                     break
1167                 case 'r2r_jitstress1':
1168                 case 'r2r_jitstress2':
1169                 case 'r2r_jitstressregs1':
1170                 case 'r2r_jitstressregs2':
1171                 case 'r2r_jitstressregs3':
1172                 case 'r2r_jitstressregs4':
1173                 case 'r2r_jitstressregs8':
1174                 case 'r2r_jitstressregs0x10':
1175                 case 'r2r_jitstressregs0x80':
1176                 case 'r2r_jitstressregs0x1000':
1177                 case 'r2r_jitminopts':
1178                 case 'r2r_jitforcerelocs':
1179                     if (configuration == 'Release' || configuration == 'Checked') {
1180                         def displayStr = getR2RStressModeDisplayName(scenario)
1181                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1182                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1183                     }
1184                     break
1185                 case 'longgc':
1186                     if (configuration == 'Release') {
1187                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1188                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1189                     }
1190                     break
1191                 case 'gcsimulator':
1192                     if (configuration == 'Release') {
1193                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1194                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1195                     }
1196                     break
1197                 case 'standalone_gc':
1198                     if (configuration == 'Release' || configuration == 'Checked') {
1199                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1200                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1201                     }
1202                     break
1203                 case 'minopts':
1204                 case 'forcerelocs':
1205                 case 'jitstress1':
1206                 case 'jitstress2':
1207                 case 'jitstressregs1':
1208                 case 'jitstressregs2':
1209                 case 'jitstressregs3':
1210                 case 'jitstressregs4':
1211                 case 'jitstressregs8':
1212                 case 'jitstressregs0x10':
1213                 case 'jitstressregs0x80':
1214                 case 'jitstressregs0x1000':
1215                 case 'jitstress2_jitstressregs1':
1216                 case 'jitstress2_jitstressregs2':
1217                 case 'jitstress2_jitstressregs3':
1218                 case 'jitstress2_jitstressregs4':
1219                 case 'jitstress2_jitstressregs8':
1220                 case 'jitstress2_jitstressregs0x10':
1221                 case 'jitstress2_jitstressregs0x80':
1222                 case 'jitstress2_jitstressregs0x1000':
1223                 case 'tailcallstress':
1224                 case 'jitsse2only':
1225                 case 'gcstress0x3':
1226                 case 'gcstress0xc':
1227                 case 'zapdisable':
1228                 case 'heapverify1':
1229                 case 'gcstress0xc_zapdisable':
1230                 case 'gcstress0xc_zapdisable_jitstress2':
1231                 case 'gcstress0xc_zapdisable_heapverify1':
1232                 case 'gcstress0xc_jitstress1':
1233                 case 'gcstress0xc_jitstress2':
1234                 case 'gcstress0xc_minopts_heapverify1':
1235                     def displayStr = getStressModeDisplayName(scenario)
1236                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1237                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1238                     break
1239                 case 'corefx_baseline':
1240                 case 'corefx_minopts':
1241                 case 'corefx_jitstress1':
1242                 case 'corefx_jitstress2':
1243                 case 'corefx_jitstressregs1':
1244                 case 'corefx_jitstressregs2':
1245                 case 'corefx_jitstressregs3':
1246                 case 'corefx_jitstressregs4':
1247                 case 'corefx_jitstressregs8':
1248                 case 'corefx_jitstressregs0x10':
1249                 case 'corefx_jitstressregs0x80':
1250                 case 'corefx_jitstressregs0x1000':
1251                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1252                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1253                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1254                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1255                     break
1256                 case 'illink':
1257                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1258                     break
1259                 default:
1260                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1261                     assert false
1262                     break
1263             }
1264             break
1265         // editor brace matching: }
1266         case 'x86compatjit': // editor brace matching: {
1267             assert (os == 'Windows_NT')
1268             def arch = 'x86'
1269             def jit = 'compatjit'
1270             switch (scenario) {
1271                 case 'default':
1272                     if (configuration == 'Checked') {
1273                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1274                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1275                     }
1276                     break
1277                 case 'pri1':
1278                     if (configuration == 'Release') {
1279                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Priority 1 Build and Test",
1280                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+Priority 1 Build and Test.*")
1281                     }
1282                     break
1283                 case 'ilrt':
1284                     if (configuration == 'Release') {
1285                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} IL RoundTrip Build and Test",
1286                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1287                     }
1288                     break
1289                 case 'r2r':
1290                     if (configuration == 'Checked' || configuration == 'Release') {
1291                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri0 Build & Test",
1292                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1293                     }
1294                     break
1295                 case 'pri1r2r':
1296                     if (configuration == 'Checked' || configuration == 'Release') {
1297                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} R2R pri1 Build & Test",
1298                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1299                     }
1300                     break
1301                 case 'gcstress15_pri1r2r':
1302                     if (configuration == 'Release' || configuration == 'Checked') {
1303                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GCStress 15 R2R pri1 Build & Test",
1304                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1305                     }
1306                     break
1307                 case 'r2r_jitstress1':
1308                 case 'r2r_jitstress2':
1309                 case 'r2r_jitstressregs1':
1310                 case 'r2r_jitstressregs2':
1311                 case 'r2r_jitstressregs3':
1312                 case 'r2r_jitstressregs4':
1313                 case 'r2r_jitstressregs8':
1314                 case 'r2r_jitstressregs0x10':
1315                 case 'r2r_jitstressregs0x80':
1316                 case 'r2r_jitstressregs0x1000':
1317                 case 'r2r_jitminopts':
1318                 case 'r2r_jitforcerelocs':
1319                     if (configuration == 'Release' || configuration == 'Checked') {
1320                         def displayStr = getR2RStressModeDisplayName(scenario)
1321                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} ${displayStr} R2R Build & Test",
1322                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1323                     }
1324                     break
1325                 case 'longgc':
1326                     if (configuration == 'Release') {
1327                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Long-Running GC Build & Test",
1328                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1329                     }
1330                     break
1331                 case 'gcsimulator':
1332                     if (configuration == 'Release') {
1333                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} GC Simulator",
1334                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1335                     }
1336                     break
1337                 case 'standalone_gc':
1338                     if (configuration == 'Release' || configuration == 'Checked') {
1339                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1340                     }
1341                     break
1342                 case 'minopts':
1343                 case 'forcerelocs':
1344                 case 'jitstress1':
1345                 case 'jitstress2':
1346                 case 'jitstressregs1':
1347                 case 'jitstressregs2':
1348                 case 'jitstressregs3':
1349                 case 'jitstressregs4':
1350                 case 'jitstressregs8':
1351                 case 'jitstressregs0x10':
1352                 case 'jitstressregs0x80':
1353                 case 'jitstressregs0x1000':
1354                 case 'jitstress2_jitstressregs1':
1355                 case 'jitstress2_jitstressregs2':
1356                 case 'jitstress2_jitstressregs3':
1357                 case 'jitstress2_jitstressregs4':
1358                 case 'jitstress2_jitstressregs8':
1359                 case 'jitstress2_jitstressregs0x10':
1360                 case 'jitstress2_jitstressregs0x80':
1361                 case 'jitstress2_jitstressregs0x1000':
1362                 case 'tailcallstress':
1363                 case 'jitsse2only':
1364                 case 'gcstress0x3':
1365                 case 'gcstress0xc':
1366                 case 'zapdisable':
1367                 case 'heapverify1':
1368                 case 'gcstress0xc_zapdisable':
1369                 case 'gcstress0xc_zapdisable_jitstress2':
1370                 case 'gcstress0xc_zapdisable_heapverify1':
1371                 case 'gcstress0xc_jitstress1':
1372                 case 'gcstress0xc_jitstress2':
1373                 case 'gcstress0xc_minopts_heapverify1':
1374                     def displayStr = getStressModeDisplayName(scenario)
1375                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - ${displayStr})",
1376                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1377                     break
1378                 case 'corefx_baseline':
1379                 case 'corefx_minopts':
1380                 case 'corefx_jitstress1':
1381                 case 'corefx_jitstress2':
1382                 case 'corefx_jitstressregs1':
1383                 case 'corefx_jitstressregs2':
1384                 case 'corefx_jitstressregs3':
1385                 case 'corefx_jitstressregs4':
1386                 case 'corefx_jitstressregs8':
1387                 case 'corefx_jitstressregs0x10':
1388                 case 'corefx_jitstressregs0x80':
1389                 case 'corefx_jitstressregs0x1000':
1390                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1391                     assert (os == 'Windows_NT')
1392                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test (Jit - ${displayName})",
1393                        "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+${scenario}.*")
1394                     break
1395                 case 'illink':
1396                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1397                     break
1398                 default:
1399                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1400                     assert false
1401                     break
1402             }
1403             break
1404         // editor brace matching: }
1405         default:
1406             println("Unknown architecture: ${architecture}");
1407             assert false
1408             break
1409     }
1410 }
1411
1412 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1413     def buildCommands = [];
1414     def osGroup = getOSGroup(os)
1415     def lowerConfiguration = configuration.toLowerCase()
1416
1417     // Calculate the build steps, archival, and xunit results
1418     switch (os) {
1419         case 'Windows_NT': // editor brace matching: {
1420             switch (architecture) {
1421                 case 'x64':
1422                 case 'x86':
1423                 case 'x86compatjit':
1424                     def arch = architecture
1425                     def buildOpts = ''
1426
1427                     // We need to explicitly run build-test.cmd with Exclude for x86compatjit, so skip tests.
1428                     if (architecture == 'x86compatjit') {
1429                         arch = 'x86'
1430                         buildOpts = 'compatjitcrossgen skiptests'
1431                     }
1432
1433                     def illinkArch = (architecture == 'x86compatjit') ? 'x86' : architecture
1434                     if (scenario == 'illink') {
1435                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${illinkArch}"
1436                     }
1437
1438                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1439                             scenario == 'default' ||
1440                             scenario == 'r2r' ||
1441                             scenario == 'jitdiff' ||
1442                             scenario == 'illink' ||
1443                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1444                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1445                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1446                     }
1447
1448                     // For Pri 1 tests, we must shorten the output test binary path names.
1449                     // if __TestIntermediateDir is already set, build-test.cmd will
1450                     // output test binaries to that directory. If it is not set, the
1451                     // binaries are sent to a default directory whose name is about
1452                     // 35 characters long.
1453
1454                     else if (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage') {
1455                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1456                     }
1457                     else if (scenario == 'ilrt') {
1458                         // First do the build with skiptests and then build the tests with ilasm roundtrip
1459                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1460                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch} -ilasmroundtrip"
1461                     }
1462                     else if (isLongGc(scenario)) {
1463                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1464                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1465                     }
1466                     else if (scenario == 'standalone_gc') {
1467                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1468                     }
1469                     else if (scenario == 'formatting') {
1470                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1471                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1472                         break
1473                     }
1474                     else {
1475                         println("Unknown scenario: ${scenario}")
1476                         assert false
1477                     }
1478
1479                     // If we are running a stress mode, we should write out the set of key
1480                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1481
1482                     if (!isBuildOnly) {
1483                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1484                         def crossgenStr = ''
1485                         def runcrossgentestsStr = ''
1486                         def runjitstressStr = ''
1487                         def runjitstressregsStr = ''
1488                         def runjitmioptsStr = ''
1489                         def runjitforcerelocsStr = ''
1490                         def runjitdisasmStr = ''
1491                         def gcstressStr = ''
1492                         def runtestArguments = ''
1493                         def gcTestArguments = ''
1494                         def illinkArguments = ''
1495
1496                         if (scenario == 'r2r' ||
1497                             scenario == 'pri1r2r' ||
1498                             scenario == 'gcstress15_pri1r2r' ||
1499                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1500                                 crossgenStr = 'crossgen'
1501                                 runcrossgentestsStr = 'runcrossgentests'
1502
1503                                 if (scenario == 'r2r_jitstress1'){
1504                                     runjitstressStr = 'jitstress 1'
1505                                 }
1506                                 else if (scenario == 'r2r_jitstress2') {
1507                                     runjitstressStr = 'jitstress 2'
1508                                 }
1509                                 else if (scenario == 'r2r_jitstressregs1'){
1510                                     runjitstressregsStr = 'jitstressregs 1'
1511                                 }
1512                                 else if (scenario == 'r2r_jitstressregs2') {
1513                                     runjitstressregsStr = 'jitstressregs 2'
1514                                 }
1515                                 else if (scenario == 'r2r_jitstressregs3') {
1516                                     runjitstressregsStr = 'jitstressregs 3'
1517                                 }
1518                                 else if (scenario == 'r2r_jitstressregs4') {
1519                                     runjitstressregsStr = 'jitstressregs 4'
1520                                 }
1521                                 else if (scenario == 'r2r_jitstressregs8') {
1522                                     runjitstressregsStr = 'jitstressregs 8'
1523                                 }
1524                                 else if (scenario == 'r2r_jitstressregs0x10') {
1525                                     runjitstressregsStr = 'jitstressregs 0x10'
1526                                 }
1527                                 else if (scenario == 'r2r_jitstressregs0x80') {
1528                                     runjitstressregsStr = 'jitstressregs 0x80'
1529                                 }
1530                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1531                                     runjitstressregsStr = 'jitstressregs 0x1000'
1532                                 }
1533                                 else if (scenario == 'r2r_jitminopts') {
1534                                     runjitmioptsStr = 'jitminopts'
1535                                 }
1536                                 else if (scenario == 'r2r_jitforcerelocs') {
1537                                     runjitforcerelocsStr = 'jitforcerelocs'
1538                                 }
1539                         }
1540                         if (scenario == 'gcstress15_pri1r2r')
1541                         {
1542                             gcstressStr = 'gcstresslevel 0xF'
1543                         }
1544
1545                         if (scenario == 'jitdiff')
1546                         {
1547                             runjitdisasmStr = 'jitdisasm crossgen'
1548                         }
1549
1550                         if (isLongGc(scenario)) {
1551                             gcTestArguments = "${scenario} sequential"
1552                         }
1553
1554                         if (scenario == 'illink')
1555                         {
1556                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Relase\\netcoreapp2.0\\win10-${illinkArch}\\publish\\illink.exe"
1557                         }
1558
1559                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1560
1561                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1562                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1563                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1564
1565                             if (enableCorefxTesting) {
1566                                 def workspaceRelativeFxRoot = "_/fx"
1567                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1568
1569                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1570
1571                                 setTestJobTimeOut(newJob, scenario)
1572
1573                                 // Archive and process (only) the test results
1574                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1575                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1576                             }
1577                             else {
1578                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1579                             }
1580                         }
1581                         else if (architecture == 'x64' || architecture == 'x86') {
1582                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1583                         }
1584                         else if (architecture == 'x86compatjit') {
1585                             def testEnvLocation = "%WORKSPACE%\\tests\\x86\\compatjit_x86_testenv.cmd"
1586                             buildCommands += "build-test.cmd ${runtestArguments}"
1587                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv ${testEnvLocation}"
1588                         }
1589                     }
1590
1591                     if (!enableCorefxTesting) {
1592                         // Run the rest of the build
1593                         // Build the mscorlib for the other OS's
1594                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1595                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1596                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1597
1598                         // Zip up the tests directory so that we don't use so much space/time copying
1599                         // 10s of thousands of files around.
1600                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1601
1602                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1603                             // For windows, pull full test results and test drops for x86/x64.
1604                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1605                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1606                         }
1607
1608                         if (scenario == 'jitdiff') {
1609                             // retrive jit-dasm output for base commit, and run jit-diff
1610                             if (!isBuildOnly) {
1611                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1612                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1613                             }
1614                         }
1615
1616                         if (!isBuildOnly) {
1617                             if (architecture == 'x64' || !isPR) {
1618                                 Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml')
1619                             }
1620                             setTestJobTimeOut(newJob, scenario)
1621                         }
1622                     }
1623                     break
1624                 case 'arm':
1625                     assert (scenario == 'default')
1626
1627                     // Set time out
1628                     setTestJobTimeOut(newJob, scenario)
1629
1630                     if ( lowerConfiguration == "debug" ) {
1631                         // For Debug builds, we will do a P1 test build
1632                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1633                     }
1634                     else {
1635                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1636                     }
1637                     // Add archival.
1638                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1639                     break
1640                 case 'arm64':
1641                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1642
1643                     // Set time out
1644                     setTestJobTimeOut(newJob, scenario)
1645
1646                     // Debug runs take too long to run. So build job only.
1647                     if (lowerConfiguration == "debug") {
1648                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1649                     }
1650                     else {
1651                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1652                        {
1653                            // Up the timeout for arm64 checked testing only.
1654                            // Keep the longer timeout for gcstress.
1655                            Utilities.setJobTimeout(newJob, 240)
1656                        }
1657
1658                        buildCommands += "set __TestIntermediateDir=int&&build.cmd skiptests ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1659                        // Test build and run are launched together.
1660                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1661                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1662                     }
1663
1664                     // Add archival.
1665                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1666                     break
1667                 default:
1668                     println("Unknown architecture: ${architecture}");
1669                     assert false
1670                     break
1671             }
1672             break
1673         // editor brace matching: }
1674         case 'Ubuntu':
1675         case 'Ubuntu16.04':
1676         case 'Ubuntu16.10':
1677         case 'Debian8.4':
1678         case 'OSX10.12':
1679         case 'FreeBSD':
1680         case 'CentOS7.1':
1681         case 'RHEL7.2':
1682         case 'OpenSUSE42.1':
1683         case 'Tizen':
1684         case 'Fedora23': // editor brace matching: {
1685             switch (architecture) {
1686                 case 'x64':
1687                 case 'x86':
1688                 case 'x86compatjit':
1689                     def arch = architecture
1690                     if (architecture == 'x86compatjit') {
1691                         arch = 'x86'
1692                     }
1693
1694                     if (architecture == 'x86' && os == 'Ubuntu') {
1695                         // build only, not test yet
1696                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1697                         break;
1698                     }
1699
1700                     if (scenario == 'formatting') {
1701                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${arch}"
1702                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1703                         break
1704                     }
1705
1706                     def standaloneGc = ''
1707                     if (scenario == 'standalone_gc') {
1708                         standaloneGc = 'buildstandalonegc'
1709                     }
1710
1711                     if (!enableCorefxTesting) {
1712                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1713                         // only on supported OS platforms.
1714                         if (os == 'FreeBSD')
1715                         {
1716                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${arch} ${standaloneGc}"
1717                         }
1718                         else
1719                         {
1720                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1721                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1722                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${arch} ${standaloneGc}"
1723                         }
1724                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${arch}.${configuration} \${WORKSPACE}/bin/paltestout"
1725
1726                         // Set time out
1727                         setTestJobTimeOut(newJob, scenario)
1728                         // Basic archiving of the build
1729                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1730                         // And pal tests
1731                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1732                     }
1733                     else {
1734                         // Corefx stress testing
1735                         assert os == 'Ubuntu'
1736                         assert architecture == 'x64'
1737                         assert lowerConfiguration == 'checked'
1738                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1739
1740                         // Build coreclr
1741                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1742
1743                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1744                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1745
1746                         // Build and text corefx
1747                         def workspaceRelativeFxRoot = "_/fx"
1748                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1749
1750                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1751
1752                         setTestJobTimeOut(newJob, scenario)
1753
1754                         // Archive and process (only) the test results
1755                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1756                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1757                     }
1758                     break
1759                 case 'arm64':
1760                     // We don't run the cross build except on Ubuntu
1761                     assert os == 'Ubuntu'
1762
1763                     buildCommands += """echo \"Using rootfs in /opt/aarch64-linux-gnu-root\"
1764                         ROOTFS_DIR=/opt/aarch64-linux-gnu-root ./build.sh skipmscorlib arm64 cross verbose ${lowerConfiguration}"""
1765
1766                     // Basic archiving of the build, no pal tests
1767                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1768                     break
1769                 case 'arm':
1770                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1771                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1772
1773                     // default values for Ubuntu
1774                     def arm_abi="arm"
1775                     def linuxCodeName="trusty"
1776                     if (os == 'Ubuntu16.04') {
1777                         linuxCodeName="xenial"
1778                     }
1779                     else if (os == 'Tizen') {
1780                         arm_abi="armel"
1781                         linuxCodeName="tizen"
1782                     }
1783
1784                     // Unzip the Windows test binaries first. Exit with 0
1785                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1786
1787                     // Unpack the corefx binaries
1788                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1789                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1790
1791                     // Call the ARM CI script to cross build and test using docker
1792                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1793                     --mode=docker \\
1794                     --${arm_abi} \\
1795                     --linuxCodeName=${linuxCodeName} \\
1796                     --buildConfig=${lowerConfiguration} \\
1797                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1798                     --coreFxBinDir=./bin/CoreFxBinDir \\
1799                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1800
1801                     // Basic archiving of the build, no pal tests
1802                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1803                     break
1804                 default:
1805                     println("Unknown architecture: ${architecture}");
1806                     assert false
1807                     break
1808             }
1809             break
1810         // editor brace matching: }
1811         default:
1812             println("Unknown os: ${os}");
1813             assert false
1814             break
1815     } // os
1816
1817     return buildCommands
1818 }
1819
1820 // Additional scenario which can alter behavior
1821
1822 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1823 combinedScenarios.each { scenario ->
1824     [true, false].each { isPR ->
1825         Constants.architectureList.each { architecture ->
1826             Constants.configurationList.each { configuration ->
1827                 Constants.osList.each { os ->
1828                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1829                     // and reset the os to Windows_NT
1830                     def isBuildOnly = false
1831                     if (os == 'Windows_NT_BuildOnly') {
1832                         isBuildOnly = true
1833                         os = 'Windows_NT'
1834                     }
1835
1836                     // WinArm32 is only built for Debug and Release
1837                     if (os == 'Windows_NT' && architecture == 'arm')
1838                     {
1839                         if (configuration == 'Checked')
1840                         {
1841                             return
1842                         }
1843                     }
1844                     // Tizen is only supported for arm architecture
1845                     if (os == 'Tizen' && architecture != 'arm') {
1846                         return
1847                     }
1848
1849                     // Skip totally unimplemented (in CI) configurations.
1850                     switch (architecture) {
1851                         case 'arm64':
1852                             // Windows only
1853                             if (os != 'Windows_NT' || isBuildOnly) {
1854                                 return
1855                             }
1856                             break
1857                         case 'arm':
1858                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1859                                 return
1860                             }
1861                             break
1862                         case 'x86':
1863                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1864                                 return
1865                             }
1866                             break
1867                         case 'x86compatjit':
1868                             // Skip non-windows
1869                             if (os != 'Windows_NT') {
1870                                 return
1871                             }
1872                             break
1873                         case 'x64':
1874                             // Everything implemented
1875                             break
1876                         default:
1877                             println("Unknown architecture: ${architecture}")
1878                             assert false
1879                             break
1880                     }
1881
1882                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1883                     // with checked builds
1884                     def enableCorefxTesting = false
1885                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1886                         if (configuration != 'Checked') {
1887                             return
1888                         }
1889
1890                         enableCorefxTesting = isCorefxTesting(scenario)
1891
1892                         // Since these are just execution time differences,
1893                         // skip platforms that don't execute the tests here (Windows_NT only)
1894                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1895                         if (!isEnabledOS || isBuildOnly) {
1896                             return
1897                         }
1898
1899                         switch (architecture) {
1900                             case 'arm64':
1901                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1902                                     return
1903                                 }
1904                                 break
1905                             case 'x64':
1906                             case 'x86':
1907                                 // x86 ubuntu: default only
1908                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1909                                     return
1910                                 }
1911                                 // Windows: Everything implemented
1912                                 break
1913                             case 'x86compatjit':
1914                                 // No stress modes for compatjit.dll.
1915                                 // (There's no technical reason we couldn't allow these.)
1916                                 return
1917                             default:
1918                                 return
1919                         }
1920                     }
1921                     else {
1922                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
1923                         // and configuration is not Checked, bail out.
1924                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1925                             return;
1926                         }
1927
1928                         // Skip scenarios
1929                         switch (scenario) {
1930                             case 'pri1':
1931                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1932                                 // the default scenario build
1933                                 if (os != 'Windows_NT') {
1934                                     return
1935                                 }
1936                                 // Only x64 for now
1937                                 if (architecture != 'x64') {
1938                                     return
1939                                 }
1940                                 break
1941                             case 'ilrt':
1942                                 // The ilrt build isn't necessary except for Windows_NT.  Non-Windows NT uses
1943                                 // the default scenario build
1944                                 if (os != 'Windows_NT') {
1945                                     return
1946                                 }
1947                                 // Only x64 for now
1948                                 if (architecture != 'x64') {
1949                                     return
1950                                 }
1951                                 // Release only
1952                                 if (configuration != 'Release') {
1953                                     return
1954                                 }
1955                                 break
1956                             case 'jitdiff':
1957                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1958                                     return
1959                                 }
1960                                 if (architecture != 'x64') {
1961                                     return
1962                                 }
1963                                 if (configuration != 'Checked') {
1964                                     return
1965                                 }
1966                                 break
1967                             case 'r2r':
1968                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1969                                 // the default scenario build
1970                                 if (os != 'Windows_NT') {
1971                                     return
1972                                 }
1973                                 if (architecture != 'x64') {
1974                                     return
1975                                 }
1976                                 break
1977                             case 'pri1r2r':
1978                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1979                                 // the default scenario build
1980                                 if (os != 'Windows_NT') {
1981                                     return
1982                                 }
1983                                 if (architecture != 'x64') {
1984                                     if (architecture != 'arm64' || configuration == 'Debug') {
1985                                         return
1986                                     }
1987                                 }
1988                                 break
1989                             case 'gcstress15_pri1r2r':
1990                             case 'r2r_jitstress1':
1991                             case 'r2r_jitstress2':
1992                             case 'r2r_jitstressregs1':
1993                             case 'r2r_jitstressregs2':
1994                             case 'r2r_jitstressregs3':
1995                             case 'r2r_jitstressregs4':
1996                             case 'r2r_jitstressregs8':
1997                             case 'r2r_jitstressregs0x10':
1998                             case 'r2r_jitstressregs0x80':
1999                             case 'r2r_jitstressregs0x1000':
2000                             case 'r2r_jitminopts':
2001                             case 'r2r_jitforcerelocs':
2002                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
2003                                 // the default scenario build
2004                                 if (os != 'Windows_NT') {
2005                                     return
2006                                 }
2007                                 if (architecture != 'x64') {
2008                                     return
2009                                 }
2010                                 break
2011                             case 'longgc':
2012                             case 'gcsimulator':
2013                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2014                                     return
2015                                 }
2016                                 if (architecture != 'x64') {
2017                                     return
2018                                 }
2019                                 if (configuration != 'Release') {
2020                                     return
2021                                 }
2022                                 break
2023                             case 'standalone_gc':
2024                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2025                                     return
2026                                 }
2027
2028                                 if (architecture != 'x64') {
2029                                     return
2030                                 }
2031
2032                                 if (configuration != 'Release' && configuration != 'Checked') {
2033                                     return
2034                                 }
2035                                 break
2036                             // We need Windows x64 Release bits for the code coverage build
2037                             case 'coverage':
2038                                 if (os != 'Windows_NT') {
2039                                     return
2040                                 }
2041                                 if (architecture != 'x64') {
2042                                     return
2043                                 }
2044                                 if (configuration != 'Release') {
2045                                     return
2046                                 }
2047                                 break
2048                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2049                             case 'formatting':
2050                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2051                                     return
2052                                 }
2053                                 if (architecture != 'x64') {
2054                                     return
2055                                 }
2056                                 if (configuration != 'Checked') {
2057                                     return
2058                                 }
2059                                 if (isBuildOnly) {
2060                                     return
2061                                 }
2062                                 break
2063                             case 'illink':
2064                                 if (os != 'Windows_NT') {
2065                                     return
2066                                 }
2067                                 if (architecture != 'x64' && architecture != 'x86' && architecture != 'x86compatjit') {
2068                                     return
2069                                 }
2070                                 if (isBuildOnly) {
2071                                     return
2072                                 }
2073                                 break
2074                             case 'default':
2075                                 // Nothing skipped
2076                                 break
2077                             default:
2078                                 println("Unknown scenario: ${scenario}")
2079                                 assert false
2080                                 break
2081                         }
2082                     }
2083
2084                     // Calculate names
2085                     def lowerConfiguration = configuration.toLowerCase()
2086                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2087                     def folderName = getJobFolder(scenario)
2088
2089                     // Create the new job
2090                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2091
2092                     setMachineAffinity(newJob, os, architecture)
2093
2094                     // Add all the standard options
2095                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2096                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2097
2098                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
2099                     def osGroup = getOSGroup(os)
2100
2101                     newJob.with {
2102                         steps {
2103                             if (os == 'Windows_NT') {
2104                                 buildCommands.each { buildCommand ->
2105                                     batchFile(buildCommand)
2106                                 }
2107                             }
2108                             else {
2109                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2110                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2111                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2112                                     // Define the Windows Tests and Corefx build job names
2113                                     def WindowTestsName = projectFolder + '/' +
2114                                                           Utilities.getFullJobName(project,
2115                                                                                    getJobName(lowerConfiguration,
2116                                                                                               'x64' ,
2117                                                                                               'windows_nt',
2118                                                                                               'default',
2119                                                                                               true),
2120                                                                                    false)
2121                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2122                                                        Utilities.getFolderName(branch)
2123
2124                                     // Copy the Windows test binaries and the Corefx build binaries
2125                                     copyArtifacts(WindowTestsName) {
2126                                         excludePatterns('**/testResults.xml', '**/*.ni.dll')
2127                                         buildSelector {
2128                                             latestSuccessful(true)
2129                                         }
2130                                     }
2131
2132                                     // Defaults for Ubuntu
2133                                     def arm_abi = 'arm'
2134                                     def corefx_os = 'ubuntu14.04'
2135                                     if (os == 'Ubuntu16.04') {
2136                                         arm_abi = 'arm'
2137                                         corefx_os = 'ubuntu16.04'
2138                                     }
2139                                     else if (os == 'Tizen') {
2140                                         arm_abi = 'armel'
2141                                         corefx_os = 'tizen'
2142                                     }
2143
2144                                     // Let's use release CoreFX to test checked CoreCLR,
2145                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2146                                     def corefx_lowerConfiguration = lowerConfiguration
2147                                     if ( lowerConfiguration == 'checked' ) {
2148                                         corefx_lowerConfiguration='release'
2149                                     }
2150                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2151                                         includePatterns('bin/build.tar.gz')
2152                                         buildSelector {
2153                                             latestSuccessful(true)
2154                                         }
2155                                     }
2156                                 }
2157
2158                                 buildCommands.each { buildCommand ->
2159                                     shell(buildCommand)
2160                                 }
2161                             }
2162                         }
2163                     } // newJob.with
2164
2165                 } // os
2166             } // configuration
2167         } // architecture
2168     } // isPR
2169 } // scenario
2170
2171
2172 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2173 combinedScenarios.each { scenario ->
2174     [true, false].each { isPR ->
2175         // Architectures.  x64 only at this point
2176         ['x64'].each { architecture ->
2177             // Put the OS's supported for coreclr cross testing here
2178             Constants.crossList.each { os ->
2179                 Constants.configurationList.each { configuration ->
2180
2181                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2182                         if (configuration != 'Checked') {
2183                             return
2184                         }
2185                         if (isCorefxTesting(scenario)) {
2186                             return
2187                         }
2188                         //Skip stress modes for these scenarios
2189                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2190                             return
2191                         }
2192                     }
2193                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2194                     // and configuration is not Checked, bail out.
2195                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2196                         return;
2197                     }
2198                     // For CentOS, we only want Checked/Release pri1 builds.
2199                     else if (os == 'CentOS7.1') {
2200                         if (scenario != 'pri1' &&
2201                             scenario != 'r2r' &&
2202                             scenario != 'pri1r2r' &&
2203                             scenario != 'gcstress15_pri1r2r' &&
2204                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2205                             return
2206                         }
2207                         if (configuration != 'Checked' && configuration != 'Release') {
2208                             return
2209                         }
2210                     }
2211                     // For RedHat and Debian, we only do Release pri1 builds.
2212                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2213                         if (scenario != 'pri1') {
2214                             return
2215                         }
2216                         if (configuration != 'Release') {
2217                             return
2218                         }
2219                     }
2220                     else {
2221                         // Skip scenarios
2222                         switch (scenario) {
2223                             case 'pri1':
2224                                 // Nothing skipped
2225                                 break
2226                             case 'ilrt':
2227                                 // Release only
2228                                 if (configuration != 'Release') {
2229                                     return
2230                                 }
2231                                 break
2232                             case 'jitdiff':
2233                                 if (configuration != 'Checked') {
2234                                     return;
2235                                 }
2236                             case 'r2r':
2237                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2238                                 if (configuration != 'Checked' && configuration != 'Release') {
2239                                     return
2240                                 }
2241                                 break
2242                             case 'pri1r2r':
2243                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2244                                 if (configuration != 'Checked' && configuration != 'Release') {
2245                                     return
2246                                 }
2247                                 break
2248                             case 'gcstress15_pri1r2r':
2249                             case 'r2r_jitstress1':
2250                             case 'r2r_jitstress2':
2251                             case 'r2r_jitstressregs1':
2252                             case 'r2r_jitstressregs2':
2253                             case 'r2r_jitstressregs3':
2254                             case 'r2r_jitstressregs4':
2255                             case 'r2r_jitstressregs8':
2256                             case 'r2r_jitstressregs0x10':
2257                             case 'r2r_jitstressregs0x80':
2258                             case 'r2r_jitstressregs0x1000':
2259                             case 'r2r_jitminopts':
2260                             case 'r2r_jitforcerelocs':
2261                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2262                                 if (configuration != 'Checked' && configuration != 'Release') {
2263                                     return
2264                                 }
2265                                 break
2266                             case 'longgc':
2267                             case 'gcsimulator':
2268                                 // Long GC tests take a long time on non-Release builds
2269                                 if (configuration != 'Release') {
2270                                     return
2271                                 }
2272                                 break
2273                             case 'standalone_gc':
2274                                 if (configuration != 'Release' && configuration != 'Checked') {
2275                                     return
2276                                 }
2277                             case 'coverage':
2278                                 //We only want Ubuntu Release for coverage
2279                                 if (os != 'Ubuntu') {
2280                                     return
2281                                 }
2282                                 if (configuration != 'Release') {
2283                                     return
2284                                 }
2285                             case 'formatting':
2286                                 return
2287                             case 'illink':
2288                                 if (os != 'Windows_NT') {
2289                                     return
2290                                 }
2291                                 break
2292                             case 'default':
2293                                 // Nothing skipped
2294                                 break
2295                             default:
2296                                 println("Unknown scenario: ${scenario}")
2297                                 assert false
2298                                 break
2299                         }
2300                     }
2301
2302                     def lowerConfiguration = configuration.toLowerCase()
2303                     def osGroup = getOSGroup(os)
2304                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2305
2306                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2307                     def inputScenario = 'default'
2308                     if (scenario == 'coverage') {
2309                         inputScenario = 'coverage'
2310                     }
2311                     def inputCoreCLRBuildName = projectFolder + '/' +
2312                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2313                     // If this is a stress scenario, there isn't any difference in the build job
2314                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2315                     // from the default scenario
2316                     def testBuildScenario = scenario
2317                     if (testBuildScenario == 'coverage' || testBuildScenario == 'pri1r2r'|| testBuildScenario == 'gcstress15_pri1r2r') {
2318                         testBuildScenario = 'pri1'
2319                     }
2320                     else if ( testBuildScenario == 'r2r' || isLongGc(testBuildScenario)) {
2321                         testBuildScenario = 'default'
2322                     }
2323                     def inputWindowTestsBuildName = ''
2324                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2325                         inputWindowTestsBuildName = projectFolder + '/' +
2326                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', 'default', true), isPR)
2327                     }
2328                     else {
2329                         inputWindowTestsBuildName = projectFolder + '/' +
2330                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', testBuildScenario, true), isPR)
2331                     }
2332                     // Enable Server GC for Ubuntu PR builds
2333                     def serverGCString = ''
2334
2335                     // Whether or not this test run should be run sequentially instead
2336                     // of in parallel. Only used for long GC tests.
2337                     def sequentialString = ''
2338
2339                     // Whether or not this test run should run a specific playlist.
2340                     // Only used for long GC tests.
2341
2342                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2343                     // for running long GC and GCSimulator tests, respectively. We don't use them
2344                     // here because using a playlist file produces much more readable output on the CI machines
2345                     // and reduces running time.
2346                     def playlistString = ''
2347
2348                     if (os == 'Ubuntu' && isPR){
2349                         serverGCString = '--useServerGC'
2350                     }
2351
2352                     // pass --crossgen to runtest.sh for crossgen builds
2353                     def crossgenStr = ''
2354                     def runcrossgentestsStr = ''
2355                     def runjitstressStr = ''
2356                     def runjitstressregsStr = ''
2357                     def runjitmioptsStr = ''
2358                     def runjitforcerelocsStr = ''
2359                     def runjitdisasmStr = ''
2360                     def gcstressStr = ''
2361
2362                     if (scenario == 'r2r' ||
2363                         scenario == 'pri1r2r' ||
2364                         scenario == 'gcstress15_pri1r2r' ||
2365                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2366                             crossgenStr = '--crossgen'
2367                             runcrossgentestsStr = '--runcrossgentests'
2368
2369                             if (scenario == 'r2r_jitstress1'){
2370                                 runjitstressStr = '--jitstress=1'
2371                             }
2372                             else if (scenario == 'r2r_jitstress2') {
2373                                 runjitstressStr = '--jitstress=2'
2374                             }
2375                             else if (scenario == 'r2r_jitstressregs1'){
2376                                 runjitstressregsStr = '--jitstressregs=1'
2377                             }
2378                             else if (scenario == 'r2r_jitstressregs2') {
2379                                 runjitstressregsStr = '--jitstressregs=2'
2380                             }
2381                             else if (scenario == 'r2r_jitstressregs3') {
2382                                 runjitstressregsStr = '--jitstressregs=3'
2383                             }
2384                             else if (scenario == 'r2r_jitstressregs4') {
2385                                 runjitstressregsStr = '--jitstressregs=4'
2386                             }
2387                             else if (scenario == 'r2r_jitstressregs8') {
2388                                 runjitstressregsStr = '--jitstressregs=8'
2389                             }
2390                             else if (scenario == 'r2r_jitstressregs0x10') {
2391                                 runjitstressregsStr = '--jitstressregs=0x10'
2392                             }
2393                             else if (scenario == 'r2r_jitstressregs0x80') {
2394                                 runjitstressregsStr = '--jitstressregs=0x80'
2395                             }
2396                             else if (scenario == 'r2r_jitstressregs0x1000') {
2397                                 runjitstressregsStr = '--jitstressregs=0x1000'
2398                             }
2399                             else if (scenario == 'r2r_jitminopts') {
2400                                 runjitmioptsStr = '--jitminopts'
2401                             }
2402                             else if (scenario == 'r2r_jitforcerelocs') {
2403                                 runjitforcerelocsStr = '--jitforcerelocs'
2404                             }
2405                     }
2406                     if  (scenario == 'gcstress15_pri1r2r')
2407                     {
2408                         gcstressStr = '--gcstresslevel=0xF'
2409                     }
2410
2411                     if (scenario == 'jitdiff')
2412                     {
2413                         runjitdisasmStr = '--jitdisasm --crossgen'
2414                     }
2415
2416                     if (isLongGc(scenario)) {
2417                         // Long GC tests behave very poorly when they are not
2418                         // the only test running (many of them allocate until OOM).
2419                         sequentialString = '--sequential'
2420
2421                         // The Long GC playlist contains all of the tests that are
2422                         // going to be run. The GCSimulator playlist contains all of
2423                         // the GC simulator tests.
2424                         if (scenario == 'longgc') {
2425                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2426                         }
2427                         else if (scenario == 'gcsimulator') {
2428                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2429                         }
2430                     }
2431
2432                     def folder = getJobFolder(scenario)
2433                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2434                         // Add parameters for the inputs
2435
2436                         parameters {
2437                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2438                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2439                         }
2440
2441                         steps {
2442                             // Set up the copies
2443
2444                             // Coreclr build containing the tests and mscorlib
2445
2446                             copyArtifacts(inputWindowTestsBuildName) {
2447                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2448                                 buildSelector {
2449                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2450                                 }
2451                             }
2452
2453                             if (scenario == 'coverage') {
2454
2455                                 // Move coreclr to clr directory
2456                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2457
2458                                 // Build coreclr
2459                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2460
2461                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2462                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2463                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2464                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2465                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2466                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2467                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2468                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2469
2470                                 // Run PAL tests
2471                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2472
2473                                 // Remove obj files for PAL tests so they're not included in coverage results
2474                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2475
2476                                 // Unzip the tests first.  Exit with 0
2477                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2478
2479                                 // Get corefx
2480                                 shell("git clone https://github.com/dotnet/corefx fx")
2481
2482                                 // Build Linux corefx
2483                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2484                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2485
2486                                 def testEnvOpt = ""
2487                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2488                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2489                                 shell("${createScriptCmds}")
2490                                 testEnvOpt = "--test-env=" + scriptFileName
2491
2492                                 // Run corefx tests
2493                                 shell("""./fx/run-test.sh \\
2494                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2495                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2496                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2497                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2498                 --configurationGroup Release""")
2499
2500
2501                                 // Run coreclr tests w/ workstation GC
2502                                 shell("""./clr/tests/runtest.sh \\
2503                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2504                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2505                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2506                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2507                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2508                 --crossgen --runcrossgentests""")
2509
2510                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2511                                 shell("""./clr/tests/runtest.sh \\
2512                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2513                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2514                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2515                 --useServerGC ${testEnvOpt}""")
2516
2517                                  // Run long-running coreclr GC tests & produce coverage reports
2518                                 shell("""./clr/tests/runtest.sh \\
2519                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2520                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2521                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2522                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2523                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2524                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2525                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2526
2527                             }
2528                             else {
2529
2530                                 // Coreclr build we are trying to test
2531
2532                                 copyArtifacts(inputCoreCLRBuildName) {
2533                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2534                                     buildSelector {
2535                                         buildNumber('${CORECLR_BUILD}')
2536                                     }
2537                                 }
2538
2539                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2540
2541                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2542                                 def osJobName
2543                                 if (os == 'Ubuntu') {
2544                                     osJobName = 'ubuntu14.04'
2545                                 }
2546                                 else {
2547                                     osJobName = os.toLowerCase()
2548                                 }
2549                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2550                                     includePatterns('bin/build.tar.gz')
2551                                     buildSelector {
2552                                         latestSuccessful(true)
2553                                     }
2554                                 }
2555
2556                                 shell ("mkdir ./bin/CoreFxBinDir")
2557                                 // Unpack the corefx binaries
2558                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2559
2560                                 // Unzip the tests first.  Exit with 0
2561                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2562
2563                                 // Execute the tests
2564                                 // If we are running a stress mode, we'll set those variables first
2565                                 def testEnvOpt = ""
2566                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2567                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2568                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2569                                     shell("${createScriptCmds}")
2570                                     testEnvOpt = "--test-env=" + scriptFileName
2571                                 }
2572
2573                                 if (isGCStressRelatedTesting(scenario)) {
2574                                     shell('./init-tools.sh')
2575                                 }
2576
2577                                 shell("""./tests/runtest.sh \\
2578                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2579                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2580                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2581                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2582                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2583                 --limitedDumpGeneration \\
2584                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${sequentialString} ${playlistString}""")
2585                             }
2586                         }
2587                     }
2588
2589                     if (scenario == 'coverage') {
2590                         // Publish coverage reports
2591                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2592                         // TODO: Add once external email sending is available again
2593                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2594                     }
2595
2596                     if (scenario == 'jitdiff') {
2597                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2598                     }
2599
2600                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2601                     if (os in ['Ubuntu']) {
2602                         SummaryBuilder summaries = new SummaryBuilder()
2603                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2604                         summaries.emit(newJob)
2605                     }
2606
2607                     setMachineAffinity(newJob, os, architecture)
2608                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2609                     // Set timeouts to 240.
2610                     setTestJobTimeOut(newJob, scenario)
2611                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2612
2613                     // Create a build flow to join together the build and tests required to run this
2614                     // test.
2615                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2616                     // Linux CoreCLR test
2617                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2618                     def fullTestJobName = projectFolder + '/' + newJob.name
2619                     // Add a reference to the input jobs for report purposes
2620                     JobReport.Report.addReference(inputCoreCLRBuildName)
2621                     JobReport.Report.addReference(inputWindowTestsBuildName)
2622                     JobReport.Report.addReference(fullTestJobName)
2623                     def newFlowJob;
2624
2625                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2626                     // so that coverage data can be preserved.
2627                     if (scenario == 'coverage') {
2628                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2629                         buildFlow("""
2630 // Build the input Windows job
2631 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2632
2633 // And then build the test build
2634 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2635 """)
2636                         }
2637                     // Normal jobs copy a Windows build & a non-Windows build
2638                     } else {
2639                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2640                         buildFlow("""
2641 // Build the input jobs in parallel
2642 parallel (
2643     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2644     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2645 )
2646
2647 // And then build the test build
2648 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2649                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2650 """)
2651                         }
2652                     }
2653
2654                     setMachineAffinity(newFlowJob, os, architecture)
2655                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2656                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2657                 } // configuration
2658             } // os
2659         } // architecture
2660     } // isPR
2661 } // scenario
2662
2663 JobReport.Report.generateJobReport(out)
2664
2665 // Make the call to generate the help job
2666 Utilities.createHelperJob(this, project, branch,
2667     "Welcome to the ${project} Repository",  // This is prepended to the help message
2668     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.