netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'FreeBSD':'FreeBSD',
  29         'CentOS7.1': 'Linux',
  30         'OpenSUSE42.1': 'Linux',
  31         'Tizen': 'Linux']
  32     def osGroup = osGroupMap.get(os, null)
  33     assert osGroup != null : "Could not find os group for ${os}"
  34     return osGroupMap[os]
  35 }
  36
  37 // We use this class (vs variables) so that the static functions can access data here.
  38 class Constants {
  39
  40     // Innerloop build OS's
  41     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  42     // test execution in the build flow runs.  It generates the exact same build
  43     // as Windows_NT but without the tests.
  44     def static osList = [
  45                'Ubuntu',
  46                'Debian8.4',
  47                'OSX10.12',
  48                'Windows_NT',
  49                'Windows_NT_BuildOnly',
  50                'FreeBSD',
  51                'CentOS7.1',
  52                'OpenSUSE42.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  67                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  68                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  69                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  70                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  71                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  72                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  73                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  74                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  75                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  76                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  77                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  78                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  79                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  80                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  81                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  82                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  83                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  84                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  85                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  86                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  87                'corefx_baseline'                : [ : ], // corefx baseline
  88                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  89                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  90                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  91                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  92                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  93                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  94                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  95                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  96                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  97                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  98                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  99                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 100                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 101                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 103                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 104                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 105                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 107                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 109                ]
 110
 111     // This is a set of r2r jit stress scenarios
 112     def static r2rJitStressScenarios = [
 113                'r2r_jitstress1',
 114                'r2r_jitstress2',
 115                'r2r_jitstressregs1',
 116                'r2r_jitstressregs2',
 117                'r2r_jitstressregs3',
 118                'r2r_jitstressregs4',
 119                'r2r_jitstressregs8',
 120                'r2r_jitstressregs0x10',
 121                'r2r_jitstressregs0x80',
 122                'r2r_jitstressregs0x1000',
 123                'r2r_jitminopts',
 124                'r2r_jitforcerelocs']
 125
 126     // This is the basic set of scenarios
 127     def static basicScenarios = [
 128                'default',
 129                'pri1',
 130                'ilrt',
 131                'r2r',
 132                'pri1r2r',
 133                'gcstress15_pri1r2r',
 134                'longgc',
 135                'coverage',
 136                'formatting',
 137                'gcsimulator',
 138                'jitdiff',
 139                'standalone_gc',
 140                'gc_reliability_framework',
 141                'illink'] + r2rJitStressScenarios
 142
 143     def static configurationList = ['Debug', 'Checked', 'Release']
 144
 145     // This is the set of architectures
 146     def static architectureList = ['arm', 'arm64', 'x64', 'x86']
 147 }
 148
 149 def static setMachineAffinity(def job, def os, def architecture) {
 150     if (architecture == 'arm64' && os == 'Windows_NT') {
 151         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 152     } else if (architecture == 'arm64' && os == 'Ubuntu') {
 153         Utilities.setMachineAffinity(job, os, 'arm-cross-latest');
 154     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 155         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 156     } else {
 157         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 158     }
 159 }
 160
 161 def static isJITStressJob(def scenario) {
 162     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 163            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 164 }
 165
 166 def static isGCStressRelatedTesting(def scenario) {
 167     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 168     // Detect it and make it a GCStress related.
 169     if (scenario == 'gcstress15_pri1r2r')
 170     {
 171         return true;
 172     }
 173
 174     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 175     def scenarioName = scenario.toLowerCase()
 176     def isGCStressTesting = false
 177     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 178         if (k in gcStressTestEnvVars) {
 179             isGCStressTesting = true;
 180         }
 181     }
 182     return isGCStressTesting
 183 }
 184
 185 def static isCorefxTesting(def scenario) {
 186     def corefx_prefix = 'corefx_'
 187     if (scenario.length() < corefx_prefix.length()) {
 188         return false
 189     }
 190     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 191 }
 192
 193 def static isR2R(def scenario) {
 194     return (scenario == 'r2r' || scenario == 'pri1r2r')
 195 }
 196
 197 def static isCoverage(def scenario) {
 198     return (scenario == 'coverage')
 199 }
 200
 201 def static isLongGc(def scenario) {
 202     return (scenario == 'longgc' || scenario == 'gcsimulator')
 203 }
 204
 205 def static isJitDiff(def scenario) {
 206     return (scenario == 'jitdiff')
 207 }
 208
 209 def static isGcReliabilityFramework(def scenario) {
 210     return (scenario == 'gc_reliability_framework')
 211 }
 212
 213 def static scenarioNeedsPri1Build(def scenario) {
 214     return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario))
 215 }
 216
 217 def static setTestJobTimeOut(newJob, scenario) {
 218     if (isGCStressRelatedTesting(scenario)) {
 219         Utilities.setJobTimeout(newJob, 4320)
 220     }
 221     else if (isCorefxTesting(scenario)) {
 222         Utilities.setJobTimeout(newJob, 360)
 223     }
 224     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 225         Utilities.setJobTimeout(newJob, 240)
 226     }
 227     else if (isR2R(scenario)) {
 228         Utilities.setJobTimeout(newJob, 240)
 229     }
 230     else if (isCoverage(scenario)) {
 231         Utilities.setJobTimeout(newJob, 1440)
 232     }
 233     else if (isLongGc(scenario)) {
 234         Utilities.setJobTimeout(newJob, 1440)
 235     }
 236     else if (isJitDiff(scenario)) {
 237         Utilities.setJobTimeout(newJob, 240)
 238     }
 239     else if (isGcReliabilityFramework(scenario)) {
 240         Utilities.setJobTimeout(newJob, 1440)
 241     }
 242     // Non-test jobs use the default timeout value.
 243 }
 244
 245 def static getJobFolder(def scenario) {
 246     if (isJITStressJob(scenario)) {
 247         return 'jitstress'
 248     }
 249     if (scenario == 'illink') {
 250         return 'illink'
 251     }
 252     return ''
 253 }
 254
 255 def static getStressModeDisplayName(def scenario) {
 256     def displayStr = ''
 257     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 258         def prefixLength = 'COMPlus_'.length()
 259         if (k.length() >= prefixLength) {
 260             def modeName = k.substring(prefixLength, k.length())
 261             displayStr += ' ' + modeName + '=' + v
 262         }
 263     }
 264     return displayStr
 265 }
 266
 267 def static getR2RStressModeDisplayName(def scenario) {
 268     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 269     // "r2r_" prefix.
 270     def displayStr = scenario
 271     def prefixLength = 'r2r_'.length()
 272     if (displayStr.length() >= prefixLength) {
 273         displayStr = displayStr.substring(prefixLength, displayStr.length())
 274     }
 275     return displayStr
 276 }
 277
 278 // Generates the string for creating a file that sets environment variables
 279 // that makes it possible to run stress modes.  Writes the script to the file
 280 // specified by the stepScriptLocation parameter.
 281 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 282     def stepScript = ''
 283     if (os == 'Windows_NT') {
 284         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 285         stepScript += "del ${stepScriptLocation}\r\n"
 286
 287         // Timeout in ms, default is 10 minutes. For stress
 288         // modes up this to 30 minutes
 289         def timeout = 1800000
 290
 291         // Set the Timeout
 292         stepScript += "set __TestTimeout=${timeout}\r\n"
 293         stepScript += "echo. > ${stepScriptLocation}\r\n"
 294         stressModeVars.each{ k, v ->
 295             // Write out what we are writing to the script file
 296             stepScript += "echo Setting ${k}=${v}\r\n"
 297             // Write out the set itself to the script file`
 298             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 299         }
 300     }
 301     else {
 302         stepScript += "echo Setting variables for ${stressModeName}\n"
 303         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 304         stressModeVars.each{ k, v ->
 305             // Write out what we are writing to the script file
 306             stepScript += "echo Setting ${k}=${v}\n"
 307             // Write out the set itself to the script file`
 308             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 309         }
 310         stepScript += "chmod +x ${stepScriptLocation}\n"
 311     }
 312     return stepScript
 313 }
 314
 315 // Calculates the name of the build job based on some typical parameters.
 316 //
 317 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 318     // If the architecture is x64, do not add that info into the build name.
 319     // Need to change around some systems and other builds to pick up the right builds
 320     // to do that.
 321
 322     def suffix = scenario != 'default' ? "_${scenario}" : '';
 323     if (isBuildOnly) {
 324         suffix += '_bld'
 325     }
 326     def baseName = ''
 327     switch (architecture) {
 328         case 'x64':
 329             if (scenario == 'default') {
 330                 // For now we leave x64 off of the name for compatibility with other jobs
 331                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 332             }
 333             else if (scenario == 'formatting') {
 334                 // we don't care about the configuration for the formatting job. It runs all configs
 335                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 336             }
 337             else {
 338                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 339             }
 340             break
 341         case 'arm64':
 342             // These are cross builds
 343             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 344             break
 345         case 'arm':
 346             // These are cross builds
 347             if (os == 'Tizen') {
 348                 // ABI: softfp
 349                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 350             }
 351             else {
 352                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 353             }
 354             break
 355         case 'x86':
 356             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 357             break
 358         default:
 359             println("Unknown architecture: ${architecture}");
 360             assert false
 361             break
 362     }
 363
 364     return baseName + suffix
 365 }
 366
 367 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 368     // Check scenario.
 369     switch (scenario) {
 370         case 'default':
 371             switch (architecture) {
 372                 case 'x64':
 373                 case 'x86':
 374                     if (architecture == 'x86' && os == 'Ubuntu') {
 375                         Utilities.addPeriodicTrigger(job, '@daily')
 376                     }
 377                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 378                         Utilities.addGithubPushTrigger(job)
 379                     }
 380                     break
 381                 case 'arm':
 382                     Utilities.addGithubPushTrigger(job)
 383                     break
 384                 case 'arm64':
 385                     if (os == 'Windows_NT') {
 386                         Utilities.addGithubPushTrigger(job)
 387                         // TODO: Add once external email sending is available again
 388                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 389                     }
 390                     break
 391                 default:
 392                     println("Unknown architecture: ${architecture}");
 393                     assert false
 394                     break
 395             }
 396             break
 397         case 'pri1':
 398             // Pri one gets a push trigger, and only for release
 399             if (architecture == 'x64') {
 400                 if (configuration == 'Release') {
 401                     // We expect release jobs to be Windows, or in the cross list
 402                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 403                     if (!os in bidailyCrossList) {
 404                         if (isFlowJob || os == 'Windows_NT') {
 405                             Utilities.addGithubPushTrigger(job)
 406                         }
 407                     }
 408                     else {
 409                         if (isFlowJob) {
 410                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 411                         }
 412                     }
 413                 }
 414             }
 415             break
 416         case 'r2r':
 417             //r2r jobs that aren't pri1 can only be triggered by phrase
 418             break
 419         case 'pri1r2r':
 420             assert !(os in bidailyCrossList)
 421             //pri1 r2r gets a push trigger for checked/release
 422             if (configuration == 'Checked' || configuration == 'Release') {
 423                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 424                 if (architecture == 'x64' && os != 'OSX10.12') {
 425                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 426                     if (isFlowJob || os == 'Windows_NT') {
 427                         Utilities.addGithubPushTrigger(job)
 428                     }
 429                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 430                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 431                     if (isFlowJob) {
 432                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 433                     }
 434                 }
 435                 // For x86, only add per-commit jobs for Windows
 436                 else if (architecture == 'x86') {
 437                     if (os == 'Windows_NT') {
 438                         Utilities.addGithubPushTrigger(job)
 439                     }
 440                 }
 441                 // arm64 pri1r2r jobs should only run every 12 hours.
 442                 else if (architecture == 'arm64') {
 443                     if (os == 'Windows_NT') {
 444                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 445                         // TODO: Add once external email sending is available again
 446                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 447                     }
 448                 }
 449             }
 450             break
 451         case 'r2r_jitstress1':
 452         case 'r2r_jitstress2':
 453         case 'r2r_jitstressregs1':
 454         case 'r2r_jitstressregs2':
 455         case 'r2r_jitstressregs3':
 456         case 'r2r_jitstressregs4':
 457         case 'r2r_jitstressregs8':
 458         case 'r2r_jitstressregs0x10':
 459         case 'r2r_jitstressregs0x80':
 460         case 'r2r_jitstressregs0x1000':
 461         case 'r2r_jitminopts':
 462         case 'r2r_jitforcerelocs':
 463         case 'gcstress15_pri1r2r':
 464             assert !(os in bidailyCrossList)
 465
 466             // GCStress=C is currently not supported on OS X
 467             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 468                 break
 469             }
 470
 471             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 472             if (configuration == 'Checked' || configuration == 'Release') {
 473                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 474                 if (architecture == 'x64') {
 475                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 476                     if (isFlowJob || os == 'Windows_NT') {
 477                         // Add a weekly periodic trigger
 478                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 479                     }
 480                 }
 481                 // For x86, only add per-commit jobs for Windows
 482                 else if (architecture == 'x86') {
 483                     if (os == 'Windows_NT') {
 484                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 485                     }
 486                 }
 487             }
 488             break
 489         case 'longgc':
 490             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 491             assert configuration == 'Release'
 492             assert architecture == 'x64'
 493             Utilities.addPeriodicTrigger(job, '@daily')
 494             // TODO: Add once external email sending is available again
 495             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 496             break
 497         case 'gcsimulator':
 498             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 499             assert configuration == 'Release'
 500             assert architecture == 'x64'
 501             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 502             // TODO: Add once external email sending is available again
 503             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 504             break
 505         case 'standalone_gc':
 506             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 507             assert (configuration == 'Release' || configuration == 'Checked')
 508             // TODO: Add once external email sending is available again
 509             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 510             Utilities.addPeriodicTrigger(job, '@weekly')
 511             break
 512         case 'gc_reliability_framework':
 513             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 514             assert (configuration == 'Release' || configuration == 'Checked')
 515             // Only triggered by phrase.
 516             break
 517         case 'ilrt':
 518             assert !(os in bidailyCrossList)
 519             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 520             if (architecture == 'x64' && configuration == 'Release') {
 521                 // We don't expect to see a job generated except in these scenarios
 522                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 523                 if (isFlowJob || os == 'Windows_NT') {
 524                     Utilities.addPeriodicTrigger(job, '@daily')
 525                 }
 526             }
 527             break
 528         case 'jitdiff':
 529             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 530             assert configuration == 'Checked'
 531             assert (architecture == 'x64' || architecture == 'x86')
 532             Utilities.addGithubPushTrigger(job)
 533             break
 534         case 'coverage':
 535             assert (os == 'Ubuntu' || os == 'Windows_NT')
 536             assert configuration == 'Release'
 537             assert architecture == 'x64'
 538             Utilities.addPeriodicTrigger(job, '@weekly')
 539             break
 540         case 'formatting':
 541             assert (os == 'Windows_NT' || os == "Ubuntu")
 542             assert architecture == 'x64'
 543             Utilities.addGithubPushTrigger(job)
 544             break
 545         case 'jitstressregs1':
 546         case 'jitstressregs2':
 547         case 'jitstressregs3':
 548         case 'jitstressregs4':
 549         case 'jitstressregs8':
 550         case 'jitstressregs0x10':
 551         case 'jitstressregs0x80':
 552         case 'jitstressregs0x1000':
 553         case 'minopts':
 554         case 'forcerelocs':
 555         case 'jitstress1':
 556         case 'jitstress2':
 557         case 'jitstress2_jitstressregs1':
 558         case 'jitstress2_jitstressregs2':
 559         case 'jitstress2_jitstressregs3':
 560         case 'jitstress2_jitstressregs4':
 561         case 'jitstress2_jitstressregs8':
 562         case 'jitstress2_jitstressregs0x10':
 563         case 'jitstress2_jitstressregs0x80':
 564         case 'jitstress2_jitstressregs0x1000':
 565         case 'tailcallstress':
 566         case 'jitsse2only':
 567         case 'corefx_baseline':
 568         case 'corefx_minopts':
 569         case 'corefx_jitstress1':
 570         case 'corefx_jitstress2':
 571         case 'corefx_jitstressregs1':
 572         case 'corefx_jitstressregs2':
 573         case 'corefx_jitstressregs3':
 574         case 'corefx_jitstressregs4':
 575         case 'corefx_jitstressregs8':
 576         case 'corefx_jitstressregs0x10':
 577         case 'corefx_jitstressregs0x80':
 578         case 'corefx_jitstressregs0x1000':
 579         case 'zapdisable':
 580             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 581             assert (os == 'Windows_NT') || (os in Constants.crossList)
 582             Utilities.addPeriodicTrigger(job, '@daily')
 583         }
 584         break
 585         case 'heapverify1':
 586         case 'gcstress0x3':
 587             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 588                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 589                 if (architecture == 'arm64') {
 590                     assert (os == 'Windows_NT')
 591                     Utilities.addPeriodicTrigger(job, '@daily')
 592                     // TODO: Add once external email sending is available again
 593                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 594                 }
 595                 else {
 596                     Utilities.addPeriodicTrigger(job, '@weekly')
 597                 }
 598             }
 599             break
 600         case 'gcstress0xc':
 601         case 'gcstress0xc_zapdisable':
 602         case 'gcstress0xc_zapdisable_jitstress2':
 603         case 'gcstress0xc_zapdisable_heapverify1':
 604         case 'gcstress0xc_jitstress1':
 605         case 'gcstress0xc_jitstress2':
 606         case 'gcstress0xc_minopts_heapverify1':
 607             // GCStress=C is currently not supported on OS X
 608             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 609                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 610                 if (architecture == 'arm64') {
 611                     assert (os == 'Windows_NT')
 612                     // TODO: Enable a periodic trigger after tests are updated.
 613                     // Utilities.addPeriodicTrigger(job, '@daily')
 614                     // TODO: Add once external email sending is available again
 615                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 616                 }
 617                 else {
 618                     Utilities.addPeriodicTrigger(job, '@weekly')
 619                 }
 620             }
 621             break
 622
 623         case 'illink':
 624             // Testing on other operating systems TBD
 625             assert (os == 'Windows_NT' || os == 'Ubuntu')
 626             if (architecture == 'x64' || architecture == 'x86') {
 627                 if (configuration == 'Checked') {
 628                     Utilities.addPeriodicTrigger(job, '@daily')
 629                 }
 630             }
 631             break
 632
 633         default:
 634             println("Unknown scenario: ${scenario}");
 635             assert false
 636             break
 637     }
 638     return
 639 }
 640
 641 // **************************
 642 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 643 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 644 // builds will do a couple extra steps.
 645 // **************************
 646
 647 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 648 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 649 // it's just used for internal builds
 650 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 651 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 652     if (isWindowsBuildOnlyJob) {
 653         return
 654     }
 655
 656     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 657     // Non pull request builds.
 658     if (!isPR) {
 659         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 660         return
 661     }
 662     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 663     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 664     def osGroup = getOSGroup(os)
 665     switch (architecture) {
 666         case 'x64': // editor brace matching: {
 667             if (scenario == 'coverage') {
 668                 assert configuration == 'Release'
 669                 if (os == 'Ubuntu') {
 670                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 671                 }
 672                 break
 673             }
 674
 675             if (scenario == 'formatting') {
 676                 assert configuration == 'Checked'
 677                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 678                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 679                 }
 680                 break
 681             }
 682
 683             switch (os) {
 684                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 685                 case 'Debian8.4':
 686                 case 'RHEL7.2':
 687                     if (scenario == 'default') {
 688                         assert !isFlowJob
 689                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 690                     }
 691                     else if (scenario == 'pri1' && isFlowJob) {
 692                         assert (configuration == 'Release')
 693                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 694                     }
 695                     break
 696                 case 'Ubuntu16.04':
 697                     assert !isFlowJob
 698                     assert scenario == 'default'
 699                     // Distinguish with the other architectures (arm and x86)
 700                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 701                     break
 702                 case 'Fedora24':
 703                 case 'Ubuntu16.10':
 704                 case 'OpenSUSE42.1':
 705                     assert !isFlowJob
 706                     assert scenario == 'default'
 707                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 708                     break
 709                 case 'Ubuntu':
 710                     if (scenario == 'illink') {
 711                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 712                         break
 713                     }
 714                     // fall through
 715                 case 'OSX10.12':
 716                     // Triggers on the non-flow jobs aren't necessary here
 717                     // Corefx testing uses non-flow jobs.
 718                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 719                         break
 720                     }
 721                     switch (scenario) {
 722                         case 'default':
 723                             // Ubuntu uses checked for default PR tests
 724                             if (configuration == 'Checked') {
 725                                 // Default trigger
 726                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 727                             }
 728                             break
 729                         case 'pri1':
 730                             if (configuration == 'Release') {
 731                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 732                             }
 733                             break
 734                         case 'jitdiff':
 735                             if (configuration == 'Checked') {
 736                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 737                             }
 738                             break
 739                         case 'ilrt':
 740                             if (configuration == 'Release') {
 741                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 742                             }
 743                             break
 744                         case 'r2r':
 745                             if (configuration == 'Release' || configuration == 'Checked') {
 746                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 747                             }
 748                             break
 749                         case 'pri1r2r':
 750                             if (configuration == 'Release' || configuration == 'Checked') {
 751                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 752                             }
 753                             break
 754                         case 'gcstress15_pri1r2r':
 755                             if (configuration == 'Release' || configuration == 'Checked') {
 756                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 757                             }
 758                             break
 759                         case 'r2r_jitstress1':
 760                         case 'r2r_jitstress2':
 761                         case 'r2r_jitstressregs1':
 762                         case 'r2r_jitstressregs2':
 763                         case 'r2r_jitstressregs3':
 764                         case 'r2r_jitstressregs4':
 765                         case 'r2r_jitstressregs8':
 766                         case 'r2r_jitstressregs0x10':
 767                         case 'r2r_jitstressregs0x80':
 768                         case 'r2r_jitstressregs0x1000':
 769                         case 'r2r_jitminopts':
 770                         case 'r2r_jitforcerelocs':
 771                             if (configuration == 'Release' || configuration == 'Checked') {
 772                                 def displayStr = getR2RStressModeDisplayName(scenario)
 773                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 774                             }
 775                             break
 776                         case 'longgc':
 777                             if (configuration == 'Release') {
 778                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 779                             }
 780                             break
 781                         case 'gcsimulator':
 782                             if (configuration == 'Release') {
 783                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 784                             }
 785                             break
 786                         case 'standalone_gc':
 787                             if (configuration == 'Release' || configuration == 'Checked') {
 788                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 789                             }
 790                             break
 791                         case 'gc_reliability_framework':
 792                             if (configuration == 'Release' || configuration == 'Checked') {
 793                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 794                             }
 795                             break
 796                         case 'minopts':
 797                         case 'forcerelocs':
 798                         case 'jitstress1':
 799                         case 'jitstress2':
 800                         case 'jitstressregs1':
 801                         case 'jitstressregs2':
 802                         case 'jitstressregs3':
 803                         case 'jitstressregs4':
 804                         case 'jitstressregs8':
 805                         case 'jitstressregs0x10':
 806                         case 'jitstressregs0x80':
 807                         case 'jitstressregs0x1000':
 808                         case 'jitstress2_jitstressregs1':
 809                         case 'jitstress2_jitstressregs2':
 810                         case 'jitstress2_jitstressregs3':
 811                         case 'jitstress2_jitstressregs4':
 812                         case 'jitstress2_jitstressregs8':
 813                         case 'jitstress2_jitstressregs0x10':
 814                         case 'jitstress2_jitstressregs0x80':
 815                         case 'jitstress2_jitstressregs0x1000':
 816                         case 'tailcallstress':
 817                         case 'jitsse2only':
 818                         case 'gcstress0x3':
 819                         case 'gcstress0xc':
 820                         case 'zapdisable':
 821                         case 'heapverify1':
 822                         case 'gcstress0xc_zapdisable':
 823                         case 'gcstress0xc_zapdisable_jitstress2':
 824                         case 'gcstress0xc_zapdisable_heapverify1':
 825                         case 'gcstress0xc_jitstress1':
 826                         case 'gcstress0xc_jitstress2':
 827                         case 'gcstress0xc_minopts_heapverify1':
 828                             def displayStr = getStressModeDisplayName(scenario)
 829                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 830                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 831                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 832                             break
 833                         case 'corefx_baseline':
 834                         case 'corefx_minopts':
 835                         case 'corefx_jitstress1':
 836                         case 'corefx_jitstress2':
 837                         case 'corefx_jitstressregs1':
 838                         case 'corefx_jitstressregs2':
 839                         case 'corefx_jitstressregs3':
 840                         case 'corefx_jitstressregs4':
 841                         case 'corefx_jitstressregs8':
 842                         case 'corefx_jitstressregs0x10':
 843                         case 'corefx_jitstressregs0x80':
 844                         case 'corefx_jitstressregs0x1000':
 845                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 846                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 847                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 848                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 849                             break
 850                         default:
 851                             println("Unknown scenario: ${scenario}");
 852                             assert false
 853                             break
 854                     }
 855                     break
 856                 case 'CentOS7.1':
 857                     switch (scenario) {
 858                         case 'pri1':
 859                             if (configuration == 'Release') {
 860                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 861                             }
 862                             break
 863                         case 'r2r':
 864                             if (configuration == 'Checked' || configuration == 'Release') {
 865                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 866                             }
 867                             break
 868                         case 'pri1r2r':
 869                             if (configuration == 'Checked' || configuration == 'Release') {
 870                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 871                             }
 872                             break
 873                         case 'gcstress15_pri1r2r':
 874                             if (configuration == 'Release' || configuration == 'Checked') {
 875                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 876                             }
 877                             break
 878                         case 'r2r_jitstress1':
 879                         case 'r2r_jitstress2':
 880                         case 'r2r_jitstressregs1':
 881                         case 'r2r_jitstressregs2':
 882                         case 'r2r_jitstressregs3':
 883                         case 'r2r_jitstressregs4':
 884                         case 'r2r_jitstressregs8':
 885                         case 'r2r_jitstressregs0x10':
 886                         case 'r2r_jitstressregs0x80':
 887                         case 'r2r_jitstressregs0x1000':
 888                         case 'r2r_jitminopts':
 889                         case 'r2r_jitforcerelocs':
 890                             if (configuration == 'Release' || configuration == 'Checked') {
 891                                 def displayStr = getR2RStressModeDisplayName(scenario)
 892                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 893                             }
 894                             break
 895                         default:
 896                             break
 897                     }
 898                 case 'Windows_NT':
 899                     switch (scenario) {
 900                         case 'default':
 901                             // Default trigger
 902                             if (configuration == 'Debug') {
 903                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 904                             }
 905                             break
 906                         case 'pri1':
 907                             // Default trigger
 908                             if (configuration == 'Release') {
 909                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 910                             }
 911                             break
 912                         case 'jitdiff':
 913                             if (configuration == 'Checked') {
 914                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 915                             }
 916                             break
 917                         case 'ilrt':
 918                             if (configuration == 'Release') {
 919                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 920                             }
 921                             break
 922                         case 'r2r':
 923                             if (configuration == 'Checked' || configuration == 'Release') {
 924                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 925                             }
 926                             break
 927                         case 'pri1r2r':
 928                             if (configuration == 'Checked' || configuration == 'Release') {
 929                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 930                             }
 931                             break
 932                         case 'gcstress15_pri1r2r':
 933                             if (configuration == 'Release' || configuration == 'Checked') {
 934                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 935                             }
 936                             break
 937                         case 'r2r_jitstress1':
 938                         case 'r2r_jitstress2':
 939                         case 'r2r_jitstressregs1':
 940                         case 'r2r_jitstressregs2':
 941                         case 'r2r_jitstressregs3':
 942                         case 'r2r_jitstressregs4':
 943                         case 'r2r_jitstressregs8':
 944                         case 'r2r_jitstressregs0x10':
 945                         case 'r2r_jitstressregs0x80':
 946                         case 'r2r_jitstressregs0x1000':
 947                         case 'r2r_jitminopts':
 948                         case 'r2r_jitforcerelocs':
 949                             if (configuration == 'Release' || configuration == 'Checked') {
 950                                 def displayStr = getR2RStressModeDisplayName(scenario)
 951                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 952                             }
 953                             break
 954                         case 'longgc':
 955                             if (configuration == 'Release') {
 956                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 957                             }
 958                             break
 959                         case 'gcsimulator':
 960                             if (configuration == 'Release') {
 961                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 962                             }
 963                             break
 964                         case 'standalone_gc':
 965                             if (configuration == 'Release' || configuration == 'Checked') {
 966                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 967                             }
 968                             break
 969                         case 'gc_reliability_framework':
 970                             if (configuration == 'Release' || configuration == 'Checked') {
 971                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 972                             }
 973                             break
 974                         case 'minopts':
 975                         case 'forcerelocs':
 976                         case 'jitstress1':
 977                         case 'jitstress2':
 978                         case 'jitstressregs1':
 979                         case 'jitstressregs2':
 980                         case 'jitstressregs3':
 981                         case 'jitstressregs4':
 982                         case 'jitstressregs8':
 983                         case 'jitstressregs0x10':
 984                         case 'jitstressregs0x80':
 985                         case 'jitstressregs0x1000':
 986                         case 'jitstress2_jitstressregs1':
 987                         case 'jitstress2_jitstressregs2':
 988                         case 'jitstress2_jitstressregs3':
 989                         case 'jitstress2_jitstressregs4':
 990                         case 'jitstress2_jitstressregs8':
 991                         case 'jitstress2_jitstressregs0x10':
 992                         case 'jitstress2_jitstressregs0x80':
 993                         case 'jitstress2_jitstressregs0x1000':
 994                         case 'tailcallstress':
 995                         case 'jitsse2only':
 996                         case 'gcstress0x3':
 997                         case 'gcstress0xc':
 998                         case 'zapdisable':
 999                         case 'heapverify1':
1000                         case 'gcstress0xc_zapdisable':
1001                         case 'gcstress0xc_zapdisable_jitstress2':
1002                         case 'gcstress0xc_zapdisable_heapverify1':
1003                         case 'gcstress0xc_jitstress1':
1004                         case 'gcstress0xc_jitstress2':
1005                         case 'gcstress0xc_minopts_heapverify1':
1006                             def displayStr = getStressModeDisplayName(scenario)
1007                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1008                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1009                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1010                             break
1011                         case 'corefx_baseline':
1012                         case 'corefx_minopts':
1013                         case 'corefx_jitstress1':
1014                         case 'corefx_jitstress2':
1015                         case 'corefx_jitstressregs1':
1016                         case 'corefx_jitstressregs2':
1017                         case 'corefx_jitstressregs3':
1018                         case 'corefx_jitstressregs4':
1019                         case 'corefx_jitstressregs8':
1020                         case 'corefx_jitstressregs0x10':
1021                         case 'corefx_jitstressregs0x80':
1022                         case 'corefx_jitstressregs0x1000':
1023                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1024                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1025                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1026                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1027                             break
1028                         case 'illink':
1029                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1030                                     break
1031                         default:
1032                             println("Unknown scenario: ${scenario}");
1033                             assert false
1034                             break
1035                     }
1036                     break
1037                 case 'FreeBSD':
1038                     assert scenario == 'default'
1039                     if (configuration == 'Checked') {
1040                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1041                     }
1042                     break
1043                 default:
1044                     println("Unknown os: ${os}");
1045                     assert false
1046                     break
1047             }
1048             break
1049         // editor brace matching: }
1050         case 'arm': // editor brace matching: {
1051             assert scenario == 'default'
1052             switch (os) {
1053                 case 'Ubuntu':
1054                 case 'Ubuntu16.04':
1055                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1056                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1057                     }
1058                     else {
1059                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1060                     }
1061                     break;
1062                 case 'Tizen':
1063                     architecture='armel'
1064                     // Removing the regex will cause this to run on each PR.
1065                     if (configuration == 'Release' || configuration == 'Debug') {
1066                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1067                     }
1068                     else {
1069                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1070                     }
1071                     break;
1072                 case 'Windows_NT':
1073                     if (configuration == 'Debug' || configuration == 'Release')
1074                     {
1075                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1076                     }
1077                     break
1078                 default:
1079                     println("NYI os: ${os}");
1080                     assert false
1081                     break
1082             }
1083             break
1084         // editor brace matching: }
1085         case 'arm64': // editor brace matching: {
1086             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1087
1088             // Set up a private trigger
1089             def contextString = "${os} ${architecture} Cross ${configuration}"
1090             if (scenario != 'default')
1091                 contextString += " ${scenario}"
1092             contextString += " Build"
1093             // Debug builds only.
1094             if (configuration != 'Debug') {
1095                contextString += " and Test"
1096             }
1097
1098             def arm64Users = [
1099                 'adiaaida',
1100                 'AndyAyersMS',
1101                 'briansull',
1102                 'BruceForstall',
1103                 'CarolEidt',
1104                 'cmckinsey',
1105                 'erozenfeld',
1106                 'jashook',
1107                 'JosephTremoulet',
1108                 'pgavlin',
1109                 'pkukol',
1110                 'russellhadley',
1111                 'RussKeldorph',
1112                 'sandreenko',
1113                 'sivarv',
1114                 'swaroop-sridhar',
1115                 'gkhanna79',
1116                 'jkotas',
1117                 'markwilkie',
1118                 'rahku',
1119                 'ramarag',
1120                 'tzwlai',
1121                 'weshaggard'
1122             ]
1123
1124             switch (os) {
1125                 case 'Windows_NT':
1126                     switch (scenario) {
1127                         case 'default':
1128                             // For now only run Debug jobs on PR Trigger.
1129                             if (configuration != 'Debug') {
1130                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1131                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1132                             }
1133                             else {
1134                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1135                                 // are at higher risk of ARM64-breaking changes.
1136                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1137                             }
1138                             break
1139                         case 'pri1r2r':
1140                         case 'gcstress0x3':
1141                         case 'gcstress0xc':
1142                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1143                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1144                             break
1145                     }
1146                     break
1147                 default:
1148                     println("NYI os: ${os}");
1149                     assert false
1150                     break
1151             }
1152             break
1153         // editor brace matching: }
1154         case 'x86': // editor brace matching: {
1155             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1156             if (os == 'Ubuntu') {
1157                 // on-demand only for ubuntu x86
1158                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1159                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1160                 break
1161             }
1162             switch (scenario) {
1163                 case 'default':
1164                     if (configuration == 'Checked') {
1165                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1166                     }
1167                     else if (configuration == 'Release') {
1168                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1169                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1170                     }
1171                     break
1172                 case 'pri1':
1173                     if (configuration == 'Release') {
1174                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1175                     }
1176                     break
1177                 case 'ilrt':
1178                     if (configuration == 'Release') {
1179                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1180                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1181                     }
1182                     break
1183                 case 'r2r':
1184                     if (configuration == 'Checked' || configuration == 'Release') {
1185                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1186                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1187                     }
1188                     break
1189                 case 'pri1r2r':
1190                     if (configuration == 'Checked' || configuration == 'Release') {
1191                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1192                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1193                     }
1194                     break
1195                 case 'gcstress15_pri1r2r':
1196                     if (configuration == 'Release' || configuration == 'Checked') {
1197                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1198                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1199                     }
1200                     break
1201                 case 'r2r_jitstress1':
1202                 case 'r2r_jitstress2':
1203                 case 'r2r_jitstressregs1':
1204                 case 'r2r_jitstressregs2':
1205                 case 'r2r_jitstressregs3':
1206                 case 'r2r_jitstressregs4':
1207                 case 'r2r_jitstressregs8':
1208                 case 'r2r_jitstressregs0x10':
1209                 case 'r2r_jitstressregs0x80':
1210                 case 'r2r_jitstressregs0x1000':
1211                 case 'r2r_jitminopts':
1212                 case 'r2r_jitforcerelocs':
1213                     if (configuration == 'Release' || configuration == 'Checked') {
1214                         def displayStr = getR2RStressModeDisplayName(scenario)
1215                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1216                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1217                     }
1218                     break
1219                 case 'longgc':
1220                     if (configuration == 'Release') {
1221                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1222                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1223                     }
1224                     break
1225                 case 'gcsimulator':
1226                     if (configuration == 'Release') {
1227                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1228                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1229                     }
1230                     break
1231                 case 'standalone_gc':
1232                     if (configuration == 'Release' || configuration == 'Checked') {
1233                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1234                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1235                     }
1236                     break
1237                 case 'minopts':
1238                 case 'forcerelocs':
1239                 case 'jitstress1':
1240                 case 'jitstress2':
1241                 case 'jitstressregs1':
1242                 case 'jitstressregs2':
1243                 case 'jitstressregs3':
1244                 case 'jitstressregs4':
1245                 case 'jitstressregs8':
1246                 case 'jitstressregs0x10':
1247                 case 'jitstressregs0x80':
1248                 case 'jitstressregs0x1000':
1249                 case 'jitstress2_jitstressregs1':
1250                 case 'jitstress2_jitstressregs2':
1251                 case 'jitstress2_jitstressregs3':
1252                 case 'jitstress2_jitstressregs4':
1253                 case 'jitstress2_jitstressregs8':
1254                 case 'jitstress2_jitstressregs0x10':
1255                 case 'jitstress2_jitstressregs0x80':
1256                 case 'jitstress2_jitstressregs0x1000':
1257                 case 'tailcallstress':
1258                 case 'jitsse2only':
1259                 case 'gcstress0x3':
1260                 case 'gcstress0xc':
1261                 case 'zapdisable':
1262                 case 'heapverify1':
1263                 case 'gcstress0xc_zapdisable':
1264                 case 'gcstress0xc_zapdisable_jitstress2':
1265                 case 'gcstress0xc_zapdisable_heapverify1':
1266                 case 'gcstress0xc_jitstress1':
1267                 case 'gcstress0xc_jitstress2':
1268                 case 'gcstress0xc_minopts_heapverify1':
1269                     def displayStr = getStressModeDisplayName(scenario)
1270                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1271                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1272                     break
1273                 case 'corefx_baseline':
1274                 case 'corefx_minopts':
1275                 case 'corefx_jitstress1':
1276                 case 'corefx_jitstress2':
1277                 case 'corefx_jitstressregs1':
1278                 case 'corefx_jitstressregs2':
1279                 case 'corefx_jitstressregs3':
1280                 case 'corefx_jitstressregs4':
1281                 case 'corefx_jitstressregs8':
1282                 case 'corefx_jitstressregs0x10':
1283                 case 'corefx_jitstressregs0x80':
1284                 case 'corefx_jitstressregs0x1000':
1285                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1286                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1287                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1288                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1289                     break
1290                 case 'illink':
1291                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1292                     break
1293                 default:
1294                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1295                     assert false
1296                     break
1297             }
1298             break
1299         // editor brace matching: }
1300         default:
1301             println("Unknown architecture: ${architecture}");
1302             assert false
1303             break
1304     }
1305 }
1306
1307 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1308     def buildCommands = [];
1309     def osGroup = getOSGroup(os)
1310     def lowerConfiguration = configuration.toLowerCase()
1311
1312     // Calculate the build steps, archival, and xunit results
1313     switch (os) {
1314         case 'Windows_NT': // editor brace matching: {
1315             switch (architecture) {
1316                 case 'x64':
1317                 case 'x86':
1318                     def arch = architecture
1319                     def buildOpts = ''
1320
1321                     if (scenario == 'illink') {
1322                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1323                     }
1324
1325                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1326                             scenario == 'default' ||
1327                             scenario == 'r2r' ||
1328                             scenario == 'jitdiff' ||
1329                             scenario == 'ilrt' ||
1330                             scenario == 'illink' ||
1331                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1332                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1333                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1334                     }
1335
1336                     // For Pri 1 tests, we must shorten the output test binary path names.
1337                     // if __TestIntermediateDir is already set, build-test.cmd will
1338                     // output test binaries to that directory. If it is not set, the
1339                     // binaries are sent to a default directory whose name is about
1340                     // 35 characters long.
1341
1342                     else if (scenarioNeedsPri1Build(scenario)) {
1343                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1344                     }
1345                     else if (isLongGc(scenario)) {
1346                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1347                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1348                     }
1349                     else if (scenario == 'standalone_gc') {
1350                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1351                     }
1352                     else if (scenario == 'formatting') {
1353                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1354                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1355                         break
1356                     }
1357                     else {
1358                         println("Unknown scenario: ${scenario}")
1359                         assert false
1360                     }
1361
1362                     // If we are running a stress mode, we should write out the set of key
1363                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1364
1365                     if (!isBuildOnly) {
1366                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1367                         def crossgenStr = ''
1368                         def runcrossgentestsStr = ''
1369                         def runjitstressStr = ''
1370                         def runjitstressregsStr = ''
1371                         def runjitmioptsStr = ''
1372                         def runjitforcerelocsStr = ''
1373                         def runjitdisasmStr = ''
1374                         def runilasmroundtripStr = ''
1375                         def gcstressStr = ''
1376                         def runtestArguments = ''
1377                         def gcTestArguments = ''
1378                         def illinkArguments = ''
1379
1380                         if (scenario == 'r2r' ||
1381                             scenario == 'pri1r2r' ||
1382                             scenario == 'gcstress15_pri1r2r' ||
1383                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1384                                 crossgenStr = 'crossgen'
1385                                 runcrossgentestsStr = 'runcrossgentests'
1386
1387                                 if (scenario == 'r2r_jitstress1'){
1388                                     runjitstressStr = 'jitstress 1'
1389                                 }
1390                                 else if (scenario == 'r2r_jitstress2') {
1391                                     runjitstressStr = 'jitstress 2'
1392                                 }
1393                                 else if (scenario == 'r2r_jitstressregs1'){
1394                                     runjitstressregsStr = 'jitstressregs 1'
1395                                 }
1396                                 else if (scenario == 'r2r_jitstressregs2') {
1397                                     runjitstressregsStr = 'jitstressregs 2'
1398                                 }
1399                                 else if (scenario == 'r2r_jitstressregs3') {
1400                                     runjitstressregsStr = 'jitstressregs 3'
1401                                 }
1402                                 else if (scenario == 'r2r_jitstressregs4') {
1403                                     runjitstressregsStr = 'jitstressregs 4'
1404                                 }
1405                                 else if (scenario == 'r2r_jitstressregs8') {
1406                                     runjitstressregsStr = 'jitstressregs 8'
1407                                 }
1408                                 else if (scenario == 'r2r_jitstressregs0x10') {
1409                                     runjitstressregsStr = 'jitstressregs 0x10'
1410                                 }
1411                                 else if (scenario == 'r2r_jitstressregs0x80') {
1412                                     runjitstressregsStr = 'jitstressregs 0x80'
1413                                 }
1414                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1415                                     runjitstressregsStr = 'jitstressregs 0x1000'
1416                                 }
1417                                 else if (scenario == 'r2r_jitminopts') {
1418                                     runjitmioptsStr = 'jitminopts'
1419                                 }
1420                                 else if (scenario == 'r2r_jitforcerelocs') {
1421                                     runjitforcerelocsStr = 'jitforcerelocs'
1422                                 }
1423                         }
1424                         if (scenario == 'gcstress15_pri1r2r')
1425                         {
1426                             gcstressStr = 'gcstresslevel 0xF'
1427                         }
1428
1429                         if (scenario == 'jitdiff')
1430                         {
1431                             runjitdisasmStr = 'jitdisasm crossgen'
1432                         }
1433
1434                         if (scenario == 'ilrt')
1435                         {
1436                             runilasmroundtripStr = 'ilasmroundtrip'
1437                         }
1438
1439                         if (isLongGc(scenario)) {
1440                             gcTestArguments = "${scenario} sequential"
1441                         }
1442
1443                         if (scenario == 'illink')
1444                         {
1445                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1446                         }
1447
1448                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1449
1450                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1451                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1452                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1453
1454                             if (enableCorefxTesting) {
1455                                 def workspaceRelativeFxRoot = "_/fx"
1456                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1457
1458                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1459
1460                                 setTestJobTimeOut(newJob, scenario)
1461
1462                                 // Archive and process (only) the test results
1463                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1464                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1465                             }
1466                             else {
1467                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1468                             }
1469                         }
1470                         else if (isGcReliabilityFramework(scenario)) {
1471                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1472                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1473                             Utilities.addArchival(newJob, "stdout.txt")
1474                             Utilities.addArchival(newJob, "Logs/**")
1475                         }
1476                         else if (architecture == 'x64' || architecture == 'x86') {
1477                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1478                         }
1479                     }
1480
1481                     if (!enableCorefxTesting) {
1482                         // Run the rest of the build
1483                         // Build the mscorlib for the other OS's
1484                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1485                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1486                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1487
1488                         // Zip up the tests directory so that we don't use so much space/time copying
1489                         // 10s of thousands of files around.
1490                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1491
1492                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1493                             // For windows, pull full test results and test drops for x86/x64.
1494                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1495                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1496                         }
1497
1498                         if (scenario == 'jitdiff') {
1499                             // retrive jit-dasm output for base commit, and run jit-diff
1500                             if (!isBuildOnly) {
1501                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1502                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1503                             }
1504                         }
1505
1506                         if (!isBuildOnly) {
1507                             if (architecture == 'x64' || !isPR) {
1508                                 Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml')
1509                             }
1510                             setTestJobTimeOut(newJob, scenario)
1511                         }
1512                     }
1513                     break
1514                 case 'arm':
1515                     assert (scenario == 'default')
1516
1517                     // Set time out
1518                     setTestJobTimeOut(newJob, scenario)
1519
1520                     if ( lowerConfiguration == "debug" ) {
1521                         // For Debug builds, we will do a P1 test build
1522                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1523                     }
1524                     else {
1525                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1526                     }
1527                     // Add archival.
1528                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1529                     break
1530                 case 'arm64':
1531                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1532
1533                     // Set time out
1534                     setTestJobTimeOut(newJob, scenario)
1535
1536                     // Debug runs take too long to run. So build job only.
1537                     if (lowerConfiguration == "debug") {
1538                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1539                     }
1540                     else {
1541                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1542                        {
1543                            // Up the timeout for arm64 checked testing only.
1544                            // Keep the longer timeout for gcstress.
1545                            Utilities.setJobTimeout(newJob, 240)
1546                        }
1547
1548                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=1"
1549                        // Test build and run are launched together.
1550                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1551                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1552                     }
1553
1554                     // Add archival.
1555                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1556                     break
1557                 default:
1558                     println("Unknown architecture: ${architecture}");
1559                     assert false
1560                     break
1561             }
1562             break
1563         // editor brace matching: }
1564         case 'Ubuntu':
1565         case 'Ubuntu16.04':
1566         case 'Ubuntu16.10':
1567         case 'Debian8.4':
1568         case 'OSX10.12':
1569         case 'FreeBSD':
1570         case 'CentOS7.1':
1571         case 'RHEL7.2':
1572         case 'OpenSUSE42.1':
1573         case 'Tizen':
1574         case 'Fedora24': // editor brace matching: {
1575             switch (architecture) {
1576                 case 'x64':
1577                 case 'x86':
1578                     if (architecture == 'x86' && os == 'Ubuntu') {
1579                         // build and PAL test
1580                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1581                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1582                         break;
1583                     }
1584
1585                     if (scenario == 'formatting') {
1586                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1587                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1588                         break
1589                     }
1590
1591                     if (scenario == 'illink') {
1592                         assert(os == 'Ubuntu')
1593                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1594                     }
1595
1596                     def standaloneGc = ''
1597                     if (scenario == 'standalone_gc') {
1598                         standaloneGc = 'buildstandalonegc'
1599                     }
1600
1601                     if (!enableCorefxTesting) {
1602                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1603                         // only on supported OS platforms.
1604                         if (os == 'FreeBSD')
1605                         {
1606                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1607                         }
1608                         else
1609                         {
1610                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1611                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1612                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1613                         }
1614                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1615
1616                         // Set time out
1617                         setTestJobTimeOut(newJob, scenario)
1618                         // Basic archiving of the build
1619                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1620                         // And pal tests
1621                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1622                     }
1623                     else {
1624                         // Corefx stress testing
1625                         assert os == 'Ubuntu'
1626                         assert architecture == 'x64'
1627                         assert lowerConfiguration == 'checked'
1628                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1629
1630                         // Build coreclr
1631                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1632
1633                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1634                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1635
1636                         // Build and text corefx
1637                         def workspaceRelativeFxRoot = "_/fx"
1638                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1639
1640                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1641
1642                         setTestJobTimeOut(newJob, scenario)
1643
1644                         // Archive and process (only) the test results
1645                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1646                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1647                     }
1648                     break
1649                 case 'arm64':
1650                     // We don't run the cross build except on Ubuntu
1651                     assert os == 'Ubuntu'
1652
1653                     buildCommands += """echo \"Using rootfs in /opt/aarch64-linux-gnu-root\"
1654                         ROOTFS_DIR=/opt/aarch64-linux-gnu-root ./build.sh skipmscorlib arm64 cross verbose ${lowerConfiguration}"""
1655
1656                     // Basic archiving of the build, no pal tests
1657                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1658                     break
1659                 case 'arm':
1660                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1661                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1662
1663                     // default values for Ubuntu
1664                     def arm_abi="arm"
1665                     def linuxCodeName="trusty"
1666                     if (os == 'Ubuntu16.04') {
1667                         linuxCodeName="xenial"
1668                     }
1669                     else if (os == 'Tizen') {
1670                         arm_abi="armel"
1671                         linuxCodeName="tizen"
1672                     }
1673
1674                     // Unzip the Windows test binaries first. Exit with 0
1675                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1676
1677                     // Unpack the corefx binaries
1678                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1679                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1680                     if (os != 'Tizen') {
1681                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1682                     }
1683                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1684                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1685                     if (os == 'Ubuntu') {
1686                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1687                     }
1688
1689                     // Call the ARM CI script to cross build and test using docker
1690                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1691                     --mode=docker \\
1692                     --${arm_abi} \\
1693                     --linuxCodeName=${linuxCodeName} \\
1694                     --buildConfig=${lowerConfiguration} \\
1695                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1696                     --coreFxBinDir=./bin/CoreFxBinDir \\
1697                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1698
1699                     // Basic archiving of the build, no pal tests
1700                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1701                     break
1702                 default:
1703                     println("Unknown architecture: ${architecture}");
1704                     assert false
1705                     break
1706             }
1707             break
1708         // editor brace matching: }
1709         default:
1710             println("Unknown os: ${os}");
1711             assert false
1712             break
1713     } // os
1714
1715     return buildCommands
1716 }
1717
1718 // Additional scenario which can alter behavior
1719
1720 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1721 combinedScenarios.each { scenario ->
1722     [true, false].each { isPR ->
1723         Constants.architectureList.each { architecture ->
1724             Constants.configurationList.each { configuration ->
1725                 Constants.osList.each { os ->
1726                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1727                     // and reset the os to Windows_NT
1728                     def isBuildOnly = false
1729                     if (os == 'Windows_NT_BuildOnly') {
1730                         isBuildOnly = true
1731                         os = 'Windows_NT'
1732                     }
1733
1734                     // WinArm32 is only built for Debug and Release
1735                     if (os == 'Windows_NT' && architecture == 'arm')
1736                     {
1737                         if (configuration == 'Checked')
1738                         {
1739                             return
1740                         }
1741                     }
1742                     // Tizen is only supported for arm architecture
1743                     if (os == 'Tizen' && architecture != 'arm') {
1744                         return
1745                     }
1746
1747                     // Skip totally unimplemented (in CI) configurations.
1748                     switch (architecture) {
1749                         case 'arm64':
1750                             // Windows only
1751                             if (os != 'Windows_NT' || isBuildOnly) {
1752                                 return
1753                             }
1754                             break
1755                         case 'arm':
1756                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1757                                 return
1758                             }
1759                             break
1760                         case 'x86':
1761                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1762                                 return
1763                             }
1764                             break
1765                         case 'x64':
1766                             // Everything implemented
1767                             break
1768                         default:
1769                             println("Unknown architecture: ${architecture}")
1770                             assert false
1771                             break
1772                     }
1773
1774                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1775                     // with checked builds
1776                     def enableCorefxTesting = false
1777                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1778                         if (configuration != 'Checked') {
1779                             return
1780                         }
1781
1782                         enableCorefxTesting = isCorefxTesting(scenario)
1783
1784                         // Since these are just execution time differences,
1785                         // skip platforms that don't execute the tests here (Windows_NT only)
1786                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1787                         if (!isEnabledOS || isBuildOnly) {
1788                             return
1789                         }
1790
1791                         switch (architecture) {
1792                             case 'arm64':
1793                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1794                                     return
1795                                 }
1796                                 break
1797                             case 'x64':
1798                             case 'x86':
1799                                 // x86 ubuntu: default only
1800                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1801                                     return
1802                                 }
1803                                 // Windows: Everything implemented
1804                                 break
1805                             default:
1806                                 return
1807                         }
1808                     }
1809                     else {
1810                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
1811                         // and configuration is not Checked, bail out.
1812                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1813                             return;
1814                         }
1815
1816                         // Skip scenarios
1817                         switch (scenario) {
1818                             case 'pri1':
1819                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1820                                 // the default scenario build
1821                                 if (os != 'Windows_NT') {
1822                                     return
1823                                 }
1824                                 // Only x64 for now
1825                                 if (architecture != 'x64') {
1826                                     return
1827                                 }
1828                                 break
1829                             case 'ilrt':
1830                                 // The ilrt build isn't necessary except for Windows_NT.  Non-Windows NT uses
1831                                 // the default scenario build
1832                                 if (os != 'Windows_NT') {
1833                                     return
1834                                 }
1835                                 // Only x64 for now
1836                                 if (architecture != 'x64') {
1837                                     return
1838                                 }
1839                                 // Release only
1840                                 if (configuration != 'Release') {
1841                                     return
1842                                 }
1843                                 break
1844                             case 'jitdiff':
1845                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1846                                     return
1847                                 }
1848                                 if (architecture != 'x64') {
1849                                     return
1850                                 }
1851                                 if (configuration != 'Checked') {
1852                                     return
1853                                 }
1854                                 break
1855                             case 'r2r':
1856                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1857                                 // the default scenario build
1858                                 if (os != 'Windows_NT') {
1859                                     return
1860                                 }
1861                                 if (architecture != 'x64') {
1862                                     return
1863                                 }
1864                                 break
1865                             case 'pri1r2r':
1866                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1867                                 // the default scenario build
1868                                 if (os != 'Windows_NT') {
1869                                     return
1870                                 }
1871                                 if (architecture != 'x64') {
1872                                     if (architecture != 'arm64' || configuration == 'Debug') {
1873                                         return
1874                                     }
1875                                 }
1876                                 break
1877                             case 'gcstress15_pri1r2r':
1878                             case 'r2r_jitstress1':
1879                             case 'r2r_jitstress2':
1880                             case 'r2r_jitstressregs1':
1881                             case 'r2r_jitstressregs2':
1882                             case 'r2r_jitstressregs3':
1883                             case 'r2r_jitstressregs4':
1884                             case 'r2r_jitstressregs8':
1885                             case 'r2r_jitstressregs0x10':
1886                             case 'r2r_jitstressregs0x80':
1887                             case 'r2r_jitstressregs0x1000':
1888                             case 'r2r_jitminopts':
1889                             case 'r2r_jitforcerelocs':
1890                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
1891                                 // the default scenario build
1892                                 if (os != 'Windows_NT') {
1893                                     return
1894                                 }
1895                                 if (architecture != 'x64' && architecture != 'x86') {
1896                                     return
1897                                 }
1898                                 break
1899                             case 'longgc':
1900                             case 'gcsimulator':
1901                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1902                                     return
1903                                 }
1904                                 if (architecture != 'x64') {
1905                                     return
1906                                 }
1907                                 if (configuration != 'Release') {
1908                                     return
1909                                 }
1910                                 break
1911                             case 'gc_reliability_framework':
1912                             case 'standalone_gc':
1913                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1914                                     return
1915                                 }
1916
1917                                 if (architecture != 'x64') {
1918                                     return
1919                                 }
1920
1921                                 if (configuration != 'Release' && configuration != 'Checked') {
1922                                     return
1923                                 }
1924                                 break
1925                             // We need Windows x64 Release bits for the code coverage build
1926                             case 'coverage':
1927                                 if (os != 'Windows_NT') {
1928                                     return
1929                                 }
1930                                 if (architecture != 'x64') {
1931                                     return
1932                                 }
1933                                 if (configuration != 'Release') {
1934                                     return
1935                                 }
1936                                 break
1937                             // We only run Windows and Ubuntu x64 Checked for formatting right now
1938                             case 'formatting':
1939                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
1940                                     return
1941                                 }
1942                                 if (architecture != 'x64') {
1943                                     return
1944                                 }
1945                                 if (configuration != 'Checked') {
1946                                     return
1947                                 }
1948                                 if (isBuildOnly) {
1949                                     return
1950                                 }
1951                                 break
1952                             case 'illink':
1953                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
1954                                     return
1955                                 }
1956                                 if (architecture != 'x64' && architecture != 'x86') {
1957                                     return
1958                                 }
1959                                 if (isBuildOnly) {
1960                                     return
1961                                 }
1962                                 break
1963                             case 'default':
1964                                 // Nothing skipped
1965                                 break
1966                             default:
1967                                 println("Unknown scenario: ${scenario}")
1968                                 assert false
1969                                 break
1970                         }
1971                     }
1972
1973                     // Calculate names
1974                     def lowerConfiguration = configuration.toLowerCase()
1975                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
1976                     def folderName = getJobFolder(scenario)
1977
1978                     // Create the new job
1979                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
1980
1981                     setMachineAffinity(newJob, os, architecture)
1982
1983                     // Add all the standard options
1984                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
1985                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
1986
1987                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
1988                     def osGroup = getOSGroup(os)
1989
1990                     newJob.with {
1991                         steps {
1992                             if (os == 'Windows_NT') {
1993                                 buildCommands.each { buildCommand ->
1994                                     batchFile(buildCommand)
1995                                 }
1996                             }
1997                             else {
1998                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
1999                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2000                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2001                                     // Define the Windows Tests and Corefx build job names
2002                                     def WindowTestsName = projectFolder + '/' +
2003                                                           Utilities.getFullJobName(project,
2004                                                                                    getJobName(lowerConfiguration,
2005                                                                                               'x64' ,
2006                                                                                               'windows_nt',
2007                                                                                               'default',
2008                                                                                               true),
2009                                                                                    false)
2010                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2011                                                        Utilities.getFolderName(branch)
2012
2013                                     // Copy the Windows test binaries and the Corefx build binaries
2014                                     copyArtifacts(WindowTestsName) {
2015                                         excludePatterns('**/testResults.xml', '**/*.ni.dll')
2016                                         buildSelector {
2017                                             latestSuccessful(true)
2018                                         }
2019                                     }
2020
2021                                     // Defaults for Ubuntu
2022                                     def arm_abi = 'arm'
2023                                     def corefx_os = 'ubuntu14.04'
2024                                     if (os == 'Ubuntu16.04') {
2025                                         arm_abi = 'arm'
2026                                         corefx_os = 'ubuntu16.04'
2027                                     }
2028                                     else if (os == 'Tizen') {
2029                                         arm_abi = 'armel'
2030                                         corefx_os = 'tizen'
2031                                     }
2032
2033                                     // Let's use release CoreFX to test checked CoreCLR,
2034                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2035                                     def corefx_lowerConfiguration = lowerConfiguration
2036                                     if ( lowerConfiguration == 'checked' ) {
2037                                         corefx_lowerConfiguration='release'
2038                                     }
2039                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2040                                         includePatterns('bin/build.tar.gz')
2041                                         buildSelector {
2042                                             latestSuccessful(true)
2043                                         }
2044                                     }
2045                                 }
2046
2047                                 buildCommands.each { buildCommand ->
2048                                     shell(buildCommand)
2049                                 }
2050                             }
2051                         }
2052                     } // newJob.with
2053
2054                 } // os
2055             } // configuration
2056         } // architecture
2057     } // isPR
2058 } // scenario
2059
2060
2061 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2062 combinedScenarios.each { scenario ->
2063     [true, false].each { isPR ->
2064         // Architectures.  x64 only at this point
2065         ['x64'].each { architecture ->
2066             // Put the OS's supported for coreclr cross testing here
2067             Constants.crossList.each { os ->
2068                 Constants.configurationList.each { configuration ->
2069
2070                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2071                         if (configuration != 'Checked') {
2072                             return
2073                         }
2074                         if (isCorefxTesting(scenario)) {
2075                             return
2076                         }
2077                         //Skip stress modes for these scenarios
2078                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2079                             return
2080                         }
2081                     }
2082                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2083                     // and configuration is not Checked, bail out.
2084                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2085                         return;
2086                     }
2087                     // For CentOS, we only want Checked/Release pri1 builds.
2088                     else if (os == 'CentOS7.1') {
2089                         if (scenario != 'pri1' &&
2090                             scenario != 'r2r' &&
2091                             scenario != 'pri1r2r' &&
2092                             scenario != 'gcstress15_pri1r2r' &&
2093                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2094                             return
2095                         }
2096                         if (configuration != 'Checked' && configuration != 'Release') {
2097                             return
2098                         }
2099                     }
2100                     // For RedHat and Debian, we only do Release pri1 builds.
2101                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2102                         if (scenario != 'pri1') {
2103                             return
2104                         }
2105                         if (configuration != 'Release') {
2106                             return
2107                         }
2108                     }
2109                     else {
2110                         // Skip scenarios
2111                         switch (scenario) {
2112                             case 'pri1':
2113                                 // Nothing skipped
2114                                 break
2115                             case 'ilrt':
2116                                 // Release only
2117                                 if (configuration != 'Release') {
2118                                     return
2119                                 }
2120                                 break
2121                             case 'jitdiff':
2122                                 if (configuration != 'Checked') {
2123                                     return;
2124                                 }
2125                             case 'r2r':
2126                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2127                                 if (configuration != 'Checked' && configuration != 'Release') {
2128                                     return
2129                                 }
2130                                 break
2131                             case 'pri1r2r':
2132                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2133                                 if (configuration != 'Checked' && configuration != 'Release') {
2134                                     return
2135                                 }
2136                                 break
2137                             case 'gcstress15_pri1r2r':
2138                             case 'r2r_jitstress1':
2139                             case 'r2r_jitstress2':
2140                             case 'r2r_jitstressregs1':
2141                             case 'r2r_jitstressregs2':
2142                             case 'r2r_jitstressregs3':
2143                             case 'r2r_jitstressregs4':
2144                             case 'r2r_jitstressregs8':
2145                             case 'r2r_jitstressregs0x10':
2146                             case 'r2r_jitstressregs0x80':
2147                             case 'r2r_jitstressregs0x1000':
2148                             case 'r2r_jitminopts':
2149                             case 'r2r_jitforcerelocs':
2150                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2151                                 if (configuration != 'Checked' && configuration != 'Release') {
2152                                     return
2153                                 }
2154                                 break
2155                             case 'longgc':
2156                             case 'gcsimulator':
2157                                 // Long GC tests take a long time on non-Release builds
2158                                 if (configuration != 'Release') {
2159                                     return
2160                                 }
2161                                 break
2162                             case 'gc_reliability_framework':
2163                             case 'standalone_gc':
2164                                 if (configuration != 'Release' && configuration != 'Checked') {
2165                                     return
2166                                 }
2167                                 break
2168                             case 'coverage':
2169                                 //We only want Ubuntu Release for coverage
2170                                 if (os != 'Ubuntu') {
2171                                     return
2172                                 }
2173                                 if (configuration != 'Release') {
2174                                     return
2175                                 }
2176                             case 'formatting':
2177                                 return
2178                             case 'illink':
2179                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2180                                     return
2181                                 }
2182                                 break
2183                             case 'default':
2184                                 // Nothing skipped
2185                                 break
2186                             default:
2187                                 println("Unknown scenario: ${scenario}")
2188                                 assert false
2189                                 break
2190                         }
2191                     }
2192
2193                     def lowerConfiguration = configuration.toLowerCase()
2194                     def osGroup = getOSGroup(os)
2195                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2196
2197                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2198                     def inputScenario = 'default'
2199                     if (scenario == 'coverage') {
2200                         inputScenario = 'coverage'
2201                     }
2202                     def inputCoreCLRBuildName = projectFolder + '/' +
2203                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2204                     // If this is a stress scenario, there isn't any difference in the build job
2205                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2206                     // from the default scenario
2207                     def testBuildScenario = scenario
2208                     if (scenarioNeedsPri1Build(scenario)) {
2209                         testBuildScenario = 'pri1'
2210                     }
2211                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2212                         testBuildScenario = 'default'
2213                     }
2214                     def inputWindowTestsBuildName = ''
2215                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2216                         inputWindowTestsBuildName = projectFolder + '/' +
2217                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', 'default', true), isPR)
2218                     }
2219                     else {
2220                         inputWindowTestsBuildName = projectFolder + '/' +
2221                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', testBuildScenario, true), isPR)
2222                     }
2223                     // Enable Server GC for Ubuntu PR builds
2224                     def serverGCString = ''
2225
2226                     // Whether or not this test run should be run sequentially instead
2227                     // of in parallel. Only used for long GC tests.
2228                     def sequentialString = ''
2229
2230                     // Whether or not this test run should run a specific playlist.
2231                     // Only used for long GC tests.
2232
2233                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2234                     // for running long GC and GCSimulator tests, respectively. We don't use them
2235                     // here because using a playlist file produces much more readable output on the CI machines
2236                     // and reduces running time.
2237                     def playlistString = ''
2238
2239                     if (os == 'Ubuntu' && isPR){
2240                         serverGCString = '--useServerGC'
2241                     }
2242
2243                     // pass --crossgen to runtest.sh for crossgen builds
2244                     def crossgenStr = ''
2245                     def runcrossgentestsStr = ''
2246                     def runjitstressStr = ''
2247                     def runjitstressregsStr = ''
2248                     def runjitmioptsStr = ''
2249                     def runjitforcerelocsStr = ''
2250                     def runjitdisasmStr = ''
2251                     def runilasmroundtripStr = ''
2252                     def gcstressStr = ''
2253                     def illinkStr = ''
2254                     def layoutOnlyStr =''
2255
2256                     if (scenario == 'r2r' ||
2257                         scenario == 'pri1r2r' ||
2258                         scenario == 'gcstress15_pri1r2r' ||
2259                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2260                             crossgenStr = '--crossgen'
2261                             runcrossgentestsStr = '--runcrossgentests'
2262
2263                             if (scenario == 'r2r_jitstress1'){
2264                                 runjitstressStr = '--jitstress=1'
2265                             }
2266                             else if (scenario == 'r2r_jitstress2') {
2267                                 runjitstressStr = '--jitstress=2'
2268                             }
2269                             else if (scenario == 'r2r_jitstressregs1'){
2270                                 runjitstressregsStr = '--jitstressregs=1'
2271                             }
2272                             else if (scenario == 'r2r_jitstressregs2') {
2273                                 runjitstressregsStr = '--jitstressregs=2'
2274                             }
2275                             else if (scenario == 'r2r_jitstressregs3') {
2276                                 runjitstressregsStr = '--jitstressregs=3'
2277                             }
2278                             else if (scenario == 'r2r_jitstressregs4') {
2279                                 runjitstressregsStr = '--jitstressregs=4'
2280                             }
2281                             else if (scenario == 'r2r_jitstressregs8') {
2282                                 runjitstressregsStr = '--jitstressregs=8'
2283                             }
2284                             else if (scenario == 'r2r_jitstressregs0x10') {
2285                                 runjitstressregsStr = '--jitstressregs=0x10'
2286                             }
2287                             else if (scenario == 'r2r_jitstressregs0x80') {
2288                                 runjitstressregsStr = '--jitstressregs=0x80'
2289                             }
2290                             else if (scenario == 'r2r_jitstressregs0x1000') {
2291                                 runjitstressregsStr = '--jitstressregs=0x1000'
2292                             }
2293                             else if (scenario == 'r2r_jitminopts') {
2294                                 runjitmioptsStr = '--jitminopts'
2295                             }
2296                             else if (scenario == 'r2r_jitforcerelocs') {
2297                                 runjitforcerelocsStr = '--jitforcerelocs'
2298                             }
2299                     }
2300                     if  (scenario == 'gcstress15_pri1r2r')
2301                     {
2302                         gcstressStr = '--gcstresslevel=0xF'
2303                     }
2304
2305                     if (scenario == 'jitdiff')
2306                     {
2307                         runjitdisasmStr = '--jitdisasm --crossgen'
2308                     }
2309
2310                     if (scenario == 'illink')
2311                     {
2312                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2313                     }
2314
2315                     if (isLongGc(scenario)) {
2316                         // Long GC tests behave very poorly when they are not
2317                         // the only test running (many of them allocate until OOM).
2318                         sequentialString = '--sequential'
2319
2320                         // The Long GC playlist contains all of the tests that are
2321                         // going to be run. The GCSimulator playlist contains all of
2322                         // the GC simulator tests.
2323                         if (scenario == 'longgc') {
2324                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2325                         }
2326                         else if (scenario == 'gcsimulator') {
2327                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2328                         }
2329                     }
2330
2331                     if (isGcReliabilityFramework(scenario)) {
2332                         layoutOnlyStr = '--build-overlay-only'
2333                     }
2334
2335                     def folder = getJobFolder(scenario)
2336                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2337                         // Add parameters for the inputs
2338
2339                         parameters {
2340                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2341                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2342                         }
2343
2344                         steps {
2345                             // Set up the copies
2346
2347                             // Coreclr build containing the tests and mscorlib
2348
2349                             copyArtifacts(inputWindowTestsBuildName) {
2350                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2351                                 buildSelector {
2352                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2353                                 }
2354                             }
2355
2356                             if (scenario == 'coverage') {
2357
2358                                 // Move coreclr to clr directory
2359                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2360
2361                                 // Build coreclr
2362                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2363
2364                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2365                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2366                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2367                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2368                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2369                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2370                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2371                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2372
2373                                 // Run PAL tests
2374                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2375
2376                                 // Remove obj files for PAL tests so they're not included in coverage results
2377                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2378
2379                                 // Unzip the tests first.  Exit with 0
2380                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2381
2382                                 // Get corefx
2383                                 shell("git clone https://github.com/dotnet/corefx fx")
2384
2385                                 // Build Linux corefx
2386                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2387                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2388
2389                                 def testEnvOpt = ""
2390                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2391                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2392                                 shell("${createScriptCmds}")
2393                                 testEnvOpt = "--test-env=" + scriptFileName
2394
2395                                 // Run corefx tests
2396                                 shell("""./fx/run-test.sh \\
2397                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2398                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2399                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2400                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2401                 --configurationGroup Release""")
2402
2403
2404                                 // Run coreclr tests w/ workstation GC
2405                                 shell("""./clr/tests/runtest.sh \\
2406                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2407                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2408                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2409                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2410                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2411                 --crossgen --runcrossgentests""")
2412
2413                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2414                                 shell("""./clr/tests/runtest.sh \\
2415                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2416                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2417                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2418                 --useServerGC ${testEnvOpt}""")
2419
2420                                  // Run long-running coreclr GC tests & produce coverage reports
2421                                 shell("""./clr/tests/runtest.sh \\
2422                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2423                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2424                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2425                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2426                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2427                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2428                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2429
2430                             }
2431                             else {
2432
2433                                 // Coreclr build we are trying to test
2434
2435                                 copyArtifacts(inputCoreCLRBuildName) {
2436                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2437                                     buildSelector {
2438                                         buildNumber('${CORECLR_BUILD}')
2439                                     }
2440                                 }
2441
2442                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2443
2444                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2445                                 def osJobName
2446                                 if (os == 'Ubuntu') {
2447                                     osJobName = 'ubuntu14.04'
2448                                 }
2449                                 else {
2450                                     osJobName = os.toLowerCase()
2451                                 }
2452                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2453                                     includePatterns('bin/build.tar.gz')
2454                                     buildSelector {
2455                                         latestSuccessful(true)
2456                                     }
2457                                 }
2458
2459                                 shell ("mkdir ./bin/CoreFxBinDir")
2460                                 // Unpack the corefx binaries
2461                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2462
2463                                 // Unzip the tests first.  Exit with 0
2464                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2465
2466                                 // Execute the tests
2467                                 // If we are running a stress mode, we'll set those variables first
2468                                 def testEnvOpt = ""
2469                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2470                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2471                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2472                                     shell("${createScriptCmds}")
2473                                     testEnvOpt = "--test-env=" + scriptFileName
2474                                 }
2475
2476                                 if (isGCStressRelatedTesting(scenario)) {
2477                                     shell('./init-tools.sh')
2478                                 }
2479
2480                                 shell("""./tests/runtest.sh \\
2481                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2482                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2483                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2484                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2485                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2486                 --limitedDumpGeneration \\
2487                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
2488                 ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
2489                 ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
2490
2491                                 if (isGcReliabilityFramework(scenario)) {
2492                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2493                                     if (serverGCString != '') {
2494                                         shell("export COMPlus_gcServer=1")
2495                                     }
2496
2497                                     shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2498                                 }
2499                             }
2500                         }
2501                     }
2502
2503                     if (isGcReliabilityFramework(scenario))
2504                     {
2505                         // Both of these are emitted by the RF
2506                         Utilities.addArchival(newJob, "stdout.txt")
2507                         Utilities.addArchival(newJob, "Logs/**")
2508                     }
2509
2510                     if (scenario == 'coverage') {
2511                         // Publish coverage reports
2512                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2513                         // TODO: Add once external email sending is available again
2514                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2515                     }
2516
2517                     if (scenario == 'jitdiff') {
2518                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2519                     }
2520
2521                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2522                     if (os in ['Ubuntu']) {
2523                         SummaryBuilder summaries = new SummaryBuilder()
2524                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2525                         summaries.emit(newJob)
2526                     }
2527
2528                     setMachineAffinity(newJob, os, architecture)
2529                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2530                     // Set timeouts to 240.
2531                     setTestJobTimeOut(newJob, scenario)
2532                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2533
2534                     // Create a build flow to join together the build and tests required to run this
2535                     // test.
2536                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2537                     // Linux CoreCLR test
2538                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2539                     def fullTestJobName = projectFolder + '/' + newJob.name
2540                     // Add a reference to the input jobs for report purposes
2541                     JobReport.Report.addReference(inputCoreCLRBuildName)
2542                     JobReport.Report.addReference(inputWindowTestsBuildName)
2543                     JobReport.Report.addReference(fullTestJobName)
2544                     def newFlowJob;
2545
2546                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2547                     // so that coverage data can be preserved.
2548                     if (scenario == 'coverage') {
2549                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2550                         buildFlow("""
2551 // Build the input Windows job
2552 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2553
2554 // And then build the test build
2555 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2556 """)
2557                         }
2558                     // Normal jobs copy a Windows build & a non-Windows build
2559                     } else {
2560                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2561                         buildFlow("""
2562 // Build the input jobs in parallel
2563 parallel (
2564     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2565     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2566 )
2567
2568 // And then build the test build
2569 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2570                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2571 """)
2572                         }
2573                     }
2574
2575                     setMachineAffinity(newFlowJob, os, architecture)
2576                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2577                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2578                 } // configuration
2579             } // os
2580         } // architecture
2581     } // isPR
2582 } // scenario
2583
2584 JobReport.Report.generateJobReport(out)
2585
2586 // Make the call to generate the help job
2587 Utilities.createHelperJob(this, project, branch,
2588     "Welcome to the ${project} Repository",  // This is prepended to the help message
2589     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2590
2591 Utilities.addCROSSCheck(this, project, branch)