netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'FreeBSD':'FreeBSD',
  29         'CentOS7.1': 'Linux',
  30         'OpenSUSE42.1': 'Linux',
  31         'Tizen': 'Linux']
  32     def osGroup = osGroupMap.get(os, null)
  33     assert osGroup != null : "Could not find os group for ${os}"
  34     return osGroupMap[os]
  35 }
  36
  37 // We use this class (vs variables) so that the static functions can access data here.
  38 class Constants {
  39
  40     // Innerloop build OS's
  41     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  42     // test execution in the build flow runs.  It generates the exact same build
  43     // as Windows_NT but without the tests.
  44     def static osList = [
  45                'Ubuntu',
  46                'Debian8.4',
  47                'OSX10.12',
  48                'Windows_NT',
  49                'Windows_NT_BuildOnly',
  50                'FreeBSD',
  51                'CentOS7.1',
  52                'OpenSUSE42.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  67                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  68                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  69                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  70                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  71                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  72                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  73                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  74                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  75                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  76                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  77                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  78                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  79                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  80                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  81                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  82                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  83                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  84                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  85                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  86                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  87                'corefx_baseline'                : [ : ], // corefx baseline
  88                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  89                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  90                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  91                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  92                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  93                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  94                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  95                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  96                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  97                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  98                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  99                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 100                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 101                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 103                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 104                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 105                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 107                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 109                ]
 110
 111     // This is a set of r2r jit stress scenarios
 112     def static r2rJitStressScenarios = [
 113                'r2r_jitstress1',
 114                'r2r_jitstress2',
 115                'r2r_jitstressregs1',
 116                'r2r_jitstressregs2',
 117                'r2r_jitstressregs3',
 118                'r2r_jitstressregs4',
 119                'r2r_jitstressregs8',
 120                'r2r_jitstressregs0x10',
 121                'r2r_jitstressregs0x80',
 122                'r2r_jitstressregs0x1000',
 123                'r2r_jitminopts',
 124                'r2r_jitforcerelocs']
 125
 126     // This is the basic set of scenarios
 127     def static basicScenarios = [
 128                'default',
 129                'pri1',
 130                'ilrt',
 131                'r2r',
 132                'pri1r2r',
 133                'gcstress15_pri1r2r',
 134                'longgc',
 135                'coverage',
 136                'formatting',
 137                'gcsimulator',
 138                'jitdiff',
 139                'standalone_gc',
 140                'gc_reliability_framework',
 141                'illink'] + r2rJitStressScenarios
 142
 143     def static configurationList = ['Debug', 'Checked', 'Release']
 144
 145     // This is the set of architectures
 146     def static architectureList = ['arm', 'arm64', 'x64', 'x86', 'x86lb']
 147 }
 148
 149 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 150     if (architecture == 'arm64' && os == 'Windows_NT') {
 151         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 152     } else if (architecture == 'arm64' && os != 'Windows_NT' && options == null) {
 153         Utilities.setMachineAffinity(job, os, 'arm64-small-page-size');
 154     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['large_pages'] == true) {
 155         Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size');
 156     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['is_build_only'] == true) {
 157         Utilities.setMachineAffinity(job, os, 'arm64-cross-latest');
 158     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 159         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 160     } else {
 161         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 162     }
 163 }
 164
 165 def static isJITStressJob(def scenario) {
 166     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 167            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 168 }
 169
 170 def static isGCStressRelatedTesting(def scenario) {
 171     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 172     // Detect it and make it a GCStress related.
 173     if (scenario == 'gcstress15_pri1r2r')
 174     {
 175         return true;
 176     }
 177
 178     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 179     def scenarioName = scenario.toLowerCase()
 180     def isGCStressTesting = false
 181     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 182         if (k in gcStressTestEnvVars) {
 183             isGCStressTesting = true;
 184         }
 185     }
 186     return isGCStressTesting
 187 }
 188
 189 def static isCorefxTesting(def scenario) {
 190     def corefx_prefix = 'corefx_'
 191     if (scenario.length() < corefx_prefix.length()) {
 192         return false
 193     }
 194     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 195 }
 196
 197 def static isR2R(def scenario) {
 198     return (scenario == 'r2r' || scenario == 'pri1r2r')
 199 }
 200
 201 def static isCoverage(def scenario) {
 202     return (scenario == 'coverage')
 203 }
 204
 205 def static isLongGc(def scenario) {
 206     return (scenario == 'longgc' || scenario == 'gcsimulator')
 207 }
 208
 209 def static isJitDiff(def scenario) {
 210     return (scenario == 'jitdiff')
 211 }
 212
 213 def static isGcReliabilityFramework(def scenario) {
 214     return (scenario == 'gc_reliability_framework')
 215 }
 216
 217 def static scenarioNeedsPri1Build(def scenario) {
 218     return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario))
 219 }
 220
 221 def static setTestJobTimeOut(newJob, scenario) {
 222     if (isGCStressRelatedTesting(scenario)) {
 223         Utilities.setJobTimeout(newJob, 4320)
 224     }
 225     else if (isCorefxTesting(scenario)) {
 226         Utilities.setJobTimeout(newJob, 360)
 227     }
 228     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 229         Utilities.setJobTimeout(newJob, 240)
 230     }
 231     else if (isR2R(scenario)) {
 232         Utilities.setJobTimeout(newJob, 240)
 233     }
 234     else if (isCoverage(scenario)) {
 235         Utilities.setJobTimeout(newJob, 1440)
 236     }
 237     else if (isLongGc(scenario)) {
 238         Utilities.setJobTimeout(newJob, 1440)
 239     }
 240     else if (isJitDiff(scenario)) {
 241         Utilities.setJobTimeout(newJob, 240)
 242     }
 243     else if (isGcReliabilityFramework(scenario)) {
 244         Utilities.setJobTimeout(newJob, 1440)
 245     }
 246     // Non-test jobs use the default timeout value.
 247 }
 248
 249 def static getJobFolder(def scenario) {
 250     if (isJITStressJob(scenario)) {
 251         return 'jitstress'
 252     }
 253     if (scenario == 'illink') {
 254         return 'illink'
 255     }
 256     return ''
 257 }
 258
 259 def static getStressModeDisplayName(def scenario) {
 260     def displayStr = ''
 261     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 262         def prefixLength = 'COMPlus_'.length()
 263         if (k.length() >= prefixLength) {
 264             def modeName = k.substring(prefixLength, k.length())
 265             displayStr += ' ' + modeName + '=' + v
 266         }
 267     }
 268     return displayStr
 269 }
 270
 271 def static getR2RStressModeDisplayName(def scenario) {
 272     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 273     // "r2r_" prefix.
 274     def displayStr = scenario
 275     def prefixLength = 'r2r_'.length()
 276     if (displayStr.length() >= prefixLength) {
 277         displayStr = displayStr.substring(prefixLength, displayStr.length())
 278     }
 279     return displayStr
 280 }
 281
 282 // Generates the string for creating a file that sets environment variables
 283 // that makes it possible to run stress modes.  Writes the script to the file
 284 // specified by the stepScriptLocation parameter.
 285 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 286     def stepScript = ''
 287     if (os == 'Windows_NT') {
 288         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 289         stepScript += "del ${stepScriptLocation}\r\n"
 290
 291         // Timeout in ms, default is 10 minutes. For stress
 292         // modes up this to 30 minutes
 293         def timeout = 1800000
 294
 295         // Set the Timeout
 296         stepScript += "set __TestTimeout=${timeout}\r\n"
 297         stepScript += "echo. > ${stepScriptLocation}\r\n"
 298         stressModeVars.each{ k, v ->
 299             // Write out what we are writing to the script file
 300             stepScript += "echo Setting ${k}=${v}\r\n"
 301             // Write out the set itself to the script file`
 302             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 303         }
 304     }
 305     else {
 306         stepScript += "echo Setting variables for ${stressModeName}\n"
 307         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 308         stressModeVars.each{ k, v ->
 309             // Write out what we are writing to the script file
 310             stepScript += "echo Setting ${k}=${v}\n"
 311             // Write out the set itself to the script file`
 312             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 313         }
 314         stepScript += "chmod +x ${stepScriptLocation}\n"
 315     }
 316     return stepScript
 317 }
 318
 319 // Calculates the name of the build job based on some typical parameters.
 320 //
 321 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 322     // If the architecture is x64, do not add that info into the build name.
 323     // Need to change around some systems and other builds to pick up the right builds
 324     // to do that.
 325
 326     def suffix = scenario != 'default' ? "_${scenario}" : '';
 327     if (isBuildOnly) {
 328         suffix += '_bld'
 329     }
 330     def baseName = ''
 331     switch (architecture) {
 332         case 'x64':
 333             if (scenario == 'default') {
 334                 // For now we leave x64 off of the name for compatibility with other jobs
 335                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 336             }
 337             else if (scenario == 'formatting') {
 338                 // we don't care about the configuration for the formatting job. It runs all configs
 339                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 340             }
 341             else {
 342                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 343             }
 344             break
 345         case 'arm64':
 346             if (os.toLowerCase() == "windows_nt") {
 347                 // These are cross builds
 348                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 349             }
 350             else {
 351                 // Defaults to a small page size set of machines.
 352                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 353             }
 354             break
 355         case 'arm':
 356             // These are cross builds
 357             if (os == 'Tizen') {
 358                 // ABI: softfp
 359                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 360             }
 361             else {
 362                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 363             }
 364             break
 365         case 'x86':
 366         case 'x86lb':
 367             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 368             break
 369         default:
 370             println("Unknown architecture: ${architecture}");
 371             assert false
 372             break
 373     }
 374
 375     return baseName + suffix
 376 }
 377
 378 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 379     // Check scenario.
 380     switch (scenario) {
 381         case 'default':
 382             switch (architecture) {
 383                 case 'x64':
 384                 case 'x86':
 385                 case 'x86lb':
 386                     if (architecture == 'x86' && os == 'Ubuntu') {
 387                         Utilities.addPeriodicTrigger(job, '@daily')
 388                     }
 389                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 390                         Utilities.addGithubPushTrigger(job)
 391                     }
 392                     break
 393                 case 'arm':
 394                     Utilities.addGithubPushTrigger(job)
 395                     break
 396                 case 'arm64':
 397                     Utilities.addGithubPushTrigger(job)
 398                     break
 399                 default:
 400                     println("Unknown architecture: ${architecture}");
 401                     assert false
 402                     break
 403             }
 404             break
 405         case 'pri1':
 406             // Pri one gets a push trigger, and only for release
 407             if (architecture == 'x64') {
 408                 if (configuration == 'Release') {
 409                     // We expect release jobs to be Windows, or in the cross list
 410                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 411                     if (!os in bidailyCrossList) {
 412                         if (isFlowJob || os == 'Windows_NT') {
 413                             Utilities.addGithubPushTrigger(job)
 414                         }
 415                     }
 416                     else {
 417                         if (isFlowJob) {
 418                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 419                         }
 420                     }
 421                 }
 422             }
 423             break
 424         case 'r2r':
 425             //r2r jobs that aren't pri1 can only be triggered by phrase
 426             break
 427         case 'pri1r2r':
 428             assert !(os in bidailyCrossList)
 429             //pri1 r2r gets a push trigger for checked/release
 430             if (configuration == 'Checked' || configuration == 'Release') {
 431                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 432                 if (architecture == 'x64' && os != 'OSX10.12') {
 433                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 434                     if (isFlowJob || os == 'Windows_NT') {
 435                         Utilities.addGithubPushTrigger(job)
 436                     }
 437                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 438                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 439                     if (isFlowJob) {
 440                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 441                     }
 442                 }
 443                 // For x86, only add per-commit jobs for Windows
 444                 else if (architecture == 'x86' || architecture == 'x86lb') {
 445                     if (os == 'Windows_NT') {
 446                         Utilities.addGithubPushTrigger(job)
 447                     }
 448                 }
 449                 // arm64 pri1r2r jobs should only run every 12 hours.
 450                 else if (architecture == 'arm64') {
 451                     if (os == 'Windows_NT') {
 452                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 453                         // TODO: Add once external email sending is available again
 454                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 455                     }
 456                 }
 457             }
 458             break
 459         case 'r2r_jitstress1':
 460         case 'r2r_jitstress2':
 461         case 'r2r_jitstressregs1':
 462         case 'r2r_jitstressregs2':
 463         case 'r2r_jitstressregs3':
 464         case 'r2r_jitstressregs4':
 465         case 'r2r_jitstressregs8':
 466         case 'r2r_jitstressregs0x10':
 467         case 'r2r_jitstressregs0x80':
 468         case 'r2r_jitstressregs0x1000':
 469         case 'r2r_jitminopts':
 470         case 'r2r_jitforcerelocs':
 471         case 'gcstress15_pri1r2r':
 472             assert !(os in bidailyCrossList)
 473
 474             // GCStress=C is currently not supported on OS X
 475             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 476                 break
 477             }
 478
 479             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 480             if (configuration == 'Checked' || configuration == 'Release') {
 481                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 482                 if (architecture == 'x64') {
 483                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 484                     if (isFlowJob || os == 'Windows_NT') {
 485                         // Add a weekly periodic trigger
 486                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 487                     }
 488                 }
 489                 // For x86, only add per-commit jobs for Windows
 490                 else if (architecture == 'x86') {
 491                     if (os == 'Windows_NT') {
 492                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 493                     }
 494                 }
 495             }
 496             break
 497         case 'longgc':
 498             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 499             assert configuration == 'Release'
 500             assert architecture == 'x64'
 501             Utilities.addPeriodicTrigger(job, '@daily')
 502             // TODO: Add once external email sending is available again
 503             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 504             break
 505         case 'gcsimulator':
 506             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 507             assert configuration == 'Release'
 508             assert architecture == 'x64'
 509             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 510             // TODO: Add once external email sending is available again
 511             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 512             break
 513         case 'standalone_gc':
 514             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 515             assert (configuration == 'Release' || configuration == 'Checked')
 516             // TODO: Add once external email sending is available again
 517             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 518             Utilities.addPeriodicTrigger(job, '@weekly')
 519             break
 520         case 'gc_reliability_framework':
 521             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 522             assert (configuration == 'Release' || configuration == 'Checked')
 523             // Only triggered by phrase.
 524             break
 525         case 'ilrt':
 526             assert !(os in bidailyCrossList)
 527             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 528             if (architecture == 'x64' && configuration == 'Release') {
 529                 // We don't expect to see a job generated except in these scenarios
 530                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 531                 if (isFlowJob || os == 'Windows_NT') {
 532                     Utilities.addPeriodicTrigger(job, '@daily')
 533                 }
 534             }
 535             break
 536         case 'jitdiff':
 537             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 538             assert configuration == 'Checked'
 539             assert (architecture == 'x64' || architecture == 'x86')
 540             Utilities.addGithubPushTrigger(job)
 541             break
 542         case 'coverage':
 543             assert (os == 'Ubuntu' || os == 'Windows_NT')
 544             assert configuration == 'Release'
 545             assert architecture == 'x64'
 546             Utilities.addPeriodicTrigger(job, '@weekly')
 547             break
 548         case 'formatting':
 549             assert (os == 'Windows_NT' || os == "Ubuntu")
 550             assert architecture == 'x64'
 551             Utilities.addGithubPushTrigger(job)
 552             break
 553         case 'jitstressregs1':
 554         case 'jitstressregs2':
 555         case 'jitstressregs3':
 556         case 'jitstressregs4':
 557         case 'jitstressregs8':
 558         case 'jitstressregs0x10':
 559         case 'jitstressregs0x80':
 560         case 'jitstressregs0x1000':
 561         case 'minopts':
 562         case 'forcerelocs':
 563         case 'jitstress1':
 564         case 'jitstress2':
 565         case 'jitstress2_jitstressregs1':
 566         case 'jitstress2_jitstressregs2':
 567         case 'jitstress2_jitstressregs3':
 568         case 'jitstress2_jitstressregs4':
 569         case 'jitstress2_jitstressregs8':
 570         case 'jitstress2_jitstressregs0x10':
 571         case 'jitstress2_jitstressregs0x80':
 572         case 'jitstress2_jitstressregs0x1000':
 573         case 'tailcallstress':
 574         case 'jitsse2only':
 575         case 'corefx_baseline':
 576         case 'corefx_minopts':
 577         case 'corefx_jitstress1':
 578         case 'corefx_jitstress2':
 579         case 'corefx_jitstressregs1':
 580         case 'corefx_jitstressregs2':
 581         case 'corefx_jitstressregs3':
 582         case 'corefx_jitstressregs4':
 583         case 'corefx_jitstressregs8':
 584         case 'corefx_jitstressregs0x10':
 585         case 'corefx_jitstressregs0x80':
 586         case 'corefx_jitstressregs0x1000':
 587         case 'zapdisable':
 588             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 589             assert (os == 'Windows_NT') || (os in Constants.crossList)
 590             Utilities.addPeriodicTrigger(job, '@daily')
 591         }
 592         break
 593         case 'heapverify1':
 594         case 'gcstress0x3':
 595             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 596                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 597                 if (architecture == 'arm64') {
 598                     if (os == 'Windows_NT') {
 599                         Utilities.addPeriodicTrigger(job, '@daily')
 600                     }
 601                     // TODO: Add once external email sending is available again
 602                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 603                 }
 604                 else {
 605                     Utilities.addPeriodicTrigger(job, '@weekly')
 606                 }
 607             }
 608             break
 609         case 'gcstress0xc':
 610         case 'gcstress0xc_zapdisable':
 611         case 'gcstress0xc_zapdisable_jitstress2':
 612         case 'gcstress0xc_zapdisable_heapverify1':
 613         case 'gcstress0xc_jitstress1':
 614         case 'gcstress0xc_jitstress2':
 615         case 'gcstress0xc_minopts_heapverify1':
 616             // GCStress=C is currently not supported on OS X
 617             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 618                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 619                 if (architecture == 'arm64') {
 620                     // TODO: Enable a periodic trigger after tests are updated.
 621                     // Utilities.addPeriodicTrigger(job, '@daily')
 622                     // TODO: Add once external email sending is available again
 623                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 624                 }
 625                 else {
 626                     Utilities.addPeriodicTrigger(job, '@weekly')
 627                 }
 628             }
 629             break
 630
 631         case 'illink':
 632             // Testing on other operating systems TBD
 633             assert (os == 'Windows_NT' || os == 'Ubuntu')
 634             if (architecture == 'x64' || architecture == 'x86') {
 635                 if (configuration == 'Checked') {
 636                     Utilities.addPeriodicTrigger(job, '@daily')
 637                 }
 638             }
 639             break
 640
 641         default:
 642             println("Unknown scenario: ${scenario}");
 643             assert false
 644             break
 645     }
 646     return
 647 }
 648
 649 // **************************
 650 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 651 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 652 // builds will do a couple extra steps.
 653 // **************************
 654
 655 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 656 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 657 // it's just used for internal builds
 658 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 659 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 660     if (isWindowsBuildOnlyJob) {
 661         return
 662     }
 663
 664     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 665     // Non pull request builds.
 666     if (!isPR) {
 667         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 668         return
 669     }
 670     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 671     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 672     def osGroup = getOSGroup(os)
 673     switch (architecture) {
 674         case 'x64': // editor brace matching: {
 675             if (scenario == 'coverage') {
 676                 assert configuration == 'Release'
 677                 if (os == 'Ubuntu') {
 678                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 679                 }
 680                 break
 681             }
 682
 683             if (scenario == 'formatting') {
 684                 assert configuration == 'Checked'
 685                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 686                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 687                 }
 688                 break
 689             }
 690
 691             switch (os) {
 692                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 693                 case 'Debian8.4':
 694                 case 'RHEL7.2':
 695                     if (scenario == 'default') {
 696                         assert !isFlowJob
 697                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 698                     }
 699                     else if (scenario == 'pri1' && isFlowJob) {
 700                         assert (configuration == 'Release')
 701                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 702                     }
 703                     break
 704                 case 'Ubuntu16.04':
 705                     assert !isFlowJob
 706                     assert scenario == 'default'
 707                     // Distinguish with the other architectures (arm and x86)
 708                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 709                     break
 710                 case 'Fedora24':
 711                 case 'Ubuntu16.10':
 712                 case 'OpenSUSE42.1':
 713                     assert !isFlowJob
 714                     assert scenario == 'default'
 715                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 716                     break
 717                 case 'Ubuntu':
 718                     if (scenario == 'illink') {
 719                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 720                         break
 721                     }
 722                     // fall through
 723                 case 'OSX10.12':
 724                     // Triggers on the non-flow jobs aren't necessary here
 725                     // Corefx testing uses non-flow jobs.
 726                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 727                         break
 728                     }
 729                     switch (scenario) {
 730                         case 'default':
 731                             // Ubuntu uses checked for default PR tests
 732                             if (configuration == 'Checked') {
 733                                 // Default trigger
 734                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 735                             }
 736                             break
 737                         case 'pri1':
 738                             if (configuration == 'Release') {
 739                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 740                             }
 741                             break
 742                         case 'jitdiff':
 743                             if (configuration == 'Checked') {
 744                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 745                             }
 746                             break
 747                         case 'ilrt':
 748                             if (configuration == 'Release') {
 749                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 750                             }
 751                             break
 752                         case 'r2r':
 753                             if (configuration == 'Release' || configuration == 'Checked') {
 754                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 755                             }
 756                             break
 757                         case 'pri1r2r':
 758                             if (configuration == 'Release' || configuration == 'Checked') {
 759                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 760                             }
 761                             break
 762                         case 'gcstress15_pri1r2r':
 763                             if (configuration == 'Release' || configuration == 'Checked') {
 764                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 765                             }
 766                             break
 767                         case 'r2r_jitstress1':
 768                         case 'r2r_jitstress2':
 769                         case 'r2r_jitstressregs1':
 770                         case 'r2r_jitstressregs2':
 771                         case 'r2r_jitstressregs3':
 772                         case 'r2r_jitstressregs4':
 773                         case 'r2r_jitstressregs8':
 774                         case 'r2r_jitstressregs0x10':
 775                         case 'r2r_jitstressregs0x80':
 776                         case 'r2r_jitstressregs0x1000':
 777                         case 'r2r_jitminopts':
 778                         case 'r2r_jitforcerelocs':
 779                             if (configuration == 'Release' || configuration == 'Checked') {
 780                                 def displayStr = getR2RStressModeDisplayName(scenario)
 781                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 782                             }
 783                             break
 784                         case 'longgc':
 785                             if (configuration == 'Release') {
 786                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 787                             }
 788                             break
 789                         case 'gcsimulator':
 790                             if (configuration == 'Release') {
 791                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 792                             }
 793                             break
 794                         case 'standalone_gc':
 795                             if (configuration == 'Release' || configuration == 'Checked') {
 796                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 797                             }
 798                             break
 799                         case 'gc_reliability_framework':
 800                             if (configuration == 'Release' || configuration == 'Checked') {
 801                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 802                             }
 803                             break
 804                         case 'minopts':
 805                         case 'forcerelocs':
 806                         case 'jitstress1':
 807                         case 'jitstress2':
 808                         case 'jitstressregs1':
 809                         case 'jitstressregs2':
 810                         case 'jitstressregs3':
 811                         case 'jitstressregs4':
 812                         case 'jitstressregs8':
 813                         case 'jitstressregs0x10':
 814                         case 'jitstressregs0x80':
 815                         case 'jitstressregs0x1000':
 816                         case 'jitstress2_jitstressregs1':
 817                         case 'jitstress2_jitstressregs2':
 818                         case 'jitstress2_jitstressregs3':
 819                         case 'jitstress2_jitstressregs4':
 820                         case 'jitstress2_jitstressregs8':
 821                         case 'jitstress2_jitstressregs0x10':
 822                         case 'jitstress2_jitstressregs0x80':
 823                         case 'jitstress2_jitstressregs0x1000':
 824                         case 'tailcallstress':
 825                         case 'jitsse2only':
 826                         case 'gcstress0x3':
 827                         case 'gcstress0xc':
 828                         case 'zapdisable':
 829                         case 'heapverify1':
 830                         case 'gcstress0xc_zapdisable':
 831                         case 'gcstress0xc_zapdisable_jitstress2':
 832                         case 'gcstress0xc_zapdisable_heapverify1':
 833                         case 'gcstress0xc_jitstress1':
 834                         case 'gcstress0xc_jitstress2':
 835                         case 'gcstress0xc_minopts_heapverify1':
 836                             def displayStr = getStressModeDisplayName(scenario)
 837                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 838                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 839                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 840                             break
 841                         case 'corefx_baseline':
 842                         case 'corefx_minopts':
 843                         case 'corefx_jitstress1':
 844                         case 'corefx_jitstress2':
 845                         case 'corefx_jitstressregs1':
 846                         case 'corefx_jitstressregs2':
 847                         case 'corefx_jitstressregs3':
 848                         case 'corefx_jitstressregs4':
 849                         case 'corefx_jitstressregs8':
 850                         case 'corefx_jitstressregs0x10':
 851                         case 'corefx_jitstressregs0x80':
 852                         case 'corefx_jitstressregs0x1000':
 853                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 854                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 855                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 856                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 857                             break
 858                         default:
 859                             println("Unknown scenario: ${scenario}");
 860                             assert false
 861                             break
 862                     }
 863                     break
 864                 case 'CentOS7.1':
 865                     switch (scenario) {
 866                         case 'pri1':
 867                             if (configuration == 'Release') {
 868                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 869                             }
 870                             break
 871                         case 'r2r':
 872                             if (configuration == 'Checked' || configuration == 'Release') {
 873                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 874                             }
 875                             break
 876                         case 'pri1r2r':
 877                             if (configuration == 'Checked' || configuration == 'Release') {
 878                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 879                             }
 880                             break
 881                         case 'gcstress15_pri1r2r':
 882                             if (configuration == 'Release' || configuration == 'Checked') {
 883                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 884                             }
 885                             break
 886                         case 'r2r_jitstress1':
 887                         case 'r2r_jitstress2':
 888                         case 'r2r_jitstressregs1':
 889                         case 'r2r_jitstressregs2':
 890                         case 'r2r_jitstressregs3':
 891                         case 'r2r_jitstressregs4':
 892                         case 'r2r_jitstressregs8':
 893                         case 'r2r_jitstressregs0x10':
 894                         case 'r2r_jitstressregs0x80':
 895                         case 'r2r_jitstressregs0x1000':
 896                         case 'r2r_jitminopts':
 897                         case 'r2r_jitforcerelocs':
 898                             if (configuration == 'Release' || configuration == 'Checked') {
 899                                 def displayStr = getR2RStressModeDisplayName(scenario)
 900                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 901                             }
 902                             break
 903                         default:
 904                             break
 905                     }
 906                 case 'Windows_NT':
 907                     switch (scenario) {
 908                         case 'default':
 909                             // Default trigger
 910                             if (configuration == 'Debug') {
 911                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 912                             }
 913                             break
 914                         case 'pri1':
 915                             // Default trigger
 916                             if (configuration == 'Release') {
 917                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 918                             }
 919                             break
 920                         case 'jitdiff':
 921                             if (configuration == 'Checked') {
 922                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 923                             }
 924                             break
 925                         case 'ilrt':
 926                             if (configuration == 'Release') {
 927                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 928                             }
 929                             break
 930                         case 'r2r':
 931                             if (configuration == 'Checked' || configuration == 'Release') {
 932                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 933                             }
 934                             break
 935                         case 'pri1r2r':
 936                             if (configuration == 'Checked' || configuration == 'Release') {
 937                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 938                             }
 939                             break
 940                         case 'gcstress15_pri1r2r':
 941                             if (configuration == 'Release' || configuration == 'Checked') {
 942                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 943                             }
 944                             break
 945                         case 'r2r_jitstress1':
 946                         case 'r2r_jitstress2':
 947                         case 'r2r_jitstressregs1':
 948                         case 'r2r_jitstressregs2':
 949                         case 'r2r_jitstressregs3':
 950                         case 'r2r_jitstressregs4':
 951                         case 'r2r_jitstressregs8':
 952                         case 'r2r_jitstressregs0x10':
 953                         case 'r2r_jitstressregs0x80':
 954                         case 'r2r_jitstressregs0x1000':
 955                         case 'r2r_jitminopts':
 956                         case 'r2r_jitforcerelocs':
 957                             if (configuration == 'Release' || configuration == 'Checked') {
 958                                 def displayStr = getR2RStressModeDisplayName(scenario)
 959                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 960                             }
 961                             break
 962                         case 'longgc':
 963                             if (configuration == 'Release') {
 964                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 965                             }
 966                             break
 967                         case 'gcsimulator':
 968                             if (configuration == 'Release') {
 969                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 970                             }
 971                             break
 972                         case 'standalone_gc':
 973                             if (configuration == 'Release' || configuration == 'Checked') {
 974                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 975                             }
 976                             break
 977                         case 'gc_reliability_framework':
 978                             if (configuration == 'Release' || configuration == 'Checked') {
 979                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 980                             }
 981                             break
 982                         case 'minopts':
 983                         case 'forcerelocs':
 984                         case 'jitstress1':
 985                         case 'jitstress2':
 986                         case 'jitstressregs1':
 987                         case 'jitstressregs2':
 988                         case 'jitstressregs3':
 989                         case 'jitstressregs4':
 990                         case 'jitstressregs8':
 991                         case 'jitstressregs0x10':
 992                         case 'jitstressregs0x80':
 993                         case 'jitstressregs0x1000':
 994                         case 'jitstress2_jitstressregs1':
 995                         case 'jitstress2_jitstressregs2':
 996                         case 'jitstress2_jitstressregs3':
 997                         case 'jitstress2_jitstressregs4':
 998                         case 'jitstress2_jitstressregs8':
 999                         case 'jitstress2_jitstressregs0x10':
1000                         case 'jitstress2_jitstressregs0x80':
1001                         case 'jitstress2_jitstressregs0x1000':
1002                         case 'tailcallstress':
1003                         case 'jitsse2only':
1004                         case 'gcstress0x3':
1005                         case 'gcstress0xc':
1006                         case 'zapdisable':
1007                         case 'heapverify1':
1008                         case 'gcstress0xc_zapdisable':
1009                         case 'gcstress0xc_zapdisable_jitstress2':
1010                         case 'gcstress0xc_zapdisable_heapverify1':
1011                         case 'gcstress0xc_jitstress1':
1012                         case 'gcstress0xc_jitstress2':
1013                         case 'gcstress0xc_minopts_heapverify1':
1014                             def displayStr = getStressModeDisplayName(scenario)
1015                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1016                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1017                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1018                             break
1019                         case 'corefx_baseline':
1020                         case 'corefx_minopts':
1021                         case 'corefx_jitstress1':
1022                         case 'corefx_jitstress2':
1023                         case 'corefx_jitstressregs1':
1024                         case 'corefx_jitstressregs2':
1025                         case 'corefx_jitstressregs3':
1026                         case 'corefx_jitstressregs4':
1027                         case 'corefx_jitstressregs8':
1028                         case 'corefx_jitstressregs0x10':
1029                         case 'corefx_jitstressregs0x80':
1030                         case 'corefx_jitstressregs0x1000':
1031                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1032                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1033                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1034                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1035                             break
1036                         case 'illink':
1037                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1038                                     break
1039                         default:
1040                             println("Unknown scenario: ${scenario}");
1041                             assert false
1042                             break
1043                     }
1044                     break
1045                 case 'FreeBSD':
1046                     assert scenario == 'default'
1047                     if (configuration == 'Checked') {
1048                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1049                     }
1050                     break
1051                 default:
1052                     println("Unknown os: ${os}");
1053                     assert false
1054                     break
1055             }
1056             break
1057         // editor brace matching: }
1058         case 'arm': // editor brace matching: {
1059             assert scenario == 'default'
1060             switch (os) {
1061                 case 'Ubuntu':
1062                 case 'Ubuntu16.04':
1063                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1064                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1065                     }
1066                     else {
1067                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1068                     }
1069                     break;
1070                 case 'Tizen':
1071                     architecture='armel'
1072                     // Removing the regex will cause this to run on each PR.
1073                     if (configuration == 'Release' || configuration == 'Debug') {
1074                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1075                     }
1076                     else {
1077                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1078                     }
1079                     break;
1080                 case 'Windows_NT':
1081                     if (configuration == 'Debug' || configuration == 'Release')
1082                     {
1083                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1084                     }
1085                     break
1086                 default:
1087                     println("NYI os: ${os}");
1088                     assert false
1089                     break
1090             }
1091             break
1092         // editor brace matching: }
1093         case 'arm64': // editor brace matching: {
1094             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1095
1096             // Set up a private trigger
1097             def contextString = "${os} ${architecture} Cross ${configuration}"
1098             if (scenario != 'default')
1099                 contextString += " ${scenario}"
1100             contextString += " Build"
1101             // Debug builds only.
1102             if (configuration != 'Debug') {
1103                contextString += " and Test"
1104             }
1105
1106             def arm64Users = [
1107                 'adiaaida',
1108                 'AndyAyersMS',
1109                 'briansull',
1110                 'BruceForstall',
1111                 'CarolEidt',
1112                 'cmckinsey',
1113                 'erozenfeld',
1114                 'jashook',
1115                 'JosephTremoulet',
1116                 'pgavlin',
1117                 'russellhadley',
1118                 'RussKeldorph',
1119                 'sandreenko',
1120                 'sdmaclea',
1121                 'sivarv',
1122                 'swaroop-sridhar',
1123                 'gkhanna79',
1124                 'jkotas',
1125                 'markwilkie',
1126                 'rahku',
1127                 'ramarag',
1128                 'tzwlai',
1129                 'weshaggard'
1130             ]
1131
1132             switch (os) {
1133                 case 'Ubuntu':
1134                 case 'Ubuntu16.04':
1135                     switch (scenario) {
1136                         case 'pri1':
1137                             if (configuration == 'Release') {
1138                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1139                             }
1140                             break
1141                         case 'r2r':
1142                             if (configuration == 'Checked' || configuration == 'Release') {
1143                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1144                             }
1145                             break
1146                         case 'pri1r2r':
1147                             if (configuration == 'Checked' || configuration == 'Release') {
1148                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1149                             }
1150                             break
1151                         case 'gcstress15_pri1r2r':
1152                             if (configuration == 'Release' || configuration == 'Checked') {
1153                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1154                             }
1155                             break
1156                         case 'r2r_jitstress1':
1157                         case 'r2r_jitstress2':
1158                         case 'r2r_jitstressregs1':
1159                         case 'r2r_jitstressregs2':
1160                         case 'r2r_jitstressregs3':
1161                         case 'r2r_jitstressregs4':
1162                         case 'r2r_jitstressregs8':
1163                         case 'r2r_jitstressregs0x10':
1164                         case 'r2r_jitstressregs0x80':
1165                         case 'r2r_jitstressregs0x1000':
1166                         case 'r2r_jitminopts':
1167                         case 'r2r_jitforcerelocs':
1168                             if (configuration == 'Release' || configuration == 'Checked') {
1169                                 def displayStr = getR2RStressModeDisplayName(scenario)
1170                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1171                             }
1172                             break
1173                         default:
1174                             break
1175                     }
1176                 case 'Windows_NT':
1177                     switch (scenario) {
1178                         case 'default':
1179                             // For now only run Debug jobs on PR Trigger.
1180                             if (configuration != 'Debug') {
1181                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1182                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1183                             }
1184                             else {
1185                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1186                                 // are at higher risk of ARM64-breaking changes.
1187                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1188                             }
1189                             break
1190                         case 'pri1r2r':
1191                         case 'gcstress0x3':
1192                         case 'gcstress0xc':
1193                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1194                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1195                             break
1196                     }
1197                     break
1198                 default:
1199                     println("NYI os: ${os}");
1200                     assert false
1201                     break
1202             }
1203             break
1204         // editor brace matching: }
1205         case 'x86': // editor brace matching: {
1206             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1207             if (os == 'Ubuntu') {
1208                 // on-demand only for ubuntu x86
1209                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1210                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1211                 break
1212             }
1213             switch (scenario) {
1214                 case 'default':
1215                     if (configuration == 'Checked') {
1216                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1217                     }
1218                     else if (configuration == 'Release') {
1219                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1220                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1221                     }
1222                     break
1223                 case 'pri1':
1224                     if (configuration == 'Release') {
1225                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1226                     }
1227                     break
1228                 case 'ilrt':
1229                     if (configuration == 'Release') {
1230                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1231                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1232                     }
1233                     break
1234                 case 'r2r':
1235                     if (configuration == 'Checked' || configuration == 'Release') {
1236                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1237                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1238                     }
1239                     break
1240                 case 'pri1r2r':
1241                     if (configuration == 'Checked' || configuration == 'Release') {
1242                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1243                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1244                     }
1245                     break
1246                 case 'gcstress15_pri1r2r':
1247                     if (configuration == 'Release' || configuration == 'Checked') {
1248                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1249                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1250                     }
1251                     break
1252                 case 'r2r_jitstress1':
1253                 case 'r2r_jitstress2':
1254                 case 'r2r_jitstressregs1':
1255                 case 'r2r_jitstressregs2':
1256                 case 'r2r_jitstressregs3':
1257                 case 'r2r_jitstressregs4':
1258                 case 'r2r_jitstressregs8':
1259                 case 'r2r_jitstressregs0x10':
1260                 case 'r2r_jitstressregs0x80':
1261                 case 'r2r_jitstressregs0x1000':
1262                 case 'r2r_jitminopts':
1263                 case 'r2r_jitforcerelocs':
1264                     if (configuration == 'Release' || configuration == 'Checked') {
1265                         def displayStr = getR2RStressModeDisplayName(scenario)
1266                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1267                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1268                     }
1269                     break
1270                 case 'longgc':
1271                     if (configuration == 'Release') {
1272                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1273                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1274                     }
1275                     break
1276                 case 'gcsimulator':
1277                     if (configuration == 'Release') {
1278                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1279                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1280                     }
1281                     break
1282                 case 'standalone_gc':
1283                     if (configuration == 'Release' || configuration == 'Checked') {
1284                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1285                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1286                     }
1287                     break
1288                 case 'minopts':
1289                 case 'forcerelocs':
1290                 case 'jitstress1':
1291                 case 'jitstress2':
1292                 case 'jitstressregs1':
1293                 case 'jitstressregs2':
1294                 case 'jitstressregs3':
1295                 case 'jitstressregs4':
1296                 case 'jitstressregs8':
1297                 case 'jitstressregs0x10':
1298                 case 'jitstressregs0x80':
1299                 case 'jitstressregs0x1000':
1300                 case 'jitstress2_jitstressregs1':
1301                 case 'jitstress2_jitstressregs2':
1302                 case 'jitstress2_jitstressregs3':
1303                 case 'jitstress2_jitstressregs4':
1304                 case 'jitstress2_jitstressregs8':
1305                 case 'jitstress2_jitstressregs0x10':
1306                 case 'jitstress2_jitstressregs0x80':
1307                 case 'jitstress2_jitstressregs0x1000':
1308                 case 'tailcallstress':
1309                 case 'jitsse2only':
1310                 case 'gcstress0x3':
1311                 case 'gcstress0xc':
1312                 case 'zapdisable':
1313                 case 'heapverify1':
1314                 case 'gcstress0xc_zapdisable':
1315                 case 'gcstress0xc_zapdisable_jitstress2':
1316                 case 'gcstress0xc_zapdisable_heapverify1':
1317                 case 'gcstress0xc_jitstress1':
1318                 case 'gcstress0xc_jitstress2':
1319                 case 'gcstress0xc_minopts_heapverify1':
1320                     def displayStr = getStressModeDisplayName(scenario)
1321                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1322                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1323                     break
1324                 case 'corefx_baseline':
1325                 case 'corefx_minopts':
1326                 case 'corefx_jitstress1':
1327                 case 'corefx_jitstress2':
1328                 case 'corefx_jitstressregs1':
1329                 case 'corefx_jitstressregs2':
1330                 case 'corefx_jitstressregs3':
1331                 case 'corefx_jitstressregs4':
1332                 case 'corefx_jitstressregs8':
1333                 case 'corefx_jitstressregs0x10':
1334                 case 'corefx_jitstressregs0x80':
1335                 case 'corefx_jitstressregs0x1000':
1336                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1337                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1338                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1339                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1340                     break
1341                 case 'illink':
1342                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1343                     break
1344                 default:
1345                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1346                     assert false
1347                     break
1348             }
1349             break
1350          // editor brace matching: }
1351         case 'x86lb': // editor brace matching: {
1352             assert (os == 'Windows_NT')
1353             assert (scenario == 'default' || Constants.r2rJitStressScenarios.indexOf(scenario) !=1)
1354
1355             def arch = 'x86'
1356             def jit = 'legacy_backend'
1357             switch (scenario) {
1358                 case 'default':
1359                     if (configuration == 'Checked') {
1360                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1361                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1362                     }
1363                     break
1364                 default:
1365                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1366                     assert false
1367                     break
1368             }
1369             break
1370         // editor brace matching: }
1371         default:
1372             println("Unknown architecture: ${architecture}");
1373             assert false
1374             break
1375     }
1376 }
1377
1378 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1379     def buildCommands = [];
1380     def osGroup = getOSGroup(os)
1381     def lowerConfiguration = configuration.toLowerCase()
1382
1383     // Calculate the build steps, archival, and xunit results
1384     switch (os) {
1385         case 'Windows_NT': // editor brace matching: {
1386             switch (architecture) {
1387                 case 'x64':
1388                 case 'x86':
1389                 case 'x86lb':
1390                     def arch = architecture
1391                     def buildOpts = ''
1392                     if (architecture == 'x86lb') {
1393                         arch = 'x86'
1394                     }
1395
1396                     if (scenario == 'illink') {
1397                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1398                     }
1399
1400                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1401                             scenario == 'default' ||
1402                             scenario == 'r2r' ||
1403                             scenario == 'jitdiff' ||
1404                             scenario == 'ilrt' ||
1405                             scenario == 'illink' ||
1406                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1407                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1408                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1409                     }
1410
1411                     // For Pri 1 tests, we must shorten the output test binary path names.
1412                     // if __TestIntermediateDir is already set, build-test.cmd will
1413                     // output test binaries to that directory. If it is not set, the
1414                     // binaries are sent to a default directory whose name is about
1415                     // 35 characters long.
1416
1417                     else if (scenarioNeedsPri1Build(scenario)) {
1418                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1419                     }
1420                     else if (isLongGc(scenario)) {
1421                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1422                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1423                     }
1424                     else if (scenario == 'standalone_gc') {
1425                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1426                     }
1427                     else if (scenario == 'formatting') {
1428                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1429                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1430                         break
1431                     }
1432                     else {
1433                         println("Unknown scenario: ${scenario}")
1434                         assert false
1435                     }
1436
1437                     // If we are running a stress mode, we should write out the set of key
1438                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1439
1440                     if (!isBuildOnly) {
1441                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1442                         def crossgenStr = ''
1443                         def runcrossgentestsStr = ''
1444                         def runjitstressStr = ''
1445                         def runjitstressregsStr = ''
1446                         def runjitmioptsStr = ''
1447                         def runjitforcerelocsStr = ''
1448                         def runjitdisasmStr = ''
1449                         def runilasmroundtripStr = ''
1450                         def gcstressStr = ''
1451                         def runtestArguments = ''
1452                         def gcTestArguments = ''
1453                         def illinkArguments = ''
1454
1455                         if (scenario == 'r2r' ||
1456                             scenario == 'pri1r2r' ||
1457                             scenario == 'gcstress15_pri1r2r' ||
1458                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1459                                 crossgenStr = 'crossgen'
1460                                 runcrossgentestsStr = 'runcrossgentests'
1461
1462                                 if (scenario == 'r2r_jitstress1'){
1463                                     runjitstressStr = 'jitstress 1'
1464                                 }
1465                                 else if (scenario == 'r2r_jitstress2') {
1466                                     runjitstressStr = 'jitstress 2'
1467                                 }
1468                                 else if (scenario == 'r2r_jitstressregs1'){
1469                                     runjitstressregsStr = 'jitstressregs 1'
1470                                 }
1471                                 else if (scenario == 'r2r_jitstressregs2') {
1472                                     runjitstressregsStr = 'jitstressregs 2'
1473                                 }
1474                                 else if (scenario == 'r2r_jitstressregs3') {
1475                                     runjitstressregsStr = 'jitstressregs 3'
1476                                 }
1477                                 else if (scenario == 'r2r_jitstressregs4') {
1478                                     runjitstressregsStr = 'jitstressregs 4'
1479                                 }
1480                                 else if (scenario == 'r2r_jitstressregs8') {
1481                                     runjitstressregsStr = 'jitstressregs 8'
1482                                 }
1483                                 else if (scenario == 'r2r_jitstressregs0x10') {
1484                                     runjitstressregsStr = 'jitstressregs 0x10'
1485                                 }
1486                                 else if (scenario == 'r2r_jitstressregs0x80') {
1487                                     runjitstressregsStr = 'jitstressregs 0x80'
1488                                 }
1489                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1490                                     runjitstressregsStr = 'jitstressregs 0x1000'
1491                                 }
1492                                 else if (scenario == 'r2r_jitminopts') {
1493                                     runjitmioptsStr = 'jitminopts'
1494                                 }
1495                                 else if (scenario == 'r2r_jitforcerelocs') {
1496                                     runjitforcerelocsStr = 'jitforcerelocs'
1497                                 }
1498                         }
1499                         if (scenario == 'gcstress15_pri1r2r')
1500                         {
1501                             gcstressStr = 'gcstresslevel 0xF'
1502                         }
1503
1504                         if (scenario == 'jitdiff')
1505                         {
1506                             runjitdisasmStr = 'jitdisasm crossgen'
1507                         }
1508
1509                         if (scenario == 'ilrt')
1510                         {
1511                             runilasmroundtripStr = 'ilasmroundtrip'
1512                         }
1513
1514                         if (isLongGc(scenario)) {
1515                             gcTestArguments = "${scenario} sequential"
1516                         }
1517
1518                         if (scenario == 'illink')
1519                         {
1520                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1521                         }
1522
1523                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1524
1525                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1526                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1527                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1528
1529                             if (enableCorefxTesting) {
1530                                 def workspaceRelativeFxRoot = "_/fx"
1531                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1532
1533                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1534
1535                                 setTestJobTimeOut(newJob, scenario)
1536
1537                                 // Archive and process (only) the test results
1538                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1539                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1540                             }
1541                             else {
1542                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1543                             }
1544                         }
1545                         else if (isGcReliabilityFramework(scenario)) {
1546                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1547                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1548                         }
1549                         else if (architecture == 'x64' || architecture == 'x86') {
1550                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1551                         }
1552                         else if (architecture == 'x86lb') {
1553                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd"
1554                         }
1555                     }
1556
1557                     if (!enableCorefxTesting) {
1558                         // Run the rest of the build
1559                         // Build the mscorlib for the other OS's
1560                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1561                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1562                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1563
1564                         if (arch == "x64") {
1565                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1566                         }
1567
1568                         // Zip up the tests directory so that we don't use so much space/time copying
1569                         // 10s of thousands of files around.
1570                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1571
1572                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1573                             // For windows, pull full test results and test drops for x86/x64.
1574                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1575                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1576                         }
1577
1578                         if (scenario == 'jitdiff') {
1579                             // retrive jit-dasm output for base commit, and run jit-diff
1580                             if (!isBuildOnly) {
1581                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1582                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1583                             }
1584                         }
1585
1586                         if (!isBuildOnly) {
1587                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1588                             setTestJobTimeOut(newJob, scenario)
1589                         }
1590                     }
1591                     break
1592                 case 'arm':
1593                     assert (scenario == 'default')
1594
1595                     // Set time out
1596                     setTestJobTimeOut(newJob, scenario)
1597
1598                     if ( lowerConfiguration == "debug" ) {
1599                         // For Debug builds, we will do a P1 test build
1600                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1601                     }
1602                     else {
1603                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1604                     }
1605                     // Add archival.
1606                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1607                     break
1608                 case 'arm64':
1609                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1610
1611                     // Set time out
1612                     setTestJobTimeOut(newJob, scenario)
1613
1614                     // Debug runs take too long to run. So build job only.
1615                     if (lowerConfiguration == "debug") {
1616                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1617                     }
1618                     else {
1619                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1620                        {
1621                            // Up the timeout for arm64 checked testing only.
1622                            // Keep the longer timeout for gcstress.
1623                            Utilities.setJobTimeout(newJob, 240)
1624                        }
1625
1626                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=1"
1627                        // Test build and run are launched together.
1628                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1629                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1630                     }
1631
1632                     // Add archival.
1633                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1634                     break
1635                 default:
1636                     println("Unknown architecture: ${architecture}");
1637                     assert false
1638                     break
1639             }
1640             break
1641         // editor brace matching: }
1642         case 'Ubuntu':
1643         case 'Ubuntu16.04':
1644         case 'Ubuntu16.10':
1645         case 'Debian8.4':
1646         case 'OSX10.12':
1647         case 'FreeBSD':
1648         case 'CentOS7.1':
1649         case 'RHEL7.2':
1650         case 'OpenSUSE42.1':
1651         case 'Tizen':
1652         case 'Fedora24': // editor brace matching: {
1653             switch (architecture) {
1654                 case 'x64':
1655                 case 'x86':
1656                     if (architecture == 'x86' && os == 'Ubuntu') {
1657                         // build and PAL test
1658                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1659                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1660                         break;
1661                     }
1662
1663                     if (scenario == 'formatting') {
1664                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1665                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1666                         break
1667                     }
1668
1669                     if (scenario == 'illink') {
1670                         assert(os == 'Ubuntu')
1671                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1672                     }
1673
1674                     def standaloneGc = ''
1675                     if (scenario == 'standalone_gc') {
1676                         standaloneGc = 'buildstandalonegc'
1677                     }
1678
1679                     if (!enableCorefxTesting) {
1680                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1681                         // only on supported OS platforms.
1682                         if (os == 'FreeBSD')
1683                         {
1684                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1685                         }
1686                         else
1687                         {
1688                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1689                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1690                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1691                         }
1692                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1693
1694                         // Set time out
1695                         setTestJobTimeOut(newJob, scenario)
1696                         // Basic archiving of the build
1697                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1698                         // And pal tests
1699                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1700                     }
1701                     else {
1702                         // Corefx stress testing
1703                         assert os == 'Ubuntu'
1704                         assert architecture == 'x64'
1705                         assert lowerConfiguration == 'checked'
1706                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1707
1708                         // Build coreclr
1709                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1710
1711                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1712                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1713
1714                         // Build and text corefx
1715                         def workspaceRelativeFxRoot = "_/fx"
1716                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1717
1718                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1719
1720                         setTestJobTimeOut(newJob, scenario)
1721
1722                         // Archive and process (only) the test results
1723                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1724                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1725                     }
1726                     break
1727                 case 'arm64':
1728                     def standaloneGc = ''
1729                     if (scenario == 'standalone_gc') {
1730                         standaloneGc = 'buildstandalonegc'
1731                     }
1732
1733                     if (!enableCorefxTesting) {
1734                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} cross clang3.8 skipnuget ${standaloneGc}"
1735
1736                         // HACK -- Arm64 does not have corefx jobs yet.
1737                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1738                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1739                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1740                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1741
1742                         // Set time out
1743                         setTestJobTimeOut(newJob, scenario)
1744                         // Basic archiving of the build
1745                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1746                     }
1747                     break
1748                 case 'arm':
1749                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1750                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1751
1752                     // default values for Ubuntu
1753                     def arm_abi="arm"
1754                     def linuxCodeName="trusty"
1755                     if (os == 'Ubuntu16.04') {
1756                         linuxCodeName="xenial"
1757                     }
1758                     else if (os == 'Tizen') {
1759                         arm_abi="armel"
1760                         linuxCodeName="tizen"
1761                     }
1762
1763                     // Unzip the Windows test binaries first. Exit with 0
1764                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1765
1766                     // Unpack the corefx binaries
1767                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1768                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1769                     if (os != 'Tizen') {
1770                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1771                     }
1772                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1773                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1774                     if (os == 'Ubuntu') {
1775                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1776                     }
1777
1778                     // Call the ARM CI script to cross build and test using docker
1779                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1780                     --mode=docker \\
1781                     --${arm_abi} \\
1782                     --linuxCodeName=${linuxCodeName} \\
1783                     --buildConfig=${lowerConfiguration} \\
1784                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1785                     --coreFxBinDir=./bin/CoreFxBinDir \\
1786                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1787
1788                     // Basic archiving of the build, no pal tests
1789                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1790                     break
1791                 default:
1792                     println("Unknown architecture: ${architecture}");
1793                     assert false
1794                     break
1795             }
1796             break
1797         // editor brace matching: }
1798         default:
1799             println("Unknown os: ${os}");
1800             assert false
1801             break
1802     } // os
1803
1804     return buildCommands
1805 }
1806
1807 // Additional scenario which can alter behavior
1808
1809 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1810 combinedScenarios.each { scenario ->
1811     [true, false].each { isPR ->
1812         Constants.architectureList.each { architecture ->
1813             Constants.configurationList.each { configuration ->
1814                 Constants.osList.each { os ->
1815                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1816                     // and reset the os to Windows_NT
1817                     def isBuildOnly = false
1818                     if (os == 'Windows_NT_BuildOnly') {
1819                         isBuildOnly = true
1820                         os = 'Windows_NT'
1821                     }
1822
1823                     // WinArm32 is only built for Debug and Release
1824                     if (os == 'Windows_NT' && architecture == 'arm')
1825                     {
1826                         if (configuration == 'Checked')
1827                         {
1828                             return
1829                         }
1830                     }
1831                     // Tizen is only supported for arm architecture
1832                     if (os == 'Tizen' && architecture != 'arm') {
1833                         return
1834                     }
1835
1836                     // Skip totally unimplemented (in CI) configurations.
1837                     switch (architecture) {
1838                         case 'arm64':
1839                             if (os == 'Ubuntu16.04') {
1840                                 os = 'Ubuntu'
1841                             }
1842
1843                             // Windows and Ubuntu only
1844                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1845                                 return
1846                             }
1847                             break
1848                         case 'arm':
1849                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1850                                 return
1851                             }
1852                             break
1853                         case 'x86':
1854                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1855                                 return
1856                             }
1857                             break
1858                         case 'x86lb':
1859                             if (os != 'Windows_NT') {
1860                                 return
1861                             }
1862                             break
1863                         case 'x64':
1864                             // Everything implemented
1865                             break
1866                         default:
1867                             println("Unknown architecture: ${architecture}")
1868                             assert false
1869                             break
1870                     }
1871
1872                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1873                     // with checked builds
1874                     def enableCorefxTesting = false
1875                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1876                         if (configuration != 'Checked') {
1877                             return
1878                         }
1879
1880                         enableCorefxTesting = isCorefxTesting(scenario)
1881
1882                         // Since these are just execution time differences,
1883                         // skip platforms that don't execute the tests here (Windows_NT only)
1884                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1885                         if (!isEnabledOS || isBuildOnly) {
1886                             return
1887                         }
1888
1889                         switch (architecture) {
1890                             case 'arm64':
1891                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1892                                     return
1893                                 }
1894                                 break
1895                             case 'x64':
1896                             case 'x86':
1897                                 // x86 ubuntu: default only
1898                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1899                                     return
1900                                 }
1901                                 // Windows: Everything implemented
1902                                 break
1903                             case 'x86lb':
1904                                 // No stress modes for legacy jit.
1905                                 // (There's no technical reason we couldn't allow these.)
1906                                 return
1907                             default:
1908                                 return
1909                         }
1910                     }
1911                     else {
1912                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
1913                         // and configuration is not Checked, bail out.
1914                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1915                             return;
1916                         }
1917
1918                         // Skip scenarios
1919                         switch (scenario) {
1920                             case 'pri1':
1921                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1922                                 // the default scenario build
1923                                 if (os != 'Windows_NT') {
1924                                     return
1925                                 }
1926                                 // Only x64 for now
1927                                 if (architecture != 'x64') {
1928                                     return
1929                                 }
1930                                 break
1931                             case 'ilrt':
1932                                 // The ilrt build isn't necessary except for Windows_NT.  Non-Windows NT uses
1933                                 // the default scenario build
1934                                 if (os != 'Windows_NT') {
1935                                     return
1936                                 }
1937                                 // Only x64 for now
1938                                 if (architecture != 'x64') {
1939                                     return
1940                                 }
1941                                 // Release only
1942                                 if (configuration != 'Release') {
1943                                     return
1944                                 }
1945                                 break
1946                             case 'jitdiff':
1947                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1948                                     return
1949                                 }
1950                                 if (architecture != 'x64') {
1951                                     return
1952                                 }
1953                                 if (configuration != 'Checked') {
1954                                     return
1955                                 }
1956                                 break
1957                             case 'r2r':
1958                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1959                                 // the default scenario build
1960                                 if (os != 'Windows_NT') {
1961                                     return
1962                                 }
1963                                 if (architecture != 'x64') {
1964                                     return
1965                                 }
1966                                 break
1967                             case 'pri1r2r':
1968                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1969                                 // the default scenario build
1970                                 if (os != 'Windows_NT') {
1971                                     return
1972                                 }
1973                                 if (architecture != 'x64') {
1974                                     if (architecture != 'arm64' || configuration == 'Debug') {
1975                                         return
1976                                     }
1977                                 }
1978                                 break
1979                             case 'gcstress15_pri1r2r':
1980                             case 'r2r_jitstress1':
1981                             case 'r2r_jitstress2':
1982                             case 'r2r_jitstressregs1':
1983                             case 'r2r_jitstressregs2':
1984                             case 'r2r_jitstressregs3':
1985                             case 'r2r_jitstressregs4':
1986                             case 'r2r_jitstressregs8':
1987                             case 'r2r_jitstressregs0x10':
1988                             case 'r2r_jitstressregs0x80':
1989                             case 'r2r_jitstressregs0x1000':
1990                             case 'r2r_jitminopts':
1991                             case 'r2r_jitforcerelocs':
1992                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
1993                                 // the default scenario build
1994                                 if (os != 'Windows_NT') {
1995                                     return
1996                                 }
1997                                 if (architecture != 'x64' && architecture != 'x86') {
1998                                     return
1999                                 }
2000                                 break
2001                             case 'longgc':
2002                             case 'gcsimulator':
2003                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2004                                     return
2005                                 }
2006                                 if (architecture != 'x64') {
2007                                     return
2008                                 }
2009                                 if (configuration != 'Release') {
2010                                     return
2011                                 }
2012                                 break
2013                             case 'gc_reliability_framework':
2014                             case 'standalone_gc':
2015                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2016                                     return
2017                                 }
2018
2019                                 if (architecture != 'x64') {
2020                                     return
2021                                 }
2022
2023                                 if (configuration != 'Release' && configuration != 'Checked') {
2024                                     return
2025                                 }
2026                                 break
2027                             // We need Windows x64 Release bits for the code coverage build
2028                             case 'coverage':
2029                                 if (os != 'Windows_NT') {
2030                                     return
2031                                 }
2032                                 if (architecture != 'x64') {
2033                                     return
2034                                 }
2035                                 if (configuration != 'Release') {
2036                                     return
2037                                 }
2038                                 break
2039                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2040                             case 'formatting':
2041                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2042                                     return
2043                                 }
2044                                 if (architecture != 'x64') {
2045                                     return
2046                                 }
2047                                 if (configuration != 'Checked') {
2048                                     return
2049                                 }
2050                                 if (isBuildOnly) {
2051                                     return
2052                                 }
2053                                 break
2054                             case 'illink':
2055                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2056                                     return
2057                                 }
2058                                 if (architecture != 'x64' && architecture != 'x86') {
2059                                     return
2060                                 }
2061                                 if (isBuildOnly) {
2062                                     return
2063                                 }
2064                                 break
2065                             case 'default':
2066                                 // Nothing skipped
2067                                 break
2068                             default:
2069                                 println("Unknown scenario: ${scenario}")
2070                                 assert false
2071                                 break
2072                         }
2073                     }
2074
2075                     // Calculate names
2076                     def lowerConfiguration = configuration.toLowerCase()
2077                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2078                     def folderName = getJobFolder(scenario)
2079
2080                     // Create the new job
2081                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2082
2083                     def machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2084                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2085
2086                     // Add all the standard options
2087                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2088                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2089
2090                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
2091                     def osGroup = getOSGroup(os)
2092
2093                     newJob.with {
2094                         steps {
2095                             if (os == 'Windows_NT') {
2096                                 buildCommands.each { buildCommand ->
2097                                     batchFile(buildCommand)
2098                                 }
2099                             }
2100                             else {
2101                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2102                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2103                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2104                                     // Define the Windows Tests and Corefx build job names
2105                                     def WindowTestsName = projectFolder + '/' +
2106                                                           Utilities.getFullJobName(project,
2107                                                                                    getJobName(lowerConfiguration,
2108                                                                                               'x64' ,
2109                                                                                               'windows_nt',
2110                                                                                               'default',
2111                                                                                               true),
2112                                                                                    false)
2113                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2114                                                        Utilities.getFolderName(branch)
2115
2116                                     // Copy the Windows test binaries and the Corefx build binaries
2117                                     copyArtifacts(WindowTestsName) {
2118                                         includePatterns('bin/tests/tests.zip')
2119                                         buildSelector {
2120                                             latestSuccessful(true)
2121                                         }
2122                                     }
2123
2124                                     def arm_abi = 'arm'
2125                                     def corefx_os = 'linux'
2126                                     if (os == 'Tizen') {
2127                                         arm_abi = 'armel'
2128                                         corefx_os = 'tizen'
2129                                     }
2130
2131                                     // Let's use release CoreFX to test checked CoreCLR,
2132                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2133                                     def corefx_lowerConfiguration = lowerConfiguration
2134                                     if ( lowerConfiguration == 'checked' ) {
2135                                         corefx_lowerConfiguration='release'
2136                                     }
2137
2138                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2139                                         includePatterns('bin/build.tar.gz')
2140                                         buildSelector {
2141                                             latestSuccessful(true)
2142                                         }
2143                                     }
2144                                 }
2145
2146                                 buildCommands.each { buildCommand ->
2147                                     shell(buildCommand)
2148                                 }
2149                             }
2150                         }
2151                     } // newJob.with
2152
2153                 } // os
2154             } // configuration
2155         } // architecture
2156     } // isPR
2157 } // scenario
2158
2159
2160 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2161 combinedScenarios.each { scenario ->
2162     [true, false].each { isPR ->
2163         // Architectures.  x64 only at this point
2164         ['x64', 'arm64'].each { architecture ->
2165             // Put the OS's supported for coreclr cross testing here
2166             Constants.crossList.each { os ->
2167                 if (architecture == 'arm64') {
2168                     if (os != "Ubuntu") {
2169                         return
2170                     }
2171                 }
2172
2173                 Constants.configurationList.each { configuration ->
2174
2175                     if (architecture == 'arm64') {
2176                         if (scenario != 'default' && scenario != 'pri1r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2177                             return
2178                         }
2179                     }
2180
2181                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2182                         if (configuration != 'Checked') {
2183                             return
2184                         }
2185                         if (isCorefxTesting(scenario)) {
2186                             return
2187                         }
2188                         //Skip stress modes for these scenarios
2189                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2190                             return
2191                         }
2192                     }
2193                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2194                     // and configuration is not Checked, bail out.
2195                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2196                         return;
2197                     }
2198                     // For CentOS, we only want Checked/Release pri1 builds.
2199                     else if (os == 'CentOS7.1') {
2200                         if (scenario != 'pri1' &&
2201                             scenario != 'r2r' &&
2202                             scenario != 'pri1r2r' &&
2203                             scenario != 'gcstress15_pri1r2r' &&
2204                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2205                             return
2206                         }
2207                         if (configuration != 'Checked' && configuration != 'Release') {
2208                             return
2209                         }
2210                     }
2211                     // For RedHat and Debian, we only do Release pri1 builds.
2212                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2213                         if (scenario != 'pri1') {
2214                             return
2215                         }
2216                         if (configuration != 'Release') {
2217                             return
2218                         }
2219                     }
2220                     else {
2221                         // Skip scenarios
2222                         switch (scenario) {
2223                             case 'pri1':
2224                                 // Nothing skipped
2225                                 break
2226                             case 'ilrt':
2227                                 // Release only
2228                                 if (configuration != 'Release') {
2229                                     return
2230                                 }
2231                                 break
2232                             case 'jitdiff':
2233                                 if (configuration != 'Checked') {
2234                                     return;
2235                                 }
2236                             case 'r2r':
2237                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2238                                 if (configuration != 'Checked' && configuration != 'Release') {
2239                                     return
2240                                 }
2241                                 break
2242                             case 'pri1r2r':
2243                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2244                                 if (configuration != 'Checked' && configuration != 'Release') {
2245                                     return
2246                                 }
2247                                 break
2248                             case 'gcstress15_pri1r2r':
2249                             case 'r2r_jitstress1':
2250                             case 'r2r_jitstress2':
2251                             case 'r2r_jitstressregs1':
2252                             case 'r2r_jitstressregs2':
2253                             case 'r2r_jitstressregs3':
2254                             case 'r2r_jitstressregs4':
2255                             case 'r2r_jitstressregs8':
2256                             case 'r2r_jitstressregs0x10':
2257                             case 'r2r_jitstressregs0x80':
2258                             case 'r2r_jitstressregs0x1000':
2259                             case 'r2r_jitminopts':
2260                             case 'r2r_jitforcerelocs':
2261                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2262                                 if (configuration != 'Checked' && configuration != 'Release') {
2263                                     return
2264                                 }
2265                                 break
2266                             case 'longgc':
2267                             case 'gcsimulator':
2268                                 // Long GC tests take a long time on non-Release builds
2269                                 if (configuration != 'Release') {
2270                                     return
2271                                 }
2272                                 break
2273                             case 'gc_reliability_framework':
2274                             case 'standalone_gc':
2275                                 if (configuration != 'Release' && configuration != 'Checked') {
2276                                     return
2277                                 }
2278                                 break
2279                             case 'coverage':
2280                                 //We only want Ubuntu Release for coverage
2281                                 if (os != 'Ubuntu') {
2282                                     return
2283                                 }
2284                                 if (configuration != 'Release') {
2285                                     return
2286                                 }
2287                             case 'formatting':
2288                                 return
2289                             case 'illink':
2290                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2291                                     return
2292                                 }
2293                                 break
2294                             case 'default':
2295                                 // Nothing skipped
2296                                 break
2297                             default:
2298                                 println("Unknown scenario: ${scenario}")
2299                                 assert false
2300                                 break
2301                         }
2302                     }
2303
2304                     def lowerConfiguration = configuration.toLowerCase()
2305                     def osGroup = getOSGroup(os)
2306                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2307
2308                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2309                     def inputScenario = 'default'
2310                     if (scenario == 'coverage') {
2311                         inputScenario = 'coverage'
2312                     }
2313                     def inputCoreCLRBuildName = projectFolder + '/' +
2314                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2315                     // If this is a stress scenario, there isn't any difference in the build job
2316                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2317                     // from the default scenario
2318                     def testBuildScenario = scenario
2319                     if (scenarioNeedsPri1Build(scenario)) {
2320                         testBuildScenario = 'pri1'
2321                     }
2322                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2323                         testBuildScenario = 'default'
2324                     }
2325                     def inputWindowTestsBuildName = ''
2326                     def inputWindowsTestBuildArch = architecture
2327                     if (architecture == "arm64") {
2328                         // Use the x64 test build for arm64 unix
2329                         inputWindowsTestBuildArch = "x64"
2330                     }
2331                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2332                         inputWindowTestsBuildName = projectFolder + '/' +
2333                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', 'default', true), isPR)
2334                     }
2335                     else {
2336                         inputWindowTestsBuildName = projectFolder + '/' +
2337                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2338                     }
2339                     // Enable Server GC for Ubuntu PR builds
2340                     def serverGCString = ''
2341
2342                     // Whether or not this test run should be run sequentially instead
2343                     // of in parallel. Only used for long GC tests.
2344                     def sequentialString = ''
2345
2346                     // Whether or not this test run should run a specific playlist.
2347                     // Only used for long GC tests.
2348
2349                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2350                     // for running long GC and GCSimulator tests, respectively. We don't use them
2351                     // here because using a playlist file produces much more readable output on the CI machines
2352                     // and reduces running time.
2353                     def playlistString = ''
2354
2355                     if (os == 'Ubuntu' && isPR){
2356                         serverGCString = '--useServerGC'
2357                     }
2358
2359                     // pass --crossgen to runtest.sh for crossgen builds
2360                     def crossgenStr = ''
2361                     def runcrossgentestsStr = ''
2362                     def runjitstressStr = ''
2363                     def runjitstressregsStr = ''
2364                     def runjitmioptsStr = ''
2365                     def runjitforcerelocsStr = ''
2366                     def runjitdisasmStr = ''
2367                     def runilasmroundtripStr = ''
2368                     def gcstressStr = ''
2369                     def illinkStr = ''
2370                     def layoutOnlyStr =''
2371
2372                     if (scenario == 'r2r' ||
2373                         scenario == 'pri1r2r' ||
2374                         scenario == 'gcstress15_pri1r2r' ||
2375                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2376                             crossgenStr = '--crossgen'
2377                             runcrossgentestsStr = '--runcrossgentests'
2378
2379                             if (scenario == 'r2r_jitstress1'){
2380                                 runjitstressStr = '--jitstress=1'
2381                             }
2382                             else if (scenario == 'r2r_jitstress2') {
2383                                 runjitstressStr = '--jitstress=2'
2384                             }
2385                             else if (scenario == 'r2r_jitstressregs1'){
2386                                 runjitstressregsStr = '--jitstressregs=1'
2387                             }
2388                             else if (scenario == 'r2r_jitstressregs2') {
2389                                 runjitstressregsStr = '--jitstressregs=2'
2390                             }
2391                             else if (scenario == 'r2r_jitstressregs3') {
2392                                 runjitstressregsStr = '--jitstressregs=3'
2393                             }
2394                             else if (scenario == 'r2r_jitstressregs4') {
2395                                 runjitstressregsStr = '--jitstressregs=4'
2396                             }
2397                             else if (scenario == 'r2r_jitstressregs8') {
2398                                 runjitstressregsStr = '--jitstressregs=8'
2399                             }
2400                             else if (scenario == 'r2r_jitstressregs0x10') {
2401                                 runjitstressregsStr = '--jitstressregs=0x10'
2402                             }
2403                             else if (scenario == 'r2r_jitstressregs0x80') {
2404                                 runjitstressregsStr = '--jitstressregs=0x80'
2405                             }
2406                             else if (scenario == 'r2r_jitstressregs0x1000') {
2407                                 runjitstressregsStr = '--jitstressregs=0x1000'
2408                             }
2409                             else if (scenario == 'r2r_jitminopts') {
2410                                 runjitmioptsStr = '--jitminopts'
2411                             }
2412                             else if (scenario == 'r2r_jitforcerelocs') {
2413                                 runjitforcerelocsStr = '--jitforcerelocs'
2414                             }
2415                     }
2416                     if  (scenario == 'gcstress15_pri1r2r')
2417                     {
2418                         gcstressStr = '--gcstresslevel=0xF'
2419                     }
2420
2421                     if (scenario == 'jitdiff')
2422                     {
2423                         runjitdisasmStr = '--jitdisasm --crossgen'
2424                     }
2425
2426                     if (scenario == 'illink')
2427                     {
2428                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2429                     }
2430
2431                     if (isLongGc(scenario)) {
2432                         // Long GC tests behave very poorly when they are not
2433                         // the only test running (many of them allocate until OOM).
2434                         sequentialString = '--sequential'
2435
2436                         // The Long GC playlist contains all of the tests that are
2437                         // going to be run. The GCSimulator playlist contains all of
2438                         // the GC simulator tests.
2439                         if (scenario == 'longgc') {
2440                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2441                         }
2442                         else if (scenario == 'gcsimulator') {
2443                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2444                         }
2445                     }
2446
2447                     if (isGcReliabilityFramework(scenario)) {
2448                         layoutOnlyStr = '--build-overlay-only'
2449                     }
2450
2451                     def folder = getJobFolder(scenario)
2452                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2453                         // Add parameters for the inputs
2454
2455                         parameters {
2456                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2457                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2458                         }
2459
2460                         steps {
2461                             // Set up the copies
2462
2463                             // Coreclr build containing the tests and mscorlib
2464
2465                             copyArtifacts(inputWindowTestsBuildName) {
2466                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2467                                 buildSelector {
2468                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2469                                 }
2470                             }
2471
2472                             if (scenario == 'coverage') {
2473
2474                                 // Move coreclr to clr directory
2475                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2476
2477                                 // Build coreclr
2478                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2479
2480                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2481                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2482                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2483                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2484                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2485                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2486                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2487                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2488
2489                                 // Run PAL tests
2490                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2491
2492                                 // Remove obj files for PAL tests so they're not included in coverage results
2493                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2494
2495                                 // Unzip the tests first.  Exit with 0
2496                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2497
2498                                 // Get corefx
2499                                 shell("git clone https://github.com/dotnet/corefx fx")
2500
2501                                 // Build Linux corefx
2502                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2503                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2504
2505                                 def testEnvOpt = ""
2506                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2507                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2508                                 shell("${createScriptCmds}")
2509                                 testEnvOpt = "--test-env=" + scriptFileName
2510
2511                                 // Run corefx tests
2512                                 shell("""./fx/run-test.sh \\
2513                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2514                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2515                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2516                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2517                 --configurationGroup Release""")
2518
2519
2520                                 // Run coreclr tests w/ workstation GC
2521                                 shell("""./clr/tests/runtest.sh \\
2522                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2523                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2524                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2525                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2526                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2527                 --crossgen --runcrossgentests""")
2528
2529                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2530                                 shell("""./clr/tests/runtest.sh \\
2531                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2532                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2533                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2534                 --useServerGC ${testEnvOpt}""")
2535
2536                                  // Run long-running coreclr GC tests & produce coverage reports
2537                                 shell("""./clr/tests/runtest.sh \\
2538                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2539                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2540                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2541                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2542                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2543                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2544                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2545
2546                             }
2547                             else {
2548
2549                                 // Coreclr build we are trying to test
2550
2551                                 copyArtifacts(inputCoreCLRBuildName) {
2552                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2553                                     buildSelector {
2554                                         buildNumber('${CORECLR_BUILD}')
2555                                     }
2556                                 }
2557
2558                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2559
2560                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2561                                 def osJobName
2562                                 if (os == 'Ubuntu') {
2563                                     osJobName = 'ubuntu14.04'
2564                                 }
2565                                 else {
2566                                     osJobName = os.toLowerCase()
2567                                 }
2568                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2569                                     includePatterns('bin/build.tar.gz')
2570                                     buildSelector {
2571                                         latestSuccessful(true)
2572                                     }
2573                                 }
2574
2575                                 shell ("mkdir ./bin/CoreFxBinDir")
2576                                 // Unpack the corefx binaries
2577                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2578
2579                                 // HACK -- Arm64 does not have corefx jobs yet.
2580                                 // Clone corefx and build the native packages overwriting the x64 packages.
2581                                 if (architecture == 'arm64') {
2582                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2583                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2584                                 }
2585
2586                                 // Unzip the tests first.  Exit with 0
2587                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2588
2589                                 // Execute the tests
2590                                 // If we are running a stress mode, we'll set those variables first
2591                                 def testEnvOpt = ""
2592                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2593                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2594                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2595                                     shell("${createScriptCmds}")
2596                                     testEnvOpt = "--test-env=" + scriptFileName
2597                                 }
2598
2599                                 if (isGCStressRelatedTesting(scenario)) {
2600                                     shell('./init-tools.sh')
2601                                 }
2602
2603                                 shell("""./tests/runtest.sh \\
2604                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2605                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2606                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2607                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2608                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2609                 --limitedDumpGeneration \\
2610                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
2611                 ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
2612                 ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
2613
2614                                 if (isGcReliabilityFramework(scenario)) {
2615                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2616                                     if (serverGCString != '') {
2617                                         shell("export COMPlus_gcServer=1")
2618                                     }
2619
2620                                     shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2621                                 }
2622                             }
2623                         }
2624                     }
2625
2626                     if (scenario == 'coverage') {
2627                         // Publish coverage reports
2628                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2629                         // TODO: Add once external email sending is available again
2630                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2631                     }
2632
2633                     if (scenario == 'jitdiff') {
2634                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2635                     }
2636
2637                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2638                     if (os in ['Ubuntu']) {
2639                         SummaryBuilder summaries = new SummaryBuilder()
2640                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2641                         summaries.emit(newJob)
2642                     }
2643
2644                     setMachineAffinity(newJob, os, architecture)
2645                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2646                     // Set timeouts to 240.
2647                     setTestJobTimeOut(newJob, scenario)
2648
2649                     if (architecture == 'arm64') {
2650                         Utilities.setJobTimeout(newJob, 480)
2651                     }
2652
2653                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2654
2655                     // Create a build flow to join together the build and tests required to run this
2656                     // test.
2657                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2658                     // Linux CoreCLR test
2659                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2660                     def fullTestJobName = projectFolder + '/' + newJob.name
2661                     // Add a reference to the input jobs for report purposes
2662                     JobReport.Report.addReference(inputCoreCLRBuildName)
2663                     JobReport.Report.addReference(inputWindowTestsBuildName)
2664                     JobReport.Report.addReference(fullTestJobName)
2665                     def newFlowJob;
2666
2667                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2668                     // so that coverage data can be preserved.
2669                     if (scenario == 'coverage') {
2670                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2671                         buildFlow("""
2672 // Build the input Windows job
2673 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2674
2675 // And then build the test build
2676 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2677 """)
2678                         }
2679                     // Normal jobs copy a Windows build & a non-Windows build
2680                     } else {
2681                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2682                         buildFlow("""
2683 // Build the input jobs in parallel
2684 parallel (
2685     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2686     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2687 )
2688
2689 // And then build the test build
2690 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2691                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2692 """)
2693                         }
2694                     }
2695
2696                     setMachineAffinity(newFlowJob, os, architecture)
2697                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2698                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2699                 } // configuration
2700             } // os
2701         } // architecture
2702     } // isPR
2703 } // scenario
2704
2705 JobReport.Report.generateJobReport(out)
2706
2707 // Make the call to generate the help job
2708 Utilities.createHelperJob(this, project, branch,
2709     "Welcome to the ${project} Repository",  // This is prepended to the help message
2710     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2711
2712 Utilities.addCROSSCheck(this, project, branch)