netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'FreeBSD':'FreeBSD',
  29         'CentOS7.1': 'Linux',
  30         'OpenSUSE42.1': 'Linux',
  31         'Tizen': 'Linux']
  32     def osGroup = osGroupMap.get(os, null)
  33     assert osGroup != null : "Could not find os group for ${os}"
  34     return osGroupMap[os]
  35 }
  36
  37 // We use this class (vs variables) so that the static functions can access data here.
  38 class Constants {
  39
  40     // Innerloop build OS's
  41     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  42     // test execution in the build flow runs.  It generates the exact same build
  43     // as Windows_NT but without the tests.
  44     def static osList = [
  45                'Ubuntu',
  46                'Debian8.4',
  47                'OSX10.12',
  48                'Windows_NT',
  49                'Windows_NT_BuildOnly',
  50                'FreeBSD',
  51                'CentOS7.1',
  52                'OpenSUSE42.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  67                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  68                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  69                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  70                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  71                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  72                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  73                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  74                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  75                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  76                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  77                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  78                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  79                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  80                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  81                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  82                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  83                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  84                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  85                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  86                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  87                'corefx_baseline'                : [ : ], // corefx baseline
  88                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  89                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  90                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  91                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  92                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  93                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  94                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  95                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  96                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  97                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  98                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  99                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 100                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 101                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 103                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 104                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 105                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 107                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 109                ]
 110
 111     // This is a set of r2r jit stress scenarios
 112     def static r2rJitStressScenarios = [
 113                'r2r_jitstress1',
 114                'r2r_jitstress2',
 115                'r2r_jitstressregs1',
 116                'r2r_jitstressregs2',
 117                'r2r_jitstressregs3',
 118                'r2r_jitstressregs4',
 119                'r2r_jitstressregs8',
 120                'r2r_jitstressregs0x10',
 121                'r2r_jitstressregs0x80',
 122                'r2r_jitstressregs0x1000',
 123                'r2r_jitminopts',
 124                'r2r_jitforcerelocs']
 125
 126     // This is the basic set of scenarios
 127     def static basicScenarios = [
 128                'default',
 129                'pri1',
 130                'ilrt',
 131                'r2r',
 132                'pri1r2r',
 133                'gcstress15_pri1r2r',
 134                'longgc',
 135                'coverage',
 136                'formatting',
 137                'gcsimulator',
 138                'jitdiff',
 139                'standalone_gc',
 140                'gc_reliability_framework',
 141                'illink'] + r2rJitStressScenarios
 142
 143     def static configurationList = ['Debug', 'Checked', 'Release']
 144
 145     // This is the set of architectures
 146     def static architectureList = ['arm', 'arm64', 'x64', 'x86', 'x86lb']
 147 }
 148
 149 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 150     if (architecture == 'arm64' && os == 'Windows_NT') {
 151         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 152     } else if (architecture == 'arm64' && os != 'Windows_NT' && options == null) {
 153         Utilities.setMachineAffinity(job, os, 'arm64-small-page-size');
 154     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['large_pages'] == true) {
 155         Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size');
 156     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['is_build_only'] == true) {
 157         Utilities.setMachineAffinity(job, os, 'arm64-cross-latest');
 158     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 159         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 160     } else if ((architecture == 'arm') && (os == 'Windows_NT') && options['use_arm64_build_machine'] == true) {
 161         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 162     }else {
 163         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 164     }
 165 }
 166
 167 def static isJITStressJob(def scenario) {
 168     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 169            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 170 }
 171
 172 def static isGCStressRelatedTesting(def scenario) {
 173     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 174     // Detect it and make it a GCStress related.
 175     if (scenario == 'gcstress15_pri1r2r')
 176     {
 177         return true;
 178     }
 179
 180     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 181     def scenarioName = scenario.toLowerCase()
 182     def isGCStressTesting = false
 183     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 184         if (k in gcStressTestEnvVars) {
 185             isGCStressTesting = true;
 186         }
 187     }
 188     return isGCStressTesting
 189 }
 190
 191 def static isCorefxTesting(def scenario) {
 192     def corefx_prefix = 'corefx_'
 193     if (scenario.length() < corefx_prefix.length()) {
 194         return false
 195     }
 196     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 197 }
 198
 199 def static isR2R(def scenario) {
 200     return (scenario == 'r2r' || scenario == 'pri1r2r')
 201 }
 202
 203 def static isCoverage(def scenario) {
 204     return (scenario == 'coverage')
 205 }
 206
 207 def static isLongGc(def scenario) {
 208     return (scenario == 'longgc' || scenario == 'gcsimulator')
 209 }
 210
 211 def static isJitDiff(def scenario) {
 212     return (scenario == 'jitdiff')
 213 }
 214
 215 def static isGcReliabilityFramework(def scenario) {
 216     return (scenario == 'gc_reliability_framework')
 217 }
 218
 219 def static scenarioNeedsPri1Build(def scenario) {
 220     return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario))
 221 }
 222
 223 def static setTestJobTimeOut(newJob, scenario) {
 224     if (isGCStressRelatedTesting(scenario)) {
 225         Utilities.setJobTimeout(newJob, 4320)
 226     }
 227     else if (isCorefxTesting(scenario)) {
 228         Utilities.setJobTimeout(newJob, 360)
 229     }
 230     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 231         Utilities.setJobTimeout(newJob, 240)
 232     }
 233     else if (isR2R(scenario)) {
 234         Utilities.setJobTimeout(newJob, 240)
 235     }
 236     else if (isCoverage(scenario)) {
 237         Utilities.setJobTimeout(newJob, 1440)
 238     }
 239     else if (isLongGc(scenario)) {
 240         Utilities.setJobTimeout(newJob, 1440)
 241     }
 242     else if (isJitDiff(scenario)) {
 243         Utilities.setJobTimeout(newJob, 240)
 244     }
 245     else if (isGcReliabilityFramework(scenario)) {
 246         Utilities.setJobTimeout(newJob, 1440)
 247     }
 248     // Non-test jobs use the default timeout value.
 249 }
 250
 251 def static getJobFolder(def scenario) {
 252     if (isJITStressJob(scenario)) {
 253         return 'jitstress'
 254     }
 255     if (scenario == 'illink') {
 256         return 'illink'
 257     }
 258     return ''
 259 }
 260
 261 def static getStressModeDisplayName(def scenario) {
 262     def displayStr = ''
 263     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 264         def prefixLength = 'COMPlus_'.length()
 265         if (k.length() >= prefixLength) {
 266             def modeName = k.substring(prefixLength, k.length())
 267             displayStr += ' ' + modeName + '=' + v
 268         }
 269     }
 270     return displayStr
 271 }
 272
 273 def static getR2RStressModeDisplayName(def scenario) {
 274     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 275     // "r2r_" prefix.
 276     def displayStr = scenario
 277     def prefixLength = 'r2r_'.length()
 278     if (displayStr.length() >= prefixLength) {
 279         displayStr = displayStr.substring(prefixLength, displayStr.length())
 280     }
 281     return displayStr
 282 }
 283
 284 // Generates the string for creating a file that sets environment variables
 285 // that makes it possible to run stress modes.  Writes the script to the file
 286 // specified by the stepScriptLocation parameter.
 287 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 288     def stepScript = ''
 289     if (os == 'Windows_NT') {
 290         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 291         stepScript += "del ${stepScriptLocation}\r\n"
 292
 293         // Timeout in ms, default is 10 minutes. For stress
 294         // modes up this to 30 minutes
 295         def timeout = 1800000
 296
 297         // Set the Timeout
 298         stepScript += "set __TestTimeout=${timeout}\r\n"
 299         stepScript += "echo. > ${stepScriptLocation}\r\n"
 300         stressModeVars.each{ k, v ->
 301             // Write out what we are writing to the script file
 302             stepScript += "echo Setting ${k}=${v}\r\n"
 303             // Write out the set itself to the script file`
 304             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 305         }
 306     }
 307     else {
 308         stepScript += "echo Setting variables for ${stressModeName}\n"
 309         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 310         stressModeVars.each{ k, v ->
 311             // Write out what we are writing to the script file
 312             stepScript += "echo Setting ${k}=${v}\n"
 313             // Write out the set itself to the script file`
 314             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 315         }
 316         stepScript += "chmod +x ${stepScriptLocation}\n"
 317     }
 318     return stepScript
 319 }
 320
 321 // Calculates the name of the build job based on some typical parameters.
 322 //
 323 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 324     // If the architecture is x64, do not add that info into the build name.
 325     // Need to change around some systems and other builds to pick up the right builds
 326     // to do that.
 327
 328     def suffix = scenario != 'default' ? "_${scenario}" : '';
 329     if (isBuildOnly) {
 330         suffix += '_bld'
 331     }
 332     def baseName = ''
 333     switch (architecture) {
 334         case 'x64':
 335             if (scenario == 'default') {
 336                 // For now we leave x64 off of the name for compatibility with other jobs
 337                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 338             }
 339             else if (scenario == 'formatting') {
 340                 // we don't care about the configuration for the formatting job. It runs all configs
 341                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 342             }
 343             else {
 344                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 345             }
 346             break
 347         case 'arm64':
 348             if (os.toLowerCase() == "windows_nt") {
 349                 // These are cross builds
 350                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 351             }
 352             else {
 353                 // Defaults to a small page size set of machines.
 354                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 355             }
 356             break
 357         case 'arm':
 358             // These are cross builds
 359             if (os == 'Tizen') {
 360                 // ABI: softfp
 361                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 362             }
 363             else {
 364                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 365             }
 366             break
 367         case 'x86':
 368         case 'x86lb':
 369             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 370             break
 371         default:
 372             println("Unknown architecture: ${architecture}");
 373             assert false
 374             break
 375     }
 376
 377     return baseName + suffix
 378 }
 379
 380 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 381     // Check scenario.
 382     switch (scenario) {
 383         case 'default':
 384             switch (architecture) {
 385                 case 'x64':
 386                 case 'x86':
 387                 case 'x86lb':
 388                     if (architecture == 'x86' && os == 'Ubuntu') {
 389                         Utilities.addPeriodicTrigger(job, '@daily')
 390                     }
 391                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 392                         Utilities.addGithubPushTrigger(job)
 393                     }
 394                     break
 395                 case 'arm':
 396                     Utilities.addGithubPushTrigger(job)
 397                     break
 398                 case 'arm64':
 399                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 400                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 401                     break
 402                 default:
 403                     println("Unknown architecture: ${architecture}");
 404                     assert false
 405                     break
 406             }
 407             break
 408         case 'pri1':
 409             // Pri one gets a push trigger, and only for release
 410             if (architecture == 'x64') {
 411                 if (configuration == 'Release') {
 412                     // We expect release jobs to be Windows, or in the cross list
 413                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 414                     if (!os in bidailyCrossList) {
 415                         if (isFlowJob || os == 'Windows_NT') {
 416                             Utilities.addGithubPushTrigger(job)
 417                         }
 418                     }
 419                     else {
 420                         if (isFlowJob) {
 421                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 422                         }
 423                     }
 424                 }
 425             }
 426             break
 427         case 'r2r':
 428             //r2r jobs that aren't pri1 can only be triggered by phrase
 429             break
 430         case 'pri1r2r':
 431             assert !(os in bidailyCrossList)
 432             //pri1 r2r gets a push trigger for checked/release
 433             if (configuration == 'Checked' || configuration == 'Release') {
 434                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 435                 if (architecture == 'x64' && os != 'OSX10.12') {
 436                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 437                     if (isFlowJob || os == 'Windows_NT') {
 438                         Utilities.addGithubPushTrigger(job)
 439                     }
 440                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 441                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 442                     if (isFlowJob) {
 443                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 444                     }
 445                 }
 446                 // For x86, only add per-commit jobs for Windows
 447                 else if (architecture == 'x86' || architecture == 'x86lb') {
 448                     if (os == 'Windows_NT') {
 449                         Utilities.addGithubPushTrigger(job)
 450                     }
 451                 }
 452                 // arm64 pri1r2r jobs should only run every 12 hours.
 453                 else if (architecture == 'arm64') {
 454                     if (os == 'Windows_NT') {
 455                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 456                         // TODO: Add once external email sending is available again
 457                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 458                     }
 459                 }
 460             }
 461             break
 462         case 'r2r_jitstress1':
 463         case 'r2r_jitstress2':
 464         case 'r2r_jitstressregs1':
 465         case 'r2r_jitstressregs2':
 466         case 'r2r_jitstressregs3':
 467         case 'r2r_jitstressregs4':
 468         case 'r2r_jitstressregs8':
 469         case 'r2r_jitstressregs0x10':
 470         case 'r2r_jitstressregs0x80':
 471         case 'r2r_jitstressregs0x1000':
 472         case 'r2r_jitminopts':
 473         case 'r2r_jitforcerelocs':
 474         case 'gcstress15_pri1r2r':
 475             assert !(os in bidailyCrossList)
 476
 477             // GCStress=C is currently not supported on OS X
 478             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 479                 break
 480             }
 481
 482             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 483             if (configuration == 'Checked' || configuration == 'Release') {
 484                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 485                 if (architecture == 'x64') {
 486                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 487                     if (isFlowJob || os == 'Windows_NT') {
 488                         // Add a weekly periodic trigger
 489                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 490                     }
 491                 }
 492                 // For x86, only add per-commit jobs for Windows
 493                 else if (architecture == 'x86') {
 494                     if (os == 'Windows_NT') {
 495                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 496                     }
 497                 }
 498             }
 499             break
 500         case 'longgc':
 501             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 502             assert configuration == 'Release'
 503             assert architecture == 'x64'
 504             Utilities.addPeriodicTrigger(job, '@daily')
 505             // TODO: Add once external email sending is available again
 506             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 507             break
 508         case 'gcsimulator':
 509             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 510             assert configuration == 'Release'
 511             assert architecture == 'x64'
 512             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 513             // TODO: Add once external email sending is available again
 514             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 515             break
 516         case 'standalone_gc':
 517             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 518             assert (configuration == 'Release' || configuration == 'Checked')
 519             // TODO: Add once external email sending is available again
 520             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 521             Utilities.addPeriodicTrigger(job, '@weekly')
 522             break
 523         case 'gc_reliability_framework':
 524             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 525             assert (configuration == 'Release' || configuration == 'Checked')
 526             // Only triggered by phrase.
 527             break
 528         case 'ilrt':
 529             assert !(os in bidailyCrossList)
 530             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 531             if (architecture == 'x64' && configuration == 'Release') {
 532                 // We don't expect to see a job generated except in these scenarios
 533                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 534                 if (isFlowJob || os == 'Windows_NT') {
 535                     Utilities.addPeriodicTrigger(job, '@daily')
 536                 }
 537             }
 538             break
 539         case 'jitdiff':
 540             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 541             assert configuration == 'Checked'
 542             assert (architecture == 'x64' || architecture == 'x86')
 543             Utilities.addGithubPushTrigger(job)
 544             break
 545         case 'coverage':
 546             assert (os == 'Ubuntu' || os == 'Windows_NT')
 547             assert configuration == 'Release'
 548             assert architecture == 'x64'
 549             Utilities.addPeriodicTrigger(job, '@weekly')
 550             break
 551         case 'formatting':
 552             assert (os == 'Windows_NT' || os == "Ubuntu")
 553             assert architecture == 'x64'
 554             Utilities.addGithubPushTrigger(job)
 555             break
 556         case 'jitstressregs1':
 557         case 'jitstressregs2':
 558         case 'jitstressregs3':
 559         case 'jitstressregs4':
 560         case 'jitstressregs8':
 561         case 'jitstressregs0x10':
 562         case 'jitstressregs0x80':
 563         case 'jitstressregs0x1000':
 564         case 'minopts':
 565         case 'forcerelocs':
 566         case 'jitstress1':
 567         case 'jitstress2':
 568         case 'jitstress2_jitstressregs1':
 569         case 'jitstress2_jitstressregs2':
 570         case 'jitstress2_jitstressregs3':
 571         case 'jitstress2_jitstressregs4':
 572         case 'jitstress2_jitstressregs8':
 573         case 'jitstress2_jitstressregs0x10':
 574         case 'jitstress2_jitstressregs0x80':
 575         case 'jitstress2_jitstressregs0x1000':
 576         case 'tailcallstress':
 577         case 'jitsse2only':
 578         case 'corefx_baseline':
 579         case 'corefx_minopts':
 580         case 'corefx_jitstress1':
 581         case 'corefx_jitstress2':
 582         case 'corefx_jitstressregs1':
 583         case 'corefx_jitstressregs2':
 584         case 'corefx_jitstressregs3':
 585         case 'corefx_jitstressregs4':
 586         case 'corefx_jitstressregs8':
 587         case 'corefx_jitstressregs0x10':
 588         case 'corefx_jitstressregs0x80':
 589         case 'corefx_jitstressregs0x1000':
 590         case 'zapdisable':
 591             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 592             assert (os == 'Windows_NT') || (os in Constants.crossList)
 593             Utilities.addPeriodicTrigger(job, '@daily')
 594         }
 595         break
 596         case 'heapverify1':
 597         case 'gcstress0x3':
 598             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 599                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 600                 if (architecture == 'arm64') {
 601                     if (os == 'Windows_NT') {
 602                         Utilities.addPeriodicTrigger(job, '@daily')
 603                     }
 604                     // TODO: Add once external email sending is available again
 605                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 606                 }
 607                 else {
 608                     Utilities.addPeriodicTrigger(job, '@weekly')
 609                 }
 610             }
 611             break
 612         case 'gcstress0xc':
 613         case 'gcstress0xc_zapdisable':
 614         case 'gcstress0xc_zapdisable_jitstress2':
 615         case 'gcstress0xc_zapdisable_heapverify1':
 616         case 'gcstress0xc_jitstress1':
 617         case 'gcstress0xc_jitstress2':
 618         case 'gcstress0xc_minopts_heapverify1':
 619             // GCStress=C is currently not supported on OS X
 620             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 621                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 622                 if (architecture == 'arm64') {
 623                     // TODO: Enable a periodic trigger after tests are updated.
 624                     // Utilities.addPeriodicTrigger(job, '@daily')
 625                     // TODO: Add once external email sending is available again
 626                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 627                 }
 628                 else {
 629                     Utilities.addPeriodicTrigger(job, '@weekly')
 630                 }
 631             }
 632             break
 633
 634         case 'illink':
 635             // Testing on other operating systems TBD
 636             assert (os == 'Windows_NT' || os == 'Ubuntu')
 637             if (architecture == 'x64' || architecture == 'x86') {
 638                 if (configuration == 'Checked') {
 639                     Utilities.addPeriodicTrigger(job, '@daily')
 640                 }
 641             }
 642             break
 643
 644         default:
 645             println("Unknown scenario: ${scenario}");
 646             assert false
 647             break
 648     }
 649     return
 650 }
 651
 652 // **************************
 653 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 654 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 655 // builds will do a couple extra steps.
 656 // **************************
 657
 658 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 659 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 660 // it's just used for internal builds
 661 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 662 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 663     if (isWindowsBuildOnlyJob) {
 664         return
 665     }
 666
 667     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 668     // Non pull request builds.
 669     if (!isPR) {
 670         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 671         return
 672     }
 673
 674      def arm64Users = [
 675         'adiaaida',
 676         'AndyAyersMS',
 677         'briansull',
 678         'BruceForstall',
 679         'CarolEidt',
 680         'cmckinsey',
 681         'erozenfeld',
 682         'jashook',
 683         'JosephTremoulet',
 684         'pgavlin',
 685         'russellhadley',
 686         'RussKeldorph',
 687         'sandreenko',
 688         'sdmaclea',
 689         'sivarv',
 690         'swaroop-sridhar',
 691         'gkhanna79',
 692         'jkotas',
 693         'markwilkie',
 694         'rahku',
 695         'ramarag',
 696         'tzwlai',
 697         'weshaggard'
 698     ]
 699
 700     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 701     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 702     def osGroup = getOSGroup(os)
 703     switch (architecture) {
 704         case 'x64': // editor brace matching: {
 705             if (scenario == 'coverage') {
 706                 assert configuration == 'Release'
 707                 if (os == 'Ubuntu') {
 708                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 709                 }
 710                 break
 711             }
 712
 713             if (scenario == 'formatting') {
 714                 assert configuration == 'Checked'
 715                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 716                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 717                 }
 718                 break
 719             }
 720
 721             switch (os) {
 722                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 723                 case 'Debian8.4':
 724                 case 'RHEL7.2':
 725                     if (scenario == 'default') {
 726                         assert !isFlowJob
 727                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 728                     }
 729                     else if (scenario == 'pri1' && isFlowJob) {
 730                         assert (configuration == 'Release')
 731                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 732                     }
 733                     break
 734                 case 'Ubuntu16.04':
 735                     assert !isFlowJob
 736                     assert scenario == 'default'
 737                     // Distinguish with the other architectures (arm and x86)
 738                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 739                     break
 740                 case 'Fedora24':
 741                 case 'Ubuntu16.10':
 742                 case 'OpenSUSE42.1':
 743                     assert !isFlowJob
 744                     assert scenario == 'default'
 745                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 746                     break
 747                 case 'Ubuntu':
 748                     if (scenario == 'illink') {
 749                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 750                         break
 751                     }
 752                     // fall through
 753                 case 'OSX10.12':
 754                     // Triggers on the non-flow jobs aren't necessary here
 755                     // Corefx testing uses non-flow jobs.
 756                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 757                         break
 758                     }
 759                     switch (scenario) {
 760                         case 'default':
 761                             // Ubuntu uses checked for default PR tests
 762                             if (configuration == 'Checked') {
 763                                 // Default trigger
 764                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 765                             }
 766                             break
 767                         case 'pri1':
 768                             if (configuration == 'Release') {
 769                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 770                             }
 771                             break
 772                         case 'jitdiff':
 773                             if (configuration == 'Checked') {
 774                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 775                             }
 776                             break
 777                         case 'ilrt':
 778                             if (configuration == 'Release') {
 779                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 780                             }
 781                             break
 782                         case 'r2r':
 783                             if (configuration == 'Release' || configuration == 'Checked') {
 784                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 785                             }
 786                             break
 787                         case 'pri1r2r':
 788                             if (configuration == 'Release' || configuration == 'Checked') {
 789                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 790                             }
 791                             break
 792                         case 'gcstress15_pri1r2r':
 793                             if (configuration == 'Release' || configuration == 'Checked') {
 794                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 795                             }
 796                             break
 797                         case 'r2r_jitstress1':
 798                         case 'r2r_jitstress2':
 799                         case 'r2r_jitstressregs1':
 800                         case 'r2r_jitstressregs2':
 801                         case 'r2r_jitstressregs3':
 802                         case 'r2r_jitstressregs4':
 803                         case 'r2r_jitstressregs8':
 804                         case 'r2r_jitstressregs0x10':
 805                         case 'r2r_jitstressregs0x80':
 806                         case 'r2r_jitstressregs0x1000':
 807                         case 'r2r_jitminopts':
 808                         case 'r2r_jitforcerelocs':
 809                             if (configuration == 'Release' || configuration == 'Checked') {
 810                                 def displayStr = getR2RStressModeDisplayName(scenario)
 811                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 812                             }
 813                             break
 814                         case 'longgc':
 815                             if (configuration == 'Release') {
 816                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 817                             }
 818                             break
 819                         case 'gcsimulator':
 820                             if (configuration == 'Release') {
 821                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 822                             }
 823                             break
 824                         case 'standalone_gc':
 825                             if (configuration == 'Release' || configuration == 'Checked') {
 826                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 827                             }
 828                             break
 829                         case 'gc_reliability_framework':
 830                             if (configuration == 'Release' || configuration == 'Checked') {
 831                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 832                             }
 833                             break
 834                         case 'minopts':
 835                         case 'forcerelocs':
 836                         case 'jitstress1':
 837                         case 'jitstress2':
 838                         case 'jitstressregs1':
 839                         case 'jitstressregs2':
 840                         case 'jitstressregs3':
 841                         case 'jitstressregs4':
 842                         case 'jitstressregs8':
 843                         case 'jitstressregs0x10':
 844                         case 'jitstressregs0x80':
 845                         case 'jitstressregs0x1000':
 846                         case 'jitstress2_jitstressregs1':
 847                         case 'jitstress2_jitstressregs2':
 848                         case 'jitstress2_jitstressregs3':
 849                         case 'jitstress2_jitstressregs4':
 850                         case 'jitstress2_jitstressregs8':
 851                         case 'jitstress2_jitstressregs0x10':
 852                         case 'jitstress2_jitstressregs0x80':
 853                         case 'jitstress2_jitstressregs0x1000':
 854                         case 'tailcallstress':
 855                         case 'jitsse2only':
 856                         case 'gcstress0x3':
 857                         case 'gcstress0xc':
 858                         case 'zapdisable':
 859                         case 'heapverify1':
 860                         case 'gcstress0xc_zapdisable':
 861                         case 'gcstress0xc_zapdisable_jitstress2':
 862                         case 'gcstress0xc_zapdisable_heapverify1':
 863                         case 'gcstress0xc_jitstress1':
 864                         case 'gcstress0xc_jitstress2':
 865                         case 'gcstress0xc_minopts_heapverify1':
 866                             def displayStr = getStressModeDisplayName(scenario)
 867                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 868                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 869                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 870                             break
 871                         case 'corefx_baseline':
 872                         case 'corefx_minopts':
 873                         case 'corefx_jitstress1':
 874                         case 'corefx_jitstress2':
 875                         case 'corefx_jitstressregs1':
 876                         case 'corefx_jitstressregs2':
 877                         case 'corefx_jitstressregs3':
 878                         case 'corefx_jitstressregs4':
 879                         case 'corefx_jitstressregs8':
 880                         case 'corefx_jitstressregs0x10':
 881                         case 'corefx_jitstressregs0x80':
 882                         case 'corefx_jitstressregs0x1000':
 883                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 884                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 885                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 886                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 887                             break
 888                         default:
 889                             println("Unknown scenario: ${scenario}");
 890                             assert false
 891                             break
 892                     }
 893                     break
 894                 case 'CentOS7.1':
 895                     switch (scenario) {
 896                         case 'pri1':
 897                             if (configuration == 'Release') {
 898                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 899                             }
 900                             break
 901                         case 'r2r':
 902                             if (configuration == 'Checked' || configuration == 'Release') {
 903                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 904                             }
 905                             break
 906                         case 'pri1r2r':
 907                             if (configuration == 'Checked' || configuration == 'Release') {
 908                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 909                             }
 910                             break
 911                         case 'gcstress15_pri1r2r':
 912                             if (configuration == 'Release' || configuration == 'Checked') {
 913                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 914                             }
 915                             break
 916                         case 'r2r_jitstress1':
 917                         case 'r2r_jitstress2':
 918                         case 'r2r_jitstressregs1':
 919                         case 'r2r_jitstressregs2':
 920                         case 'r2r_jitstressregs3':
 921                         case 'r2r_jitstressregs4':
 922                         case 'r2r_jitstressregs8':
 923                         case 'r2r_jitstressregs0x10':
 924                         case 'r2r_jitstressregs0x80':
 925                         case 'r2r_jitstressregs0x1000':
 926                         case 'r2r_jitminopts':
 927                         case 'r2r_jitforcerelocs':
 928                             if (configuration == 'Release' || configuration == 'Checked') {
 929                                 def displayStr = getR2RStressModeDisplayName(scenario)
 930                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 931                             }
 932                             break
 933                         default:
 934                             break
 935                     }
 936                 case 'Windows_NT':
 937                     switch (scenario) {
 938                         case 'default':
 939                             // Default trigger
 940                             if (configuration == 'Debug') {
 941                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 942                             }
 943                             break
 944                         case 'pri1':
 945                             // Default trigger
 946                             if (configuration == 'Release') {
 947                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 948                             }
 949                             break
 950                         case 'jitdiff':
 951                             if (configuration == 'Checked') {
 952                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 953                             }
 954                             break
 955                         case 'ilrt':
 956                             if (configuration == 'Release') {
 957                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 958                             }
 959                             break
 960                         case 'r2r':
 961                             if (configuration == 'Checked' || configuration == 'Release') {
 962                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 963                             }
 964                             break
 965                         case 'pri1r2r':
 966                             if (configuration == 'Checked' || configuration == 'Release') {
 967                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 968                             }
 969                             break
 970                         case 'gcstress15_pri1r2r':
 971                             if (configuration == 'Release' || configuration == 'Checked') {
 972                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 973                             }
 974                             break
 975                         case 'r2r_jitstress1':
 976                         case 'r2r_jitstress2':
 977                         case 'r2r_jitstressregs1':
 978                         case 'r2r_jitstressregs2':
 979                         case 'r2r_jitstressregs3':
 980                         case 'r2r_jitstressregs4':
 981                         case 'r2r_jitstressregs8':
 982                         case 'r2r_jitstressregs0x10':
 983                         case 'r2r_jitstressregs0x80':
 984                         case 'r2r_jitstressregs0x1000':
 985                         case 'r2r_jitminopts':
 986                         case 'r2r_jitforcerelocs':
 987                             if (configuration == 'Release' || configuration == 'Checked') {
 988                                 def displayStr = getR2RStressModeDisplayName(scenario)
 989                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 990                             }
 991                             break
 992                         case 'longgc':
 993                             if (configuration == 'Release') {
 994                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 995                             }
 996                             break
 997                         case 'gcsimulator':
 998                             if (configuration == 'Release') {
 999                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1000                             }
1001                             break
1002                         case 'standalone_gc':
1003                             if (configuration == 'Release' || configuration == 'Checked') {
1004                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1005                             }
1006                             break
1007                         case 'gc_reliability_framework':
1008                             if (configuration == 'Release' || configuration == 'Checked') {
1009                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1010                             }
1011                             break
1012                         case 'minopts':
1013                         case 'forcerelocs':
1014                         case 'jitstress1':
1015                         case 'jitstress2':
1016                         case 'jitstressregs1':
1017                         case 'jitstressregs2':
1018                         case 'jitstressregs3':
1019                         case 'jitstressregs4':
1020                         case 'jitstressregs8':
1021                         case 'jitstressregs0x10':
1022                         case 'jitstressregs0x80':
1023                         case 'jitstressregs0x1000':
1024                         case 'jitstress2_jitstressregs1':
1025                         case 'jitstress2_jitstressregs2':
1026                         case 'jitstress2_jitstressregs3':
1027                         case 'jitstress2_jitstressregs4':
1028                         case 'jitstress2_jitstressregs8':
1029                         case 'jitstress2_jitstressregs0x10':
1030                         case 'jitstress2_jitstressregs0x80':
1031                         case 'jitstress2_jitstressregs0x1000':
1032                         case 'tailcallstress':
1033                         case 'jitsse2only':
1034                         case 'gcstress0x3':
1035                         case 'gcstress0xc':
1036                         case 'zapdisable':
1037                         case 'heapverify1':
1038                         case 'gcstress0xc_zapdisable':
1039                         case 'gcstress0xc_zapdisable_jitstress2':
1040                         case 'gcstress0xc_zapdisable_heapverify1':
1041                         case 'gcstress0xc_jitstress1':
1042                         case 'gcstress0xc_jitstress2':
1043                         case 'gcstress0xc_minopts_heapverify1':
1044                             def displayStr = getStressModeDisplayName(scenario)
1045                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1046                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1047                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1048                             break
1049                         case 'corefx_baseline':
1050                         case 'corefx_minopts':
1051                         case 'corefx_jitstress1':
1052                         case 'corefx_jitstress2':
1053                         case 'corefx_jitstressregs1':
1054                         case 'corefx_jitstressregs2':
1055                         case 'corefx_jitstressregs3':
1056                         case 'corefx_jitstressregs4':
1057                         case 'corefx_jitstressregs8':
1058                         case 'corefx_jitstressregs0x10':
1059                         case 'corefx_jitstressregs0x80':
1060                         case 'corefx_jitstressregs0x1000':
1061                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1062                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1063                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1064                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1065                             break
1066                         case 'illink':
1067                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1068                                     break
1069                         default:
1070                             println("Unknown scenario: ${scenario}");
1071                             assert false
1072                             break
1073                     }
1074                     break
1075                 case 'FreeBSD':
1076                     assert scenario == 'default'
1077                     if (configuration == 'Checked') {
1078                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1079                     }
1080                     break
1081                 default:
1082                     println("Unknown os: ${os}");
1083                     assert false
1084                     break
1085             }
1086             break
1087         // editor brace matching: }
1088         case 'arm': // editor brace matching: {
1089             switch (os) {
1090                 case 'Ubuntu':
1091                 case 'Ubuntu16.04':
1092                     assert scenario == 'default'
1093                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1094                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1095                     }
1096                     else {
1097                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1098                     }
1099                     break;
1100                 case 'Tizen':
1101                     architecture='armel'
1102                     // Removing the regex will cause this to run on each PR.
1103                     if (configuration == 'Release' || configuration == 'Debug') {
1104                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1105                     }
1106                     else {
1107                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1108                     }
1109                     break;
1110                 case 'Windows_NT':
1111                     // Set up a private trigger
1112                     def contextString = "${os} ${architecture} Cross ${configuration}"
1113                     if (scenario != 'default')
1114                         contextString += " ${scenario}"
1115                     contextString += " Build"
1116                     // Debug builds only.
1117                     if (configuration != 'Debug') {
1118                         contextString += " and Test"
1119                     }
1120                     switch (scenario) {
1121                         case 'default':
1122                             // For now only run Debug and Release build jobs on PR Trigger. Note this is not a private trigger.
1123                             if (configuration == 'Debug' || configuration == 'Release')
1124                             {
1125                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1126                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1127                             }
1128                             else
1129                             {
1130                                 // Checked jobs will run on private trigger and run tests.
1131                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1132                             }
1133                             break
1134                         case 'pri1r2r':
1135                         case 'gcstress0x3':
1136                         case 'gcstress0xc':
1137                             // Stress jobs will will run on private trigger and run tests.
1138                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1139                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1140                             break
1141                     }
1142                     break
1143                 default:
1144                     println("NYI os: ${os}");
1145                     assert false
1146                     break
1147             }
1148             break
1149         // editor brace matching: }
1150         case 'arm64': // editor brace matching: {
1151             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1152
1153             // Set up a private trigger
1154             def contextString = "${os} ${architecture} Cross ${configuration}"
1155             if (scenario != 'default')
1156                 contextString += " ${scenario}"
1157             contextString += " Build"
1158             // Debug builds only.
1159             if (configuration != 'Debug') {
1160                contextString += " and Test"
1161             }
1162
1163             switch (os) {
1164                 case 'Ubuntu':
1165                 case 'Ubuntu16.04':
1166                     switch (scenario) {
1167                         case 'pri1':
1168                             if (configuration == 'Release') {
1169                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1170                             }
1171                             break
1172                         case 'r2r':
1173                             if (configuration == 'Checked' || configuration == 'Release') {
1174                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1175                             }
1176                             break
1177                         case 'pri1r2r':
1178                             if (configuration == 'Checked' || configuration == 'Release') {
1179                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1180                             }
1181                             break
1182                         case 'gcstress15_pri1r2r':
1183                             if (configuration == 'Release' || configuration == 'Checked') {
1184                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1185                             }
1186                             break
1187                         case 'r2r_jitstress1':
1188                         case 'r2r_jitstress2':
1189                         case 'r2r_jitstressregs1':
1190                         case 'r2r_jitstressregs2':
1191                         case 'r2r_jitstressregs3':
1192                         case 'r2r_jitstressregs4':
1193                         case 'r2r_jitstressregs8':
1194                         case 'r2r_jitstressregs0x10':
1195                         case 'r2r_jitstressregs0x80':
1196                         case 'r2r_jitstressregs0x1000':
1197                         case 'r2r_jitminopts':
1198                         case 'r2r_jitforcerelocs':
1199                             if (configuration == 'Release' || configuration == 'Checked') {
1200                                 def displayStr = getR2RStressModeDisplayName(scenario)
1201                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1202                             }
1203                             break
1204                         default:
1205                             break
1206                     }
1207                 case 'Windows_NT':
1208                     switch (scenario) {
1209                         case 'default':
1210                             if (isFlowJob == true) {
1211                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration}", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1212                             }
1213
1214                             // For now only run Debug jobs on PR Trigger.
1215                             else if (configuration != 'Debug') {
1216                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1217                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1218                             }
1219                             else {
1220                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1221                                 // are at higher risk of ARM64-breaking changes.
1222                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1223                             }
1224                             break
1225                         case 'pri1r2r':
1226                         case 'gcstress0x3':
1227                         case 'gcstress0xc':
1228                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1229                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1230                             break
1231                     }
1232                     break
1233                 default:
1234                     println("NYI os: ${os}");
1235                     assert false
1236                     break
1237             }
1238             break
1239         // editor brace matching: }
1240         case 'x86': // editor brace matching: {
1241             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1242             if (os == 'Ubuntu') {
1243                 // on-demand only for ubuntu x86
1244                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1245                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1246                 break
1247             }
1248             switch (scenario) {
1249                 case 'default':
1250                     if (configuration == 'Checked') {
1251                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1252                     }
1253                     else if (configuration == 'Release') {
1254                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1255                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1256                     }
1257                     break
1258                 case 'pri1':
1259                     if (configuration == 'Release') {
1260                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1261                     }
1262                     break
1263                 case 'ilrt':
1264                     if (configuration == 'Release') {
1265                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1266                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1267                     }
1268                     break
1269                 case 'r2r':
1270                     if (configuration == 'Checked' || configuration == 'Release') {
1271                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1272                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1273                     }
1274                     break
1275                 case 'pri1r2r':
1276                     if (configuration == 'Checked' || configuration == 'Release') {
1277                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1278                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1279                     }
1280                     break
1281                 case 'gcstress15_pri1r2r':
1282                     if (configuration == 'Release' || configuration == 'Checked') {
1283                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1284                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1285                     }
1286                     break
1287                 case 'r2r_jitstress1':
1288                 case 'r2r_jitstress2':
1289                 case 'r2r_jitstressregs1':
1290                 case 'r2r_jitstressregs2':
1291                 case 'r2r_jitstressregs3':
1292                 case 'r2r_jitstressregs4':
1293                 case 'r2r_jitstressregs8':
1294                 case 'r2r_jitstressregs0x10':
1295                 case 'r2r_jitstressregs0x80':
1296                 case 'r2r_jitstressregs0x1000':
1297                 case 'r2r_jitminopts':
1298                 case 'r2r_jitforcerelocs':
1299                     if (configuration == 'Release' || configuration == 'Checked') {
1300                         def displayStr = getR2RStressModeDisplayName(scenario)
1301                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1302                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1303                     }
1304                     break
1305                 case 'longgc':
1306                     if (configuration == 'Release') {
1307                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1308                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1309                     }
1310                     break
1311                 case 'gcsimulator':
1312                     if (configuration == 'Release') {
1313                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1314                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1315                     }
1316                     break
1317                 case 'standalone_gc':
1318                     if (configuration == 'Release' || configuration == 'Checked') {
1319                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1320                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1321                     }
1322                     break
1323                 case 'minopts':
1324                 case 'forcerelocs':
1325                 case 'jitstress1':
1326                 case 'jitstress2':
1327                 case 'jitstressregs1':
1328                 case 'jitstressregs2':
1329                 case 'jitstressregs3':
1330                 case 'jitstressregs4':
1331                 case 'jitstressregs8':
1332                 case 'jitstressregs0x10':
1333                 case 'jitstressregs0x80':
1334                 case 'jitstressregs0x1000':
1335                 case 'jitstress2_jitstressregs1':
1336                 case 'jitstress2_jitstressregs2':
1337                 case 'jitstress2_jitstressregs3':
1338                 case 'jitstress2_jitstressregs4':
1339                 case 'jitstress2_jitstressregs8':
1340                 case 'jitstress2_jitstressregs0x10':
1341                 case 'jitstress2_jitstressregs0x80':
1342                 case 'jitstress2_jitstressregs0x1000':
1343                 case 'tailcallstress':
1344                 case 'jitsse2only':
1345                 case 'gcstress0x3':
1346                 case 'gcstress0xc':
1347                 case 'zapdisable':
1348                 case 'heapverify1':
1349                 case 'gcstress0xc_zapdisable':
1350                 case 'gcstress0xc_zapdisable_jitstress2':
1351                 case 'gcstress0xc_zapdisable_heapverify1':
1352                 case 'gcstress0xc_jitstress1':
1353                 case 'gcstress0xc_jitstress2':
1354                 case 'gcstress0xc_minopts_heapverify1':
1355                     def displayStr = getStressModeDisplayName(scenario)
1356                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1357                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1358                     break
1359                 case 'corefx_baseline':
1360                 case 'corefx_minopts':
1361                 case 'corefx_jitstress1':
1362                 case 'corefx_jitstress2':
1363                 case 'corefx_jitstressregs1':
1364                 case 'corefx_jitstressregs2':
1365                 case 'corefx_jitstressregs3':
1366                 case 'corefx_jitstressregs4':
1367                 case 'corefx_jitstressregs8':
1368                 case 'corefx_jitstressregs0x10':
1369                 case 'corefx_jitstressregs0x80':
1370                 case 'corefx_jitstressregs0x1000':
1371                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1372                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1373                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1374                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1375                     break
1376                 case 'illink':
1377                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1378                     break
1379                 default:
1380                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1381                     assert false
1382                     break
1383             }
1384             break
1385          // editor brace matching: }
1386         case 'x86lb': // editor brace matching: {
1387             assert (os == 'Windows_NT')
1388             assert (scenario == 'default' || Constants.r2rJitStressScenarios.indexOf(scenario) !=1)
1389
1390             def arch = 'x86'
1391             def jit = 'legacy_backend'
1392             switch (scenario) {
1393                 case 'default':
1394                     if (configuration == 'Checked') {
1395                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1396                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1397                     }
1398                     break
1399                 default:
1400                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1401                     assert false
1402                     break
1403             }
1404             break
1405         // editor brace matching: }
1406         default:
1407             println("Unknown architecture: ${architecture}");
1408             assert false
1409             break
1410     }
1411 }
1412
1413 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1414     def buildCommands = [];
1415     def osGroup = getOSGroup(os)
1416     def lowerConfiguration = configuration.toLowerCase()
1417
1418     // Calculate the build steps, archival, and xunit results
1419     switch (os) {
1420         case 'Windows_NT': // editor brace matching: {
1421             switch (architecture) {
1422                 case 'x64':
1423                 case 'x86':
1424                 case 'x86lb':
1425                     def arch = architecture
1426                     def buildOpts = ''
1427                     if (architecture == 'x86lb') {
1428                         arch = 'x86'
1429                     }
1430
1431                     if (scenario == 'illink') {
1432                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1433                     }
1434
1435                     // If it is a release build for windows, ensure PGO is used, else fail the build
1436                     if ((lowerConfiguration == 'release') && (scenario in Constants.basicScenarios) && (architecture != 'x86lb')) {
1437                         buildOpts += ' enforcepgo'
1438                     }
1439
1440                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1441                             scenario == 'default' ||
1442                             scenario == 'r2r' ||
1443                             scenario == 'jitdiff' ||
1444                             scenario == 'ilrt' ||
1445                             scenario == 'illink' ||
1446                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1447                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1448                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1449                     }
1450
1451                     // For Pri 1 tests, we must shorten the output test binary path names.
1452                     // if __TestIntermediateDir is already set, build-test.cmd will
1453                     // output test binaries to that directory. If it is not set, the
1454                     // binaries are sent to a default directory whose name is about
1455                     // 35 characters long.
1456
1457                     else if (scenarioNeedsPri1Build(scenario)) {
1458                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1459                     }
1460                     else if (isLongGc(scenario)) {
1461                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1462                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1463                     }
1464                     else if (scenario == 'standalone_gc') {
1465                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1466                     }
1467                     else if (scenario == 'formatting') {
1468                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1469                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1470                         break
1471                     }
1472                     else {
1473                         println("Unknown scenario: ${scenario}")
1474                         assert false
1475                     }
1476
1477                     // If we are running a stress mode, we should write out the set of key
1478                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1479
1480                     if (!isBuildOnly) {
1481                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1482                         def crossgenStr = ''
1483                         def runcrossgentestsStr = ''
1484                         def runjitstressStr = ''
1485                         def runjitstressregsStr = ''
1486                         def runjitmioptsStr = ''
1487                         def runjitforcerelocsStr = ''
1488                         def runjitdisasmStr = ''
1489                         def runilasmroundtripStr = ''
1490                         def gcstressStr = ''
1491                         def runtestArguments = ''
1492                         def gcTestArguments = ''
1493                         def illinkArguments = ''
1494
1495                         if (scenario == 'r2r' ||
1496                             scenario == 'pri1r2r' ||
1497                             scenario == 'gcstress15_pri1r2r' ||
1498                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1499                                 crossgenStr = 'crossgen'
1500                                 runcrossgentestsStr = 'runcrossgentests'
1501
1502                                 if (scenario == 'r2r_jitstress1'){
1503                                     runjitstressStr = 'jitstress 1'
1504                                 }
1505                                 else if (scenario == 'r2r_jitstress2') {
1506                                     runjitstressStr = 'jitstress 2'
1507                                 }
1508                                 else if (scenario == 'r2r_jitstressregs1'){
1509                                     runjitstressregsStr = 'jitstressregs 1'
1510                                 }
1511                                 else if (scenario == 'r2r_jitstressregs2') {
1512                                     runjitstressregsStr = 'jitstressregs 2'
1513                                 }
1514                                 else if (scenario == 'r2r_jitstressregs3') {
1515                                     runjitstressregsStr = 'jitstressregs 3'
1516                                 }
1517                                 else if (scenario == 'r2r_jitstressregs4') {
1518                                     runjitstressregsStr = 'jitstressregs 4'
1519                                 }
1520                                 else if (scenario == 'r2r_jitstressregs8') {
1521                                     runjitstressregsStr = 'jitstressregs 8'
1522                                 }
1523                                 else if (scenario == 'r2r_jitstressregs0x10') {
1524                                     runjitstressregsStr = 'jitstressregs 0x10'
1525                                 }
1526                                 else if (scenario == 'r2r_jitstressregs0x80') {
1527                                     runjitstressregsStr = 'jitstressregs 0x80'
1528                                 }
1529                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1530                                     runjitstressregsStr = 'jitstressregs 0x1000'
1531                                 }
1532                                 else if (scenario == 'r2r_jitminopts') {
1533                                     runjitmioptsStr = 'jitminopts'
1534                                 }
1535                                 else if (scenario == 'r2r_jitforcerelocs') {
1536                                     runjitforcerelocsStr = 'jitforcerelocs'
1537                                 }
1538                         }
1539                         if (scenario == 'gcstress15_pri1r2r')
1540                         {
1541                             gcstressStr = 'gcstresslevel 0xF'
1542                         }
1543
1544                         if (scenario == 'jitdiff')
1545                         {
1546                             runjitdisasmStr = 'jitdisasm crossgen'
1547                         }
1548
1549                         if (scenario == 'ilrt')
1550                         {
1551                             runilasmroundtripStr = 'ilasmroundtrip'
1552                         }
1553
1554                         if (isLongGc(scenario)) {
1555                             gcTestArguments = "${scenario} sequential"
1556                         }
1557
1558                         if (scenario == 'illink')
1559                         {
1560                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1561                         }
1562
1563                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1564
1565                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1566                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1567                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1568
1569                             if (enableCorefxTesting) {
1570                                 def workspaceRelativeFxRoot = "_/fx"
1571                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1572
1573                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1574
1575                                 setTestJobTimeOut(newJob, scenario)
1576
1577                                 // Archive and process (only) the test results
1578                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1579                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1580                             }
1581                             else {
1582                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1583                             }
1584                         }
1585                         else if (isGcReliabilityFramework(scenario)) {
1586                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1587                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1588                         }
1589                         else if (architecture == 'x64' || architecture == 'x86') {
1590                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1591                         }
1592                         else if (architecture == 'x86lb') {
1593                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd"
1594                         }
1595                     }
1596
1597                     if (!enableCorefxTesting) {
1598                         // Run the rest of the build
1599                         // Build the mscorlib for the other OS's
1600                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1601                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1602                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1603
1604                         if (arch == "x64") {
1605                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1606                         }
1607
1608                         // Zip up the tests directory so that we don't use so much space/time copying
1609                         // 10s of thousands of files around.
1610                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1611
1612                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1613                             // For windows, pull full test results and test drops for x86/x64.
1614                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1615                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1616                         }
1617
1618                         if (scenario == 'jitdiff') {
1619                             // retrive jit-dasm output for base commit, and run jit-diff
1620                             if (!isBuildOnly) {
1621                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1622                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1623                             }
1624                         }
1625
1626                         if (!isBuildOnly) {
1627                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1628                             setTestJobTimeOut(newJob, scenario)
1629                         }
1630                     }
1631                     break
1632                 case 'arm':
1633                     def validArmWindowsScenarios = [ "default",
1634                                                      "pri1r2r",
1635                                                      "zapdisable",
1636                                                      "minopts",
1637                                                      "tailcallstress",
1638                                                      "jitstress1",
1639                                                      "jitstress2",
1640                                                      "gcstress0x3",
1641                                                      "gcstress0xc",
1642                                                      "jitstressregs1",
1643                                                      "jitstressregs2",
1644                                                      "gcstress0xc_jitstress1",
1645                                                      "gcstress0xc_jitstress2"]
1646
1647                     assert validArmWindowsScenarios.contains(scenario)
1648
1649                     // Set time out
1650                     setTestJobTimeOut(newJob, scenario)
1651
1652                     if ( lowerConfiguration == "debug" ) {
1653                         // For Debug builds, we will do a P1 test build
1654                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1655                     }
1656                     else if (lowerConfiguration == "checked") {
1657
1658                         if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1659                         {
1660                            // Up the timeout for arm checked testing only.
1661                            // Keep the longer timeout for gcstress.
1662                            Utilities.setJobTimeout(newJob, 240)
1663                         }
1664
1665                         def machineAffinityOptions = ['use_arm64_build_machine' : true]
1666                         setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1667                         // For checked runs we will also run testing.
1668                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1669                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1670                     }
1671                     else if (lowerConfiguration == "release") {
1672                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1673                     }
1674                     // Add archival.
1675                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1676                     break
1677                 case 'arm64':
1678                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1679
1680                     // Set time out
1681                     setTestJobTimeOut(newJob, scenario)
1682
1683                     // Debug runs take too long to run. So build job only.
1684                     if (lowerConfiguration == "debug") {
1685                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1686                     }
1687                     else {
1688                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1689                        {
1690                            // Up the timeout for arm64 checked testing only.
1691                            // Keep the longer timeout for gcstress.
1692                            Utilities.setJobTimeout(newJob, 240)
1693                        }
1694
1695                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=1"
1696                        // Test build and run are launched together.
1697                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1698                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1699                     }
1700
1701                     // Add archival.
1702                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1703                     break
1704                 default:
1705                     println("Unknown architecture: ${architecture}");
1706                     assert false
1707                     break
1708             }
1709             break
1710         // editor brace matching: }
1711         case 'Ubuntu':
1712         case 'Ubuntu16.04':
1713         case 'Ubuntu16.10':
1714         case 'Debian8.4':
1715         case 'OSX10.12':
1716         case 'FreeBSD':
1717         case 'CentOS7.1':
1718         case 'RHEL7.2':
1719         case 'OpenSUSE42.1':
1720         case 'Tizen':
1721         case 'Fedora24': // editor brace matching: {
1722             switch (architecture) {
1723                 case 'x64':
1724                 case 'x86':
1725                     if (architecture == 'x86' && os == 'Ubuntu') {
1726                         // build and PAL test
1727                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1728                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1729                         break;
1730                     }
1731
1732                     if (scenario == 'formatting') {
1733                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1734                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1735                         break
1736                     }
1737
1738                     if (scenario == 'illink') {
1739                         assert(os == 'Ubuntu')
1740                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1741                     }
1742
1743                     def standaloneGc = ''
1744                     if (scenario == 'standalone_gc') {
1745                         standaloneGc = 'buildstandalonegc'
1746                     }
1747
1748                     if (!enableCorefxTesting) {
1749                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1750                         // only on supported OS platforms.
1751                         if (os == 'FreeBSD')
1752                         {
1753                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1754                         }
1755                         else
1756                         {
1757                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1758                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1759                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1760                         }
1761                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1762
1763                         // Set time out
1764                         setTestJobTimeOut(newJob, scenario)
1765                         // Basic archiving of the build
1766                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1767                         // And pal tests
1768                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1769                     }
1770                     else {
1771                         // Corefx stress testing
1772                         assert os == 'Ubuntu'
1773                         assert architecture == 'x64'
1774                         assert lowerConfiguration == 'checked'
1775                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1776
1777                         // Build coreclr
1778                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1779
1780                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1781                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1782
1783                         // Build and text corefx
1784                         def workspaceRelativeFxRoot = "_/fx"
1785                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1786
1787                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1788
1789                         setTestJobTimeOut(newJob, scenario)
1790
1791                         // Archive and process (only) the test results
1792                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1793                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1794                     }
1795                     break
1796                 case 'arm64':
1797                     def standaloneGc = ''
1798                     if (scenario == 'standalone_gc') {
1799                         standaloneGc = 'buildstandalonegc'
1800                     }
1801
1802                     if (!enableCorefxTesting) {
1803                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8 ${standaloneGc}"
1804
1805                         // HACK -- Arm64 does not have corefx jobs yet.
1806                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1807                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1808                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1809                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1810
1811                         // Set time out
1812                         setTestJobTimeOut(newJob, scenario)
1813                         // Basic archiving of the build
1814                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1815                     }
1816                     break
1817                 case 'arm':
1818                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1819                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1820
1821                     // default values for Ubuntu
1822                     def arm_abi="arm"
1823                     def linuxCodeName="trusty"
1824                     if (os == 'Ubuntu16.04') {
1825                         linuxCodeName="xenial"
1826                     }
1827                     else if (os == 'Tizen') {
1828                         arm_abi="armel"
1829                         linuxCodeName="tizen"
1830                     }
1831
1832                     // Unzip the Windows test binaries first. Exit with 0
1833                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1834
1835                     // Unpack the corefx binaries
1836                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1837                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1838                     if (os != 'Tizen') {
1839                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1840                     }
1841                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1842                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1843                     if (os == 'Ubuntu') {
1844                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1845                     }
1846
1847                     // Call the ARM CI script to cross build and test using docker
1848                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1849                     --mode=docker \\
1850                     --${arm_abi} \\
1851                     --linuxCodeName=${linuxCodeName} \\
1852                     --buildConfig=${lowerConfiguration} \\
1853                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1854                     --coreFxBinDir=./bin/CoreFxBinDir \\
1855                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1856
1857                     // Basic archiving of the build, no pal tests
1858                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1859                     break
1860                 default:
1861                     println("Unknown architecture: ${architecture}");
1862                     assert false
1863                     break
1864             }
1865             break
1866         // editor brace matching: }
1867         default:
1868             println("Unknown os: ${os}");
1869             assert false
1870             break
1871     } // os
1872
1873     return buildCommands
1874 }
1875
1876 // Additional scenario which can alter behavior
1877
1878 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1879 combinedScenarios.each { scenario ->
1880     [true, false].each { isPR ->
1881         Constants.architectureList.each { architecture ->
1882             Constants.configurationList.each { configuration ->
1883                 Constants.osList.each { os ->
1884                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1885                     // and reset the os to Windows_NT
1886                     def isBuildOnly = false
1887                     if (os == 'Windows_NT_BuildOnly') {
1888                         isBuildOnly = true
1889                         os = 'Windows_NT'
1890                     }
1891
1892                     // Tizen is only supported for arm architecture
1893                     if (os == 'Tizen' && architecture != 'arm') {
1894                         return
1895                     }
1896
1897                     // Skip totally unimplemented (in CI) configurations.
1898                     switch (architecture) {
1899                         case 'arm64':
1900                             if (os == 'Ubuntu16.04') {
1901                                 os = 'Ubuntu'
1902                             }
1903
1904                             // Windows and Ubuntu only
1905                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1906                                 return
1907                             }
1908                             break
1909                         case 'arm':
1910                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1911                                 return
1912                             }
1913                             break
1914                         case 'x86':
1915                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1916                                 return
1917                             }
1918                             break
1919                         case 'x86lb':
1920                             if (os != 'Windows_NT') {
1921                                 return
1922                             }
1923                             break
1924                         case 'x64':
1925                             // Everything implemented
1926                             break
1927                         default:
1928                             println("Unknown architecture: ${architecture}")
1929                             assert false
1930                             break
1931                     }
1932
1933                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1934                     // with checked builds
1935                     def enableCorefxTesting = false
1936                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1937                         if (configuration != 'Checked') {
1938                             return
1939                         }
1940
1941                         enableCorefxTesting = isCorefxTesting(scenario)
1942
1943                         // Since these are just execution time differences,
1944                         // skip platforms that don't execute the tests here (Windows_NT only)
1945                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1946                         if (!isEnabledOS || isBuildOnly) {
1947                             return
1948                         }
1949
1950                         switch (architecture) {
1951                             case 'arm':
1952                                 if ((scenario != 'gcstress0x3') &&
1953                                     (scenario != 'gcstress0xc') &&
1954                                     (scenario != 'jitstress1') &&
1955                                     (scenario != 'jitstress2') &&
1956                                     (scenario != 'jitstressregs1') &&
1957                                     (scenario != 'jitstressregs2') &&
1958                                     (scenario != 'gcstress0xc_jitstress1') &&
1959                                     (scenario != 'gcstress0xc_jitstress2') &&
1960                                     (scenario != 'minopts') &&
1961                                     (scenario != 'tailcallstress') &&
1962                                     (scenario != 'zapdisable')) {
1963                                         return
1964                                     }
1965                                     break
1966                             case 'arm64':
1967                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1968                                     return
1969                                 }
1970                                 break
1971                             case 'x64':
1972                             case 'x86':
1973                                 // x86 ubuntu: default only
1974                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1975                                     return
1976                                 }
1977                                 // Windows: Everything implemented
1978                                 break
1979                             case 'x86lb':
1980                                 // No stress modes for legacy jit.
1981                                 // (There's no technical reason we couldn't allow these.)
1982                                 return
1983                             default:
1984                                 return
1985                         }
1986                     }
1987                     else {
1988                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
1989                         // and configuration is not Checked, bail out.
1990                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1991                             return;
1992                         }
1993
1994                         // Skip scenarios
1995                         switch (scenario) {
1996                             case 'pri1':
1997                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1998                                 // the default scenario build
1999                                 if (os != 'Windows_NT') {
2000                                     return
2001                                 }
2002                                 // Only x64 for now
2003                                 if (architecture != 'x64') {
2004                                     return
2005                                 }
2006                                 break
2007                             case 'ilrt':
2008                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2009                                 // the default scenario build
2010                                 if (os != 'Windows_NT') {
2011                                     return
2012                                 }
2013                                 // Only x64 for now
2014                                 if (architecture != 'x64') {
2015                                     return
2016                                 }
2017                                 // Release only
2018                                 if (configuration != 'Release') {
2019                                     return
2020                                 }
2021                                 break
2022                             case 'jitdiff':
2023                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2024                                     return
2025                                 }
2026                                 if (architecture != 'x64') {
2027                                     return
2028                                 }
2029                                 if (configuration != 'Checked') {
2030                                     return
2031                                 }
2032                                 break
2033                             case 'r2r':
2034                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2035                                 // the default scenario build
2036                                 if (os != 'Windows_NT') {
2037                                     return
2038                                 }
2039                                 if (architecture != 'x64') {
2040                                     return
2041                                 }
2042                                 break
2043                             case 'pri1r2r':
2044                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2045                                 // the default scenario build
2046                                 if (os != 'Windows_NT') {
2047                                     return
2048                                 }
2049                                 if (architecture != 'x64') {
2050                                     if ((architecture != 'arm64' && architecture != 'arm') || (configuration == 'Debug')) {
2051                                         return
2052                                     }
2053                                 }
2054                                 break
2055                             case 'gcstress15_pri1r2r':
2056                             case 'r2r_jitstress1':
2057                             case 'r2r_jitstress2':
2058                             case 'r2r_jitstressregs1':
2059                             case 'r2r_jitstressregs2':
2060                             case 'r2r_jitstressregs3':
2061                             case 'r2r_jitstressregs4':
2062                             case 'r2r_jitstressregs8':
2063                             case 'r2r_jitstressregs0x10':
2064                             case 'r2r_jitstressregs0x80':
2065                             case 'r2r_jitstressregs0x1000':
2066                             case 'r2r_jitminopts':
2067                             case 'r2r_jitforcerelocs':
2068                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
2069                                 // the default scenario build
2070                                 if (os != 'Windows_NT') {
2071                                     return
2072                                 }
2073                                 if (architecture != 'x64' && architecture != 'x86') {
2074                                     return
2075                                 }
2076                                 break
2077                             case 'longgc':
2078                             case 'gcsimulator':
2079                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2080                                     return
2081                                 }
2082                                 if (architecture != 'x64') {
2083                                     return
2084                                 }
2085                                 if (configuration != 'Release') {
2086                                     return
2087                                 }
2088                                 break
2089                             case 'gc_reliability_framework':
2090                             case 'standalone_gc':
2091                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2092                                     return
2093                                 }
2094
2095                                 if (architecture != 'x64') {
2096                                     return
2097                                 }
2098
2099                                 if (configuration != 'Release' && configuration != 'Checked') {
2100                                     return
2101                                 }
2102                                 break
2103                             // We need Windows x64 Release bits for the code coverage build
2104                             case 'coverage':
2105                                 if (os != 'Windows_NT') {
2106                                     return
2107                                 }
2108                                 if (architecture != 'x64') {
2109                                     return
2110                                 }
2111                                 if (configuration != 'Release') {
2112                                     return
2113                                 }
2114                                 break
2115                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2116                             case 'formatting':
2117                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2118                                     return
2119                                 }
2120                                 if (architecture != 'x64') {
2121                                     return
2122                                 }
2123                                 if (configuration != 'Checked') {
2124                                     return
2125                                 }
2126                                 if (isBuildOnly) {
2127                                     return
2128                                 }
2129                                 break
2130                             case 'illink':
2131                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2132                                     return
2133                                 }
2134                                 if (architecture != 'x64' && architecture != 'x86') {
2135                                     return
2136                                 }
2137                                 if (isBuildOnly) {
2138                                     return
2139                                 }
2140                                 break
2141                             case 'default':
2142                                 // Nothing skipped
2143                                 break
2144                             default:
2145                                 println("Unknown scenario: ${scenario}")
2146                                 assert false
2147                                 break
2148                         }
2149                     }
2150
2151                     // Calculate names
2152                     def lowerConfiguration = configuration.toLowerCase()
2153                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2154                     def folderName = getJobFolder(scenario)
2155
2156                     // Create the new job
2157                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2158
2159                     def machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2160                     machineAffinityOptions = architecture == 'arm' ? ['use_arm64_build_machine': false] : machineAffinityOptions
2161                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2162
2163                     // Add all the standard options
2164                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2165                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2166
2167                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
2168                     def osGroup = getOSGroup(os)
2169
2170                     newJob.with {
2171                         steps {
2172                             if (os == 'Windows_NT') {
2173                                 buildCommands.each { buildCommand ->
2174                                     batchFile(buildCommand)
2175                                 }
2176                             }
2177                             else {
2178                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2179                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2180                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2181                                     // Define the Windows Tests and Corefx build job names
2182                                     def WindowTestsName = projectFolder + '/' +
2183                                                           Utilities.getFullJobName(project,
2184                                                                                    getJobName(lowerConfiguration,
2185                                                                                               'x64' ,
2186                                                                                               'windows_nt',
2187                                                                                               'default',
2188                                                                                               true),
2189                                                                                    false)
2190                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2191                                                        Utilities.getFolderName(branch)
2192
2193                                     // Copy the Windows test binaries and the Corefx build binaries
2194                                     copyArtifacts(WindowTestsName) {
2195                                         includePatterns('bin/tests/tests.zip')
2196                                         buildSelector {
2197                                             latestSuccessful(true)
2198                                         }
2199                                     }
2200
2201                                     def arm_abi = 'arm'
2202                                     def corefx_os = 'linux'
2203                                     if (os == 'Tizen') {
2204                                         arm_abi = 'armel'
2205                                         corefx_os = 'tizen'
2206                                     }
2207
2208                                     // Let's use release CoreFX to test checked CoreCLR,
2209                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2210                                     def corefx_lowerConfiguration = lowerConfiguration
2211                                     if ( lowerConfiguration == 'checked' ) {
2212                                         corefx_lowerConfiguration='release'
2213                                     }
2214
2215                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2216                                         includePatterns('bin/build.tar.gz')
2217                                         buildSelector {
2218                                             latestSuccessful(true)
2219                                         }
2220                                     }
2221                                 }
2222
2223                                 buildCommands.each { buildCommand ->
2224                                     shell(buildCommand)
2225                                 }
2226                             }
2227                         }
2228                     } // newJob.with
2229
2230                 } // os
2231             } // configuration
2232         } // architecture
2233     } // isPR
2234 } // scenario
2235
2236
2237 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2238 combinedScenarios.each { scenario ->
2239     [true, false].each { isPR ->
2240         // Architectures.  x64 only at this point
2241         ['x64', 'arm64'].each { architecture ->
2242             // Put the OS's supported for coreclr cross testing here
2243             Constants.crossList.each { os ->
2244                 if (architecture == 'arm64') {
2245                     if (os != "Ubuntu") {
2246                         return
2247                     }
2248                 }
2249
2250                 Constants.configurationList.each { configuration ->
2251
2252                     if (architecture == 'arm64') {
2253                         if (scenario != 'default' && scenario != 'pri1r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2254                             return
2255                         }
2256                     }
2257
2258                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2259                         if (configuration != 'Checked') {
2260                             return
2261                         }
2262                         if (isCorefxTesting(scenario)) {
2263                             return
2264                         }
2265                         //Skip stress modes for these scenarios
2266                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2267                             return
2268                         }
2269                     }
2270                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2271                     // and configuration is not Checked, bail out.
2272                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2273                         return;
2274                     }
2275                     // For CentOS, we only want Checked/Release pri1 builds.
2276                     else if (os == 'CentOS7.1') {
2277                         if (scenario != 'pri1' &&
2278                             scenario != 'r2r' &&
2279                             scenario != 'pri1r2r' &&
2280                             scenario != 'gcstress15_pri1r2r' &&
2281                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2282                             return
2283                         }
2284                         if (configuration != 'Checked' && configuration != 'Release') {
2285                             return
2286                         }
2287                     }
2288                     // For RedHat and Debian, we only do Release pri1 builds.
2289                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2290                         if (scenario != 'pri1') {
2291                             return
2292                         }
2293                         if (configuration != 'Release') {
2294                             return
2295                         }
2296                     }
2297                     else {
2298                         // Skip scenarios
2299                         switch (scenario) {
2300                             case 'pri1':
2301                                 // Nothing skipped
2302                                 break
2303                             case 'ilrt':
2304                                 // Release only
2305                                 if (configuration != 'Release') {
2306                                     return
2307                                 }
2308                                 break
2309                             case 'jitdiff':
2310                                 if (configuration != 'Checked') {
2311                                     return;
2312                                 }
2313                             case 'r2r':
2314                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2315                                 if (configuration != 'Checked' && configuration != 'Release') {
2316                                     return
2317                                 }
2318                                 break
2319                             case 'pri1r2r':
2320                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2321                                 if (configuration != 'Checked' && configuration != 'Release') {
2322                                     return
2323                                 }
2324                                 break
2325                             case 'gcstress15_pri1r2r':
2326                             case 'r2r_jitstress1':
2327                             case 'r2r_jitstress2':
2328                             case 'r2r_jitstressregs1':
2329                             case 'r2r_jitstressregs2':
2330                             case 'r2r_jitstressregs3':
2331                             case 'r2r_jitstressregs4':
2332                             case 'r2r_jitstressregs8':
2333                             case 'r2r_jitstressregs0x10':
2334                             case 'r2r_jitstressregs0x80':
2335                             case 'r2r_jitstressregs0x1000':
2336                             case 'r2r_jitminopts':
2337                             case 'r2r_jitforcerelocs':
2338                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2339                                 if (configuration != 'Checked' && configuration != 'Release') {
2340                                     return
2341                                 }
2342                                 break
2343                             case 'longgc':
2344                             case 'gcsimulator':
2345                                 // Long GC tests take a long time on non-Release builds
2346                                 if (configuration != 'Release') {
2347                                     return
2348                                 }
2349                                 break
2350                             case 'gc_reliability_framework':
2351                             case 'standalone_gc':
2352                                 if (configuration != 'Release' && configuration != 'Checked') {
2353                                     return
2354                                 }
2355                                 break
2356                             case 'coverage':
2357                                 //We only want Ubuntu Release for coverage
2358                                 if (os != 'Ubuntu') {
2359                                     return
2360                                 }
2361                                 if (configuration != 'Release') {
2362                                     return
2363                                 }
2364                             case 'formatting':
2365                                 return
2366                             case 'illink':
2367                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2368                                     return
2369                                 }
2370                                 break
2371                             case 'default':
2372                                 // Nothing skipped
2373                                 break
2374                             default:
2375                                 println("Unknown scenario: ${scenario}")
2376                                 assert false
2377                                 break
2378                         }
2379                     }
2380
2381                     def lowerConfiguration = configuration.toLowerCase()
2382                     def osGroup = getOSGroup(os)
2383                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2384
2385                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2386                     def inputScenario = 'default'
2387                     if (scenario == 'coverage') {
2388                         inputScenario = 'coverage'
2389                     }
2390                     def inputCoreCLRBuildName = projectFolder + '/' +
2391                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2392                     // If this is a stress scenario, there isn't any difference in the build job
2393                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2394                     // from the default scenario
2395                     def testBuildScenario = scenario
2396                     if (scenarioNeedsPri1Build(scenario)) {
2397                         testBuildScenario = 'pri1'
2398                     }
2399                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2400                         testBuildScenario = 'default'
2401                     }
2402                     def inputWindowTestsBuildName = ''
2403                     def inputWindowsTestBuildArch = architecture
2404                     if (architecture == "arm64") {
2405                         // Use the x64 test build for arm64 unix
2406                         inputWindowsTestBuildArch = "x64"
2407                     }
2408                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2409                         inputWindowTestsBuildName = projectFolder + '/' +
2410                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', 'default', true), isPR)
2411                     }
2412                     else {
2413                         inputWindowTestsBuildName = projectFolder + '/' +
2414                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2415                     }
2416                     // Enable Server GC for Ubuntu PR builds
2417                     def serverGCString = ''
2418
2419                     // Whether or not this test run should be run sequentially instead
2420                     // of in parallel. Only used for long GC tests.
2421                     def sequentialString = ''
2422
2423                     // Whether or not this test run should run a specific playlist.
2424                     // Only used for long GC tests.
2425
2426                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2427                     // for running long GC and GCSimulator tests, respectively. We don't use them
2428                     // here because using a playlist file produces much more readable output on the CI machines
2429                     // and reduces running time.
2430                     def playlistString = ''
2431
2432                     if (os == 'Ubuntu' && isPR){
2433                         serverGCString = '--useServerGC'
2434                     }
2435
2436                     // pass --crossgen to runtest.sh for crossgen builds
2437                     def crossgenStr = ''
2438                     def runcrossgentestsStr = ''
2439                     def runjitstressStr = ''
2440                     def runjitstressregsStr = ''
2441                     def runjitmioptsStr = ''
2442                     def runjitforcerelocsStr = ''
2443                     def runjitdisasmStr = ''
2444                     def runilasmroundtripStr = ''
2445                     def gcstressStr = ''
2446                     def illinkStr = ''
2447                     def layoutOnlyStr =''
2448
2449                     if (scenario == 'r2r' ||
2450                         scenario == 'pri1r2r' ||
2451                         scenario == 'gcstress15_pri1r2r' ||
2452                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2453                             crossgenStr = '--crossgen'
2454                             runcrossgentestsStr = '--runcrossgentests'
2455
2456                             if (scenario == 'r2r_jitstress1'){
2457                                 runjitstressStr = '--jitstress=1'
2458                             }
2459                             else if (scenario == 'r2r_jitstress2') {
2460                                 runjitstressStr = '--jitstress=2'
2461                             }
2462                             else if (scenario == 'r2r_jitstressregs1'){
2463                                 runjitstressregsStr = '--jitstressregs=1'
2464                             }
2465                             else if (scenario == 'r2r_jitstressregs2') {
2466                                 runjitstressregsStr = '--jitstressregs=2'
2467                             }
2468                             else if (scenario == 'r2r_jitstressregs3') {
2469                                 runjitstressregsStr = '--jitstressregs=3'
2470                             }
2471                             else if (scenario == 'r2r_jitstressregs4') {
2472                                 runjitstressregsStr = '--jitstressregs=4'
2473                             }
2474                             else if (scenario == 'r2r_jitstressregs8') {
2475                                 runjitstressregsStr = '--jitstressregs=8'
2476                             }
2477                             else if (scenario == 'r2r_jitstressregs0x10') {
2478                                 runjitstressregsStr = '--jitstressregs=0x10'
2479                             }
2480                             else if (scenario == 'r2r_jitstressregs0x80') {
2481                                 runjitstressregsStr = '--jitstressregs=0x80'
2482                             }
2483                             else if (scenario == 'r2r_jitstressregs0x1000') {
2484                                 runjitstressregsStr = '--jitstressregs=0x1000'
2485                             }
2486                             else if (scenario == 'r2r_jitminopts') {
2487                                 runjitmioptsStr = '--jitminopts'
2488                             }
2489                             else if (scenario == 'r2r_jitforcerelocs') {
2490                                 runjitforcerelocsStr = '--jitforcerelocs'
2491                             }
2492                     }
2493                     if  (scenario == 'gcstress15_pri1r2r')
2494                     {
2495                         gcstressStr = '--gcstresslevel=0xF'
2496                     }
2497
2498                     if (scenario == 'jitdiff')
2499                     {
2500                         runjitdisasmStr = '--jitdisasm --crossgen'
2501                     }
2502
2503                     if (scenario == 'illink')
2504                     {
2505                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2506                     }
2507
2508                     if (isLongGc(scenario)) {
2509                         // Long GC tests behave very poorly when they are not
2510                         // the only test running (many of them allocate until OOM).
2511                         sequentialString = '--sequential'
2512
2513                         // The Long GC playlist contains all of the tests that are
2514                         // going to be run. The GCSimulator playlist contains all of
2515                         // the GC simulator tests.
2516                         if (scenario == 'longgc') {
2517                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2518                         }
2519                         else if (scenario == 'gcsimulator') {
2520                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2521                         }
2522                     }
2523
2524                     if (isGcReliabilityFramework(scenario)) {
2525                         layoutOnlyStr = '--build-overlay-only'
2526                     }
2527
2528                     def folder = getJobFolder(scenario)
2529                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2530                         // Add parameters for the inputs
2531
2532                         parameters {
2533                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2534                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2535                         }
2536
2537                         steps {
2538                             // Set up the copies
2539
2540                             // Coreclr build containing the tests and mscorlib
2541
2542                             copyArtifacts(inputWindowTestsBuildName) {
2543                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2544                                 buildSelector {
2545                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2546                                 }
2547                             }
2548
2549                             if (scenario == 'coverage') {
2550
2551                                 // Move coreclr to clr directory
2552                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2553
2554                                 // Build coreclr
2555                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2556
2557                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2558                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2559                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2560                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2561                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2562                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2563                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2564                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2565
2566                                 // Run PAL tests
2567                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2568
2569                                 // Remove obj files for PAL tests so they're not included in coverage results
2570                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2571
2572                                 // Unzip the tests first.  Exit with 0
2573                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2574
2575                                 // Get corefx
2576                                 shell("git clone https://github.com/dotnet/corefx fx")
2577
2578                                 // Build Linux corefx
2579                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2580                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2581
2582                                 def testEnvOpt = ""
2583                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2584                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2585                                 shell("${createScriptCmds}")
2586                                 testEnvOpt = "--test-env=" + scriptFileName
2587
2588                                 // Run corefx tests
2589                                 shell("""./fx/run-test.sh \\
2590                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2591                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2592                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2593                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2594                 --configurationGroup Release""")
2595
2596
2597                                 // Run coreclr tests w/ workstation GC
2598                                 shell("""./clr/tests/runtest.sh \\
2599                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2600                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2601                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2602                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2603                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2604                 --crossgen --runcrossgentests""")
2605
2606                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2607                                 shell("""./clr/tests/runtest.sh \\
2608                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2609                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2610                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2611                 --useServerGC ${testEnvOpt}""")
2612
2613                                  // Run long-running coreclr GC tests & produce coverage reports
2614                                 shell("""./clr/tests/runtest.sh \\
2615                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2616                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2617                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2618                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2619                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2620                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2621                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2622
2623                             }
2624                             else {
2625
2626                                 // Coreclr build we are trying to test
2627
2628                                 copyArtifacts(inputCoreCLRBuildName) {
2629                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2630                                     buildSelector {
2631                                         buildNumber('${CORECLR_BUILD}')
2632                                     }
2633                                 }
2634
2635                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2636
2637                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2638                                 def osJobName
2639                                 if (os == 'Ubuntu') {
2640                                     osJobName = 'ubuntu14.04'
2641                                 }
2642                                 else {
2643                                     osJobName = os.toLowerCase()
2644                                 }
2645                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2646                                     includePatterns('bin/build.tar.gz')
2647                                     buildSelector {
2648                                         latestSuccessful(true)
2649                                     }
2650                                 }
2651
2652                                 shell ("mkdir ./bin/CoreFxBinDir")
2653                                 // Unpack the corefx binaries
2654                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2655
2656                                 // HACK -- Arm64 does not have corefx jobs yet.
2657                                 // Clone corefx and build the native packages overwriting the x64 packages.
2658                                 if (architecture == 'arm64') {
2659                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2660                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2661                                 }
2662
2663                                 // Unzip the tests first.  Exit with 0
2664                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2665
2666                                 // Execute the tests
2667                                 // If we are running a stress mode, we'll set those variables first
2668                                 def testEnvOpt = ""
2669                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2670                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2671                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2672                                     shell("${createScriptCmds}")
2673                                     testEnvOpt = "--test-env=" + scriptFileName
2674                                 }
2675
2676                                 if (isGCStressRelatedTesting(scenario)) {
2677                                     shell('./init-tools.sh')
2678                                 }
2679
2680                                 shell("""./tests/runtest.sh \\
2681                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2682                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2683                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2684                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2685                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2686                 --limitedDumpGeneration \\
2687                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
2688                 ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
2689                 ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
2690
2691                                 if (isGcReliabilityFramework(scenario)) {
2692                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2693                                     if (serverGCString != '') {
2694                                         shell("export COMPlus_gcServer=1")
2695                                     }
2696
2697                                     shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2698                                 }
2699                             }
2700                         }
2701                     }
2702
2703                     if (scenario == 'coverage') {
2704                         // Publish coverage reports
2705                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2706                         // TODO: Add once external email sending is available again
2707                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2708                     }
2709
2710                     if (scenario == 'jitdiff') {
2711                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2712                     }
2713
2714                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2715                     if (os in ['Ubuntu']) {
2716                         SummaryBuilder summaries = new SummaryBuilder()
2717                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2718                         summaries.emit(newJob)
2719                     }
2720
2721                     setMachineAffinity(newJob, os, architecture)
2722                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2723                     // Set timeouts to 240.
2724                     setTestJobTimeOut(newJob, scenario)
2725
2726                     if (architecture == 'arm64') {
2727                         Utilities.setJobTimeout(newJob, 240)
2728                     }
2729
2730                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2731
2732                     // Create a build flow to join together the build and tests required to run this
2733                     // test.
2734                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2735                     // Linux CoreCLR test
2736                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2737                     def fullTestJobName = projectFolder + '/' + newJob.name
2738                     // Add a reference to the input jobs for report purposes
2739                     JobReport.Report.addReference(inputCoreCLRBuildName)
2740                     JobReport.Report.addReference(inputWindowTestsBuildName)
2741                     JobReport.Report.addReference(fullTestJobName)
2742                     def newFlowJob;
2743
2744                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2745                     // so that coverage data can be preserved.
2746                     if (scenario == 'coverage') {
2747                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2748                         buildFlow("""
2749 // Build the input Windows job
2750 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2751
2752 // And then build the test build
2753 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2754 """)
2755                         }
2756                     // Normal jobs copy a Windows build & a non-Windows build
2757                     } else {
2758                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2759                         buildFlow("""
2760 // Build the input jobs in parallel
2761 parallel (
2762     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2763     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2764 )
2765
2766 // And then build the test build
2767 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2768                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2769 """)
2770                         }
2771                     }
2772
2773                     setMachineAffinity(newFlowJob, os, architecture)
2774                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2775                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2776                 } // configuration
2777             } // os
2778         } // architecture
2779     } // isPR
2780 } // scenario
2781
2782 JobReport.Report.generateJobReport(out)
2783
2784 // Make the call to generate the help job
2785 Utilities.createHelperJob(this, project, branch,
2786     "Welcome to the ${project} Repository",  // This is prepended to the help message
2787     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2788
2789 Utilities.addCROSSCheck(this, project, branch)