netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'FreeBSD':'FreeBSD',
  29         'CentOS7.1': 'Linux',
  30         'OpenSUSE42.1': 'Linux',
  31         'Tizen': 'Linux']
  32     def osGroup = osGroupMap.get(os, null)
  33     assert osGroup != null : "Could not find os group for ${os}"
  34     return osGroupMap[os]
  35 }
  36
  37 // We use this class (vs variables) so that the static functions can access data here.
  38 class Constants {
  39
  40     // Innerloop build OS's
  41     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  42     // test execution in the build flow runs.  It generates the exact same build
  43     // as Windows_NT but without the tests.
  44     def static osList = [
  45                'Ubuntu',
  46                'Debian8.4',
  47                'OSX10.12',
  48                'Windows_NT',
  49                'Windows_NT_BuildOnly',
  50                'FreeBSD',
  51                'CentOS7.1',
  52                'OpenSUSE42.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  67                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  68                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  69                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  70                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  71                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  72                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  73                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  74                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  75                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  76                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  77                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  78                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  79                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  80                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  81                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  82                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  83                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  84                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  85                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  86                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  87                'corefx_baseline'                : [ : ], // corefx baseline
  88                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  89                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  90                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  91                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  92                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  93                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  94                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  95                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  96                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  97                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  98                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  99                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 100                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 101                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 103                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 104                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 105                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 107                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 109                ]
 110
 111     // This is a set of r2r jit stress scenarios
 112     def static r2rJitStressScenarios = [
 113                'r2r_jitstress1',
 114                'r2r_jitstress2',
 115                'r2r_jitstressregs1',
 116                'r2r_jitstressregs2',
 117                'r2r_jitstressregs3',
 118                'r2r_jitstressregs4',
 119                'r2r_jitstressregs8',
 120                'r2r_jitstressregs0x10',
 121                'r2r_jitstressregs0x80',
 122                'r2r_jitstressregs0x1000',
 123                'r2r_jitminopts',
 124                'r2r_jitforcerelocs']
 125
 126     // This is the basic set of scenarios
 127     def static basicScenarios = [
 128                'default',
 129                'pri1',
 130                'ilrt',
 131                'r2r',
 132                'pri1r2r',
 133                'gcstress15_pri1r2r',
 134                'longgc',
 135                'coverage',
 136                'formatting',
 137                'gcsimulator',
 138                'jitdiff',
 139                'standalone_gc',
 140                'gc_reliability_framework',
 141                'illink'] + r2rJitStressScenarios
 142
 143     def static configurationList = ['Debug', 'Checked', 'Release']
 144
 145     // This is the set of architectures
 146     def static architectureList = ['arm', 'arm64', 'x64', 'x86', 'x86lb']
 147 }
 148
 149 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 150     if (architecture == 'arm64' && os == 'Windows_NT') {
 151         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 152     } else if (architecture == 'arm64' && os != 'Windows_NT' && options == null) {
 153         Utilities.setMachineAffinity(job, os, 'arm64-small-page-size');
 154     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['large_pages'] == true) {
 155         Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size');
 156     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['is_build_only'] == true) {
 157         Utilities.setMachineAffinity(job, os, 'arm64-cross-latest');
 158     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 159         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 160     } else if ((architecture == 'arm') && (os == 'Windows_NT') && options['use_arm64_build_machine'] == true) {
 161         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 162     }else {
 163         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 164     }
 165 }
 166
 167 def static isJITStressJob(def scenario) {
 168     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 169            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 170 }
 171
 172 def static isGCStressRelatedTesting(def scenario) {
 173     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 174     // Detect it and make it a GCStress related.
 175     if (scenario == 'gcstress15_pri1r2r')
 176     {
 177         return true;
 178     }
 179
 180     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 181     def scenarioName = scenario.toLowerCase()
 182     def isGCStressTesting = false
 183     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 184         if (k in gcStressTestEnvVars) {
 185             isGCStressTesting = true;
 186         }
 187     }
 188     return isGCStressTesting
 189 }
 190
 191 def static isCorefxTesting(def scenario) {
 192     def corefx_prefix = 'corefx_'
 193     if (scenario.length() < corefx_prefix.length()) {
 194         return false
 195     }
 196     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 197 }
 198
 199 def static isR2R(def scenario) {
 200     return (scenario == 'r2r' || scenario == 'pri1r2r')
 201 }
 202
 203 def static isCoverage(def scenario) {
 204     return (scenario == 'coverage')
 205 }
 206
 207 def static isLongGc(def scenario) {
 208     return (scenario == 'longgc' || scenario == 'gcsimulator')
 209 }
 210
 211 def static isJitDiff(def scenario) {
 212     return (scenario == 'jitdiff')
 213 }
 214
 215 def static isGcReliabilityFramework(def scenario) {
 216     return (scenario == 'gc_reliability_framework')
 217 }
 218
 219 def static scenarioNeedsPri1Build(def scenario) {
 220     return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario))
 221 }
 222
 223 def static setTestJobTimeOut(newJob, scenario) {
 224     if (isGCStressRelatedTesting(scenario)) {
 225         Utilities.setJobTimeout(newJob, 4320)
 226     }
 227     else if (isCorefxTesting(scenario)) {
 228         Utilities.setJobTimeout(newJob, 360)
 229     }
 230     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 231         Utilities.setJobTimeout(newJob, 240)
 232     }
 233     else if (isR2R(scenario)) {
 234         Utilities.setJobTimeout(newJob, 240)
 235     }
 236     else if (isCoverage(scenario)) {
 237         Utilities.setJobTimeout(newJob, 1440)
 238     }
 239     else if (isLongGc(scenario)) {
 240         Utilities.setJobTimeout(newJob, 1440)
 241     }
 242     else if (isJitDiff(scenario)) {
 243         Utilities.setJobTimeout(newJob, 240)
 244     }
 245     else if (isGcReliabilityFramework(scenario)) {
 246         Utilities.setJobTimeout(newJob, 1440)
 247     }
 248     // Non-test jobs use the default timeout value.
 249 }
 250
 251 def static getJobFolder(def scenario) {
 252     if (isJITStressJob(scenario)) {
 253         return 'jitstress'
 254     }
 255     if (scenario == 'illink') {
 256         return 'illink'
 257     }
 258     return ''
 259 }
 260
 261 def static getStressModeDisplayName(def scenario) {
 262     def displayStr = ''
 263     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 264         def prefixLength = 'COMPlus_'.length()
 265         if (k.length() >= prefixLength) {
 266             def modeName = k.substring(prefixLength, k.length())
 267             displayStr += ' ' + modeName + '=' + v
 268         }
 269     }
 270     return displayStr
 271 }
 272
 273 def static getR2RStressModeDisplayName(def scenario) {
 274     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 275     // "r2r_" prefix.
 276     def displayStr = scenario
 277     def prefixLength = 'r2r_'.length()
 278     if (displayStr.length() >= prefixLength) {
 279         displayStr = displayStr.substring(prefixLength, displayStr.length())
 280     }
 281     return displayStr
 282 }
 283
 284 // Generates the string for creating a file that sets environment variables
 285 // that makes it possible to run stress modes.  Writes the script to the file
 286 // specified by the stepScriptLocation parameter.
 287 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 288     def stepScript = ''
 289     if (os == 'Windows_NT') {
 290         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 291         stepScript += "del ${stepScriptLocation}\r\n"
 292
 293         // Timeout in ms, default is 10 minutes. For stress
 294         // modes up this to 30 minutes
 295         def timeout = 1800000
 296
 297         // Set the Timeout
 298         stepScript += "set __TestTimeout=${timeout}\r\n"
 299         stepScript += "echo. > ${stepScriptLocation}\r\n"
 300         stressModeVars.each{ k, v ->
 301             // Write out what we are writing to the script file
 302             stepScript += "echo Setting ${k}=${v}\r\n"
 303             // Write out the set itself to the script file`
 304             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 305         }
 306     }
 307     else {
 308         stepScript += "echo Setting variables for ${stressModeName}\n"
 309         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 310         stressModeVars.each{ k, v ->
 311             // Write out what we are writing to the script file
 312             stepScript += "echo Setting ${k}=${v}\n"
 313             // Write out the set itself to the script file`
 314             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 315         }
 316         stepScript += "chmod +x ${stepScriptLocation}\n"
 317     }
 318     return stepScript
 319 }
 320
 321 // Calculates the name of the build job based on some typical parameters.
 322 //
 323 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 324     // If the architecture is x64, do not add that info into the build name.
 325     // Need to change around some systems and other builds to pick up the right builds
 326     // to do that.
 327
 328     def suffix = scenario != 'default' ? "_${scenario}" : '';
 329     if (isBuildOnly) {
 330         suffix += '_bld'
 331     }
 332     def baseName = ''
 333     switch (architecture) {
 334         case 'x64':
 335             if (scenario == 'default') {
 336                 // For now we leave x64 off of the name for compatibility with other jobs
 337                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 338             }
 339             else if (scenario == 'formatting') {
 340                 // we don't care about the configuration for the formatting job. It runs all configs
 341                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 342             }
 343             else {
 344                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 345             }
 346             break
 347         case 'arm64':
 348             if (os.toLowerCase() == "windows_nt") {
 349                 // These are cross builds
 350                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 351             }
 352             else {
 353                 // Defaults to a small page size set of machines.
 354                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 355             }
 356             break
 357         case 'arm':
 358             // These are cross builds
 359             if (os == 'Tizen') {
 360                 // ABI: softfp
 361                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 362             }
 363             else {
 364                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 365             }
 366             break
 367         case 'x86':
 368         case 'x86lb':
 369             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 370             break
 371         default:
 372             println("Unknown architecture: ${architecture}");
 373             assert false
 374             break
 375     }
 376
 377     return baseName + suffix
 378 }
 379
 380 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 381     // Check scenario.
 382     switch (scenario) {
 383         case 'default':
 384             switch (architecture) {
 385                 case 'x64':
 386                 case 'x86':
 387                 case 'x86lb':
 388                     if (architecture == 'x86' && os == 'Ubuntu') {
 389                         Utilities.addPeriodicTrigger(job, '@daily')
 390                     }
 391                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 392                         Utilities.addGithubPushTrigger(job)
 393                     }
 394                     break
 395                 case 'arm':
 396                     Utilities.addGithubPushTrigger(job)
 397                     break
 398                 case 'arm64':
 399                     Utilities.addGithubPushTrigger(job)
 400                     break
 401                 default:
 402                     println("Unknown architecture: ${architecture}");
 403                     assert false
 404                     break
 405             }
 406             break
 407         case 'pri1':
 408             // Pri one gets a push trigger, and only for release
 409             if (architecture == 'x64') {
 410                 if (configuration == 'Release') {
 411                     // We expect release jobs to be Windows, or in the cross list
 412                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 413                     if (!os in bidailyCrossList) {
 414                         if (isFlowJob || os == 'Windows_NT') {
 415                             Utilities.addGithubPushTrigger(job)
 416                         }
 417                     }
 418                     else {
 419                         if (isFlowJob) {
 420                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 421                         }
 422                     }
 423                 }
 424             }
 425             break
 426         case 'r2r':
 427             //r2r jobs that aren't pri1 can only be triggered by phrase
 428             break
 429         case 'pri1r2r':
 430             assert !(os in bidailyCrossList)
 431             //pri1 r2r gets a push trigger for checked/release
 432             if (configuration == 'Checked' || configuration == 'Release') {
 433                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 434                 if (architecture == 'x64' && os != 'OSX10.12') {
 435                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 436                     if (isFlowJob || os == 'Windows_NT') {
 437                         Utilities.addGithubPushTrigger(job)
 438                     }
 439                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 440                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 441                     if (isFlowJob) {
 442                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 443                     }
 444                 }
 445                 // For x86, only add per-commit jobs for Windows
 446                 else if (architecture == 'x86' || architecture == 'x86lb') {
 447                     if (os == 'Windows_NT') {
 448                         Utilities.addGithubPushTrigger(job)
 449                     }
 450                 }
 451                 // arm64 pri1r2r jobs should only run every 12 hours.
 452                 else if (architecture == 'arm64') {
 453                     if (os == 'Windows_NT') {
 454                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 455                         // TODO: Add once external email sending is available again
 456                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 457                     }
 458                 }
 459             }
 460             break
 461         case 'r2r_jitstress1':
 462         case 'r2r_jitstress2':
 463         case 'r2r_jitstressregs1':
 464         case 'r2r_jitstressregs2':
 465         case 'r2r_jitstressregs3':
 466         case 'r2r_jitstressregs4':
 467         case 'r2r_jitstressregs8':
 468         case 'r2r_jitstressregs0x10':
 469         case 'r2r_jitstressregs0x80':
 470         case 'r2r_jitstressregs0x1000':
 471         case 'r2r_jitminopts':
 472         case 'r2r_jitforcerelocs':
 473         case 'gcstress15_pri1r2r':
 474             assert !(os in bidailyCrossList)
 475
 476             // GCStress=C is currently not supported on OS X
 477             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 478                 break
 479             }
 480
 481             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 482             if (configuration == 'Checked' || configuration == 'Release') {
 483                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 484                 if (architecture == 'x64') {
 485                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 486                     if (isFlowJob || os == 'Windows_NT') {
 487                         // Add a weekly periodic trigger
 488                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 489                     }
 490                 }
 491                 // For x86, only add per-commit jobs for Windows
 492                 else if (architecture == 'x86') {
 493                     if (os == 'Windows_NT') {
 494                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 495                     }
 496                 }
 497             }
 498             break
 499         case 'longgc':
 500             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 501             assert configuration == 'Release'
 502             assert architecture == 'x64'
 503             Utilities.addPeriodicTrigger(job, '@daily')
 504             // TODO: Add once external email sending is available again
 505             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 506             break
 507         case 'gcsimulator':
 508             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 509             assert configuration == 'Release'
 510             assert architecture == 'x64'
 511             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 512             // TODO: Add once external email sending is available again
 513             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 514             break
 515         case 'standalone_gc':
 516             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 517             assert (configuration == 'Release' || configuration == 'Checked')
 518             // TODO: Add once external email sending is available again
 519             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 520             Utilities.addPeriodicTrigger(job, '@weekly')
 521             break
 522         case 'gc_reliability_framework':
 523             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 524             assert (configuration == 'Release' || configuration == 'Checked')
 525             // Only triggered by phrase.
 526             break
 527         case 'ilrt':
 528             assert !(os in bidailyCrossList)
 529             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 530             if (architecture == 'x64' && configuration == 'Release') {
 531                 // We don't expect to see a job generated except in these scenarios
 532                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 533                 if (isFlowJob || os == 'Windows_NT') {
 534                     Utilities.addPeriodicTrigger(job, '@daily')
 535                 }
 536             }
 537             break
 538         case 'jitdiff':
 539             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 540             assert configuration == 'Checked'
 541             assert (architecture == 'x64' || architecture == 'x86')
 542             Utilities.addGithubPushTrigger(job)
 543             break
 544         case 'coverage':
 545             assert (os == 'Ubuntu' || os == 'Windows_NT')
 546             assert configuration == 'Release'
 547             assert architecture == 'x64'
 548             Utilities.addPeriodicTrigger(job, '@weekly')
 549             break
 550         case 'formatting':
 551             assert (os == 'Windows_NT' || os == "Ubuntu")
 552             assert architecture == 'x64'
 553             Utilities.addGithubPushTrigger(job)
 554             break
 555         case 'jitstressregs1':
 556         case 'jitstressregs2':
 557         case 'jitstressregs3':
 558         case 'jitstressregs4':
 559         case 'jitstressregs8':
 560         case 'jitstressregs0x10':
 561         case 'jitstressregs0x80':
 562         case 'jitstressregs0x1000':
 563         case 'minopts':
 564         case 'forcerelocs':
 565         case 'jitstress1':
 566         case 'jitstress2':
 567         case 'jitstress2_jitstressregs1':
 568         case 'jitstress2_jitstressregs2':
 569         case 'jitstress2_jitstressregs3':
 570         case 'jitstress2_jitstressregs4':
 571         case 'jitstress2_jitstressregs8':
 572         case 'jitstress2_jitstressregs0x10':
 573         case 'jitstress2_jitstressregs0x80':
 574         case 'jitstress2_jitstressregs0x1000':
 575         case 'tailcallstress':
 576         case 'jitsse2only':
 577         case 'corefx_baseline':
 578         case 'corefx_minopts':
 579         case 'corefx_jitstress1':
 580         case 'corefx_jitstress2':
 581         case 'corefx_jitstressregs1':
 582         case 'corefx_jitstressregs2':
 583         case 'corefx_jitstressregs3':
 584         case 'corefx_jitstressregs4':
 585         case 'corefx_jitstressregs8':
 586         case 'corefx_jitstressregs0x10':
 587         case 'corefx_jitstressregs0x80':
 588         case 'corefx_jitstressregs0x1000':
 589         case 'zapdisable':
 590             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 591             assert (os == 'Windows_NT') || (os in Constants.crossList)
 592             Utilities.addPeriodicTrigger(job, '@daily')
 593         }
 594         break
 595         case 'heapverify1':
 596         case 'gcstress0x3':
 597             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 598                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 599                 if (architecture == 'arm64') {
 600                     if (os == 'Windows_NT') {
 601                         Utilities.addPeriodicTrigger(job, '@daily')
 602                     }
 603                     // TODO: Add once external email sending is available again
 604                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 605                 }
 606                 else {
 607                     Utilities.addPeriodicTrigger(job, '@weekly')
 608                 }
 609             }
 610             break
 611         case 'gcstress0xc':
 612         case 'gcstress0xc_zapdisable':
 613         case 'gcstress0xc_zapdisable_jitstress2':
 614         case 'gcstress0xc_zapdisable_heapverify1':
 615         case 'gcstress0xc_jitstress1':
 616         case 'gcstress0xc_jitstress2':
 617         case 'gcstress0xc_minopts_heapverify1':
 618             // GCStress=C is currently not supported on OS X
 619             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 620                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 621                 if (architecture == 'arm64') {
 622                     // TODO: Enable a periodic trigger after tests are updated.
 623                     // Utilities.addPeriodicTrigger(job, '@daily')
 624                     // TODO: Add once external email sending is available again
 625                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 626                 }
 627                 else {
 628                     Utilities.addPeriodicTrigger(job, '@weekly')
 629                 }
 630             }
 631             break
 632
 633         case 'illink':
 634             // Testing on other operating systems TBD
 635             assert (os == 'Windows_NT' || os == 'Ubuntu')
 636             if (architecture == 'x64' || architecture == 'x86') {
 637                 if (configuration == 'Checked') {
 638                     Utilities.addPeriodicTrigger(job, '@daily')
 639                 }
 640             }
 641             break
 642
 643         default:
 644             println("Unknown scenario: ${scenario}");
 645             assert false
 646             break
 647     }
 648     return
 649 }
 650
 651 // **************************
 652 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 653 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 654 // builds will do a couple extra steps.
 655 // **************************
 656
 657 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 658 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 659 // it's just used for internal builds
 660 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 661 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 662     if (isWindowsBuildOnlyJob) {
 663         return
 664     }
 665
 666     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 667     // Non pull request builds.
 668     if (!isPR) {
 669         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 670         return
 671     }
 672
 673      def arm64Users = [
 674         'adiaaida',
 675         'AndyAyersMS',
 676         'briansull',
 677         'BruceForstall',
 678         'CarolEidt',
 679         'cmckinsey',
 680         'erozenfeld',
 681         'jashook',
 682         'JosephTremoulet',
 683         'pgavlin',
 684         'russellhadley',
 685         'RussKeldorph',
 686         'sandreenko',
 687         'sdmaclea',
 688         'sivarv',
 689         'swaroop-sridhar',
 690         'gkhanna79',
 691         'jkotas',
 692         'markwilkie',
 693         'rahku',
 694         'ramarag',
 695         'tzwlai',
 696         'weshaggard'
 697     ]
 698
 699     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 700     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 701     def osGroup = getOSGroup(os)
 702     switch (architecture) {
 703         case 'x64': // editor brace matching: {
 704             if (scenario == 'coverage') {
 705                 assert configuration == 'Release'
 706                 if (os == 'Ubuntu') {
 707                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 708                 }
 709                 break
 710             }
 711
 712             if (scenario == 'formatting') {
 713                 assert configuration == 'Checked'
 714                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 715                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 716                 }
 717                 break
 718             }
 719
 720             switch (os) {
 721                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 722                 case 'Debian8.4':
 723                 case 'RHEL7.2':
 724                     if (scenario == 'default') {
 725                         assert !isFlowJob
 726                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 727                     }
 728                     else if (scenario == 'pri1' && isFlowJob) {
 729                         assert (configuration == 'Release')
 730                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 731                     }
 732                     break
 733                 case 'Ubuntu16.04':
 734                     assert !isFlowJob
 735                     assert scenario == 'default'
 736                     // Distinguish with the other architectures (arm and x86)
 737                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 738                     break
 739                 case 'Fedora24':
 740                 case 'Ubuntu16.10':
 741                 case 'OpenSUSE42.1':
 742                     assert !isFlowJob
 743                     assert scenario == 'default'
 744                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 745                     break
 746                 case 'Ubuntu':
 747                     if (scenario == 'illink') {
 748                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 749                         break
 750                     }
 751                     // fall through
 752                 case 'OSX10.12':
 753                     // Triggers on the non-flow jobs aren't necessary here
 754                     // Corefx testing uses non-flow jobs.
 755                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 756                         break
 757                     }
 758                     switch (scenario) {
 759                         case 'default':
 760                             // Ubuntu uses checked for default PR tests
 761                             if (configuration == 'Checked') {
 762                                 // Default trigger
 763                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 764                             }
 765                             break
 766                         case 'pri1':
 767                             if (configuration == 'Release') {
 768                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 769                             }
 770                             break
 771                         case 'jitdiff':
 772                             if (configuration == 'Checked') {
 773                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 774                             }
 775                             break
 776                         case 'ilrt':
 777                             if (configuration == 'Release') {
 778                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 779                             }
 780                             break
 781                         case 'r2r':
 782                             if (configuration == 'Release' || configuration == 'Checked') {
 783                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 784                             }
 785                             break
 786                         case 'pri1r2r':
 787                             if (configuration == 'Release' || configuration == 'Checked') {
 788                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 789                             }
 790                             break
 791                         case 'gcstress15_pri1r2r':
 792                             if (configuration == 'Release' || configuration == 'Checked') {
 793                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 794                             }
 795                             break
 796                         case 'r2r_jitstress1':
 797                         case 'r2r_jitstress2':
 798                         case 'r2r_jitstressregs1':
 799                         case 'r2r_jitstressregs2':
 800                         case 'r2r_jitstressregs3':
 801                         case 'r2r_jitstressregs4':
 802                         case 'r2r_jitstressregs8':
 803                         case 'r2r_jitstressregs0x10':
 804                         case 'r2r_jitstressregs0x80':
 805                         case 'r2r_jitstressregs0x1000':
 806                         case 'r2r_jitminopts':
 807                         case 'r2r_jitforcerelocs':
 808                             if (configuration == 'Release' || configuration == 'Checked') {
 809                                 def displayStr = getR2RStressModeDisplayName(scenario)
 810                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 811                             }
 812                             break
 813                         case 'longgc':
 814                             if (configuration == 'Release') {
 815                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 816                             }
 817                             break
 818                         case 'gcsimulator':
 819                             if (configuration == 'Release') {
 820                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 821                             }
 822                             break
 823                         case 'standalone_gc':
 824                             if (configuration == 'Release' || configuration == 'Checked') {
 825                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 826                             }
 827                             break
 828                         case 'gc_reliability_framework':
 829                             if (configuration == 'Release' || configuration == 'Checked') {
 830                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 831                             }
 832                             break
 833                         case 'minopts':
 834                         case 'forcerelocs':
 835                         case 'jitstress1':
 836                         case 'jitstress2':
 837                         case 'jitstressregs1':
 838                         case 'jitstressregs2':
 839                         case 'jitstressregs3':
 840                         case 'jitstressregs4':
 841                         case 'jitstressregs8':
 842                         case 'jitstressregs0x10':
 843                         case 'jitstressregs0x80':
 844                         case 'jitstressregs0x1000':
 845                         case 'jitstress2_jitstressregs1':
 846                         case 'jitstress2_jitstressregs2':
 847                         case 'jitstress2_jitstressregs3':
 848                         case 'jitstress2_jitstressregs4':
 849                         case 'jitstress2_jitstressregs8':
 850                         case 'jitstress2_jitstressregs0x10':
 851                         case 'jitstress2_jitstressregs0x80':
 852                         case 'jitstress2_jitstressregs0x1000':
 853                         case 'tailcallstress':
 854                         case 'jitsse2only':
 855                         case 'gcstress0x3':
 856                         case 'gcstress0xc':
 857                         case 'zapdisable':
 858                         case 'heapverify1':
 859                         case 'gcstress0xc_zapdisable':
 860                         case 'gcstress0xc_zapdisable_jitstress2':
 861                         case 'gcstress0xc_zapdisable_heapverify1':
 862                         case 'gcstress0xc_jitstress1':
 863                         case 'gcstress0xc_jitstress2':
 864                         case 'gcstress0xc_minopts_heapverify1':
 865                             def displayStr = getStressModeDisplayName(scenario)
 866                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 867                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 868                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 869                             break
 870                         case 'corefx_baseline':
 871                         case 'corefx_minopts':
 872                         case 'corefx_jitstress1':
 873                         case 'corefx_jitstress2':
 874                         case 'corefx_jitstressregs1':
 875                         case 'corefx_jitstressregs2':
 876                         case 'corefx_jitstressregs3':
 877                         case 'corefx_jitstressregs4':
 878                         case 'corefx_jitstressregs8':
 879                         case 'corefx_jitstressregs0x10':
 880                         case 'corefx_jitstressregs0x80':
 881                         case 'corefx_jitstressregs0x1000':
 882                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 883                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 884                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 885                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 886                             break
 887                         default:
 888                             println("Unknown scenario: ${scenario}");
 889                             assert false
 890                             break
 891                     }
 892                     break
 893                 case 'CentOS7.1':
 894                     switch (scenario) {
 895                         case 'pri1':
 896                             if (configuration == 'Release') {
 897                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 898                             }
 899                             break
 900                         case 'r2r':
 901                             if (configuration == 'Checked' || configuration == 'Release') {
 902                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 903                             }
 904                             break
 905                         case 'pri1r2r':
 906                             if (configuration == 'Checked' || configuration == 'Release') {
 907                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 908                             }
 909                             break
 910                         case 'gcstress15_pri1r2r':
 911                             if (configuration == 'Release' || configuration == 'Checked') {
 912                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 913                             }
 914                             break
 915                         case 'r2r_jitstress1':
 916                         case 'r2r_jitstress2':
 917                         case 'r2r_jitstressregs1':
 918                         case 'r2r_jitstressregs2':
 919                         case 'r2r_jitstressregs3':
 920                         case 'r2r_jitstressregs4':
 921                         case 'r2r_jitstressregs8':
 922                         case 'r2r_jitstressregs0x10':
 923                         case 'r2r_jitstressregs0x80':
 924                         case 'r2r_jitstressregs0x1000':
 925                         case 'r2r_jitminopts':
 926                         case 'r2r_jitforcerelocs':
 927                             if (configuration == 'Release' || configuration == 'Checked') {
 928                                 def displayStr = getR2RStressModeDisplayName(scenario)
 929                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 930                             }
 931                             break
 932                         default:
 933                             break
 934                     }
 935                 case 'Windows_NT':
 936                     switch (scenario) {
 937                         case 'default':
 938                             // Default trigger
 939                             if (configuration == 'Debug') {
 940                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 941                             }
 942                             break
 943                         case 'pri1':
 944                             // Default trigger
 945                             if (configuration == 'Release') {
 946                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 947                             }
 948                             break
 949                         case 'jitdiff':
 950                             if (configuration == 'Checked') {
 951                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 952                             }
 953                             break
 954                         case 'ilrt':
 955                             if (configuration == 'Release') {
 956                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 957                             }
 958                             break
 959                         case 'r2r':
 960                             if (configuration == 'Checked' || configuration == 'Release') {
 961                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 962                             }
 963                             break
 964                         case 'pri1r2r':
 965                             if (configuration == 'Checked' || configuration == 'Release') {
 966                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 967                             }
 968                             break
 969                         case 'gcstress15_pri1r2r':
 970                             if (configuration == 'Release' || configuration == 'Checked') {
 971                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 972                             }
 973                             break
 974                         case 'r2r_jitstress1':
 975                         case 'r2r_jitstress2':
 976                         case 'r2r_jitstressregs1':
 977                         case 'r2r_jitstressregs2':
 978                         case 'r2r_jitstressregs3':
 979                         case 'r2r_jitstressregs4':
 980                         case 'r2r_jitstressregs8':
 981                         case 'r2r_jitstressregs0x10':
 982                         case 'r2r_jitstressregs0x80':
 983                         case 'r2r_jitstressregs0x1000':
 984                         case 'r2r_jitminopts':
 985                         case 'r2r_jitforcerelocs':
 986                             if (configuration == 'Release' || configuration == 'Checked') {
 987                                 def displayStr = getR2RStressModeDisplayName(scenario)
 988                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 989                             }
 990                             break
 991                         case 'longgc':
 992                             if (configuration == 'Release') {
 993                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 994                             }
 995                             break
 996                         case 'gcsimulator':
 997                             if (configuration == 'Release') {
 998                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 999                             }
1000                             break
1001                         case 'standalone_gc':
1002                             if (configuration == 'Release' || configuration == 'Checked') {
1003                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1004                             }
1005                             break
1006                         case 'gc_reliability_framework':
1007                             if (configuration == 'Release' || configuration == 'Checked') {
1008                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1009                             }
1010                             break
1011                         case 'minopts':
1012                         case 'forcerelocs':
1013                         case 'jitstress1':
1014                         case 'jitstress2':
1015                         case 'jitstressregs1':
1016                         case 'jitstressregs2':
1017                         case 'jitstressregs3':
1018                         case 'jitstressregs4':
1019                         case 'jitstressregs8':
1020                         case 'jitstressregs0x10':
1021                         case 'jitstressregs0x80':
1022                         case 'jitstressregs0x1000':
1023                         case 'jitstress2_jitstressregs1':
1024                         case 'jitstress2_jitstressregs2':
1025                         case 'jitstress2_jitstressregs3':
1026                         case 'jitstress2_jitstressregs4':
1027                         case 'jitstress2_jitstressregs8':
1028                         case 'jitstress2_jitstressregs0x10':
1029                         case 'jitstress2_jitstressregs0x80':
1030                         case 'jitstress2_jitstressregs0x1000':
1031                         case 'tailcallstress':
1032                         case 'jitsse2only':
1033                         case 'gcstress0x3':
1034                         case 'gcstress0xc':
1035                         case 'zapdisable':
1036                         case 'heapverify1':
1037                         case 'gcstress0xc_zapdisable':
1038                         case 'gcstress0xc_zapdisable_jitstress2':
1039                         case 'gcstress0xc_zapdisable_heapverify1':
1040                         case 'gcstress0xc_jitstress1':
1041                         case 'gcstress0xc_jitstress2':
1042                         case 'gcstress0xc_minopts_heapverify1':
1043                             def displayStr = getStressModeDisplayName(scenario)
1044                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1045                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1046                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1047                             break
1048                         case 'corefx_baseline':
1049                         case 'corefx_minopts':
1050                         case 'corefx_jitstress1':
1051                         case 'corefx_jitstress2':
1052                         case 'corefx_jitstressregs1':
1053                         case 'corefx_jitstressregs2':
1054                         case 'corefx_jitstressregs3':
1055                         case 'corefx_jitstressregs4':
1056                         case 'corefx_jitstressregs8':
1057                         case 'corefx_jitstressregs0x10':
1058                         case 'corefx_jitstressregs0x80':
1059                         case 'corefx_jitstressregs0x1000':
1060                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1061                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1062                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1063                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1064                             break
1065                         case 'illink':
1066                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1067                                     break
1068                         default:
1069                             println("Unknown scenario: ${scenario}");
1070                             assert false
1071                             break
1072                     }
1073                     break
1074                 case 'FreeBSD':
1075                     assert scenario == 'default'
1076                     if (configuration == 'Checked') {
1077                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1078                     }
1079                     break
1080                 default:
1081                     println("Unknown os: ${os}");
1082                     assert false
1083                     break
1084             }
1085             break
1086         // editor brace matching: }
1087         case 'arm': // editor brace matching: {
1088             switch (os) {
1089                 case 'Ubuntu':
1090                 case 'Ubuntu16.04':
1091                     assert scenario == 'default'
1092                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1093                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1094                     }
1095                     else {
1096                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1097                     }
1098                     break;
1099                 case 'Tizen':
1100                     architecture='armel'
1101                     // Removing the regex will cause this to run on each PR.
1102                     if (configuration == 'Release' || configuration == 'Debug') {
1103                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1104                     }
1105                     else {
1106                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1107                     }
1108                     break;
1109                 case 'Windows_NT':
1110                     // Set up a private trigger
1111                     def contextString = "${os} ${architecture} Cross ${configuration}"
1112                     if (scenario != 'default')
1113                         contextString += " ${scenario}"
1114                     contextString += " Build"
1115                     // Debug builds only.
1116                     if (configuration != 'Debug') {
1117                         contextString += " and Test"
1118                     }
1119                     switch (scenario) {
1120                         case 'default':
1121                             // For now only run Debug and Release build jobs on PR Trigger. Note this is not a private trigger.
1122                             if (configuration == 'Debug' || configuration == 'Release')
1123                             {
1124                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1125                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1126                             }
1127                             else
1128                             {
1129                                 // Checked jobs will run on private trigger and run tests.
1130                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1131                             }
1132                             break
1133                         case 'pri1r2r':
1134                         case 'gcstress0x3':
1135                         case 'gcstress0xc':
1136                             // Stress jobs will will run on private trigger and run tests.
1137                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1138                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1139                             break
1140                     }
1141                     break
1142                 default:
1143                     println("NYI os: ${os}");
1144                     assert false
1145                     break
1146             }
1147             break
1148         // editor brace matching: }
1149         case 'arm64': // editor brace matching: {
1150             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1151
1152             // Set up a private trigger
1153             def contextString = "${os} ${architecture} Cross ${configuration}"
1154             if (scenario != 'default')
1155                 contextString += " ${scenario}"
1156             contextString += " Build"
1157             // Debug builds only.
1158             if (configuration != 'Debug') {
1159                contextString += " and Test"
1160             }
1161
1162             switch (os) {
1163                 case 'Ubuntu':
1164                 case 'Ubuntu16.04':
1165                     switch (scenario) {
1166                         case 'pri1':
1167                             if (configuration == 'Release') {
1168                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1169                             }
1170                             break
1171                         case 'r2r':
1172                             if (configuration == 'Checked' || configuration == 'Release') {
1173                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1174                             }
1175                             break
1176                         case 'pri1r2r':
1177                             if (configuration == 'Checked' || configuration == 'Release') {
1178                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1179                             }
1180                             break
1181                         case 'gcstress15_pri1r2r':
1182                             if (configuration == 'Release' || configuration == 'Checked') {
1183                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1184                             }
1185                             break
1186                         case 'r2r_jitstress1':
1187                         case 'r2r_jitstress2':
1188                         case 'r2r_jitstressregs1':
1189                         case 'r2r_jitstressregs2':
1190                         case 'r2r_jitstressregs3':
1191                         case 'r2r_jitstressregs4':
1192                         case 'r2r_jitstressregs8':
1193                         case 'r2r_jitstressregs0x10':
1194                         case 'r2r_jitstressregs0x80':
1195                         case 'r2r_jitstressregs0x1000':
1196                         case 'r2r_jitminopts':
1197                         case 'r2r_jitforcerelocs':
1198                             if (configuration == 'Release' || configuration == 'Checked') {
1199                                 def displayStr = getR2RStressModeDisplayName(scenario)
1200                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1201                             }
1202                             break
1203                         default:
1204                             break
1205                     }
1206                 case 'Windows_NT':
1207                     switch (scenario) {
1208                         case 'default':
1209                             if (isFlowJob == true) {
1210                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration}", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1211                             }
1212
1213                             // For now only run Debug jobs on PR Trigger.
1214                             else if (configuration != 'Debug') {
1215                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1216                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1217                             }
1218                             else {
1219                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1220                                 // are at higher risk of ARM64-breaking changes.
1221                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1222                             }
1223                             break
1224                         case 'pri1r2r':
1225                         case 'gcstress0x3':
1226                         case 'gcstress0xc':
1227                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1228                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1229                             break
1230                     }
1231                     break
1232                 default:
1233                     println("NYI os: ${os}");
1234                     assert false
1235                     break
1236             }
1237             break
1238         // editor brace matching: }
1239         case 'x86': // editor brace matching: {
1240             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1241             if (os == 'Ubuntu') {
1242                 // on-demand only for ubuntu x86
1243                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1244                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1245                 break
1246             }
1247             switch (scenario) {
1248                 case 'default':
1249                     if (configuration == 'Checked') {
1250                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1251                     }
1252                     else if (configuration == 'Release') {
1253                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1254                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1255                     }
1256                     break
1257                 case 'pri1':
1258                     if (configuration == 'Release') {
1259                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1260                     }
1261                     break
1262                 case 'ilrt':
1263                     if (configuration == 'Release') {
1264                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1265                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1266                     }
1267                     break
1268                 case 'r2r':
1269                     if (configuration == 'Checked' || configuration == 'Release') {
1270                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1271                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1272                     }
1273                     break
1274                 case 'pri1r2r':
1275                     if (configuration == 'Checked' || configuration == 'Release') {
1276                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1277                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1278                     }
1279                     break
1280                 case 'gcstress15_pri1r2r':
1281                     if (configuration == 'Release' || configuration == 'Checked') {
1282                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1283                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1284                     }
1285                     break
1286                 case 'r2r_jitstress1':
1287                 case 'r2r_jitstress2':
1288                 case 'r2r_jitstressregs1':
1289                 case 'r2r_jitstressregs2':
1290                 case 'r2r_jitstressregs3':
1291                 case 'r2r_jitstressregs4':
1292                 case 'r2r_jitstressregs8':
1293                 case 'r2r_jitstressregs0x10':
1294                 case 'r2r_jitstressregs0x80':
1295                 case 'r2r_jitstressregs0x1000':
1296                 case 'r2r_jitminopts':
1297                 case 'r2r_jitforcerelocs':
1298                     if (configuration == 'Release' || configuration == 'Checked') {
1299                         def displayStr = getR2RStressModeDisplayName(scenario)
1300                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1301                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1302                     }
1303                     break
1304                 case 'longgc':
1305                     if (configuration == 'Release') {
1306                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1307                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1308                     }
1309                     break
1310                 case 'gcsimulator':
1311                     if (configuration == 'Release') {
1312                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1313                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1314                     }
1315                     break
1316                 case 'standalone_gc':
1317                     if (configuration == 'Release' || configuration == 'Checked') {
1318                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1319                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1320                     }
1321                     break
1322                 case 'minopts':
1323                 case 'forcerelocs':
1324                 case 'jitstress1':
1325                 case 'jitstress2':
1326                 case 'jitstressregs1':
1327                 case 'jitstressregs2':
1328                 case 'jitstressregs3':
1329                 case 'jitstressregs4':
1330                 case 'jitstressregs8':
1331                 case 'jitstressregs0x10':
1332                 case 'jitstressregs0x80':
1333                 case 'jitstressregs0x1000':
1334                 case 'jitstress2_jitstressregs1':
1335                 case 'jitstress2_jitstressregs2':
1336                 case 'jitstress2_jitstressregs3':
1337                 case 'jitstress2_jitstressregs4':
1338                 case 'jitstress2_jitstressregs8':
1339                 case 'jitstress2_jitstressregs0x10':
1340                 case 'jitstress2_jitstressregs0x80':
1341                 case 'jitstress2_jitstressregs0x1000':
1342                 case 'tailcallstress':
1343                 case 'jitsse2only':
1344                 case 'gcstress0x3':
1345                 case 'gcstress0xc':
1346                 case 'zapdisable':
1347                 case 'heapverify1':
1348                 case 'gcstress0xc_zapdisable':
1349                 case 'gcstress0xc_zapdisable_jitstress2':
1350                 case 'gcstress0xc_zapdisable_heapverify1':
1351                 case 'gcstress0xc_jitstress1':
1352                 case 'gcstress0xc_jitstress2':
1353                 case 'gcstress0xc_minopts_heapverify1':
1354                     def displayStr = getStressModeDisplayName(scenario)
1355                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1356                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1357                     break
1358                 case 'corefx_baseline':
1359                 case 'corefx_minopts':
1360                 case 'corefx_jitstress1':
1361                 case 'corefx_jitstress2':
1362                 case 'corefx_jitstressregs1':
1363                 case 'corefx_jitstressregs2':
1364                 case 'corefx_jitstressregs3':
1365                 case 'corefx_jitstressregs4':
1366                 case 'corefx_jitstressregs8':
1367                 case 'corefx_jitstressregs0x10':
1368                 case 'corefx_jitstressregs0x80':
1369                 case 'corefx_jitstressregs0x1000':
1370                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1371                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1372                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1373                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1374                     break
1375                 case 'illink':
1376                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1377                     break
1378                 default:
1379                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1380                     assert false
1381                     break
1382             }
1383             break
1384          // editor brace matching: }
1385         case 'x86lb': // editor brace matching: {
1386             assert (os == 'Windows_NT')
1387             assert (scenario == 'default' || Constants.r2rJitStressScenarios.indexOf(scenario) !=1)
1388
1389             def arch = 'x86'
1390             def jit = 'legacy_backend'
1391             switch (scenario) {
1392                 case 'default':
1393                     if (configuration == 'Checked') {
1394                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1395                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1396                     }
1397                     break
1398                 default:
1399                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1400                     assert false
1401                     break
1402             }
1403             break
1404         // editor brace matching: }
1405         default:
1406             println("Unknown architecture: ${architecture}");
1407             assert false
1408             break
1409     }
1410 }
1411
1412 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1413     def buildCommands = [];
1414     def osGroup = getOSGroup(os)
1415     def lowerConfiguration = configuration.toLowerCase()
1416
1417     // Calculate the build steps, archival, and xunit results
1418     switch (os) {
1419         case 'Windows_NT': // editor brace matching: {
1420             switch (architecture) {
1421                 case 'x64':
1422                 case 'x86':
1423                 case 'x86lb':
1424                     def arch = architecture
1425                     def buildOpts = ''
1426                     if (architecture == 'x86lb') {
1427                         arch = 'x86'
1428                     }
1429
1430                     if (scenario == 'illink') {
1431                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1432                     }
1433
1434                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1435                             scenario == 'default' ||
1436                             scenario == 'r2r' ||
1437                             scenario == 'jitdiff' ||
1438                             scenario == 'ilrt' ||
1439                             scenario == 'illink' ||
1440                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1441                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1442                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1443                     }
1444
1445                     // For Pri 1 tests, we must shorten the output test binary path names.
1446                     // if __TestIntermediateDir is already set, build-test.cmd will
1447                     // output test binaries to that directory. If it is not set, the
1448                     // binaries are sent to a default directory whose name is about
1449                     // 35 characters long.
1450
1451                     else if (scenarioNeedsPri1Build(scenario)) {
1452                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1453                     }
1454                     else if (isLongGc(scenario)) {
1455                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1456                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1457                     }
1458                     else if (scenario == 'standalone_gc') {
1459                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1460                     }
1461                     else if (scenario == 'formatting') {
1462                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1463                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1464                         break
1465                     }
1466                     else {
1467                         println("Unknown scenario: ${scenario}")
1468                         assert false
1469                     }
1470
1471                     // If we are running a stress mode, we should write out the set of key
1472                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1473
1474                     if (!isBuildOnly) {
1475                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1476                         def crossgenStr = ''
1477                         def runcrossgentestsStr = ''
1478                         def runjitstressStr = ''
1479                         def runjitstressregsStr = ''
1480                         def runjitmioptsStr = ''
1481                         def runjitforcerelocsStr = ''
1482                         def runjitdisasmStr = ''
1483                         def runilasmroundtripStr = ''
1484                         def gcstressStr = ''
1485                         def runtestArguments = ''
1486                         def gcTestArguments = ''
1487                         def illinkArguments = ''
1488
1489                         if (scenario == 'r2r' ||
1490                             scenario == 'pri1r2r' ||
1491                             scenario == 'gcstress15_pri1r2r' ||
1492                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1493                                 crossgenStr = 'crossgen'
1494                                 runcrossgentestsStr = 'runcrossgentests'
1495
1496                                 if (scenario == 'r2r_jitstress1'){
1497                                     runjitstressStr = 'jitstress 1'
1498                                 }
1499                                 else if (scenario == 'r2r_jitstress2') {
1500                                     runjitstressStr = 'jitstress 2'
1501                                 }
1502                                 else if (scenario == 'r2r_jitstressregs1'){
1503                                     runjitstressregsStr = 'jitstressregs 1'
1504                                 }
1505                                 else if (scenario == 'r2r_jitstressregs2') {
1506                                     runjitstressregsStr = 'jitstressregs 2'
1507                                 }
1508                                 else if (scenario == 'r2r_jitstressregs3') {
1509                                     runjitstressregsStr = 'jitstressregs 3'
1510                                 }
1511                                 else if (scenario == 'r2r_jitstressregs4') {
1512                                     runjitstressregsStr = 'jitstressregs 4'
1513                                 }
1514                                 else if (scenario == 'r2r_jitstressregs8') {
1515                                     runjitstressregsStr = 'jitstressregs 8'
1516                                 }
1517                                 else if (scenario == 'r2r_jitstressregs0x10') {
1518                                     runjitstressregsStr = 'jitstressregs 0x10'
1519                                 }
1520                                 else if (scenario == 'r2r_jitstressregs0x80') {
1521                                     runjitstressregsStr = 'jitstressregs 0x80'
1522                                 }
1523                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1524                                     runjitstressregsStr = 'jitstressregs 0x1000'
1525                                 }
1526                                 else if (scenario == 'r2r_jitminopts') {
1527                                     runjitmioptsStr = 'jitminopts'
1528                                 }
1529                                 else if (scenario == 'r2r_jitforcerelocs') {
1530                                     runjitforcerelocsStr = 'jitforcerelocs'
1531                                 }
1532                         }
1533                         if (scenario == 'gcstress15_pri1r2r')
1534                         {
1535                             gcstressStr = 'gcstresslevel 0xF'
1536                         }
1537
1538                         if (scenario == 'jitdiff')
1539                         {
1540                             runjitdisasmStr = 'jitdisasm crossgen'
1541                         }
1542
1543                         if (scenario == 'ilrt')
1544                         {
1545                             runilasmroundtripStr = 'ilasmroundtrip'
1546                         }
1547
1548                         if (isLongGc(scenario)) {
1549                             gcTestArguments = "${scenario} sequential"
1550                         }
1551
1552                         if (scenario == 'illink')
1553                         {
1554                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1555                         }
1556
1557                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1558
1559                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1560                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1561                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1562
1563                             if (enableCorefxTesting) {
1564                                 def workspaceRelativeFxRoot = "_/fx"
1565                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1566
1567                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1568
1569                                 setTestJobTimeOut(newJob, scenario)
1570
1571                                 // Archive and process (only) the test results
1572                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1573                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1574                             }
1575                             else {
1576                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1577                             }
1578                         }
1579                         else if (isGcReliabilityFramework(scenario)) {
1580                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1581                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1582                         }
1583                         else if (architecture == 'x64' || architecture == 'x86') {
1584                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1585                         }
1586                         else if (architecture == 'x86lb') {
1587                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd"
1588                         }
1589                     }
1590
1591                     if (!enableCorefxTesting) {
1592                         // Run the rest of the build
1593                         // Build the mscorlib for the other OS's
1594                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1595                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1596                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1597
1598                         if (arch == "x64") {
1599                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1600                         }
1601
1602                         // Zip up the tests directory so that we don't use so much space/time copying
1603                         // 10s of thousands of files around.
1604                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1605
1606                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1607                             // For windows, pull full test results and test drops for x86/x64.
1608                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1609                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1610                         }
1611
1612                         if (scenario == 'jitdiff') {
1613                             // retrive jit-dasm output for base commit, and run jit-diff
1614                             if (!isBuildOnly) {
1615                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1616                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1617                             }
1618                         }
1619
1620                         if (!isBuildOnly) {
1621                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1622                             setTestJobTimeOut(newJob, scenario)
1623                         }
1624                     }
1625                     break
1626                 case 'arm':
1627                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1628
1629                     // Set time out
1630                     setTestJobTimeOut(newJob, scenario)
1631
1632                     if ( lowerConfiguration == "debug" ) {
1633                         // For Debug builds, we will do a P1 test build
1634                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1635                     }
1636                     else if (lowerConfiguration == "checked") {
1637
1638                         if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1639                         {
1640                            // Up the timeout for arm checked testing only.
1641                            // Keep the longer timeout for gcstress.
1642                            Utilities.setJobTimeout(newJob, 240)
1643                         }
1644
1645                         def machineAffinityOptions = ['use_arm64_build_machine' : true]
1646                         setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1647                         // For checked runs we will also run testing.
1648                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1649                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1650                     }
1651                     else if (lowerConfiguration == "release") {
1652                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1653                     }
1654                     // Add archival.
1655                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1656                     break
1657                 case 'arm64':
1658                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1659
1660                     // Set time out
1661                     setTestJobTimeOut(newJob, scenario)
1662
1663                     // Debug runs take too long to run. So build job only.
1664                     if (lowerConfiguration == "debug") {
1665                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1666                     }
1667                     else {
1668                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1669                        {
1670                            // Up the timeout for arm64 checked testing only.
1671                            // Keep the longer timeout for gcstress.
1672                            Utilities.setJobTimeout(newJob, 240)
1673                        }
1674
1675                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=1"
1676                        // Test build and run are launched together.
1677                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1678                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1679                     }
1680
1681                     // Add archival.
1682                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1683                     break
1684                 default:
1685                     println("Unknown architecture: ${architecture}");
1686                     assert false
1687                     break
1688             }
1689             break
1690         // editor brace matching: }
1691         case 'Ubuntu':
1692         case 'Ubuntu16.04':
1693         case 'Ubuntu16.10':
1694         case 'Debian8.4':
1695         case 'OSX10.12':
1696         case 'FreeBSD':
1697         case 'CentOS7.1':
1698         case 'RHEL7.2':
1699         case 'OpenSUSE42.1':
1700         case 'Tizen':
1701         case 'Fedora24': // editor brace matching: {
1702             switch (architecture) {
1703                 case 'x64':
1704                 case 'x86':
1705                     if (architecture == 'x86' && os == 'Ubuntu') {
1706                         // build and PAL test
1707                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1708                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1709                         break;
1710                     }
1711
1712                     if (scenario == 'formatting') {
1713                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1714                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1715                         break
1716                     }
1717
1718                     if (scenario == 'illink') {
1719                         assert(os == 'Ubuntu')
1720                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1721                     }
1722
1723                     def standaloneGc = ''
1724                     if (scenario == 'standalone_gc') {
1725                         standaloneGc = 'buildstandalonegc'
1726                     }
1727
1728                     if (!enableCorefxTesting) {
1729                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1730                         // only on supported OS platforms.
1731                         if (os == 'FreeBSD')
1732                         {
1733                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1734                         }
1735                         else
1736                         {
1737                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1738                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1739                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1740                         }
1741                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1742
1743                         // Set time out
1744                         setTestJobTimeOut(newJob, scenario)
1745                         // Basic archiving of the build
1746                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1747                         // And pal tests
1748                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1749                     }
1750                     else {
1751                         // Corefx stress testing
1752                         assert os == 'Ubuntu'
1753                         assert architecture == 'x64'
1754                         assert lowerConfiguration == 'checked'
1755                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1756
1757                         // Build coreclr
1758                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1759
1760                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1761                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1762
1763                         // Build and text corefx
1764                         def workspaceRelativeFxRoot = "_/fx"
1765                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1766
1767                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1768
1769                         setTestJobTimeOut(newJob, scenario)
1770
1771                         // Archive and process (only) the test results
1772                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1773                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1774                     }
1775                     break
1776                 case 'arm64':
1777                     def standaloneGc = ''
1778                     if (scenario == 'standalone_gc') {
1779                         standaloneGc = 'buildstandalonegc'
1780                     }
1781
1782                     if (!enableCorefxTesting) {
1783                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} cross clang3.8 skipnuget ${standaloneGc}"
1784
1785                         // HACK -- Arm64 does not have corefx jobs yet.
1786                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1787                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1788                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1789                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1790
1791                         // Set time out
1792                         setTestJobTimeOut(newJob, scenario)
1793                         // Basic archiving of the build
1794                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1795                     }
1796                     break
1797                 case 'arm':
1798                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1799                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1800
1801                     // default values for Ubuntu
1802                     def arm_abi="arm"
1803                     def linuxCodeName="trusty"
1804                     if (os == 'Ubuntu16.04') {
1805                         linuxCodeName="xenial"
1806                     }
1807                     else if (os == 'Tizen') {
1808                         arm_abi="armel"
1809                         linuxCodeName="tizen"
1810                     }
1811
1812                     // Unzip the Windows test binaries first. Exit with 0
1813                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1814
1815                     // Unpack the corefx binaries
1816                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1817                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1818                     if (os != 'Tizen') {
1819                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1820                     }
1821                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1822                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1823                     if (os == 'Ubuntu') {
1824                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1825                     }
1826
1827                     // Call the ARM CI script to cross build and test using docker
1828                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1829                     --mode=docker \\
1830                     --${arm_abi} \\
1831                     --linuxCodeName=${linuxCodeName} \\
1832                     --buildConfig=${lowerConfiguration} \\
1833                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1834                     --coreFxBinDir=./bin/CoreFxBinDir \\
1835                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1836
1837                     // Basic archiving of the build, no pal tests
1838                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1839                     break
1840                 default:
1841                     println("Unknown architecture: ${architecture}");
1842                     assert false
1843                     break
1844             }
1845             break
1846         // editor brace matching: }
1847         default:
1848             println("Unknown os: ${os}");
1849             assert false
1850             break
1851     } // os
1852
1853     return buildCommands
1854 }
1855
1856 // Additional scenario which can alter behavior
1857
1858 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1859 combinedScenarios.each { scenario ->
1860     [true, false].each { isPR ->
1861         Constants.architectureList.each { architecture ->
1862             Constants.configurationList.each { configuration ->
1863                 Constants.osList.each { os ->
1864                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1865                     // and reset the os to Windows_NT
1866                     def isBuildOnly = false
1867                     if (os == 'Windows_NT_BuildOnly') {
1868                         isBuildOnly = true
1869                         os = 'Windows_NT'
1870                     }
1871
1872                     // Tizen is only supported for arm architecture
1873                     if (os == 'Tizen' && architecture != 'arm') {
1874                         return
1875                     }
1876
1877                     // Skip totally unimplemented (in CI) configurations.
1878                     switch (architecture) {
1879                         case 'arm64':
1880                             if (os == 'Ubuntu16.04') {
1881                                 os = 'Ubuntu'
1882                             }
1883
1884                             // Windows and Ubuntu only
1885                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1886                                 return
1887                             }
1888                             break
1889                         case 'arm':
1890                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1891                                 return
1892                             }
1893                             break
1894                         case 'x86':
1895                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1896                                 return
1897                             }
1898                             break
1899                         case 'x86lb':
1900                             if (os != 'Windows_NT') {
1901                                 return
1902                             }
1903                             break
1904                         case 'x64':
1905                             // Everything implemented
1906                             break
1907                         default:
1908                             println("Unknown architecture: ${architecture}")
1909                             assert false
1910                             break
1911                     }
1912
1913                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1914                     // with checked builds
1915                     def enableCorefxTesting = false
1916                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1917                         if (configuration != 'Checked') {
1918                             return
1919                         }
1920
1921                         enableCorefxTesting = isCorefxTesting(scenario)
1922
1923                         // Since these are just execution time differences,
1924                         // skip platforms that don't execute the tests here (Windows_NT only)
1925                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1926                         if (!isEnabledOS || isBuildOnly) {
1927                             return
1928                         }
1929
1930                         switch (architecture) {
1931                             case 'arm':
1932                             case 'arm64':
1933                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1934                                     return
1935                                 }
1936                                 break
1937                             case 'x64':
1938                             case 'x86':
1939                                 // x86 ubuntu: default only
1940                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1941                                     return
1942                                 }
1943                                 // Windows: Everything implemented
1944                                 break
1945                             case 'x86lb':
1946                                 // No stress modes for legacy jit.
1947                                 // (There's no technical reason we couldn't allow these.)
1948                                 return
1949                             default:
1950                                 return
1951                         }
1952                     }
1953                     else {
1954                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
1955                         // and configuration is not Checked, bail out.
1956                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1957                             return;
1958                         }
1959
1960                         // Skip scenarios
1961                         switch (scenario) {
1962                             case 'pri1':
1963                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1964                                 // the default scenario build
1965                                 if (os != 'Windows_NT') {
1966                                     return
1967                                 }
1968                                 // Only x64 for now
1969                                 if (architecture != 'x64') {
1970                                     return
1971                                 }
1972                                 break
1973                             case 'ilrt':
1974                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1975                                 // the default scenario build
1976                                 if (os != 'Windows_NT') {
1977                                     return
1978                                 }
1979                                 // Only x64 for now
1980                                 if (architecture != 'x64') {
1981                                     return
1982                                 }
1983                                 // Release only
1984                                 if (configuration != 'Release') {
1985                                     return
1986                                 }
1987                                 break
1988                             case 'jitdiff':
1989                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1990                                     return
1991                                 }
1992                                 if (architecture != 'x64') {
1993                                     return
1994                                 }
1995                                 if (configuration != 'Checked') {
1996                                     return
1997                                 }
1998                                 break
1999                             case 'r2r':
2000                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2001                                 // the default scenario build
2002                                 if (os != 'Windows_NT') {
2003                                     return
2004                                 }
2005                                 if (architecture != 'x64') {
2006                                     return
2007                                 }
2008                                 break
2009                             case 'pri1r2r':
2010                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2011                                 // the default scenario build
2012                                 if (os != 'Windows_NT') {
2013                                     return
2014                                 }
2015                                 if (architecture != 'x64') {
2016                                     if ((architecture != 'arm64' && architecture != 'arm') || (configuration == 'Debug')) {
2017                                         return
2018                                     }
2019                                 }
2020                                 break
2021                             case 'gcstress15_pri1r2r':
2022                             case 'r2r_jitstress1':
2023                             case 'r2r_jitstress2':
2024                             case 'r2r_jitstressregs1':
2025                             case 'r2r_jitstressregs2':
2026                             case 'r2r_jitstressregs3':
2027                             case 'r2r_jitstressregs4':
2028                             case 'r2r_jitstressregs8':
2029                             case 'r2r_jitstressregs0x10':
2030                             case 'r2r_jitstressregs0x80':
2031                             case 'r2r_jitstressregs0x1000':
2032                             case 'r2r_jitminopts':
2033                             case 'r2r_jitforcerelocs':
2034                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
2035                                 // the default scenario build
2036                                 if (os != 'Windows_NT') {
2037                                     return
2038                                 }
2039                                 if (architecture != 'x64' && architecture != 'x86') {
2040                                     return
2041                                 }
2042                                 break
2043                             case 'longgc':
2044                             case 'gcsimulator':
2045                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2046                                     return
2047                                 }
2048                                 if (architecture != 'x64') {
2049                                     return
2050                                 }
2051                                 if (configuration != 'Release') {
2052                                     return
2053                                 }
2054                                 break
2055                             case 'gc_reliability_framework':
2056                             case 'standalone_gc':
2057                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2058                                     return
2059                                 }
2060
2061                                 if (architecture != 'x64') {
2062                                     return
2063                                 }
2064
2065                                 if (configuration != 'Release' && configuration != 'Checked') {
2066                                     return
2067                                 }
2068                                 break
2069                             // We need Windows x64 Release bits for the code coverage build
2070                             case 'coverage':
2071                                 if (os != 'Windows_NT') {
2072                                     return
2073                                 }
2074                                 if (architecture != 'x64') {
2075                                     return
2076                                 }
2077                                 if (configuration != 'Release') {
2078                                     return
2079                                 }
2080                                 break
2081                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2082                             case 'formatting':
2083                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2084                                     return
2085                                 }
2086                                 if (architecture != 'x64') {
2087                                     return
2088                                 }
2089                                 if (configuration != 'Checked') {
2090                                     return
2091                                 }
2092                                 if (isBuildOnly) {
2093                                     return
2094                                 }
2095                                 break
2096                             case 'illink':
2097                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2098                                     return
2099                                 }
2100                                 if (architecture != 'x64' && architecture != 'x86') {
2101                                     return
2102                                 }
2103                                 if (isBuildOnly) {
2104                                     return
2105                                 }
2106                                 break
2107                             case 'default':
2108                                 // Nothing skipped
2109                                 break
2110                             default:
2111                                 println("Unknown scenario: ${scenario}")
2112                                 assert false
2113                                 break
2114                         }
2115                     }
2116
2117                     // Calculate names
2118                     def lowerConfiguration = configuration.toLowerCase()
2119                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2120                     def folderName = getJobFolder(scenario)
2121
2122                     // Create the new job
2123                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2124
2125                     def machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2126                     machineAffinityOptions = architecture == 'arm' ? ['use_arm64_build_machine': false] : machineAffinityOptions
2127                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2128
2129                     // Add all the standard options
2130                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2131                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2132
2133                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
2134                     def osGroup = getOSGroup(os)
2135
2136                     newJob.with {
2137                         steps {
2138                             if (os == 'Windows_NT') {
2139                                 buildCommands.each { buildCommand ->
2140                                     batchFile(buildCommand)
2141                                 }
2142                             }
2143                             else {
2144                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2145                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2146                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2147                                     // Define the Windows Tests and Corefx build job names
2148                                     def WindowTestsName = projectFolder + '/' +
2149                                                           Utilities.getFullJobName(project,
2150                                                                                    getJobName(lowerConfiguration,
2151                                                                                               'x64' ,
2152                                                                                               'windows_nt',
2153                                                                                               'default',
2154                                                                                               true),
2155                                                                                    false)
2156                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2157                                                        Utilities.getFolderName(branch)
2158
2159                                     // Copy the Windows test binaries and the Corefx build binaries
2160                                     copyArtifacts(WindowTestsName) {
2161                                         includePatterns('bin/tests/tests.zip')
2162                                         buildSelector {
2163                                             latestSuccessful(true)
2164                                         }
2165                                     }
2166
2167                                     def arm_abi = 'arm'
2168                                     def corefx_os = 'linux'
2169                                     if (os == 'Tizen') {
2170                                         arm_abi = 'armel'
2171                                         corefx_os = 'tizen'
2172                                     }
2173
2174                                     // Let's use release CoreFX to test checked CoreCLR,
2175                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2176                                     def corefx_lowerConfiguration = lowerConfiguration
2177                                     if ( lowerConfiguration == 'checked' ) {
2178                                         corefx_lowerConfiguration='release'
2179                                     }
2180
2181                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2182                                         includePatterns('bin/build.tar.gz')
2183                                         buildSelector {
2184                                             latestSuccessful(true)
2185                                         }
2186                                     }
2187                                 }
2188
2189                                 buildCommands.each { buildCommand ->
2190                                     shell(buildCommand)
2191                                 }
2192                             }
2193                         }
2194                     } // newJob.with
2195
2196                 } // os
2197             } // configuration
2198         } // architecture
2199     } // isPR
2200 } // scenario
2201
2202
2203 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2204 combinedScenarios.each { scenario ->
2205     [true, false].each { isPR ->
2206         // Architectures.  x64 only at this point
2207         ['x64', 'arm64'].each { architecture ->
2208             // Put the OS's supported for coreclr cross testing here
2209             Constants.crossList.each { os ->
2210                 if (architecture == 'arm64') {
2211                     if (os != "Ubuntu") {
2212                         return
2213                     }
2214                 }
2215
2216                 Constants.configurationList.each { configuration ->
2217
2218                     if (architecture == 'arm64') {
2219                         if (scenario != 'default' && scenario != 'pri1r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2220                             return
2221                         }
2222                     }
2223
2224                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2225                         if (configuration != 'Checked') {
2226                             return
2227                         }
2228                         if (isCorefxTesting(scenario)) {
2229                             return
2230                         }
2231                         //Skip stress modes for these scenarios
2232                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2233                             return
2234                         }
2235                     }
2236                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2237                     // and configuration is not Checked, bail out.
2238                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2239                         return;
2240                     }
2241                     // For CentOS, we only want Checked/Release pri1 builds.
2242                     else if (os == 'CentOS7.1') {
2243                         if (scenario != 'pri1' &&
2244                             scenario != 'r2r' &&
2245                             scenario != 'pri1r2r' &&
2246                             scenario != 'gcstress15_pri1r2r' &&
2247                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2248                             return
2249                         }
2250                         if (configuration != 'Checked' && configuration != 'Release') {
2251                             return
2252                         }
2253                     }
2254                     // For RedHat and Debian, we only do Release pri1 builds.
2255                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2256                         if (scenario != 'pri1') {
2257                             return
2258                         }
2259                         if (configuration != 'Release') {
2260                             return
2261                         }
2262                     }
2263                     else {
2264                         // Skip scenarios
2265                         switch (scenario) {
2266                             case 'pri1':
2267                                 // Nothing skipped
2268                                 break
2269                             case 'ilrt':
2270                                 // Release only
2271                                 if (configuration != 'Release') {
2272                                     return
2273                                 }
2274                                 break
2275                             case 'jitdiff':
2276                                 if (configuration != 'Checked') {
2277                                     return;
2278                                 }
2279                             case 'r2r':
2280                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2281                                 if (configuration != 'Checked' && configuration != 'Release') {
2282                                     return
2283                                 }
2284                                 break
2285                             case 'pri1r2r':
2286                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2287                                 if (configuration != 'Checked' && configuration != 'Release') {
2288                                     return
2289                                 }
2290                                 break
2291                             case 'gcstress15_pri1r2r':
2292                             case 'r2r_jitstress1':
2293                             case 'r2r_jitstress2':
2294                             case 'r2r_jitstressregs1':
2295                             case 'r2r_jitstressregs2':
2296                             case 'r2r_jitstressregs3':
2297                             case 'r2r_jitstressregs4':
2298                             case 'r2r_jitstressregs8':
2299                             case 'r2r_jitstressregs0x10':
2300                             case 'r2r_jitstressregs0x80':
2301                             case 'r2r_jitstressregs0x1000':
2302                             case 'r2r_jitminopts':
2303                             case 'r2r_jitforcerelocs':
2304                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2305                                 if (configuration != 'Checked' && configuration != 'Release') {
2306                                     return
2307                                 }
2308                                 break
2309                             case 'longgc':
2310                             case 'gcsimulator':
2311                                 // Long GC tests take a long time on non-Release builds
2312                                 if (configuration != 'Release') {
2313                                     return
2314                                 }
2315                                 break
2316                             case 'gc_reliability_framework':
2317                             case 'standalone_gc':
2318                                 if (configuration != 'Release' && configuration != 'Checked') {
2319                                     return
2320                                 }
2321                                 break
2322                             case 'coverage':
2323                                 //We only want Ubuntu Release for coverage
2324                                 if (os != 'Ubuntu') {
2325                                     return
2326                                 }
2327                                 if (configuration != 'Release') {
2328                                     return
2329                                 }
2330                             case 'formatting':
2331                                 return
2332                             case 'illink':
2333                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2334                                     return
2335                                 }
2336                                 break
2337                             case 'default':
2338                                 // Nothing skipped
2339                                 break
2340                             default:
2341                                 println("Unknown scenario: ${scenario}")
2342                                 assert false
2343                                 break
2344                         }
2345                     }
2346
2347                     def lowerConfiguration = configuration.toLowerCase()
2348                     def osGroup = getOSGroup(os)
2349                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2350
2351                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2352                     def inputScenario = 'default'
2353                     if (scenario == 'coverage') {
2354                         inputScenario = 'coverage'
2355                     }
2356                     def inputCoreCLRBuildName = projectFolder + '/' +
2357                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2358                     // If this is a stress scenario, there isn't any difference in the build job
2359                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2360                     // from the default scenario
2361                     def testBuildScenario = scenario
2362                     if (scenarioNeedsPri1Build(scenario)) {
2363                         testBuildScenario = 'pri1'
2364                     }
2365                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2366                         testBuildScenario = 'default'
2367                     }
2368                     def inputWindowTestsBuildName = ''
2369                     def inputWindowsTestBuildArch = architecture
2370                     if (architecture == "arm64") {
2371                         // Use the x64 test build for arm64 unix
2372                         inputWindowsTestBuildArch = "x64"
2373                     }
2374                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2375                         inputWindowTestsBuildName = projectFolder + '/' +
2376                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', 'default', true), isPR)
2377                     }
2378                     else {
2379                         inputWindowTestsBuildName = projectFolder + '/' +
2380                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2381                     }
2382                     // Enable Server GC for Ubuntu PR builds
2383                     def serverGCString = ''
2384
2385                     // Whether or not this test run should be run sequentially instead
2386                     // of in parallel. Only used for long GC tests.
2387                     def sequentialString = ''
2388
2389                     // Whether or not this test run should run a specific playlist.
2390                     // Only used for long GC tests.
2391
2392                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2393                     // for running long GC and GCSimulator tests, respectively. We don't use them
2394                     // here because using a playlist file produces much more readable output on the CI machines
2395                     // and reduces running time.
2396                     def playlistString = ''
2397
2398                     if (os == 'Ubuntu' && isPR){
2399                         serverGCString = '--useServerGC'
2400                     }
2401
2402                     // pass --crossgen to runtest.sh for crossgen builds
2403                     def crossgenStr = ''
2404                     def runcrossgentestsStr = ''
2405                     def runjitstressStr = ''
2406                     def runjitstressregsStr = ''
2407                     def runjitmioptsStr = ''
2408                     def runjitforcerelocsStr = ''
2409                     def runjitdisasmStr = ''
2410                     def runilasmroundtripStr = ''
2411                     def gcstressStr = ''
2412                     def illinkStr = ''
2413                     def layoutOnlyStr =''
2414
2415                     if (scenario == 'r2r' ||
2416                         scenario == 'pri1r2r' ||
2417                         scenario == 'gcstress15_pri1r2r' ||
2418                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2419                             crossgenStr = '--crossgen'
2420                             runcrossgentestsStr = '--runcrossgentests'
2421
2422                             if (scenario == 'r2r_jitstress1'){
2423                                 runjitstressStr = '--jitstress=1'
2424                             }
2425                             else if (scenario == 'r2r_jitstress2') {
2426                                 runjitstressStr = '--jitstress=2'
2427                             }
2428                             else if (scenario == 'r2r_jitstressregs1'){
2429                                 runjitstressregsStr = '--jitstressregs=1'
2430                             }
2431                             else if (scenario == 'r2r_jitstressregs2') {
2432                                 runjitstressregsStr = '--jitstressregs=2'
2433                             }
2434                             else if (scenario == 'r2r_jitstressregs3') {
2435                                 runjitstressregsStr = '--jitstressregs=3'
2436                             }
2437                             else if (scenario == 'r2r_jitstressregs4') {
2438                                 runjitstressregsStr = '--jitstressregs=4'
2439                             }
2440                             else if (scenario == 'r2r_jitstressregs8') {
2441                                 runjitstressregsStr = '--jitstressregs=8'
2442                             }
2443                             else if (scenario == 'r2r_jitstressregs0x10') {
2444                                 runjitstressregsStr = '--jitstressregs=0x10'
2445                             }
2446                             else if (scenario == 'r2r_jitstressregs0x80') {
2447                                 runjitstressregsStr = '--jitstressregs=0x80'
2448                             }
2449                             else if (scenario == 'r2r_jitstressregs0x1000') {
2450                                 runjitstressregsStr = '--jitstressregs=0x1000'
2451                             }
2452                             else if (scenario == 'r2r_jitminopts') {
2453                                 runjitmioptsStr = '--jitminopts'
2454                             }
2455                             else if (scenario == 'r2r_jitforcerelocs') {
2456                                 runjitforcerelocsStr = '--jitforcerelocs'
2457                             }
2458                     }
2459                     if  (scenario == 'gcstress15_pri1r2r')
2460                     {
2461                         gcstressStr = '--gcstresslevel=0xF'
2462                     }
2463
2464                     if (scenario == 'jitdiff')
2465                     {
2466                         runjitdisasmStr = '--jitdisasm --crossgen'
2467                     }
2468
2469                     if (scenario == 'illink')
2470                     {
2471                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2472                     }
2473
2474                     if (isLongGc(scenario)) {
2475                         // Long GC tests behave very poorly when they are not
2476                         // the only test running (many of them allocate until OOM).
2477                         sequentialString = '--sequential'
2478
2479                         // The Long GC playlist contains all of the tests that are
2480                         // going to be run. The GCSimulator playlist contains all of
2481                         // the GC simulator tests.
2482                         if (scenario == 'longgc') {
2483                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2484                         }
2485                         else if (scenario == 'gcsimulator') {
2486                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2487                         }
2488                     }
2489
2490                     if (isGcReliabilityFramework(scenario)) {
2491                         layoutOnlyStr = '--build-overlay-only'
2492                     }
2493
2494                     def folder = getJobFolder(scenario)
2495                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2496                         // Add parameters for the inputs
2497
2498                         parameters {
2499                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2500                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2501                         }
2502
2503                         steps {
2504                             // Set up the copies
2505
2506                             // Coreclr build containing the tests and mscorlib
2507
2508                             copyArtifacts(inputWindowTestsBuildName) {
2509                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2510                                 buildSelector {
2511                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2512                                 }
2513                             }
2514
2515                             if (scenario == 'coverage') {
2516
2517                                 // Move coreclr to clr directory
2518                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2519
2520                                 // Build coreclr
2521                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2522
2523                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2524                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2525                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2526                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2527                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2528                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2529                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2530                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2531
2532                                 // Run PAL tests
2533                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2534
2535                                 // Remove obj files for PAL tests so they're not included in coverage results
2536                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2537
2538                                 // Unzip the tests first.  Exit with 0
2539                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2540
2541                                 // Get corefx
2542                                 shell("git clone https://github.com/dotnet/corefx fx")
2543
2544                                 // Build Linux corefx
2545                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2546                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2547
2548                                 def testEnvOpt = ""
2549                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2550                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2551                                 shell("${createScriptCmds}")
2552                                 testEnvOpt = "--test-env=" + scriptFileName
2553
2554                                 // Run corefx tests
2555                                 shell("""./fx/run-test.sh \\
2556                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2557                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2558                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2559                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2560                 --configurationGroup Release""")
2561
2562
2563                                 // Run coreclr tests w/ workstation GC
2564                                 shell("""./clr/tests/runtest.sh \\
2565                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2566                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2567                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2568                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2569                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2570                 --crossgen --runcrossgentests""")
2571
2572                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2573                                 shell("""./clr/tests/runtest.sh \\
2574                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2575                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2576                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2577                 --useServerGC ${testEnvOpt}""")
2578
2579                                  // Run long-running coreclr GC tests & produce coverage reports
2580                                 shell("""./clr/tests/runtest.sh \\
2581                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2582                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2583                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2584                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2585                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2586                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2587                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2588
2589                             }
2590                             else {
2591
2592                                 // Coreclr build we are trying to test
2593
2594                                 copyArtifacts(inputCoreCLRBuildName) {
2595                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2596                                     buildSelector {
2597                                         buildNumber('${CORECLR_BUILD}')
2598                                     }
2599                                 }
2600
2601                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2602
2603                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2604                                 def osJobName
2605                                 if (os == 'Ubuntu') {
2606                                     osJobName = 'ubuntu14.04'
2607                                 }
2608                                 else {
2609                                     osJobName = os.toLowerCase()
2610                                 }
2611                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2612                                     includePatterns('bin/build.tar.gz')
2613                                     buildSelector {
2614                                         latestSuccessful(true)
2615                                     }
2616                                 }
2617
2618                                 shell ("mkdir ./bin/CoreFxBinDir")
2619                                 // Unpack the corefx binaries
2620                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2621
2622                                 // HACK -- Arm64 does not have corefx jobs yet.
2623                                 // Clone corefx and build the native packages overwriting the x64 packages.
2624                                 if (architecture == 'arm64') {
2625                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2626                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2627                                 }
2628
2629                                 // Unzip the tests first.  Exit with 0
2630                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2631
2632                                 // Execute the tests
2633                                 // If we are running a stress mode, we'll set those variables first
2634                                 def testEnvOpt = ""
2635                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2636                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2637                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2638                                     shell("${createScriptCmds}")
2639                                     testEnvOpt = "--test-env=" + scriptFileName
2640                                 }
2641
2642                                 if (isGCStressRelatedTesting(scenario)) {
2643                                     shell('./init-tools.sh')
2644                                 }
2645
2646                                 shell("""./tests/runtest.sh \\
2647                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2648                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2649                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2650                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2651                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2652                 --limitedDumpGeneration \\
2653                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
2654                 ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
2655                 ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
2656
2657                                 if (isGcReliabilityFramework(scenario)) {
2658                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2659                                     if (serverGCString != '') {
2660                                         shell("export COMPlus_gcServer=1")
2661                                     }
2662
2663                                     shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2664                                 }
2665                             }
2666                         }
2667                     }
2668
2669                     if (scenario == 'coverage') {
2670                         // Publish coverage reports
2671                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2672                         // TODO: Add once external email sending is available again
2673                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2674                     }
2675
2676                     if (scenario == 'jitdiff') {
2677                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2678                     }
2679
2680                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2681                     if (os in ['Ubuntu']) {
2682                         SummaryBuilder summaries = new SummaryBuilder()
2683                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2684                         summaries.emit(newJob)
2685                     }
2686
2687                     setMachineAffinity(newJob, os, architecture)
2688                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2689                     // Set timeouts to 240.
2690                     setTestJobTimeOut(newJob, scenario)
2691
2692                     if (architecture == 'arm64') {
2693                         Utilities.setJobTimeout(newJob, 480)
2694                     }
2695
2696                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2697
2698                     // Create a build flow to join together the build and tests required to run this
2699                     // test.
2700                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2701                     // Linux CoreCLR test
2702                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2703                     def fullTestJobName = projectFolder + '/' + newJob.name
2704                     // Add a reference to the input jobs for report purposes
2705                     JobReport.Report.addReference(inputCoreCLRBuildName)
2706                     JobReport.Report.addReference(inputWindowTestsBuildName)
2707                     JobReport.Report.addReference(fullTestJobName)
2708                     def newFlowJob;
2709
2710                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2711                     // so that coverage data can be preserved.
2712                     if (scenario == 'coverage') {
2713                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2714                         buildFlow("""
2715 // Build the input Windows job
2716 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2717
2718 // And then build the test build
2719 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2720 """)
2721                         }
2722                     // Normal jobs copy a Windows build & a non-Windows build
2723                     } else {
2724                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2725                         buildFlow("""
2726 // Build the input jobs in parallel
2727 parallel (
2728     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2729     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2730 )
2731
2732 // And then build the test build
2733 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2734                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2735 """)
2736                         }
2737                     }
2738
2739                     setMachineAffinity(newFlowJob, os, architecture)
2740                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2741                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2742                 } // configuration
2743             } // os
2744         } // architecture
2745     } // isPR
2746 } // scenario
2747
2748 JobReport.Report.generateJobReport(out)
2749
2750 // Make the call to generate the help job
2751 Utilities.createHelperJob(this, project, branch,
2752     "Welcome to the ${project} Repository",  // This is prepended to the help message
2753     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2754
2755 Utilities.addCROSSCheck(this, project, branch)