netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'FreeBSD':'FreeBSD',
  29         'CentOS7.1': 'Linux',
  30         'OpenSUSE42.1': 'Linux',
  31         'Tizen': 'Linux']
  32     def osGroup = osGroupMap.get(os, null)
  33     assert osGroup != null : "Could not find os group for ${os}"
  34     return osGroupMap[os]
  35 }
  36
  37 // We use this class (vs variables) so that the static functions can access data here.
  38 class Constants {
  39
  40     // Innerloop build OS's
  41     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  42     // test execution in the build flow runs.  It generates the exact same build
  43     // as Windows_NT but without the tests.
  44     def static osList = [
  45                'Ubuntu',
  46                'Debian8.4',
  47                'OSX10.12',
  48                'Windows_NT',
  49                'Windows_NT_BuildOnly',
  50                'FreeBSD',
  51                'CentOS7.1',
  52                'OpenSUSE42.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  67                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  68                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  69                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  70                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  71                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  72                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  73                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  74                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  75                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  76                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  77                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  78                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  79                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  80                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  81                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  82                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  83                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  84                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  85                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  86                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  87                'corefx_baseline'                : [ : ], // corefx baseline
  88                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  89                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  90                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  91                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  92                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  93                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  94                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  95                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  96                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  97                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  98                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  99                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 100                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 101                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 103                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 104                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 105                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 107                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 109                ]
 110
 111     // This is a set of r2r jit stress scenarios
 112     def static r2rJitStressScenarios = [
 113                'r2r_jitstress1',
 114                'r2r_jitstress2',
 115                'r2r_jitstressregs1',
 116                'r2r_jitstressregs2',
 117                'r2r_jitstressregs3',
 118                'r2r_jitstressregs4',
 119                'r2r_jitstressregs8',
 120                'r2r_jitstressregs0x10',
 121                'r2r_jitstressregs0x80',
 122                'r2r_jitstressregs0x1000',
 123                'r2r_jitminopts',
 124                'r2r_jitforcerelocs']
 125
 126     // This is the basic set of scenarios
 127     def static basicScenarios = [
 128                'default',
 129                'pri1',
 130                'ilrt',
 131                'r2r',
 132                'pri1r2r',
 133                'gcstress15_pri1r2r',
 134                'longgc',
 135                'coverage',
 136                'formatting',
 137                'gcsimulator',
 138                'jitdiff',
 139                'standalone_gc',
 140                'gc_reliability_framework',
 141                'illink'] + r2rJitStressScenarios
 142
 143     def static configurationList = ['Debug', 'Checked', 'Release']
 144
 145     // This is the set of architectures
 146     def static architectureList = ['arm', 'arm64', 'x64', 'x86', 'x86lb']
 147 }
 148
 149 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 150     if (architecture == 'arm64' && os == 'Windows_NT') {
 151         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 152     } else if (architecture == 'arm64' && os != 'Windows_NT' && options == null) {
 153         Utilities.setMachineAffinity(job, os, 'arm64-small-page-size');
 154     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['large_pages'] == true) {
 155         Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size');
 156     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['is_build_only'] == true) {
 157         Utilities.setMachineAffinity(job, os, 'arm64-cross-latest');
 158     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 159         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 160     } else if ((architecture == 'arm') && (os == 'Windows_NT') && options['use_arm64_build_machine'] == true) {
 161         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 162     }else {
 163         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 164     }
 165 }
 166
 167 def static isJITStressJob(def scenario) {
 168     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 169            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 170 }
 171
 172 def static isGCStressRelatedTesting(def scenario) {
 173     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 174     // Detect it and make it a GCStress related.
 175     if (scenario == 'gcstress15_pri1r2r')
 176     {
 177         return true;
 178     }
 179
 180     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 181     def scenarioName = scenario.toLowerCase()
 182     def isGCStressTesting = false
 183     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 184         if (k in gcStressTestEnvVars) {
 185             isGCStressTesting = true;
 186         }
 187     }
 188     return isGCStressTesting
 189 }
 190
 191 def static isCorefxTesting(def scenario) {
 192     def corefx_prefix = 'corefx_'
 193     if (scenario.length() < corefx_prefix.length()) {
 194         return false
 195     }
 196     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 197 }
 198
 199 def static isR2R(def scenario) {
 200     return (scenario == 'r2r' || scenario == 'pri1r2r')
 201 }
 202
 203 def static isCoverage(def scenario) {
 204     return (scenario == 'coverage')
 205 }
 206
 207 def static isLongGc(def scenario) {
 208     return (scenario == 'longgc' || scenario == 'gcsimulator')
 209 }
 210
 211 def static isJitDiff(def scenario) {
 212     return (scenario == 'jitdiff')
 213 }
 214
 215 def static isGcReliabilityFramework(def scenario) {
 216     return (scenario == 'gc_reliability_framework')
 217 }
 218
 219 def static scenarioNeedsPri1Build(def scenario) {
 220     return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario))
 221 }
 222
 223 def static setTestJobTimeOut(newJob, scenario) {
 224     if (isGCStressRelatedTesting(scenario)) {
 225         Utilities.setJobTimeout(newJob, 4320)
 226     }
 227     else if (isCorefxTesting(scenario)) {
 228         Utilities.setJobTimeout(newJob, 360)
 229     }
 230     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 231         Utilities.setJobTimeout(newJob, 240)
 232     }
 233     else if (isR2R(scenario)) {
 234         Utilities.setJobTimeout(newJob, 240)
 235     }
 236     else if (isCoverage(scenario)) {
 237         Utilities.setJobTimeout(newJob, 1440)
 238     }
 239     else if (isLongGc(scenario)) {
 240         Utilities.setJobTimeout(newJob, 1440)
 241     }
 242     else if (isJitDiff(scenario)) {
 243         Utilities.setJobTimeout(newJob, 240)
 244     }
 245     else if (isGcReliabilityFramework(scenario)) {
 246         Utilities.setJobTimeout(newJob, 1440)
 247     }
 248     // Non-test jobs use the default timeout value.
 249 }
 250
 251 def static getJobFolder(def scenario) {
 252     if (isJITStressJob(scenario)) {
 253         return 'jitstress'
 254     }
 255     if (scenario == 'illink') {
 256         return 'illink'
 257     }
 258     return ''
 259 }
 260
 261 def static getStressModeDisplayName(def scenario) {
 262     def displayStr = ''
 263     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 264         def prefixLength = 'COMPlus_'.length()
 265         if (k.length() >= prefixLength) {
 266             def modeName = k.substring(prefixLength, k.length())
 267             displayStr += ' ' + modeName + '=' + v
 268         }
 269     }
 270     return displayStr
 271 }
 272
 273 def static getR2RStressModeDisplayName(def scenario) {
 274     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 275     // "r2r_" prefix.
 276     def displayStr = scenario
 277     def prefixLength = 'r2r_'.length()
 278     if (displayStr.length() >= prefixLength) {
 279         displayStr = displayStr.substring(prefixLength, displayStr.length())
 280     }
 281     return displayStr
 282 }
 283
 284 // Generates the string for creating a file that sets environment variables
 285 // that makes it possible to run stress modes.  Writes the script to the file
 286 // specified by the stepScriptLocation parameter.
 287 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 288     def stepScript = ''
 289     if (os == 'Windows_NT') {
 290         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 291         stepScript += "del ${stepScriptLocation}\r\n"
 292
 293         // Timeout in ms, default is 10 minutes. For stress
 294         // modes up this to 30 minutes
 295         def timeout = 1800000
 296
 297         // Set the Timeout
 298         stepScript += "set __TestTimeout=${timeout}\r\n"
 299         stepScript += "echo. > ${stepScriptLocation}\r\n"
 300         stressModeVars.each{ k, v ->
 301             // Write out what we are writing to the script file
 302             stepScript += "echo Setting ${k}=${v}\r\n"
 303             // Write out the set itself to the script file`
 304             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 305         }
 306     }
 307     else {
 308         stepScript += "echo Setting variables for ${stressModeName}\n"
 309         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 310         stressModeVars.each{ k, v ->
 311             // Write out what we are writing to the script file
 312             stepScript += "echo Setting ${k}=${v}\n"
 313             // Write out the set itself to the script file`
 314             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 315         }
 316         stepScript += "chmod +x ${stepScriptLocation}\n"
 317     }
 318     return stepScript
 319 }
 320
 321 // Calculates the name of the build job based on some typical parameters.
 322 //
 323 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 324     // If the architecture is x64, do not add that info into the build name.
 325     // Need to change around some systems and other builds to pick up the right builds
 326     // to do that.
 327
 328     def suffix = scenario != 'default' ? "_${scenario}" : '';
 329     if (isBuildOnly) {
 330         suffix += '_bld'
 331     }
 332     def baseName = ''
 333     switch (architecture) {
 334         case 'x64':
 335             if (scenario == 'default') {
 336                 // For now we leave x64 off of the name for compatibility with other jobs
 337                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 338             }
 339             else if (scenario == 'formatting') {
 340                 // we don't care about the configuration for the formatting job. It runs all configs
 341                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 342             }
 343             else {
 344                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 345             }
 346             break
 347         case 'arm64':
 348             if (os.toLowerCase() == "windows_nt") {
 349                 // These are cross builds
 350                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 351             }
 352             else {
 353                 // Defaults to a small page size set of machines.
 354                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 355             }
 356             break
 357         case 'arm':
 358             // These are cross builds
 359             if (os == 'Tizen') {
 360                 // ABI: softfp
 361                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 362             }
 363             else {
 364                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 365             }
 366             break
 367         case 'x86':
 368         case 'x86lb':
 369             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 370             break
 371         default:
 372             println("Unknown architecture: ${architecture}");
 373             assert false
 374             break
 375     }
 376
 377     return baseName + suffix
 378 }
 379
 380 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 381     // Check scenario.
 382     switch (scenario) {
 383         case 'default':
 384             switch (architecture) {
 385                 case 'x64':
 386                 case 'x86':
 387                 case 'x86lb':
 388                     if (architecture == 'x86' && os == 'Ubuntu') {
 389                         Utilities.addPeriodicTrigger(job, '@daily')
 390                     }
 391                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 392                         Utilities.addGithubPushTrigger(job)
 393                     }
 394                     break
 395                 case 'arm':
 396                     Utilities.addGithubPushTrigger(job)
 397                     break
 398                 case 'arm64':
 399                     Utilities.addGithubPushTrigger(job)
 400                     break
 401                 default:
 402                     println("Unknown architecture: ${architecture}");
 403                     assert false
 404                     break
 405             }
 406             break
 407         case 'pri1':
 408             // Pri one gets a push trigger, and only for release
 409             if (architecture == 'x64') {
 410                 if (configuration == 'Release') {
 411                     // We expect release jobs to be Windows, or in the cross list
 412                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 413                     if (!os in bidailyCrossList) {
 414                         if (isFlowJob || os == 'Windows_NT') {
 415                             Utilities.addGithubPushTrigger(job)
 416                         }
 417                     }
 418                     else {
 419                         if (isFlowJob) {
 420                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 421                         }
 422                     }
 423                 }
 424             }
 425             break
 426         case 'r2r':
 427             //r2r jobs that aren't pri1 can only be triggered by phrase
 428             break
 429         case 'pri1r2r':
 430             assert !(os in bidailyCrossList)
 431             //pri1 r2r gets a push trigger for checked/release
 432             if (configuration == 'Checked' || configuration == 'Release') {
 433                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 434                 if (architecture == 'x64' && os != 'OSX10.12') {
 435                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 436                     if (isFlowJob || os == 'Windows_NT') {
 437                         Utilities.addGithubPushTrigger(job)
 438                     }
 439                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 440                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 441                     if (isFlowJob) {
 442                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 443                     }
 444                 }
 445                 // For x86, only add per-commit jobs for Windows
 446                 else if (architecture == 'x86' || architecture == 'x86lb') {
 447                     if (os == 'Windows_NT') {
 448                         Utilities.addGithubPushTrigger(job)
 449                     }
 450                 }
 451                 // arm64 pri1r2r jobs should only run every 12 hours.
 452                 else if (architecture == 'arm64') {
 453                     if (os == 'Windows_NT') {
 454                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 455                         // TODO: Add once external email sending is available again
 456                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 457                     }
 458                 }
 459             }
 460             break
 461         case 'r2r_jitstress1':
 462         case 'r2r_jitstress2':
 463         case 'r2r_jitstressregs1':
 464         case 'r2r_jitstressregs2':
 465         case 'r2r_jitstressregs3':
 466         case 'r2r_jitstressregs4':
 467         case 'r2r_jitstressregs8':
 468         case 'r2r_jitstressregs0x10':
 469         case 'r2r_jitstressregs0x80':
 470         case 'r2r_jitstressregs0x1000':
 471         case 'r2r_jitminopts':
 472         case 'r2r_jitforcerelocs':
 473         case 'gcstress15_pri1r2r':
 474             assert !(os in bidailyCrossList)
 475
 476             // GCStress=C is currently not supported on OS X
 477             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 478                 break
 479             }
 480
 481             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 482             if (configuration == 'Checked' || configuration == 'Release') {
 483                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 484                 if (architecture == 'x64') {
 485                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 486                     if (isFlowJob || os == 'Windows_NT') {
 487                         // Add a weekly periodic trigger
 488                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 489                     }
 490                 }
 491                 // For x86, only add per-commit jobs for Windows
 492                 else if (architecture == 'x86') {
 493                     if (os == 'Windows_NT') {
 494                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 495                     }
 496                 }
 497             }
 498             break
 499         case 'longgc':
 500             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 501             assert configuration == 'Release'
 502             assert architecture == 'x64'
 503             Utilities.addPeriodicTrigger(job, '@daily')
 504             // TODO: Add once external email sending is available again
 505             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 506             break
 507         case 'gcsimulator':
 508             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 509             assert configuration == 'Release'
 510             assert architecture == 'x64'
 511             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 512             // TODO: Add once external email sending is available again
 513             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 514             break
 515         case 'standalone_gc':
 516             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 517             assert (configuration == 'Release' || configuration == 'Checked')
 518             // TODO: Add once external email sending is available again
 519             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 520             Utilities.addPeriodicTrigger(job, '@weekly')
 521             break
 522         case 'gc_reliability_framework':
 523             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 524             assert (configuration == 'Release' || configuration == 'Checked')
 525             // Only triggered by phrase.
 526             break
 527         case 'ilrt':
 528             assert !(os in bidailyCrossList)
 529             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 530             if (architecture == 'x64' && configuration == 'Release') {
 531                 // We don't expect to see a job generated except in these scenarios
 532                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 533                 if (isFlowJob || os == 'Windows_NT') {
 534                     Utilities.addPeriodicTrigger(job, '@daily')
 535                 }
 536             }
 537             break
 538         case 'jitdiff':
 539             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 540             assert configuration == 'Checked'
 541             assert (architecture == 'x64' || architecture == 'x86')
 542             Utilities.addGithubPushTrigger(job)
 543             break
 544         case 'coverage':
 545             assert (os == 'Ubuntu' || os == 'Windows_NT')
 546             assert configuration == 'Release'
 547             assert architecture == 'x64'
 548             Utilities.addPeriodicTrigger(job, '@weekly')
 549             break
 550         case 'formatting':
 551             assert (os == 'Windows_NT' || os == "Ubuntu")
 552             assert architecture == 'x64'
 553             Utilities.addGithubPushTrigger(job)
 554             break
 555         case 'jitstressregs1':
 556         case 'jitstressregs2':
 557         case 'jitstressregs3':
 558         case 'jitstressregs4':
 559         case 'jitstressregs8':
 560         case 'jitstressregs0x10':
 561         case 'jitstressregs0x80':
 562         case 'jitstressregs0x1000':
 563         case 'minopts':
 564         case 'forcerelocs':
 565         case 'jitstress1':
 566         case 'jitstress2':
 567         case 'jitstress2_jitstressregs1':
 568         case 'jitstress2_jitstressregs2':
 569         case 'jitstress2_jitstressregs3':
 570         case 'jitstress2_jitstressregs4':
 571         case 'jitstress2_jitstressregs8':
 572         case 'jitstress2_jitstressregs0x10':
 573         case 'jitstress2_jitstressregs0x80':
 574         case 'jitstress2_jitstressregs0x1000':
 575         case 'tailcallstress':
 576         case 'jitsse2only':
 577         case 'corefx_baseline':
 578         case 'corefx_minopts':
 579         case 'corefx_jitstress1':
 580         case 'corefx_jitstress2':
 581         case 'corefx_jitstressregs1':
 582         case 'corefx_jitstressregs2':
 583         case 'corefx_jitstressregs3':
 584         case 'corefx_jitstressregs4':
 585         case 'corefx_jitstressregs8':
 586         case 'corefx_jitstressregs0x10':
 587         case 'corefx_jitstressregs0x80':
 588         case 'corefx_jitstressregs0x1000':
 589         case 'zapdisable':
 590             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 591             assert (os == 'Windows_NT') || (os in Constants.crossList)
 592             Utilities.addPeriodicTrigger(job, '@daily')
 593         }
 594         break
 595         case 'heapverify1':
 596         case 'gcstress0x3':
 597             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 598                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 599                 if (architecture == 'arm64') {
 600                     if (os == 'Windows_NT') {
 601                         Utilities.addPeriodicTrigger(job, '@daily')
 602                     }
 603                     // TODO: Add once external email sending is available again
 604                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 605                 }
 606                 else {
 607                     Utilities.addPeriodicTrigger(job, '@weekly')
 608                 }
 609             }
 610             break
 611         case 'gcstress0xc':
 612         case 'gcstress0xc_zapdisable':
 613         case 'gcstress0xc_zapdisable_jitstress2':
 614         case 'gcstress0xc_zapdisable_heapverify1':
 615         case 'gcstress0xc_jitstress1':
 616         case 'gcstress0xc_jitstress2':
 617         case 'gcstress0xc_minopts_heapverify1':
 618             // GCStress=C is currently not supported on OS X
 619             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 620                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 621                 if (architecture == 'arm64') {
 622                     // TODO: Enable a periodic trigger after tests are updated.
 623                     // Utilities.addPeriodicTrigger(job, '@daily')
 624                     // TODO: Add once external email sending is available again
 625                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 626                 }
 627                 else {
 628                     Utilities.addPeriodicTrigger(job, '@weekly')
 629                 }
 630             }
 631             break
 632
 633         case 'illink':
 634             // Testing on other operating systems TBD
 635             assert (os == 'Windows_NT' || os == 'Ubuntu')
 636             if (architecture == 'x64' || architecture == 'x86') {
 637                 if (configuration == 'Checked') {
 638                     Utilities.addPeriodicTrigger(job, '@daily')
 639                 }
 640             }
 641             break
 642
 643         default:
 644             println("Unknown scenario: ${scenario}");
 645             assert false
 646             break
 647     }
 648     return
 649 }
 650
 651 // **************************
 652 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 653 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 654 // builds will do a couple extra steps.
 655 // **************************
 656
 657 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 658 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 659 // it's just used for internal builds
 660 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 661 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 662     if (isWindowsBuildOnlyJob) {
 663         return
 664     }
 665
 666     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 667     // Non pull request builds.
 668     if (!isPR) {
 669         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 670         return
 671     }
 672
 673      def arm64Users = [
 674         'adiaaida',
 675         'AndyAyersMS',
 676         'briansull',
 677         'BruceForstall',
 678         'CarolEidt',
 679         'cmckinsey',
 680         'erozenfeld',
 681         'jashook',
 682         'JosephTremoulet',
 683         'pgavlin',
 684         'russellhadley',
 685         'RussKeldorph',
 686         'sandreenko',
 687         'sdmaclea',
 688         'sivarv',
 689         'swaroop-sridhar',
 690         'gkhanna79',
 691         'jkotas',
 692         'markwilkie',
 693         'rahku',
 694         'ramarag',
 695         'tzwlai',
 696         'weshaggard'
 697     ]
 698
 699     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 700     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 701     def osGroup = getOSGroup(os)
 702     switch (architecture) {
 703         case 'x64': // editor brace matching: {
 704             if (scenario == 'coverage') {
 705                 assert configuration == 'Release'
 706                 if (os == 'Ubuntu') {
 707                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 708                 }
 709                 break
 710             }
 711
 712             if (scenario == 'formatting') {
 713                 assert configuration == 'Checked'
 714                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 715                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 716                 }
 717                 break
 718             }
 719
 720             switch (os) {
 721                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 722                 case 'Debian8.4':
 723                 case 'RHEL7.2':
 724                     if (scenario == 'default') {
 725                         assert !isFlowJob
 726                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 727                     }
 728                     else if (scenario == 'pri1' && isFlowJob) {
 729                         assert (configuration == 'Release')
 730                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 731                     }
 732                     break
 733                 case 'Ubuntu16.04':
 734                     assert !isFlowJob
 735                     assert scenario == 'default'
 736                     // Distinguish with the other architectures (arm and x86)
 737                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 738                     break
 739                 case 'Fedora24':
 740                 case 'Ubuntu16.10':
 741                 case 'OpenSUSE42.1':
 742                     assert !isFlowJob
 743                     assert scenario == 'default'
 744                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 745                     break
 746                 case 'Ubuntu':
 747                     if (scenario == 'illink') {
 748                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 749                         break
 750                     }
 751                     // fall through
 752                 case 'OSX10.12':
 753                     // Triggers on the non-flow jobs aren't necessary here
 754                     // Corefx testing uses non-flow jobs.
 755                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 756                         break
 757                     }
 758                     switch (scenario) {
 759                         case 'default':
 760                             // Ubuntu uses checked for default PR tests
 761                             if (configuration == 'Checked') {
 762                                 // Default trigger
 763                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 764                             }
 765                             break
 766                         case 'pri1':
 767                             if (configuration == 'Release') {
 768                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 769                             }
 770                             break
 771                         case 'jitdiff':
 772                             if (configuration == 'Checked') {
 773                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 774                             }
 775                             break
 776                         case 'ilrt':
 777                             if (configuration == 'Release') {
 778                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 779                             }
 780                             break
 781                         case 'r2r':
 782                             if (configuration == 'Release' || configuration == 'Checked') {
 783                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 784                             }
 785                             break
 786                         case 'pri1r2r':
 787                             if (configuration == 'Release' || configuration == 'Checked') {
 788                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 789                             }
 790                             break
 791                         case 'gcstress15_pri1r2r':
 792                             if (configuration == 'Release' || configuration == 'Checked') {
 793                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 794                             }
 795                             break
 796                         case 'r2r_jitstress1':
 797                         case 'r2r_jitstress2':
 798                         case 'r2r_jitstressregs1':
 799                         case 'r2r_jitstressregs2':
 800                         case 'r2r_jitstressregs3':
 801                         case 'r2r_jitstressregs4':
 802                         case 'r2r_jitstressregs8':
 803                         case 'r2r_jitstressregs0x10':
 804                         case 'r2r_jitstressregs0x80':
 805                         case 'r2r_jitstressregs0x1000':
 806                         case 'r2r_jitminopts':
 807                         case 'r2r_jitforcerelocs':
 808                             if (configuration == 'Release' || configuration == 'Checked') {
 809                                 def displayStr = getR2RStressModeDisplayName(scenario)
 810                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 811                             }
 812                             break
 813                         case 'longgc':
 814                             if (configuration == 'Release') {
 815                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 816                             }
 817                             break
 818                         case 'gcsimulator':
 819                             if (configuration == 'Release') {
 820                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 821                             }
 822                             break
 823                         case 'standalone_gc':
 824                             if (configuration == 'Release' || configuration == 'Checked') {
 825                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 826                             }
 827                             break
 828                         case 'gc_reliability_framework':
 829                             if (configuration == 'Release' || configuration == 'Checked') {
 830                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 831                             }
 832                             break
 833                         case 'minopts':
 834                         case 'forcerelocs':
 835                         case 'jitstress1':
 836                         case 'jitstress2':
 837                         case 'jitstressregs1':
 838                         case 'jitstressregs2':
 839                         case 'jitstressregs3':
 840                         case 'jitstressregs4':
 841                         case 'jitstressregs8':
 842                         case 'jitstressregs0x10':
 843                         case 'jitstressregs0x80':
 844                         case 'jitstressregs0x1000':
 845                         case 'jitstress2_jitstressregs1':
 846                         case 'jitstress2_jitstressregs2':
 847                         case 'jitstress2_jitstressregs3':
 848                         case 'jitstress2_jitstressregs4':
 849                         case 'jitstress2_jitstressregs8':
 850                         case 'jitstress2_jitstressregs0x10':
 851                         case 'jitstress2_jitstressregs0x80':
 852                         case 'jitstress2_jitstressregs0x1000':
 853                         case 'tailcallstress':
 854                         case 'jitsse2only':
 855                         case 'gcstress0x3':
 856                         case 'gcstress0xc':
 857                         case 'zapdisable':
 858                         case 'heapverify1':
 859                         case 'gcstress0xc_zapdisable':
 860                         case 'gcstress0xc_zapdisable_jitstress2':
 861                         case 'gcstress0xc_zapdisable_heapverify1':
 862                         case 'gcstress0xc_jitstress1':
 863                         case 'gcstress0xc_jitstress2':
 864                         case 'gcstress0xc_minopts_heapverify1':
 865                             def displayStr = getStressModeDisplayName(scenario)
 866                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 867                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 868                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 869                             break
 870                         case 'corefx_baseline':
 871                         case 'corefx_minopts':
 872                         case 'corefx_jitstress1':
 873                         case 'corefx_jitstress2':
 874                         case 'corefx_jitstressregs1':
 875                         case 'corefx_jitstressregs2':
 876                         case 'corefx_jitstressregs3':
 877                         case 'corefx_jitstressregs4':
 878                         case 'corefx_jitstressregs8':
 879                         case 'corefx_jitstressregs0x10':
 880                         case 'corefx_jitstressregs0x80':
 881                         case 'corefx_jitstressregs0x1000':
 882                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 883                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 884                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 885                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 886                             break
 887                         default:
 888                             println("Unknown scenario: ${scenario}");
 889                             assert false
 890                             break
 891                     }
 892                     break
 893                 case 'CentOS7.1':
 894                     switch (scenario) {
 895                         case 'pri1':
 896                             if (configuration == 'Release') {
 897                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 898                             }
 899                             break
 900                         case 'r2r':
 901                             if (configuration == 'Checked' || configuration == 'Release') {
 902                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 903                             }
 904                             break
 905                         case 'pri1r2r':
 906                             if (configuration == 'Checked' || configuration == 'Release') {
 907                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 908                             }
 909                             break
 910                         case 'gcstress15_pri1r2r':
 911                             if (configuration == 'Release' || configuration == 'Checked') {
 912                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 913                             }
 914                             break
 915                         case 'r2r_jitstress1':
 916                         case 'r2r_jitstress2':
 917                         case 'r2r_jitstressregs1':
 918                         case 'r2r_jitstressregs2':
 919                         case 'r2r_jitstressregs3':
 920                         case 'r2r_jitstressregs4':
 921                         case 'r2r_jitstressregs8':
 922                         case 'r2r_jitstressregs0x10':
 923                         case 'r2r_jitstressregs0x80':
 924                         case 'r2r_jitstressregs0x1000':
 925                         case 'r2r_jitminopts':
 926                         case 'r2r_jitforcerelocs':
 927                             if (configuration == 'Release' || configuration == 'Checked') {
 928                                 def displayStr = getR2RStressModeDisplayName(scenario)
 929                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 930                             }
 931                             break
 932                         default:
 933                             break
 934                     }
 935                 case 'Windows_NT':
 936                     switch (scenario) {
 937                         case 'default':
 938                             // Default trigger
 939                             if (configuration == 'Debug') {
 940                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 941                             }
 942                             break
 943                         case 'pri1':
 944                             // Default trigger
 945                             if (configuration == 'Release') {
 946                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 947                             }
 948                             break
 949                         case 'jitdiff':
 950                             if (configuration == 'Checked') {
 951                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 952                             }
 953                             break
 954                         case 'ilrt':
 955                             if (configuration == 'Release') {
 956                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 957                             }
 958                             break
 959                         case 'r2r':
 960                             if (configuration == 'Checked' || configuration == 'Release') {
 961                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 962                             }
 963                             break
 964                         case 'pri1r2r':
 965                             if (configuration == 'Checked' || configuration == 'Release') {
 966                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 967                             }
 968                             break
 969                         case 'gcstress15_pri1r2r':
 970                             if (configuration == 'Release' || configuration == 'Checked') {
 971                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 972                             }
 973                             break
 974                         case 'r2r_jitstress1':
 975                         case 'r2r_jitstress2':
 976                         case 'r2r_jitstressregs1':
 977                         case 'r2r_jitstressregs2':
 978                         case 'r2r_jitstressregs3':
 979                         case 'r2r_jitstressregs4':
 980                         case 'r2r_jitstressregs8':
 981                         case 'r2r_jitstressregs0x10':
 982                         case 'r2r_jitstressregs0x80':
 983                         case 'r2r_jitstressregs0x1000':
 984                         case 'r2r_jitminopts':
 985                         case 'r2r_jitforcerelocs':
 986                             if (configuration == 'Release' || configuration == 'Checked') {
 987                                 def displayStr = getR2RStressModeDisplayName(scenario)
 988                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 989                             }
 990                             break
 991                         case 'longgc':
 992                             if (configuration == 'Release') {
 993                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 994                             }
 995                             break
 996                         case 'gcsimulator':
 997                             if (configuration == 'Release') {
 998                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 999                             }
1000                             break
1001                         case 'standalone_gc':
1002                             if (configuration == 'Release' || configuration == 'Checked') {
1003                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1004                             }
1005                             break
1006                         case 'gc_reliability_framework':
1007                             if (configuration == 'Release' || configuration == 'Checked') {
1008                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1009                             }
1010                             break
1011                         case 'minopts':
1012                         case 'forcerelocs':
1013                         case 'jitstress1':
1014                         case 'jitstress2':
1015                         case 'jitstressregs1':
1016                         case 'jitstressregs2':
1017                         case 'jitstressregs3':
1018                         case 'jitstressregs4':
1019                         case 'jitstressregs8':
1020                         case 'jitstressregs0x10':
1021                         case 'jitstressregs0x80':
1022                         case 'jitstressregs0x1000':
1023                         case 'jitstress2_jitstressregs1':
1024                         case 'jitstress2_jitstressregs2':
1025                         case 'jitstress2_jitstressregs3':
1026                         case 'jitstress2_jitstressregs4':
1027                         case 'jitstress2_jitstressregs8':
1028                         case 'jitstress2_jitstressregs0x10':
1029                         case 'jitstress2_jitstressregs0x80':
1030                         case 'jitstress2_jitstressregs0x1000':
1031                         case 'tailcallstress':
1032                         case 'jitsse2only':
1033                         case 'gcstress0x3':
1034                         case 'gcstress0xc':
1035                         case 'zapdisable':
1036                         case 'heapverify1':
1037                         case 'gcstress0xc_zapdisable':
1038                         case 'gcstress0xc_zapdisable_jitstress2':
1039                         case 'gcstress0xc_zapdisable_heapverify1':
1040                         case 'gcstress0xc_jitstress1':
1041                         case 'gcstress0xc_jitstress2':
1042                         case 'gcstress0xc_minopts_heapverify1':
1043                             def displayStr = getStressModeDisplayName(scenario)
1044                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1045                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1046                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1047                             break
1048                         case 'corefx_baseline':
1049                         case 'corefx_minopts':
1050                         case 'corefx_jitstress1':
1051                         case 'corefx_jitstress2':
1052                         case 'corefx_jitstressregs1':
1053                         case 'corefx_jitstressregs2':
1054                         case 'corefx_jitstressregs3':
1055                         case 'corefx_jitstressregs4':
1056                         case 'corefx_jitstressregs8':
1057                         case 'corefx_jitstressregs0x10':
1058                         case 'corefx_jitstressregs0x80':
1059                         case 'corefx_jitstressregs0x1000':
1060                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1061                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1062                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1063                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1064                             break
1065                         case 'illink':
1066                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1067                                     break
1068                         default:
1069                             println("Unknown scenario: ${scenario}");
1070                             assert false
1071                             break
1072                     }
1073                     break
1074                 case 'FreeBSD':
1075                     assert scenario == 'default'
1076                     if (configuration == 'Checked') {
1077                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1078                     }
1079                     break
1080                 default:
1081                     println("Unknown os: ${os}");
1082                     assert false
1083                     break
1084             }
1085             break
1086         // editor brace matching: }
1087         case 'arm': // editor brace matching: {
1088             switch (os) {
1089                 case 'Ubuntu':
1090                 case 'Ubuntu16.04':
1091                     assert scenario == 'default'
1092                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1093                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1094                     }
1095                     else {
1096                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1097                     }
1098                     break;
1099                 case 'Tizen':
1100                     architecture='armel'
1101                     // Removing the regex will cause this to run on each PR.
1102                     if (configuration == 'Release' || configuration == 'Debug') {
1103                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1104                     }
1105                     else {
1106                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1107                     }
1108                     break;
1109                 case 'Windows_NT':
1110                     // Set up a private trigger
1111                     def contextString = "${os} ${architecture} Cross ${configuration}"
1112                     if (scenario != 'default')
1113                         contextString += " ${scenario}"
1114                     contextString += " Build"
1115                     // Debug builds only.
1116                     if (configuration != 'Debug') {
1117                         contextString += " and Test"
1118                     }
1119                     switch (scenario) {
1120                         case 'default':
1121                             // For now only run Debug and Release build jobs on PR Trigger. Note this is not a private trigger.
1122                             if (configuration == 'Debug' || configuration == 'Release')
1123                             {
1124                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1125                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1126                             }
1127                             else
1128                             {
1129                                 // Checked jobs will run on private trigger and run tests.
1130                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1131                             }
1132                             break
1133                         case 'pri1r2r':
1134                         case 'gcstress0x3':
1135                         case 'gcstress0xc':
1136                             // Stress jobs will will run on private trigger and run tests.
1137                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1138                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1139                             break
1140                     }
1141                     break
1142                 default:
1143                     println("NYI os: ${os}");
1144                     assert false
1145                     break
1146             }
1147             break
1148         // editor brace matching: }
1149         case 'arm64': // editor brace matching: {
1150             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1151
1152             // Set up a private trigger
1153             def contextString = "${os} ${architecture} Cross ${configuration}"
1154             if (scenario != 'default')
1155                 contextString += " ${scenario}"
1156             contextString += " Build"
1157             // Debug builds only.
1158             if (configuration != 'Debug') {
1159                contextString += " and Test"
1160             }
1161
1162             switch (os) {
1163                 case 'Ubuntu':
1164                 case 'Ubuntu16.04':
1165                     switch (scenario) {
1166                         case 'pri1':
1167                             if (configuration == 'Release') {
1168                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1169                             }
1170                             break
1171                         case 'r2r':
1172                             if (configuration == 'Checked' || configuration == 'Release') {
1173                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1174                             }
1175                             break
1176                         case 'pri1r2r':
1177                             if (configuration == 'Checked' || configuration == 'Release') {
1178                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1179                             }
1180                             break
1181                         case 'gcstress15_pri1r2r':
1182                             if (configuration == 'Release' || configuration == 'Checked') {
1183                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1184                             }
1185                             break
1186                         case 'r2r_jitstress1':
1187                         case 'r2r_jitstress2':
1188                         case 'r2r_jitstressregs1':
1189                         case 'r2r_jitstressregs2':
1190                         case 'r2r_jitstressregs3':
1191                         case 'r2r_jitstressregs4':
1192                         case 'r2r_jitstressregs8':
1193                         case 'r2r_jitstressregs0x10':
1194                         case 'r2r_jitstressregs0x80':
1195                         case 'r2r_jitstressregs0x1000':
1196                         case 'r2r_jitminopts':
1197                         case 'r2r_jitforcerelocs':
1198                             if (configuration == 'Release' || configuration == 'Checked') {
1199                                 def displayStr = getR2RStressModeDisplayName(scenario)
1200                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1201                             }
1202                             break
1203                         default:
1204                             break
1205                     }
1206                 case 'Windows_NT':
1207                     switch (scenario) {
1208                         case 'default':
1209                             if (isFlowJob == true) {
1210                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration}", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1211                             }
1212
1213                             // For now only run Debug jobs on PR Trigger.
1214                             else if (configuration != 'Debug') {
1215                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1216                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1217                             }
1218                             else {
1219                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1220                                 // are at higher risk of ARM64-breaking changes.
1221                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1222                             }
1223                             break
1224                         case 'pri1r2r':
1225                         case 'gcstress0x3':
1226                         case 'gcstress0xc':
1227                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1228                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1229                             break
1230                     }
1231                     break
1232                 default:
1233                     println("NYI os: ${os}");
1234                     assert false
1235                     break
1236             }
1237             break
1238         // editor brace matching: }
1239         case 'x86': // editor brace matching: {
1240             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1241             if (os == 'Ubuntu') {
1242                 // on-demand only for ubuntu x86
1243                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1244                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1245                 break
1246             }
1247             switch (scenario) {
1248                 case 'default':
1249                     if (configuration == 'Checked') {
1250                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1251                     }
1252                     else if (configuration == 'Release') {
1253                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1254                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1255                     }
1256                     break
1257                 case 'pri1':
1258                     if (configuration == 'Release') {
1259                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1260                     }
1261                     break
1262                 case 'ilrt':
1263                     if (configuration == 'Release') {
1264                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1265                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1266                     }
1267                     break
1268                 case 'r2r':
1269                     if (configuration == 'Checked' || configuration == 'Release') {
1270                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1271                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1272                     }
1273                     break
1274                 case 'pri1r2r':
1275                     if (configuration == 'Checked' || configuration == 'Release') {
1276                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1277                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1278                     }
1279                     break
1280                 case 'gcstress15_pri1r2r':
1281                     if (configuration == 'Release' || configuration == 'Checked') {
1282                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1283                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1284                     }
1285                     break
1286                 case 'r2r_jitstress1':
1287                 case 'r2r_jitstress2':
1288                 case 'r2r_jitstressregs1':
1289                 case 'r2r_jitstressregs2':
1290                 case 'r2r_jitstressregs3':
1291                 case 'r2r_jitstressregs4':
1292                 case 'r2r_jitstressregs8':
1293                 case 'r2r_jitstressregs0x10':
1294                 case 'r2r_jitstressregs0x80':
1295                 case 'r2r_jitstressregs0x1000':
1296                 case 'r2r_jitminopts':
1297                 case 'r2r_jitforcerelocs':
1298                     if (configuration == 'Release' || configuration == 'Checked') {
1299                         def displayStr = getR2RStressModeDisplayName(scenario)
1300                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1301                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1302                     }
1303                     break
1304                 case 'longgc':
1305                     if (configuration == 'Release') {
1306                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1307                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1308                     }
1309                     break
1310                 case 'gcsimulator':
1311                     if (configuration == 'Release') {
1312                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1313                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1314                     }
1315                     break
1316                 case 'standalone_gc':
1317                     if (configuration == 'Release' || configuration == 'Checked') {
1318                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1319                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1320                     }
1321                     break
1322                 case 'minopts':
1323                 case 'forcerelocs':
1324                 case 'jitstress1':
1325                 case 'jitstress2':
1326                 case 'jitstressregs1':
1327                 case 'jitstressregs2':
1328                 case 'jitstressregs3':
1329                 case 'jitstressregs4':
1330                 case 'jitstressregs8':
1331                 case 'jitstressregs0x10':
1332                 case 'jitstressregs0x80':
1333                 case 'jitstressregs0x1000':
1334                 case 'jitstress2_jitstressregs1':
1335                 case 'jitstress2_jitstressregs2':
1336                 case 'jitstress2_jitstressregs3':
1337                 case 'jitstress2_jitstressregs4':
1338                 case 'jitstress2_jitstressregs8':
1339                 case 'jitstress2_jitstressregs0x10':
1340                 case 'jitstress2_jitstressregs0x80':
1341                 case 'jitstress2_jitstressregs0x1000':
1342                 case 'tailcallstress':
1343                 case 'jitsse2only':
1344                 case 'gcstress0x3':
1345                 case 'gcstress0xc':
1346                 case 'zapdisable':
1347                 case 'heapverify1':
1348                 case 'gcstress0xc_zapdisable':
1349                 case 'gcstress0xc_zapdisable_jitstress2':
1350                 case 'gcstress0xc_zapdisable_heapverify1':
1351                 case 'gcstress0xc_jitstress1':
1352                 case 'gcstress0xc_jitstress2':
1353                 case 'gcstress0xc_minopts_heapverify1':
1354                     def displayStr = getStressModeDisplayName(scenario)
1355                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1356                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1357                     break
1358                 case 'corefx_baseline':
1359                 case 'corefx_minopts':
1360                 case 'corefx_jitstress1':
1361                 case 'corefx_jitstress2':
1362                 case 'corefx_jitstressregs1':
1363                 case 'corefx_jitstressregs2':
1364                 case 'corefx_jitstressregs3':
1365                 case 'corefx_jitstressregs4':
1366                 case 'corefx_jitstressregs8':
1367                 case 'corefx_jitstressregs0x10':
1368                 case 'corefx_jitstressregs0x80':
1369                 case 'corefx_jitstressregs0x1000':
1370                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1371                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1372                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1373                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1374                     break
1375                 case 'illink':
1376                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1377                     break
1378                 default:
1379                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1380                     assert false
1381                     break
1382             }
1383             break
1384          // editor brace matching: }
1385         case 'x86lb': // editor brace matching: {
1386             assert (os == 'Windows_NT')
1387             assert (scenario == 'default' || Constants.r2rJitStressScenarios.indexOf(scenario) !=1)
1388
1389             def arch = 'x86'
1390             def jit = 'legacy_backend'
1391             switch (scenario) {
1392                 case 'default':
1393                     if (configuration == 'Checked') {
1394                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1395                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1396                     }
1397                     break
1398                 default:
1399                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1400                     assert false
1401                     break
1402             }
1403             break
1404         // editor brace matching: }
1405         default:
1406             println("Unknown architecture: ${architecture}");
1407             assert false
1408             break
1409     }
1410 }
1411
1412 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1413     def buildCommands = [];
1414     def osGroup = getOSGroup(os)
1415     def lowerConfiguration = configuration.toLowerCase()
1416
1417     // Calculate the build steps, archival, and xunit results
1418     switch (os) {
1419         case 'Windows_NT': // editor brace matching: {
1420             switch (architecture) {
1421                 case 'x64':
1422                 case 'x86':
1423                 case 'x86lb':
1424                     def arch = architecture
1425                     def buildOpts = ''
1426                     if (architecture == 'x86lb') {
1427                         arch = 'x86'
1428                     }
1429
1430                     if (scenario == 'illink') {
1431                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1432                     }
1433
1434                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1435                             scenario == 'default' ||
1436                             scenario == 'r2r' ||
1437                             scenario == 'jitdiff' ||
1438                             scenario == 'ilrt' ||
1439                             scenario == 'illink' ||
1440                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1441                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1442                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1443                     }
1444
1445                     // For Pri 1 tests, we must shorten the output test binary path names.
1446                     // if __TestIntermediateDir is already set, build-test.cmd will
1447                     // output test binaries to that directory. If it is not set, the
1448                     // binaries are sent to a default directory whose name is about
1449                     // 35 characters long.
1450
1451                     else if (scenarioNeedsPri1Build(scenario)) {
1452                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1453                     }
1454                     else if (isLongGc(scenario)) {
1455                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1456                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1457                     }
1458                     else if (scenario == 'standalone_gc') {
1459                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1460                     }
1461                     else if (scenario == 'formatting') {
1462                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1463                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1464                         break
1465                     }
1466                     else {
1467                         println("Unknown scenario: ${scenario}")
1468                         assert false
1469                     }
1470
1471                     // If we are running a stress mode, we should write out the set of key
1472                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1473
1474                     if (!isBuildOnly) {
1475                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1476                         def crossgenStr = ''
1477                         def runcrossgentestsStr = ''
1478                         def runjitstressStr = ''
1479                         def runjitstressregsStr = ''
1480                         def runjitmioptsStr = ''
1481                         def runjitforcerelocsStr = ''
1482                         def runjitdisasmStr = ''
1483                         def runilasmroundtripStr = ''
1484                         def gcstressStr = ''
1485                         def runtestArguments = ''
1486                         def gcTestArguments = ''
1487                         def illinkArguments = ''
1488
1489                         if (scenario == 'r2r' ||
1490                             scenario == 'pri1r2r' ||
1491                             scenario == 'gcstress15_pri1r2r' ||
1492                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1493                                 crossgenStr = 'crossgen'
1494                                 runcrossgentestsStr = 'runcrossgentests'
1495
1496                                 if (scenario == 'r2r_jitstress1'){
1497                                     runjitstressStr = 'jitstress 1'
1498                                 }
1499                                 else if (scenario == 'r2r_jitstress2') {
1500                                     runjitstressStr = 'jitstress 2'
1501                                 }
1502                                 else if (scenario == 'r2r_jitstressregs1'){
1503                                     runjitstressregsStr = 'jitstressregs 1'
1504                                 }
1505                                 else if (scenario == 'r2r_jitstressregs2') {
1506                                     runjitstressregsStr = 'jitstressregs 2'
1507                                 }
1508                                 else if (scenario == 'r2r_jitstressregs3') {
1509                                     runjitstressregsStr = 'jitstressregs 3'
1510                                 }
1511                                 else if (scenario == 'r2r_jitstressregs4') {
1512                                     runjitstressregsStr = 'jitstressregs 4'
1513                                 }
1514                                 else if (scenario == 'r2r_jitstressregs8') {
1515                                     runjitstressregsStr = 'jitstressregs 8'
1516                                 }
1517                                 else if (scenario == 'r2r_jitstressregs0x10') {
1518                                     runjitstressregsStr = 'jitstressregs 0x10'
1519                                 }
1520                                 else if (scenario == 'r2r_jitstressregs0x80') {
1521                                     runjitstressregsStr = 'jitstressregs 0x80'
1522                                 }
1523                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1524                                     runjitstressregsStr = 'jitstressregs 0x1000'
1525                                 }
1526                                 else if (scenario == 'r2r_jitminopts') {
1527                                     runjitmioptsStr = 'jitminopts'
1528                                 }
1529                                 else if (scenario == 'r2r_jitforcerelocs') {
1530                                     runjitforcerelocsStr = 'jitforcerelocs'
1531                                 }
1532                         }
1533                         if (scenario == 'gcstress15_pri1r2r')
1534                         {
1535                             gcstressStr = 'gcstresslevel 0xF'
1536                         }
1537
1538                         if (scenario == 'jitdiff')
1539                         {
1540                             runjitdisasmStr = 'jitdisasm crossgen'
1541                         }
1542
1543                         if (scenario == 'ilrt')
1544                         {
1545                             runilasmroundtripStr = 'ilasmroundtrip'
1546                         }
1547
1548                         if (isLongGc(scenario)) {
1549                             gcTestArguments = "${scenario} sequential"
1550                         }
1551
1552                         if (scenario == 'illink')
1553                         {
1554                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1555                         }
1556
1557                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1558
1559                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1560                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1561                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1562
1563                             if (enableCorefxTesting) {
1564                                 def workspaceRelativeFxRoot = "_/fx"
1565                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1566
1567                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1568
1569                                 setTestJobTimeOut(newJob, scenario)
1570
1571                                 // Archive and process (only) the test results
1572                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1573                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1574                             }
1575                             else {
1576                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1577                             }
1578                         }
1579                         else if (isGcReliabilityFramework(scenario)) {
1580                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1581                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1582                         }
1583                         else if (architecture == 'x64' || architecture == 'x86') {
1584                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1585                         }
1586                         else if (architecture == 'x86lb') {
1587                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd"
1588                         }
1589                     }
1590
1591                     if (!enableCorefxTesting) {
1592                         // Run the rest of the build
1593                         // Build the mscorlib for the other OS's
1594                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1595                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1596                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1597
1598                         if (arch == "x64") {
1599                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1600                         }
1601
1602                         // Zip up the tests directory so that we don't use so much space/time copying
1603                         // 10s of thousands of files around.
1604                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1605
1606                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1607                             // For windows, pull full test results and test drops for x86/x64.
1608                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1609                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1610                         }
1611
1612                         if (scenario == 'jitdiff') {
1613                             // retrive jit-dasm output for base commit, and run jit-diff
1614                             if (!isBuildOnly) {
1615                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1616                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1617                             }
1618                         }
1619
1620                         if (!isBuildOnly) {
1621                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1622                             setTestJobTimeOut(newJob, scenario)
1623                         }
1624                     }
1625                     break
1626                 case 'arm':
1627                     def validArmWindowsScenarios = [ "default",
1628                                                      "pri1r2r",
1629                                                      "zapdisable",
1630                                                      "minopts",
1631                                                      "tailcallstress",
1632                                                      "jitstress1",
1633                                                      "jitstress2",
1634                                                      "gcstress0x3",
1635                                                      "gcstress0xc",
1636                                                      "jitstressregs1",
1637                                                      "jitstressregs2",
1638                                                      "gcstress0xc_jitstress1",
1639                                                      "gcstress0xc_jitstress2"]
1640
1641                     assert validArmWindowsScenarios.contains(scenario)
1642
1643                     // Set time out
1644                     setTestJobTimeOut(newJob, scenario)
1645
1646                     if ( lowerConfiguration == "debug" ) {
1647                         // For Debug builds, we will do a P1 test build
1648                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1649                     }
1650                     else if (lowerConfiguration == "checked") {
1651
1652                         if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1653                         {
1654                            // Up the timeout for arm checked testing only.
1655                            // Keep the longer timeout for gcstress.
1656                            Utilities.setJobTimeout(newJob, 240)
1657                         }
1658
1659                         def machineAffinityOptions = ['use_arm64_build_machine' : true]
1660                         setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1661                         // For checked runs we will also run testing.
1662                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1663                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1664                     }
1665                     else if (lowerConfiguration == "release") {
1666                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1667                     }
1668                     // Add archival.
1669                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1670                     break
1671                 case 'arm64':
1672                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1673
1674                     // Set time out
1675                     setTestJobTimeOut(newJob, scenario)
1676
1677                     // Debug runs take too long to run. So build job only.
1678                     if (lowerConfiguration == "debug") {
1679                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1680                     }
1681                     else {
1682                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1683                        {
1684                            // Up the timeout for arm64 checked testing only.
1685                            // Keep the longer timeout for gcstress.
1686                            Utilities.setJobTimeout(newJob, 240)
1687                        }
1688
1689                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=1"
1690                        // Test build and run are launched together.
1691                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1692                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1693                     }
1694
1695                     // Add archival.
1696                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1697                     break
1698                 default:
1699                     println("Unknown architecture: ${architecture}");
1700                     assert false
1701                     break
1702             }
1703             break
1704         // editor brace matching: }
1705         case 'Ubuntu':
1706         case 'Ubuntu16.04':
1707         case 'Ubuntu16.10':
1708         case 'Debian8.4':
1709         case 'OSX10.12':
1710         case 'FreeBSD':
1711         case 'CentOS7.1':
1712         case 'RHEL7.2':
1713         case 'OpenSUSE42.1':
1714         case 'Tizen':
1715         case 'Fedora24': // editor brace matching: {
1716             switch (architecture) {
1717                 case 'x64':
1718                 case 'x86':
1719                     if (architecture == 'x86' && os == 'Ubuntu') {
1720                         // build and PAL test
1721                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1722                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1723                         break;
1724                     }
1725
1726                     if (scenario == 'formatting') {
1727                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1728                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1729                         break
1730                     }
1731
1732                     if (scenario == 'illink') {
1733                         assert(os == 'Ubuntu')
1734                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1735                     }
1736
1737                     def standaloneGc = ''
1738                     if (scenario == 'standalone_gc') {
1739                         standaloneGc = 'buildstandalonegc'
1740                     }
1741
1742                     if (!enableCorefxTesting) {
1743                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1744                         // only on supported OS platforms.
1745                         if (os == 'FreeBSD')
1746                         {
1747                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1748                         }
1749                         else
1750                         {
1751                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1752                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1753                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1754                         }
1755                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1756
1757                         // Set time out
1758                         setTestJobTimeOut(newJob, scenario)
1759                         // Basic archiving of the build
1760                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1761                         // And pal tests
1762                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1763                     }
1764                     else {
1765                         // Corefx stress testing
1766                         assert os == 'Ubuntu'
1767                         assert architecture == 'x64'
1768                         assert lowerConfiguration == 'checked'
1769                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1770
1771                         // Build coreclr
1772                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1773
1774                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1775                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1776
1777                         // Build and text corefx
1778                         def workspaceRelativeFxRoot = "_/fx"
1779                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1780
1781                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1782
1783                         setTestJobTimeOut(newJob, scenario)
1784
1785                         // Archive and process (only) the test results
1786                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1787                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1788                     }
1789                     break
1790                 case 'arm64':
1791                     def standaloneGc = ''
1792                     if (scenario == 'standalone_gc') {
1793                         standaloneGc = 'buildstandalonegc'
1794                     }
1795
1796                     if (!enableCorefxTesting) {
1797                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8 ${standaloneGc}"
1798
1799                         // HACK -- Arm64 does not have corefx jobs yet.
1800                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1801                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1802                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1803                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1804
1805                         // Set time out
1806                         setTestJobTimeOut(newJob, scenario)
1807                         // Basic archiving of the build
1808                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1809                     }
1810                     break
1811                 case 'arm':
1812                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1813                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1814
1815                     // default values for Ubuntu
1816                     def arm_abi="arm"
1817                     def linuxCodeName="trusty"
1818                     if (os == 'Ubuntu16.04') {
1819                         linuxCodeName="xenial"
1820                     }
1821                     else if (os == 'Tizen') {
1822                         arm_abi="armel"
1823                         linuxCodeName="tizen"
1824                     }
1825
1826                     // Unzip the Windows test binaries first. Exit with 0
1827                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1828
1829                     // Unpack the corefx binaries
1830                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1831                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1832                     if (os != 'Tizen') {
1833                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1834                     }
1835                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1836                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1837                     if (os == 'Ubuntu') {
1838                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1839                     }
1840
1841                     // Call the ARM CI script to cross build and test using docker
1842                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1843                     --mode=docker \\
1844                     --${arm_abi} \\
1845                     --linuxCodeName=${linuxCodeName} \\
1846                     --buildConfig=${lowerConfiguration} \\
1847                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1848                     --coreFxBinDir=./bin/CoreFxBinDir \\
1849                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1850
1851                     // Basic archiving of the build, no pal tests
1852                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1853                     break
1854                 default:
1855                     println("Unknown architecture: ${architecture}");
1856                     assert false
1857                     break
1858             }
1859             break
1860         // editor brace matching: }
1861         default:
1862             println("Unknown os: ${os}");
1863             assert false
1864             break
1865     } // os
1866
1867     return buildCommands
1868 }
1869
1870 // Additional scenario which can alter behavior
1871
1872 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1873 combinedScenarios.each { scenario ->
1874     [true, false].each { isPR ->
1875         Constants.architectureList.each { architecture ->
1876             Constants.configurationList.each { configuration ->
1877                 Constants.osList.each { os ->
1878                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1879                     // and reset the os to Windows_NT
1880                     def isBuildOnly = false
1881                     if (os == 'Windows_NT_BuildOnly') {
1882                         isBuildOnly = true
1883                         os = 'Windows_NT'
1884                     }
1885
1886                     // Tizen is only supported for arm architecture
1887                     if (os == 'Tizen' && architecture != 'arm') {
1888                         return
1889                     }
1890
1891                     // Skip totally unimplemented (in CI) configurations.
1892                     switch (architecture) {
1893                         case 'arm64':
1894                             if (os == 'Ubuntu16.04') {
1895                                 os = 'Ubuntu'
1896                             }
1897
1898                             // Windows and Ubuntu only
1899                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1900                                 return
1901                             }
1902                             break
1903                         case 'arm':
1904                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1905                                 return
1906                             }
1907                             break
1908                         case 'x86':
1909                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1910                                 return
1911                             }
1912                             break
1913                         case 'x86lb':
1914                             if (os != 'Windows_NT') {
1915                                 return
1916                             }
1917                             break
1918                         case 'x64':
1919                             // Everything implemented
1920                             break
1921                         default:
1922                             println("Unknown architecture: ${architecture}")
1923                             assert false
1924                             break
1925                     }
1926
1927                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1928                     // with checked builds
1929                     def enableCorefxTesting = false
1930                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1931                         if (configuration != 'Checked') {
1932                             return
1933                         }
1934
1935                         enableCorefxTesting = isCorefxTesting(scenario)
1936
1937                         // Since these are just execution time differences,
1938                         // skip platforms that don't execute the tests here (Windows_NT only)
1939                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1940                         if (!isEnabledOS || isBuildOnly) {
1941                             return
1942                         }
1943
1944                         switch (architecture) {
1945                             case 'arm':
1946                                 if ((scenario != 'gcstress0x3') &&
1947                                     (scenario != 'gcstress0xc') &&
1948                                     (scenario != 'jitstress1') &&
1949                                     (scenario != 'jitstress2') &&
1950                                     (scenario != 'jitstressregs1') &&
1951                                     (scenario != 'jitstressregs2') &&
1952                                     (scenario != 'gcstress0xc_jitstress1') &&
1953                                     (scenario != 'gcstress0xc_jitstress2') &&
1954                                     (scenario != 'minopts') &&
1955                                     (scenario != 'tailcallstress') &&
1956                                     (scenario != 'zapdisable')) {
1957                                         return
1958                                     }
1959                                     break
1960                             case 'arm64':
1961                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1962                                     return
1963                                 }
1964                                 break
1965                             case 'x64':
1966                             case 'x86':
1967                                 // x86 ubuntu: default only
1968                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1969                                     return
1970                                 }
1971                                 // Windows: Everything implemented
1972                                 break
1973                             case 'x86lb':
1974                                 // No stress modes for legacy jit.
1975                                 // (There's no technical reason we couldn't allow these.)
1976                                 return
1977                             default:
1978                                 return
1979                         }
1980                     }
1981                     else {
1982                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
1983                         // and configuration is not Checked, bail out.
1984                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1985                             return;
1986                         }
1987
1988                         // Skip scenarios
1989                         switch (scenario) {
1990                             case 'pri1':
1991                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1992                                 // the default scenario build
1993                                 if (os != 'Windows_NT') {
1994                                     return
1995                                 }
1996                                 // Only x64 for now
1997                                 if (architecture != 'x64') {
1998                                     return
1999                                 }
2000                                 break
2001                             case 'ilrt':
2002                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2003                                 // the default scenario build
2004                                 if (os != 'Windows_NT') {
2005                                     return
2006                                 }
2007                                 // Only x64 for now
2008                                 if (architecture != 'x64') {
2009                                     return
2010                                 }
2011                                 // Release only
2012                                 if (configuration != 'Release') {
2013                                     return
2014                                 }
2015                                 break
2016                             case 'jitdiff':
2017                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2018                                     return
2019                                 }
2020                                 if (architecture != 'x64') {
2021                                     return
2022                                 }
2023                                 if (configuration != 'Checked') {
2024                                     return
2025                                 }
2026                                 break
2027                             case 'r2r':
2028                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2029                                 // the default scenario build
2030                                 if (os != 'Windows_NT') {
2031                                     return
2032                                 }
2033                                 if (architecture != 'x64') {
2034                                     return
2035                                 }
2036                                 break
2037                             case 'pri1r2r':
2038                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2039                                 // the default scenario build
2040                                 if (os != 'Windows_NT') {
2041                                     return
2042                                 }
2043                                 if (architecture != 'x64') {
2044                                     if ((architecture != 'arm64' && architecture != 'arm') || (configuration == 'Debug')) {
2045                                         return
2046                                     }
2047                                 }
2048                                 break
2049                             case 'gcstress15_pri1r2r':
2050                             case 'r2r_jitstress1':
2051                             case 'r2r_jitstress2':
2052                             case 'r2r_jitstressregs1':
2053                             case 'r2r_jitstressregs2':
2054                             case 'r2r_jitstressregs3':
2055                             case 'r2r_jitstressregs4':
2056                             case 'r2r_jitstressregs8':
2057                             case 'r2r_jitstressregs0x10':
2058                             case 'r2r_jitstressregs0x80':
2059                             case 'r2r_jitstressregs0x1000':
2060                             case 'r2r_jitminopts':
2061                             case 'r2r_jitforcerelocs':
2062                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
2063                                 // the default scenario build
2064                                 if (os != 'Windows_NT') {
2065                                     return
2066                                 }
2067                                 if (architecture != 'x64' && architecture != 'x86') {
2068                                     return
2069                                 }
2070                                 break
2071                             case 'longgc':
2072                             case 'gcsimulator':
2073                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2074                                     return
2075                                 }
2076                                 if (architecture != 'x64') {
2077                                     return
2078                                 }
2079                                 if (configuration != 'Release') {
2080                                     return
2081                                 }
2082                                 break
2083                             case 'gc_reliability_framework':
2084                             case 'standalone_gc':
2085                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2086                                     return
2087                                 }
2088
2089                                 if (architecture != 'x64') {
2090                                     return
2091                                 }
2092
2093                                 if (configuration != 'Release' && configuration != 'Checked') {
2094                                     return
2095                                 }
2096                                 break
2097                             // We need Windows x64 Release bits for the code coverage build
2098                             case 'coverage':
2099                                 if (os != 'Windows_NT') {
2100                                     return
2101                                 }
2102                                 if (architecture != 'x64') {
2103                                     return
2104                                 }
2105                                 if (configuration != 'Release') {
2106                                     return
2107                                 }
2108                                 break
2109                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2110                             case 'formatting':
2111                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2112                                     return
2113                                 }
2114                                 if (architecture != 'x64') {
2115                                     return
2116                                 }
2117                                 if (configuration != 'Checked') {
2118                                     return
2119                                 }
2120                                 if (isBuildOnly) {
2121                                     return
2122                                 }
2123                                 break
2124                             case 'illink':
2125                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2126                                     return
2127                                 }
2128                                 if (architecture != 'x64' && architecture != 'x86') {
2129                                     return
2130                                 }
2131                                 if (isBuildOnly) {
2132                                     return
2133                                 }
2134                                 break
2135                             case 'default':
2136                                 // Nothing skipped
2137                                 break
2138                             default:
2139                                 println("Unknown scenario: ${scenario}")
2140                                 assert false
2141                                 break
2142                         }
2143                     }
2144
2145                     // Calculate names
2146                     def lowerConfiguration = configuration.toLowerCase()
2147                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2148                     def folderName = getJobFolder(scenario)
2149
2150                     // Create the new job
2151                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2152
2153                     def machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2154                     machineAffinityOptions = architecture == 'arm' ? ['use_arm64_build_machine': false] : machineAffinityOptions
2155                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2156
2157                     // Add all the standard options
2158                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2159                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2160
2161                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
2162                     def osGroup = getOSGroup(os)
2163
2164                     newJob.with {
2165                         steps {
2166                             if (os == 'Windows_NT') {
2167                                 buildCommands.each { buildCommand ->
2168                                     batchFile(buildCommand)
2169                                 }
2170                             }
2171                             else {
2172                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2173                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2174                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2175                                     // Define the Windows Tests and Corefx build job names
2176                                     def WindowTestsName = projectFolder + '/' +
2177                                                           Utilities.getFullJobName(project,
2178                                                                                    getJobName(lowerConfiguration,
2179                                                                                               'x64' ,
2180                                                                                               'windows_nt',
2181                                                                                               'default',
2182                                                                                               true),
2183                                                                                    false)
2184                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2185                                                        Utilities.getFolderName(branch)
2186
2187                                     // Copy the Windows test binaries and the Corefx build binaries
2188                                     copyArtifacts(WindowTestsName) {
2189                                         includePatterns('bin/tests/tests.zip')
2190                                         buildSelector {
2191                                             latestSuccessful(true)
2192                                         }
2193                                     }
2194
2195                                     def arm_abi = 'arm'
2196                                     def corefx_os = 'linux'
2197                                     if (os == 'Tizen') {
2198                                         arm_abi = 'armel'
2199                                         corefx_os = 'tizen'
2200                                     }
2201
2202                                     // Let's use release CoreFX to test checked CoreCLR,
2203                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2204                                     def corefx_lowerConfiguration = lowerConfiguration
2205                                     if ( lowerConfiguration == 'checked' ) {
2206                                         corefx_lowerConfiguration='release'
2207                                     }
2208
2209                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2210                                         includePatterns('bin/build.tar.gz')
2211                                         buildSelector {
2212                                             latestSuccessful(true)
2213                                         }
2214                                     }
2215                                 }
2216
2217                                 buildCommands.each { buildCommand ->
2218                                     shell(buildCommand)
2219                                 }
2220                             }
2221                         }
2222                     } // newJob.with
2223
2224                 } // os
2225             } // configuration
2226         } // architecture
2227     } // isPR
2228 } // scenario
2229
2230
2231 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2232 combinedScenarios.each { scenario ->
2233     [true, false].each { isPR ->
2234         // Architectures.  x64 only at this point
2235         ['x64', 'arm64'].each { architecture ->
2236             // Put the OS's supported for coreclr cross testing here
2237             Constants.crossList.each { os ->
2238                 if (architecture == 'arm64') {
2239                     if (os != "Ubuntu") {
2240                         return
2241                     }
2242                 }
2243
2244                 Constants.configurationList.each { configuration ->
2245
2246                     if (architecture == 'arm64') {
2247                         if (scenario != 'default' && scenario != 'pri1r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2248                             return
2249                         }
2250                     }
2251
2252                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2253                         if (configuration != 'Checked') {
2254                             return
2255                         }
2256                         if (isCorefxTesting(scenario)) {
2257                             return
2258                         }
2259                         //Skip stress modes for these scenarios
2260                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2261                             return
2262                         }
2263                     }
2264                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2265                     // and configuration is not Checked, bail out.
2266                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2267                         return;
2268                     }
2269                     // For CentOS, we only want Checked/Release pri1 builds.
2270                     else if (os == 'CentOS7.1') {
2271                         if (scenario != 'pri1' &&
2272                             scenario != 'r2r' &&
2273                             scenario != 'pri1r2r' &&
2274                             scenario != 'gcstress15_pri1r2r' &&
2275                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2276                             return
2277                         }
2278                         if (configuration != 'Checked' && configuration != 'Release') {
2279                             return
2280                         }
2281                     }
2282                     // For RedHat and Debian, we only do Release pri1 builds.
2283                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2284                         if (scenario != 'pri1') {
2285                             return
2286                         }
2287                         if (configuration != 'Release') {
2288                             return
2289                         }
2290                     }
2291                     else {
2292                         // Skip scenarios
2293                         switch (scenario) {
2294                             case 'pri1':
2295                                 // Nothing skipped
2296                                 break
2297                             case 'ilrt':
2298                                 // Release only
2299                                 if (configuration != 'Release') {
2300                                     return
2301                                 }
2302                                 break
2303                             case 'jitdiff':
2304                                 if (configuration != 'Checked') {
2305                                     return;
2306                                 }
2307                             case 'r2r':
2308                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2309                                 if (configuration != 'Checked' && configuration != 'Release') {
2310                                     return
2311                                 }
2312                                 break
2313                             case 'pri1r2r':
2314                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2315                                 if (configuration != 'Checked' && configuration != 'Release') {
2316                                     return
2317                                 }
2318                                 break
2319                             case 'gcstress15_pri1r2r':
2320                             case 'r2r_jitstress1':
2321                             case 'r2r_jitstress2':
2322                             case 'r2r_jitstressregs1':
2323                             case 'r2r_jitstressregs2':
2324                             case 'r2r_jitstressregs3':
2325                             case 'r2r_jitstressregs4':
2326                             case 'r2r_jitstressregs8':
2327                             case 'r2r_jitstressregs0x10':
2328                             case 'r2r_jitstressregs0x80':
2329                             case 'r2r_jitstressregs0x1000':
2330                             case 'r2r_jitminopts':
2331                             case 'r2r_jitforcerelocs':
2332                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2333                                 if (configuration != 'Checked' && configuration != 'Release') {
2334                                     return
2335                                 }
2336                                 break
2337                             case 'longgc':
2338                             case 'gcsimulator':
2339                                 // Long GC tests take a long time on non-Release builds
2340                                 if (configuration != 'Release') {
2341                                     return
2342                                 }
2343                                 break
2344                             case 'gc_reliability_framework':
2345                             case 'standalone_gc':
2346                                 if (configuration != 'Release' && configuration != 'Checked') {
2347                                     return
2348                                 }
2349                                 break
2350                             case 'coverage':
2351                                 //We only want Ubuntu Release for coverage
2352                                 if (os != 'Ubuntu') {
2353                                     return
2354                                 }
2355                                 if (configuration != 'Release') {
2356                                     return
2357                                 }
2358                             case 'formatting':
2359                                 return
2360                             case 'illink':
2361                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2362                                     return
2363                                 }
2364                                 break
2365                             case 'default':
2366                                 // Nothing skipped
2367                                 break
2368                             default:
2369                                 println("Unknown scenario: ${scenario}")
2370                                 assert false
2371                                 break
2372                         }
2373                     }
2374
2375                     def lowerConfiguration = configuration.toLowerCase()
2376                     def osGroup = getOSGroup(os)
2377                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2378
2379                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2380                     def inputScenario = 'default'
2381                     if (scenario == 'coverage') {
2382                         inputScenario = 'coverage'
2383                     }
2384                     def inputCoreCLRBuildName = projectFolder + '/' +
2385                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2386                     // If this is a stress scenario, there isn't any difference in the build job
2387                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2388                     // from the default scenario
2389                     def testBuildScenario = scenario
2390                     if (scenarioNeedsPri1Build(scenario)) {
2391                         testBuildScenario = 'pri1'
2392                     }
2393                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2394                         testBuildScenario = 'default'
2395                     }
2396                     def inputWindowTestsBuildName = ''
2397                     def inputWindowsTestBuildArch = architecture
2398                     if (architecture == "arm64") {
2399                         // Use the x64 test build for arm64 unix
2400                         inputWindowsTestBuildArch = "x64"
2401                     }
2402                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2403                         inputWindowTestsBuildName = projectFolder + '/' +
2404                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', 'default', true), isPR)
2405                     }
2406                     else {
2407                         inputWindowTestsBuildName = projectFolder + '/' +
2408                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2409                     }
2410                     // Enable Server GC for Ubuntu PR builds
2411                     def serverGCString = ''
2412
2413                     // Whether or not this test run should be run sequentially instead
2414                     // of in parallel. Only used for long GC tests.
2415                     def sequentialString = ''
2416
2417                     // Whether or not this test run should run a specific playlist.
2418                     // Only used for long GC tests.
2419
2420                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2421                     // for running long GC and GCSimulator tests, respectively. We don't use them
2422                     // here because using a playlist file produces much more readable output on the CI machines
2423                     // and reduces running time.
2424                     def playlistString = ''
2425
2426                     if (os == 'Ubuntu' && isPR){
2427                         serverGCString = '--useServerGC'
2428                     }
2429
2430                     // pass --crossgen to runtest.sh for crossgen builds
2431                     def crossgenStr = ''
2432                     def runcrossgentestsStr = ''
2433                     def runjitstressStr = ''
2434                     def runjitstressregsStr = ''
2435                     def runjitmioptsStr = ''
2436                     def runjitforcerelocsStr = ''
2437                     def runjitdisasmStr = ''
2438                     def runilasmroundtripStr = ''
2439                     def gcstressStr = ''
2440                     def illinkStr = ''
2441                     def layoutOnlyStr =''
2442
2443                     if (scenario == 'r2r' ||
2444                         scenario == 'pri1r2r' ||
2445                         scenario == 'gcstress15_pri1r2r' ||
2446                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2447                             crossgenStr = '--crossgen'
2448                             runcrossgentestsStr = '--runcrossgentests'
2449
2450                             if (scenario == 'r2r_jitstress1'){
2451                                 runjitstressStr = '--jitstress=1'
2452                             }
2453                             else if (scenario == 'r2r_jitstress2') {
2454                                 runjitstressStr = '--jitstress=2'
2455                             }
2456                             else if (scenario == 'r2r_jitstressregs1'){
2457                                 runjitstressregsStr = '--jitstressregs=1'
2458                             }
2459                             else if (scenario == 'r2r_jitstressregs2') {
2460                                 runjitstressregsStr = '--jitstressregs=2'
2461                             }
2462                             else if (scenario == 'r2r_jitstressregs3') {
2463                                 runjitstressregsStr = '--jitstressregs=3'
2464                             }
2465                             else if (scenario == 'r2r_jitstressregs4') {
2466                                 runjitstressregsStr = '--jitstressregs=4'
2467                             }
2468                             else if (scenario == 'r2r_jitstressregs8') {
2469                                 runjitstressregsStr = '--jitstressregs=8'
2470                             }
2471                             else if (scenario == 'r2r_jitstressregs0x10') {
2472                                 runjitstressregsStr = '--jitstressregs=0x10'
2473                             }
2474                             else if (scenario == 'r2r_jitstressregs0x80') {
2475                                 runjitstressregsStr = '--jitstressregs=0x80'
2476                             }
2477                             else if (scenario == 'r2r_jitstressregs0x1000') {
2478                                 runjitstressregsStr = '--jitstressregs=0x1000'
2479                             }
2480                             else if (scenario == 'r2r_jitminopts') {
2481                                 runjitmioptsStr = '--jitminopts'
2482                             }
2483                             else if (scenario == 'r2r_jitforcerelocs') {
2484                                 runjitforcerelocsStr = '--jitforcerelocs'
2485                             }
2486                     }
2487                     if  (scenario == 'gcstress15_pri1r2r')
2488                     {
2489                         gcstressStr = '--gcstresslevel=0xF'
2490                     }
2491
2492                     if (scenario == 'jitdiff')
2493                     {
2494                         runjitdisasmStr = '--jitdisasm --crossgen'
2495                     }
2496
2497                     if (scenario == 'illink')
2498                     {
2499                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2500                     }
2501
2502                     if (isLongGc(scenario)) {
2503                         // Long GC tests behave very poorly when they are not
2504                         // the only test running (many of them allocate until OOM).
2505                         sequentialString = '--sequential'
2506
2507                         // The Long GC playlist contains all of the tests that are
2508                         // going to be run. The GCSimulator playlist contains all of
2509                         // the GC simulator tests.
2510                         if (scenario == 'longgc') {
2511                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2512                         }
2513                         else if (scenario == 'gcsimulator') {
2514                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2515                         }
2516                     }
2517
2518                     if (isGcReliabilityFramework(scenario)) {
2519                         layoutOnlyStr = '--build-overlay-only'
2520                     }
2521
2522                     def folder = getJobFolder(scenario)
2523                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2524                         // Add parameters for the inputs
2525
2526                         parameters {
2527                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2528                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2529                         }
2530
2531                         steps {
2532                             // Set up the copies
2533
2534                             // Coreclr build containing the tests and mscorlib
2535
2536                             copyArtifacts(inputWindowTestsBuildName) {
2537                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2538                                 buildSelector {
2539                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2540                                 }
2541                             }
2542
2543                             if (scenario == 'coverage') {
2544
2545                                 // Move coreclr to clr directory
2546                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2547
2548                                 // Build coreclr
2549                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2550
2551                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2552                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2553                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2554                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2555                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2556                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2557                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2558                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2559
2560                                 // Run PAL tests
2561                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2562
2563                                 // Remove obj files for PAL tests so they're not included in coverage results
2564                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2565
2566                                 // Unzip the tests first.  Exit with 0
2567                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2568
2569                                 // Get corefx
2570                                 shell("git clone https://github.com/dotnet/corefx fx")
2571
2572                                 // Build Linux corefx
2573                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2574                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2575
2576                                 def testEnvOpt = ""
2577                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2578                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2579                                 shell("${createScriptCmds}")
2580                                 testEnvOpt = "--test-env=" + scriptFileName
2581
2582                                 // Run corefx tests
2583                                 shell("""./fx/run-test.sh \\
2584                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2585                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2586                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2587                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2588                 --configurationGroup Release""")
2589
2590
2591                                 // Run coreclr tests w/ workstation GC
2592                                 shell("""./clr/tests/runtest.sh \\
2593                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2594                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2595                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2596                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2597                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2598                 --crossgen --runcrossgentests""")
2599
2600                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2601                                 shell("""./clr/tests/runtest.sh \\
2602                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2603                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2604                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2605                 --useServerGC ${testEnvOpt}""")
2606
2607                                  // Run long-running coreclr GC tests & produce coverage reports
2608                                 shell("""./clr/tests/runtest.sh \\
2609                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2610                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2611                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2612                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2613                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2614                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2615                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2616
2617                             }
2618                             else {
2619
2620                                 // Coreclr build we are trying to test
2621
2622                                 copyArtifacts(inputCoreCLRBuildName) {
2623                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2624                                     buildSelector {
2625                                         buildNumber('${CORECLR_BUILD}')
2626                                     }
2627                                 }
2628
2629                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2630
2631                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2632                                 def osJobName
2633                                 if (os == 'Ubuntu') {
2634                                     osJobName = 'ubuntu14.04'
2635                                 }
2636                                 else {
2637                                     osJobName = os.toLowerCase()
2638                                 }
2639                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2640                                     includePatterns('bin/build.tar.gz')
2641                                     buildSelector {
2642                                         latestSuccessful(true)
2643                                     }
2644                                 }
2645
2646                                 shell ("mkdir ./bin/CoreFxBinDir")
2647                                 // Unpack the corefx binaries
2648                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2649
2650                                 // HACK -- Arm64 does not have corefx jobs yet.
2651                                 // Clone corefx and build the native packages overwriting the x64 packages.
2652                                 if (architecture == 'arm64') {
2653                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2654                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2655                                 }
2656
2657                                 // Unzip the tests first.  Exit with 0
2658                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2659
2660                                 // Execute the tests
2661                                 // If we are running a stress mode, we'll set those variables first
2662                                 def testEnvOpt = ""
2663                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2664                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2665                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2666                                     shell("${createScriptCmds}")
2667                                     testEnvOpt = "--test-env=" + scriptFileName
2668                                 }
2669
2670                                 if (isGCStressRelatedTesting(scenario)) {
2671                                     shell('./init-tools.sh')
2672                                 }
2673
2674                                 shell("""./tests/runtest.sh \\
2675                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2676                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2677                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2678                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2679                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2680                 --limitedDumpGeneration \\
2681                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
2682                 ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
2683                 ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
2684
2685                                 if (isGcReliabilityFramework(scenario)) {
2686                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2687                                     if (serverGCString != '') {
2688                                         shell("export COMPlus_gcServer=1")
2689                                     }
2690
2691                                     shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2692                                 }
2693                             }
2694                         }
2695                     }
2696
2697                     if (scenario == 'coverage') {
2698                         // Publish coverage reports
2699                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2700                         // TODO: Add once external email sending is available again
2701                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2702                     }
2703
2704                     if (scenario == 'jitdiff') {
2705                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2706                     }
2707
2708                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2709                     if (os in ['Ubuntu']) {
2710                         SummaryBuilder summaries = new SummaryBuilder()
2711                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2712                         summaries.emit(newJob)
2713                     }
2714
2715                     setMachineAffinity(newJob, os, architecture)
2716                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2717                     // Set timeouts to 240.
2718                     setTestJobTimeOut(newJob, scenario)
2719
2720                     if (architecture == 'arm64') {
2721                         Utilities.setJobTimeout(newJob, 480)
2722                     }
2723
2724                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2725
2726                     // Create a build flow to join together the build and tests required to run this
2727                     // test.
2728                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2729                     // Linux CoreCLR test
2730                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2731                     def fullTestJobName = projectFolder + '/' + newJob.name
2732                     // Add a reference to the input jobs for report purposes
2733                     JobReport.Report.addReference(inputCoreCLRBuildName)
2734                     JobReport.Report.addReference(inputWindowTestsBuildName)
2735                     JobReport.Report.addReference(fullTestJobName)
2736                     def newFlowJob;
2737
2738                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2739                     // so that coverage data can be preserved.
2740                     if (scenario == 'coverage') {
2741                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2742                         buildFlow("""
2743 // Build the input Windows job
2744 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2745
2746 // And then build the test build
2747 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2748 """)
2749                         }
2750                     // Normal jobs copy a Windows build & a non-Windows build
2751                     } else {
2752                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2753                         buildFlow("""
2754 // Build the input jobs in parallel
2755 parallel (
2756     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2757     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2758 )
2759
2760 // And then build the test build
2761 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2762                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2763 """)
2764                         }
2765                     }
2766
2767                     setMachineAffinity(newFlowJob, os, architecture)
2768                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2769                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2770                 } // configuration
2771             } // os
2772         } // architecture
2773     } // isPR
2774 } // scenario
2775
2776 JobReport.Report.generateJobReport(out)
2777
2778 // Make the call to generate the help job
2779 Utilities.createHelperJob(this, project, branch,
2780     "Welcome to the ${project} Repository",  // This is prepended to the help message
2781     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2782
2783 Utilities.addCROSSCheck(this, project, branch)