netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'FreeBSD':'FreeBSD',
  29         'CentOS7.1': 'Linux',
  30         'OpenSUSE42.1': 'Linux',
  31         'Tizen': 'Linux']
  32     def osGroup = osGroupMap.get(os, null)
  33     assert osGroup != null : "Could not find os group for ${os}"
  34     return osGroupMap[os]
  35 }
  36
  37 // We use this class (vs variables) so that the static functions can access data here.
  38 class Constants {
  39
  40     // Innerloop build OS's
  41     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  42     // test execution in the build flow runs.  It generates the exact same build
  43     // as Windows_NT but without the tests.
  44     def static osList = [
  45                'Ubuntu',
  46                'Debian8.4',
  47                'OSX10.12',
  48                'Windows_NT',
  49                'Windows_NT_BuildOnly',
  50                'FreeBSD',
  51                'CentOS7.1',
  52                'OpenSUSE42.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  67                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  68                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  69                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  70                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  71                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  72                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  73                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  74                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  75                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  76                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  77                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  78                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  79                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  80                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  81                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  82                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  83                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  84                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  85                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  86                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  87                'corefx_baseline'                : [ : ], // corefx baseline
  88                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  89                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  90                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  91                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  92                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  93                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  94                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  95                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  96                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  97                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  98                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  99                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 100                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 101                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 103                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 104                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 105                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 107                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 109                ]
 110
 111     // This is a set of r2r jit stress scenarios
 112     def static r2rJitStressScenarios = [
 113                'r2r_jitstress1',
 114                'r2r_jitstress2',
 115                'r2r_jitstressregs1',
 116                'r2r_jitstressregs2',
 117                'r2r_jitstressregs3',
 118                'r2r_jitstressregs4',
 119                'r2r_jitstressregs8',
 120                'r2r_jitstressregs0x10',
 121                'r2r_jitstressregs0x80',
 122                'r2r_jitstressregs0x1000',
 123                'r2r_jitminopts',
 124                'r2r_jitforcerelocs']
 125
 126     // This is the basic set of scenarios
 127     def static basicScenarios = [
 128                'default',
 129                'pri1',
 130                'ilrt',
 131                'r2r',
 132                'pri1r2r',
 133                'gcstress15_pri1r2r',
 134                'longgc',
 135                'coverage',
 136                'formatting',
 137                'gcsimulator',
 138                'jitdiff',
 139                'standalone_gc',
 140                'gc_reliability_framework',
 141                'illink'] + r2rJitStressScenarios
 142
 143     def static configurationList = ['Debug', 'Checked', 'Release']
 144
 145     // This is the set of architectures
 146     def static architectureList = ['arm', 'arm64', 'x64', 'x86', 'x86lb']
 147 }
 148
 149 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 150     if (architecture == 'arm64' && os == 'Windows_NT') {
 151         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 152     } else if (architecture == 'arm64' && os != 'Windows_NT' && options == null) {
 153         Utilities.setMachineAffinity(job, os, 'arm64-small-page-size');
 154     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['large_pages'] == true) {
 155         Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size');
 156     } else if (architecture == 'arm64' && os != 'Windows_NT' && options['is_build_only'] == true) {
 157         Utilities.setMachineAffinity(job, os, 'arm64-cross-latest');
 158     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 159         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 160     } else if ((architecture == 'arm') && (os == 'Windows_NT') && options['use_arm64_build_machine'] == true) {
 161         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 162     }else {
 163         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 164     }
 165 }
 166
 167 def static isJITStressJob(def scenario) {
 168     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 169            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 170 }
 171
 172 def static isGCStressRelatedTesting(def scenario) {
 173     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 174     // Detect it and make it a GCStress related.
 175     if (scenario == 'gcstress15_pri1r2r')
 176     {
 177         return true;
 178     }
 179
 180     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 181     def scenarioName = scenario.toLowerCase()
 182     def isGCStressTesting = false
 183     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 184         if (k in gcStressTestEnvVars) {
 185             isGCStressTesting = true;
 186         }
 187     }
 188     return isGCStressTesting
 189 }
 190
 191 def static isCorefxTesting(def scenario) {
 192     def corefx_prefix = 'corefx_'
 193     if (scenario.length() < corefx_prefix.length()) {
 194         return false
 195     }
 196     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 197 }
 198
 199 def static isR2R(def scenario) {
 200     return (scenario == 'r2r' || scenario == 'pri1r2r')
 201 }
 202
 203 def static isCoverage(def scenario) {
 204     return (scenario == 'coverage')
 205 }
 206
 207 def static isLongGc(def scenario) {
 208     return (scenario == 'longgc' || scenario == 'gcsimulator')
 209 }
 210
 211 def static isJitDiff(def scenario) {
 212     return (scenario == 'jitdiff')
 213 }
 214
 215 def static isGcReliabilityFramework(def scenario) {
 216     return (scenario == 'gc_reliability_framework')
 217 }
 218
 219 def static scenarioNeedsPri1Build(def scenario) {
 220     return (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage' || isGcReliabilityFramework(scenario))
 221 }
 222
 223 def static setTestJobTimeOut(newJob, scenario) {
 224     if (isGCStressRelatedTesting(scenario)) {
 225         Utilities.setJobTimeout(newJob, 4320)
 226     }
 227     else if (isCorefxTesting(scenario)) {
 228         Utilities.setJobTimeout(newJob, 360)
 229     }
 230     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 231         Utilities.setJobTimeout(newJob, 240)
 232     }
 233     else if (isR2R(scenario)) {
 234         Utilities.setJobTimeout(newJob, 240)
 235     }
 236     else if (isCoverage(scenario)) {
 237         Utilities.setJobTimeout(newJob, 1440)
 238     }
 239     else if (isLongGc(scenario)) {
 240         Utilities.setJobTimeout(newJob, 1440)
 241     }
 242     else if (isJitDiff(scenario)) {
 243         Utilities.setJobTimeout(newJob, 240)
 244     }
 245     else if (isGcReliabilityFramework(scenario)) {
 246         Utilities.setJobTimeout(newJob, 1440)
 247     }
 248     // Non-test jobs use the default timeout value.
 249 }
 250
 251 def static getJobFolder(def scenario) {
 252     if (isJITStressJob(scenario)) {
 253         return 'jitstress'
 254     }
 255     if (scenario == 'illink') {
 256         return 'illink'
 257     }
 258     return ''
 259 }
 260
 261 def static getStressModeDisplayName(def scenario) {
 262     def displayStr = ''
 263     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 264         def prefixLength = 'COMPlus_'.length()
 265         if (k.length() >= prefixLength) {
 266             def modeName = k.substring(prefixLength, k.length())
 267             displayStr += ' ' + modeName + '=' + v
 268         }
 269     }
 270     return displayStr
 271 }
 272
 273 def static getR2RStressModeDisplayName(def scenario) {
 274     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 275     // "r2r_" prefix.
 276     def displayStr = scenario
 277     def prefixLength = 'r2r_'.length()
 278     if (displayStr.length() >= prefixLength) {
 279         displayStr = displayStr.substring(prefixLength, displayStr.length())
 280     }
 281     return displayStr
 282 }
 283
 284 // Generates the string for creating a file that sets environment variables
 285 // that makes it possible to run stress modes.  Writes the script to the file
 286 // specified by the stepScriptLocation parameter.
 287 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 288     def stepScript = ''
 289     if (os == 'Windows_NT') {
 290         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 291         stepScript += "del ${stepScriptLocation}\r\n"
 292
 293         // Timeout in ms, default is 10 minutes. For stress
 294         // modes up this to 30 minutes
 295         def timeout = 1800000
 296
 297         // Set the Timeout
 298         stepScript += "set __TestTimeout=${timeout}\r\n"
 299         stepScript += "echo. > ${stepScriptLocation}\r\n"
 300         stressModeVars.each{ k, v ->
 301             // Write out what we are writing to the script file
 302             stepScript += "echo Setting ${k}=${v}\r\n"
 303             // Write out the set itself to the script file`
 304             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 305         }
 306     }
 307     else {
 308         stepScript += "echo Setting variables for ${stressModeName}\n"
 309         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 310         stressModeVars.each{ k, v ->
 311             // Write out what we are writing to the script file
 312             stepScript += "echo Setting ${k}=${v}\n"
 313             // Write out the set itself to the script file`
 314             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 315         }
 316         stepScript += "chmod +x ${stepScriptLocation}\n"
 317     }
 318     return stepScript
 319 }
 320
 321 // Calculates the name of the build job based on some typical parameters.
 322 //
 323 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 324     // If the architecture is x64, do not add that info into the build name.
 325     // Need to change around some systems and other builds to pick up the right builds
 326     // to do that.
 327
 328     def suffix = scenario != 'default' ? "_${scenario}" : '';
 329     if (isBuildOnly) {
 330         suffix += '_bld'
 331     }
 332     def baseName = ''
 333     switch (architecture) {
 334         case 'x64':
 335             if (scenario == 'default') {
 336                 // For now we leave x64 off of the name for compatibility with other jobs
 337                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 338             }
 339             else if (scenario == 'formatting') {
 340                 // we don't care about the configuration for the formatting job. It runs all configs
 341                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 342             }
 343             else {
 344                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 345             }
 346             break
 347         case 'arm64':
 348             if (os.toLowerCase() == "windows_nt") {
 349                 // These are cross builds
 350                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 351             }
 352             else {
 353                 // Defaults to a small page size set of machines.
 354                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 355             }
 356             break
 357         case 'arm':
 358             // These are cross builds
 359             if (os == 'Tizen') {
 360                 // ABI: softfp
 361                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 362             }
 363             else {
 364                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 365             }
 366             break
 367         case 'x86':
 368         case 'x86lb':
 369             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 370             break
 371         default:
 372             println("Unknown architecture: ${architecture}");
 373             assert false
 374             break
 375     }
 376
 377     return baseName + suffix
 378 }
 379
 380 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 381     // Check scenario.
 382     switch (scenario) {
 383         case 'default':
 384             switch (architecture) {
 385                 case 'x64':
 386                 case 'x86':
 387                 case 'x86lb':
 388                     if (architecture == 'x86' && os == 'Ubuntu') {
 389                         Utilities.addPeriodicTrigger(job, '@daily')
 390                     }
 391                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 392                         Utilities.addGithubPushTrigger(job)
 393                     }
 394                     break
 395                 case 'arm':
 396                     Utilities.addGithubPushTrigger(job)
 397                     break
 398                 case 'arm64':
 399                     Utilities.addGithubPushTrigger(job)
 400                     break
 401                 default:
 402                     println("Unknown architecture: ${architecture}");
 403                     assert false
 404                     break
 405             }
 406             break
 407         case 'pri1':
 408             // Pri one gets a push trigger, and only for release
 409             if (architecture == 'x64') {
 410                 if (configuration == 'Release') {
 411                     // We expect release jobs to be Windows, or in the cross list
 412                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 413                     if (!os in bidailyCrossList) {
 414                         if (isFlowJob || os == 'Windows_NT') {
 415                             Utilities.addGithubPushTrigger(job)
 416                         }
 417                     }
 418                     else {
 419                         if (isFlowJob) {
 420                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 421                         }
 422                     }
 423                 }
 424             }
 425             break
 426         case 'r2r':
 427             //r2r jobs that aren't pri1 can only be triggered by phrase
 428             break
 429         case 'pri1r2r':
 430             assert !(os in bidailyCrossList)
 431             //pri1 r2r gets a push trigger for checked/release
 432             if (configuration == 'Checked' || configuration == 'Release') {
 433                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 434                 if (architecture == 'x64' && os != 'OSX10.12') {
 435                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 436                     if (isFlowJob || os == 'Windows_NT') {
 437                         Utilities.addGithubPushTrigger(job)
 438                     }
 439                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 440                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 441                     if (isFlowJob) {
 442                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 443                     }
 444                 }
 445                 // For x86, only add per-commit jobs for Windows
 446                 else if (architecture == 'x86' || architecture == 'x86lb') {
 447                     if (os == 'Windows_NT') {
 448                         Utilities.addGithubPushTrigger(job)
 449                     }
 450                 }
 451                 // arm64 pri1r2r jobs should only run every 12 hours.
 452                 else if (architecture == 'arm64') {
 453                     if (os == 'Windows_NT') {
 454                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 455                         // TODO: Add once external email sending is available again
 456                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 457                     }
 458                 }
 459             }
 460             break
 461         case 'r2r_jitstress1':
 462         case 'r2r_jitstress2':
 463         case 'r2r_jitstressregs1':
 464         case 'r2r_jitstressregs2':
 465         case 'r2r_jitstressregs3':
 466         case 'r2r_jitstressregs4':
 467         case 'r2r_jitstressregs8':
 468         case 'r2r_jitstressregs0x10':
 469         case 'r2r_jitstressregs0x80':
 470         case 'r2r_jitstressregs0x1000':
 471         case 'r2r_jitminopts':
 472         case 'r2r_jitforcerelocs':
 473         case 'gcstress15_pri1r2r':
 474             assert !(os in bidailyCrossList)
 475
 476             // GCStress=C is currently not supported on OS X
 477             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 478                 break
 479             }
 480
 481             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 482             if (configuration == 'Checked' || configuration == 'Release') {
 483                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 484                 if (architecture == 'x64') {
 485                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 486                     if (isFlowJob || os == 'Windows_NT') {
 487                         // Add a weekly periodic trigger
 488                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 489                     }
 490                 }
 491                 // For x86, only add per-commit jobs for Windows
 492                 else if (architecture == 'x86') {
 493                     if (os == 'Windows_NT') {
 494                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 495                     }
 496                 }
 497             }
 498             break
 499         case 'longgc':
 500             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 501             assert configuration == 'Release'
 502             assert architecture == 'x64'
 503             Utilities.addPeriodicTrigger(job, '@daily')
 504             // TODO: Add once external email sending is available again
 505             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 506             break
 507         case 'gcsimulator':
 508             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 509             assert configuration == 'Release'
 510             assert architecture == 'x64'
 511             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 512             // TODO: Add once external email sending is available again
 513             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 514             break
 515         case 'standalone_gc':
 516             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 517             assert (configuration == 'Release' || configuration == 'Checked')
 518             // TODO: Add once external email sending is available again
 519             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 520             Utilities.addPeriodicTrigger(job, '@weekly')
 521             break
 522         case 'gc_reliability_framework':
 523             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 524             assert (configuration == 'Release' || configuration == 'Checked')
 525             // Only triggered by phrase.
 526             break
 527         case 'ilrt':
 528             assert !(os in bidailyCrossList)
 529             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 530             if (architecture == 'x64' && configuration == 'Release') {
 531                 // We don't expect to see a job generated except in these scenarios
 532                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 533                 if (isFlowJob || os == 'Windows_NT') {
 534                     Utilities.addPeriodicTrigger(job, '@daily')
 535                 }
 536             }
 537             break
 538         case 'jitdiff':
 539             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 540             assert configuration == 'Checked'
 541             assert (architecture == 'x64' || architecture == 'x86')
 542             Utilities.addGithubPushTrigger(job)
 543             break
 544         case 'coverage':
 545             assert (os == 'Ubuntu' || os == 'Windows_NT')
 546             assert configuration == 'Release'
 547             assert architecture == 'x64'
 548             Utilities.addPeriodicTrigger(job, '@weekly')
 549             break
 550         case 'formatting':
 551             assert (os == 'Windows_NT' || os == "Ubuntu")
 552             assert architecture == 'x64'
 553             Utilities.addGithubPushTrigger(job)
 554             break
 555         case 'jitstressregs1':
 556         case 'jitstressregs2':
 557         case 'jitstressregs3':
 558         case 'jitstressregs4':
 559         case 'jitstressregs8':
 560         case 'jitstressregs0x10':
 561         case 'jitstressregs0x80':
 562         case 'jitstressregs0x1000':
 563         case 'minopts':
 564         case 'forcerelocs':
 565         case 'jitstress1':
 566         case 'jitstress2':
 567         case 'jitstress2_jitstressregs1':
 568         case 'jitstress2_jitstressregs2':
 569         case 'jitstress2_jitstressregs3':
 570         case 'jitstress2_jitstressregs4':
 571         case 'jitstress2_jitstressregs8':
 572         case 'jitstress2_jitstressregs0x10':
 573         case 'jitstress2_jitstressregs0x80':
 574         case 'jitstress2_jitstressregs0x1000':
 575         case 'tailcallstress':
 576         case 'jitsse2only':
 577         case 'corefx_baseline':
 578         case 'corefx_minopts':
 579         case 'corefx_jitstress1':
 580         case 'corefx_jitstress2':
 581         case 'corefx_jitstressregs1':
 582         case 'corefx_jitstressregs2':
 583         case 'corefx_jitstressregs3':
 584         case 'corefx_jitstressregs4':
 585         case 'corefx_jitstressregs8':
 586         case 'corefx_jitstressregs0x10':
 587         case 'corefx_jitstressregs0x80':
 588         case 'corefx_jitstressregs0x1000':
 589         case 'zapdisable':
 590             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 591             assert (os == 'Windows_NT') || (os in Constants.crossList)
 592             Utilities.addPeriodicTrigger(job, '@daily')
 593         }
 594         break
 595         case 'heapverify1':
 596         case 'gcstress0x3':
 597             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 598                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 599                 if (architecture == 'arm64') {
 600                     if (os == 'Windows_NT') {
 601                         Utilities.addPeriodicTrigger(job, '@daily')
 602                     }
 603                     // TODO: Add once external email sending is available again
 604                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 605                 }
 606                 else {
 607                     Utilities.addPeriodicTrigger(job, '@weekly')
 608                 }
 609             }
 610             break
 611         case 'gcstress0xc':
 612         case 'gcstress0xc_zapdisable':
 613         case 'gcstress0xc_zapdisable_jitstress2':
 614         case 'gcstress0xc_zapdisable_heapverify1':
 615         case 'gcstress0xc_jitstress1':
 616         case 'gcstress0xc_jitstress2':
 617         case 'gcstress0xc_minopts_heapverify1':
 618             // GCStress=C is currently not supported on OS X
 619             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 620                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 621                 if (architecture == 'arm64') {
 622                     // TODO: Enable a periodic trigger after tests are updated.
 623                     // Utilities.addPeriodicTrigger(job, '@daily')
 624                     // TODO: Add once external email sending is available again
 625                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 626                 }
 627                 else {
 628                     Utilities.addPeriodicTrigger(job, '@weekly')
 629                 }
 630             }
 631             break
 632
 633         case 'illink':
 634             // Testing on other operating systems TBD
 635             assert (os == 'Windows_NT' || os == 'Ubuntu')
 636             if (architecture == 'x64' || architecture == 'x86') {
 637                 if (configuration == 'Checked') {
 638                     Utilities.addPeriodicTrigger(job, '@daily')
 639                 }
 640             }
 641             break
 642
 643         default:
 644             println("Unknown scenario: ${scenario}");
 645             assert false
 646             break
 647     }
 648     return
 649 }
 650
 651 // **************************
 652 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 653 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 654 // builds will do a couple extra steps.
 655 // **************************
 656
 657 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 658 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 659 // it's just used for internal builds
 660 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 661 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 662     if (isWindowsBuildOnlyJob) {
 663         return
 664     }
 665
 666     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 667     // Non pull request builds.
 668     if (!isPR) {
 669         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 670         return
 671     }
 672
 673      def arm64Users = [
 674         'adiaaida',
 675         'AndyAyersMS',
 676         'briansull',
 677         'BruceForstall',
 678         'CarolEidt',
 679         'cmckinsey',
 680         'erozenfeld',
 681         'jashook',
 682         'JosephTremoulet',
 683         'pgavlin',
 684         'russellhadley',
 685         'RussKeldorph',
 686         'sandreenko',
 687         'sdmaclea',
 688         'sivarv',
 689         'swaroop-sridhar',
 690         'gkhanna79',
 691         'jkotas',
 692         'markwilkie',
 693         'rahku',
 694         'ramarag',
 695         'tzwlai',
 696         'weshaggard'
 697     ]
 698
 699     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 700     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 701     def osGroup = getOSGroup(os)
 702     switch (architecture) {
 703         case 'x64': // editor brace matching: {
 704             if (scenario == 'coverage') {
 705                 assert configuration == 'Release'
 706                 if (os == 'Ubuntu') {
 707                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 708                 }
 709                 break
 710             }
 711
 712             if (scenario == 'formatting') {
 713                 assert configuration == 'Checked'
 714                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 715                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 716                 }
 717                 break
 718             }
 719
 720             switch (os) {
 721                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 722                 case 'Debian8.4':
 723                 case 'RHEL7.2':
 724                     if (scenario == 'default') {
 725                         assert !isFlowJob
 726                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 727                     }
 728                     else if (scenario == 'pri1' && isFlowJob) {
 729                         assert (configuration == 'Release')
 730                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 731                     }
 732                     break
 733                 case 'Ubuntu16.04':
 734                     assert !isFlowJob
 735                     assert scenario == 'default'
 736                     // Distinguish with the other architectures (arm and x86)
 737                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 738                     break
 739                 case 'Fedora24':
 740                 case 'Ubuntu16.10':
 741                 case 'OpenSUSE42.1':
 742                     assert !isFlowJob
 743                     assert scenario == 'default'
 744                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 745                     break
 746                 case 'Ubuntu':
 747                     if (scenario == 'illink') {
 748                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 749                         break
 750                     }
 751                     // fall through
 752                 case 'OSX10.12':
 753                     // Triggers on the non-flow jobs aren't necessary here
 754                     // Corefx testing uses non-flow jobs.
 755                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 756                         break
 757                     }
 758                     switch (scenario) {
 759                         case 'default':
 760                             // Ubuntu uses checked for default PR tests
 761                             if (configuration == 'Checked') {
 762                                 // Default trigger
 763                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 764                             }
 765                             break
 766                         case 'pri1':
 767                             if (configuration == 'Release') {
 768                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 769                             }
 770                             break
 771                         case 'jitdiff':
 772                             if (configuration == 'Checked') {
 773                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 774                             }
 775                             break
 776                         case 'ilrt':
 777                             if (configuration == 'Release') {
 778                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 779                             }
 780                             break
 781                         case 'r2r':
 782                             if (configuration == 'Release' || configuration == 'Checked') {
 783                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 784                             }
 785                             break
 786                         case 'pri1r2r':
 787                             if (configuration == 'Release' || configuration == 'Checked') {
 788                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 789                             }
 790                             break
 791                         case 'gcstress15_pri1r2r':
 792                             if (configuration == 'Release' || configuration == 'Checked') {
 793                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 794                             }
 795                             break
 796                         case 'r2r_jitstress1':
 797                         case 'r2r_jitstress2':
 798                         case 'r2r_jitstressregs1':
 799                         case 'r2r_jitstressregs2':
 800                         case 'r2r_jitstressregs3':
 801                         case 'r2r_jitstressregs4':
 802                         case 'r2r_jitstressregs8':
 803                         case 'r2r_jitstressregs0x10':
 804                         case 'r2r_jitstressregs0x80':
 805                         case 'r2r_jitstressregs0x1000':
 806                         case 'r2r_jitminopts':
 807                         case 'r2r_jitforcerelocs':
 808                             if (configuration == 'Release' || configuration == 'Checked') {
 809                                 def displayStr = getR2RStressModeDisplayName(scenario)
 810                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 811                             }
 812                             break
 813                         case 'longgc':
 814                             if (configuration == 'Release') {
 815                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 816                             }
 817                             break
 818                         case 'gcsimulator':
 819                             if (configuration == 'Release') {
 820                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 821                             }
 822                             break
 823                         case 'standalone_gc':
 824                             if (configuration == 'Release' || configuration == 'Checked') {
 825                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 826                             }
 827                             break
 828                         case 'gc_reliability_framework':
 829                             if (configuration == 'Release' || configuration == 'Checked') {
 830                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 831                             }
 832                             break
 833                         case 'minopts':
 834                         case 'forcerelocs':
 835                         case 'jitstress1':
 836                         case 'jitstress2':
 837                         case 'jitstressregs1':
 838                         case 'jitstressregs2':
 839                         case 'jitstressregs3':
 840                         case 'jitstressregs4':
 841                         case 'jitstressregs8':
 842                         case 'jitstressregs0x10':
 843                         case 'jitstressregs0x80':
 844                         case 'jitstressregs0x1000':
 845                         case 'jitstress2_jitstressregs1':
 846                         case 'jitstress2_jitstressregs2':
 847                         case 'jitstress2_jitstressregs3':
 848                         case 'jitstress2_jitstressregs4':
 849                         case 'jitstress2_jitstressregs8':
 850                         case 'jitstress2_jitstressregs0x10':
 851                         case 'jitstress2_jitstressregs0x80':
 852                         case 'jitstress2_jitstressregs0x1000':
 853                         case 'tailcallstress':
 854                         case 'jitsse2only':
 855                         case 'gcstress0x3':
 856                         case 'gcstress0xc':
 857                         case 'zapdisable':
 858                         case 'heapverify1':
 859                         case 'gcstress0xc_zapdisable':
 860                         case 'gcstress0xc_zapdisable_jitstress2':
 861                         case 'gcstress0xc_zapdisable_heapverify1':
 862                         case 'gcstress0xc_jitstress1':
 863                         case 'gcstress0xc_jitstress2':
 864                         case 'gcstress0xc_minopts_heapverify1':
 865                             def displayStr = getStressModeDisplayName(scenario)
 866                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 867                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 868                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 869                             break
 870                         case 'corefx_baseline':
 871                         case 'corefx_minopts':
 872                         case 'corefx_jitstress1':
 873                         case 'corefx_jitstress2':
 874                         case 'corefx_jitstressregs1':
 875                         case 'corefx_jitstressregs2':
 876                         case 'corefx_jitstressregs3':
 877                         case 'corefx_jitstressregs4':
 878                         case 'corefx_jitstressregs8':
 879                         case 'corefx_jitstressregs0x10':
 880                         case 'corefx_jitstressregs0x80':
 881                         case 'corefx_jitstressregs0x1000':
 882                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 883                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 884                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 885                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 886                             break
 887                         default:
 888                             println("Unknown scenario: ${scenario}");
 889                             assert false
 890                             break
 891                     }
 892                     break
 893                 case 'CentOS7.1':
 894                     switch (scenario) {
 895                         case 'pri1':
 896                             if (configuration == 'Release') {
 897                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 898                             }
 899                             break
 900                         case 'r2r':
 901                             if (configuration == 'Checked' || configuration == 'Release') {
 902                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 903                             }
 904                             break
 905                         case 'pri1r2r':
 906                             if (configuration == 'Checked' || configuration == 'Release') {
 907                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 908                             }
 909                             break
 910                         case 'gcstress15_pri1r2r':
 911                             if (configuration == 'Release' || configuration == 'Checked') {
 912                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 913                             }
 914                             break
 915                         case 'r2r_jitstress1':
 916                         case 'r2r_jitstress2':
 917                         case 'r2r_jitstressregs1':
 918                         case 'r2r_jitstressregs2':
 919                         case 'r2r_jitstressregs3':
 920                         case 'r2r_jitstressregs4':
 921                         case 'r2r_jitstressregs8':
 922                         case 'r2r_jitstressregs0x10':
 923                         case 'r2r_jitstressregs0x80':
 924                         case 'r2r_jitstressregs0x1000':
 925                         case 'r2r_jitminopts':
 926                         case 'r2r_jitforcerelocs':
 927                             if (configuration == 'Release' || configuration == 'Checked') {
 928                                 def displayStr = getR2RStressModeDisplayName(scenario)
 929                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 930                             }
 931                             break
 932                         default:
 933                             break
 934                     }
 935                 case 'Windows_NT':
 936                     switch (scenario) {
 937                         case 'default':
 938                             // Default trigger
 939                             if (configuration == 'Debug') {
 940                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 941                             }
 942                             break
 943                         case 'pri1':
 944                             // Default trigger
 945                             if (configuration == 'Release') {
 946                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 947                             }
 948                             break
 949                         case 'jitdiff':
 950                             if (configuration == 'Checked') {
 951                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 952                             }
 953                             break
 954                         case 'ilrt':
 955                             if (configuration == 'Release') {
 956                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 957                             }
 958                             break
 959                         case 'r2r':
 960                             if (configuration == 'Checked' || configuration == 'Release') {
 961                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 962                             }
 963                             break
 964                         case 'pri1r2r':
 965                             if (configuration == 'Checked' || configuration == 'Release') {
 966                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 967                             }
 968                             break
 969                         case 'gcstress15_pri1r2r':
 970                             if (configuration == 'Release' || configuration == 'Checked') {
 971                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 972                             }
 973                             break
 974                         case 'r2r_jitstress1':
 975                         case 'r2r_jitstress2':
 976                         case 'r2r_jitstressregs1':
 977                         case 'r2r_jitstressregs2':
 978                         case 'r2r_jitstressregs3':
 979                         case 'r2r_jitstressregs4':
 980                         case 'r2r_jitstressregs8':
 981                         case 'r2r_jitstressregs0x10':
 982                         case 'r2r_jitstressregs0x80':
 983                         case 'r2r_jitstressregs0x1000':
 984                         case 'r2r_jitminopts':
 985                         case 'r2r_jitforcerelocs':
 986                             if (configuration == 'Release' || configuration == 'Checked') {
 987                                 def displayStr = getR2RStressModeDisplayName(scenario)
 988                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 989                             }
 990                             break
 991                         case 'longgc':
 992                             if (configuration == 'Release') {
 993                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 994                             }
 995                             break
 996                         case 'gcsimulator':
 997                             if (configuration == 'Release') {
 998                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 999                             }
1000                             break
1001                         case 'standalone_gc':
1002                             if (configuration == 'Release' || configuration == 'Checked') {
1003                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1004                             }
1005                             break
1006                         case 'gc_reliability_framework':
1007                             if (configuration == 'Release' || configuration == 'Checked') {
1008                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1009                             }
1010                             break
1011                         case 'minopts':
1012                         case 'forcerelocs':
1013                         case 'jitstress1':
1014                         case 'jitstress2':
1015                         case 'jitstressregs1':
1016                         case 'jitstressregs2':
1017                         case 'jitstressregs3':
1018                         case 'jitstressregs4':
1019                         case 'jitstressregs8':
1020                         case 'jitstressregs0x10':
1021                         case 'jitstressregs0x80':
1022                         case 'jitstressregs0x1000':
1023                         case 'jitstress2_jitstressregs1':
1024                         case 'jitstress2_jitstressregs2':
1025                         case 'jitstress2_jitstressregs3':
1026                         case 'jitstress2_jitstressregs4':
1027                         case 'jitstress2_jitstressregs8':
1028                         case 'jitstress2_jitstressregs0x10':
1029                         case 'jitstress2_jitstressregs0x80':
1030                         case 'jitstress2_jitstressregs0x1000':
1031                         case 'tailcallstress':
1032                         case 'jitsse2only':
1033                         case 'gcstress0x3':
1034                         case 'gcstress0xc':
1035                         case 'zapdisable':
1036                         case 'heapverify1':
1037                         case 'gcstress0xc_zapdisable':
1038                         case 'gcstress0xc_zapdisable_jitstress2':
1039                         case 'gcstress0xc_zapdisable_heapverify1':
1040                         case 'gcstress0xc_jitstress1':
1041                         case 'gcstress0xc_jitstress2':
1042                         case 'gcstress0xc_minopts_heapverify1':
1043                             def displayStr = getStressModeDisplayName(scenario)
1044                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1045                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1046                                "(?i).*test\\W+${os}\\W+${scenario}.*")
1047                             break
1048                         case 'corefx_baseline':
1049                         case 'corefx_minopts':
1050                         case 'corefx_jitstress1':
1051                         case 'corefx_jitstress2':
1052                         case 'corefx_jitstressregs1':
1053                         case 'corefx_jitstressregs2':
1054                         case 'corefx_jitstressregs3':
1055                         case 'corefx_jitstressregs4':
1056                         case 'corefx_jitstressregs8':
1057                         case 'corefx_jitstressregs0x10':
1058                         case 'corefx_jitstressregs0x80':
1059                         case 'corefx_jitstressregs0x1000':
1060                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1061                             assert (os == 'Windows_NT') || (os in Constants.crossList)
1062                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1063                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1064                             break
1065                         case 'illink':
1066                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1067                                     break
1068                         default:
1069                             println("Unknown scenario: ${scenario}");
1070                             assert false
1071                             break
1072                     }
1073                     break
1074                 case 'FreeBSD':
1075                     assert scenario == 'default'
1076                     if (configuration == 'Checked') {
1077                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1078                     }
1079                     break
1080                 default:
1081                     println("Unknown os: ${os}");
1082                     assert false
1083                     break
1084             }
1085             break
1086         // editor brace matching: }
1087         case 'arm': // editor brace matching: {
1088             switch (os) {
1089                 case 'Ubuntu':
1090                 case 'Ubuntu16.04':
1091                     assert scenario == 'default'
1092                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1093                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1094                     }
1095                     else {
1096                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1097                     }
1098                     break;
1099                 case 'Tizen':
1100                     architecture='armel'
1101                     // Removing the regex will cause this to run on each PR.
1102                     if (configuration == 'Release' || configuration == 'Debug') {
1103                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1104                     }
1105                     else {
1106                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1107                     }
1108                     break;
1109                 case 'Windows_NT':
1110                     // Set up a private trigger
1111                     def contextString = "${os} ${architecture} Cross ${configuration}"
1112                     if (scenario != 'default')
1113                         contextString += " ${scenario}"
1114                     contextString += " Build"
1115                     // Debug builds only.
1116                     if (configuration != 'Debug') {
1117                         contextString += " and Test"
1118                     }
1119                     switch (scenario) {
1120                         case 'default':
1121                             // For now only run Debug and Release build jobs on PR Trigger. Note this is not a private trigger.
1122                             if (configuration == 'Debug' || configuration == 'Release')
1123                             {
1124                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1125                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1126                             }
1127                             else
1128                             {
1129                                 // Checked jobs will run on private trigger and run tests.
1130                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1131                             }
1132                             break
1133                         case 'pri1r2r':
1134                         case 'gcstress0x3':
1135                         case 'gcstress0xc':
1136                             // Stress jobs will will run on private trigger and run tests.
1137                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1138                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1139                             break
1140                     }
1141                     break
1142                 default:
1143                     println("NYI os: ${os}");
1144                     assert false
1145                     break
1146             }
1147             break
1148         // editor brace matching: }
1149         case 'arm64': // editor brace matching: {
1150             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1151
1152             // Set up a private trigger
1153             def contextString = "${os} ${architecture} Cross ${configuration}"
1154             if (scenario != 'default')
1155                 contextString += " ${scenario}"
1156             contextString += " Build"
1157             // Debug builds only.
1158             if (configuration != 'Debug') {
1159                contextString += " and Test"
1160             }
1161
1162             switch (os) {
1163                 case 'Ubuntu':
1164                 case 'Ubuntu16.04':
1165                     switch (scenario) {
1166                         case 'pri1':
1167                             if (configuration == 'Release') {
1168                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1169                             }
1170                             break
1171                         case 'r2r':
1172                             if (configuration == 'Checked' || configuration == 'Release') {
1173                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1174                             }
1175                             break
1176                         case 'pri1r2r':
1177                             if (configuration == 'Checked' || configuration == 'Release') {
1178                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1179                             }
1180                             break
1181                         case 'gcstress15_pri1r2r':
1182                             if (configuration == 'Release' || configuration == 'Checked') {
1183                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1184                             }
1185                             break
1186                         case 'r2r_jitstress1':
1187                         case 'r2r_jitstress2':
1188                         case 'r2r_jitstressregs1':
1189                         case 'r2r_jitstressregs2':
1190                         case 'r2r_jitstressregs3':
1191                         case 'r2r_jitstressregs4':
1192                         case 'r2r_jitstressregs8':
1193                         case 'r2r_jitstressregs0x10':
1194                         case 'r2r_jitstressregs0x80':
1195                         case 'r2r_jitstressregs0x1000':
1196                         case 'r2r_jitminopts':
1197                         case 'r2r_jitforcerelocs':
1198                             if (configuration == 'Release' || configuration == 'Checked') {
1199                                 def displayStr = getR2RStressModeDisplayName(scenario)
1200                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1201                             }
1202                             break
1203                         default:
1204                             break
1205                     }
1206                 case 'Windows_NT':
1207                     switch (scenario) {
1208                         case 'default':
1209                             if (isFlowJob == true) {
1210                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration}", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1211                             }
1212
1213                             // For now only run Debug jobs on PR Trigger.
1214                             else if (configuration != 'Debug') {
1215                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1216                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1217                             }
1218                             else {
1219                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1220                                 // are at higher risk of ARM64-breaking changes.
1221                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1222                             }
1223                             break
1224                         case 'pri1r2r':
1225                         case 'gcstress0x3':
1226                         case 'gcstress0xc':
1227                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1228                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1229                             break
1230                     }
1231                     break
1232                 default:
1233                     println("NYI os: ${os}");
1234                     assert false
1235                     break
1236             }
1237             break
1238         // editor brace matching: }
1239         case 'x86': // editor brace matching: {
1240             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1241             if (os == 'Ubuntu') {
1242                 // on-demand only for ubuntu x86
1243                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1244                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1245                 break
1246             }
1247             switch (scenario) {
1248                 case 'default':
1249                     if (configuration == 'Checked') {
1250                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1251                     }
1252                     else if (configuration == 'Release') {
1253                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1254                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1255                     }
1256                     break
1257                 case 'pri1':
1258                     if (configuration == 'Release') {
1259                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1260                     }
1261                     break
1262                 case 'ilrt':
1263                     if (configuration == 'Release') {
1264                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1265                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1266                     }
1267                     break
1268                 case 'r2r':
1269                     if (configuration == 'Checked' || configuration == 'Release') {
1270                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1271                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1272                     }
1273                     break
1274                 case 'pri1r2r':
1275                     if (configuration == 'Checked' || configuration == 'Release') {
1276                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1277                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1278                     }
1279                     break
1280                 case 'gcstress15_pri1r2r':
1281                     if (configuration == 'Release' || configuration == 'Checked') {
1282                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1283                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1284                     }
1285                     break
1286                 case 'r2r_jitstress1':
1287                 case 'r2r_jitstress2':
1288                 case 'r2r_jitstressregs1':
1289                 case 'r2r_jitstressregs2':
1290                 case 'r2r_jitstressregs3':
1291                 case 'r2r_jitstressregs4':
1292                 case 'r2r_jitstressregs8':
1293                 case 'r2r_jitstressregs0x10':
1294                 case 'r2r_jitstressregs0x80':
1295                 case 'r2r_jitstressregs0x1000':
1296                 case 'r2r_jitminopts':
1297                 case 'r2r_jitforcerelocs':
1298                     if (configuration == 'Release' || configuration == 'Checked') {
1299                         def displayStr = getR2RStressModeDisplayName(scenario)
1300                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1301                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1302                     }
1303                     break
1304                 case 'longgc':
1305                     if (configuration == 'Release') {
1306                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1307                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1308                     }
1309                     break
1310                 case 'gcsimulator':
1311                     if (configuration == 'Release') {
1312                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1313                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1314                     }
1315                     break
1316                 case 'standalone_gc':
1317                     if (configuration == 'Release' || configuration == 'Checked') {
1318                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1319                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1320                     }
1321                     break
1322                 case 'minopts':
1323                 case 'forcerelocs':
1324                 case 'jitstress1':
1325                 case 'jitstress2':
1326                 case 'jitstressregs1':
1327                 case 'jitstressregs2':
1328                 case 'jitstressregs3':
1329                 case 'jitstressregs4':
1330                 case 'jitstressregs8':
1331                 case 'jitstressregs0x10':
1332                 case 'jitstressregs0x80':
1333                 case 'jitstressregs0x1000':
1334                 case 'jitstress2_jitstressregs1':
1335                 case 'jitstress2_jitstressregs2':
1336                 case 'jitstress2_jitstressregs3':
1337                 case 'jitstress2_jitstressregs4':
1338                 case 'jitstress2_jitstressregs8':
1339                 case 'jitstress2_jitstressregs0x10':
1340                 case 'jitstress2_jitstressregs0x80':
1341                 case 'jitstress2_jitstressregs0x1000':
1342                 case 'tailcallstress':
1343                 case 'jitsse2only':
1344                 case 'gcstress0x3':
1345                 case 'gcstress0xc':
1346                 case 'zapdisable':
1347                 case 'heapverify1':
1348                 case 'gcstress0xc_zapdisable':
1349                 case 'gcstress0xc_zapdisable_jitstress2':
1350                 case 'gcstress0xc_zapdisable_heapverify1':
1351                 case 'gcstress0xc_jitstress1':
1352                 case 'gcstress0xc_jitstress2':
1353                 case 'gcstress0xc_minopts_heapverify1':
1354                     def displayStr = getStressModeDisplayName(scenario)
1355                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1356                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1357                     break
1358                 case 'corefx_baseline':
1359                 case 'corefx_minopts':
1360                 case 'corefx_jitstress1':
1361                 case 'corefx_jitstress2':
1362                 case 'corefx_jitstressregs1':
1363                 case 'corefx_jitstressregs2':
1364                 case 'corefx_jitstressregs3':
1365                 case 'corefx_jitstressregs4':
1366                 case 'corefx_jitstressregs8':
1367                 case 'corefx_jitstressregs0x10':
1368                 case 'corefx_jitstressregs0x80':
1369                 case 'corefx_jitstressregs0x1000':
1370                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1371                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1372                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1373                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1374                     break
1375                 case 'illink':
1376                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1377                     break
1378                 default:
1379                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1380                     assert false
1381                     break
1382             }
1383             break
1384          // editor brace matching: }
1385         case 'x86lb': // editor brace matching: {
1386             assert (os == 'Windows_NT')
1387             assert (scenario == 'default' || Constants.r2rJitStressScenarios.indexOf(scenario) !=1)
1388
1389             def arch = 'x86'
1390             def jit = 'legacy_backend'
1391             switch (scenario) {
1392                 case 'default':
1393                     if (configuration == 'Checked') {
1394                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1395                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}.*")
1396                     }
1397                     break
1398                 default:
1399                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1400                     assert false
1401                     break
1402             }
1403             break
1404         // editor brace matching: }
1405         default:
1406             println("Unknown architecture: ${architecture}");
1407             assert false
1408             break
1409     }
1410 }
1411
1412 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1413     def buildCommands = [];
1414     def osGroup = getOSGroup(os)
1415     def lowerConfiguration = configuration.toLowerCase()
1416
1417     // Calculate the build steps, archival, and xunit results
1418     switch (os) {
1419         case 'Windows_NT': // editor brace matching: {
1420             switch (architecture) {
1421                 case 'x64':
1422                 case 'x86':
1423                 case 'x86lb':
1424                     def arch = architecture
1425                     def buildOpts = ''
1426                     if (architecture == 'x86lb') {
1427                         arch = 'x86'
1428                     }
1429
1430                     if (scenario == 'illink') {
1431                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1432                     }
1433
1434                     // If it is a release build for windows, ensure PGO is used, else fail the build
1435                     if ((lowerConfiguration == 'release') && (scenario in Constants.basicScenarios) && (architecture != 'x86lb')) {
1436                         buildOpts += ' enforcepgo'
1437                     }
1438
1439                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1440                             scenario == 'default' ||
1441                             scenario == 'r2r' ||
1442                             scenario == 'jitdiff' ||
1443                             scenario == 'ilrt' ||
1444                             scenario == 'illink' ||
1445                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1446                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1447                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1448                     }
1449
1450                     // For Pri 1 tests, we must shorten the output test binary path names.
1451                     // if __TestIntermediateDir is already set, build-test.cmd will
1452                     // output test binaries to that directory. If it is not set, the
1453                     // binaries are sent to a default directory whose name is about
1454                     // 35 characters long.
1455
1456                     else if (scenarioNeedsPri1Build(scenario)) {
1457                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1458                     }
1459                     else if (isLongGc(scenario)) {
1460                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1461                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1462                     }
1463                     else if (scenario == 'standalone_gc') {
1464                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1465                     }
1466                     else if (scenario == 'formatting') {
1467                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1468                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1469                         break
1470                     }
1471                     else {
1472                         println("Unknown scenario: ${scenario}")
1473                         assert false
1474                     }
1475
1476                     // If we are running a stress mode, we should write out the set of key
1477                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1478
1479                     if (!isBuildOnly) {
1480                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1481                         def crossgenStr = ''
1482                         def runcrossgentestsStr = ''
1483                         def runjitstressStr = ''
1484                         def runjitstressregsStr = ''
1485                         def runjitmioptsStr = ''
1486                         def runjitforcerelocsStr = ''
1487                         def runjitdisasmStr = ''
1488                         def runilasmroundtripStr = ''
1489                         def gcstressStr = ''
1490                         def runtestArguments = ''
1491                         def gcTestArguments = ''
1492                         def illinkArguments = ''
1493
1494                         if (scenario == 'r2r' ||
1495                             scenario == 'pri1r2r' ||
1496                             scenario == 'gcstress15_pri1r2r' ||
1497                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1498                                 crossgenStr = 'crossgen'
1499                                 runcrossgentestsStr = 'runcrossgentests'
1500
1501                                 if (scenario == 'r2r_jitstress1'){
1502                                     runjitstressStr = 'jitstress 1'
1503                                 }
1504                                 else if (scenario == 'r2r_jitstress2') {
1505                                     runjitstressStr = 'jitstress 2'
1506                                 }
1507                                 else if (scenario == 'r2r_jitstressregs1'){
1508                                     runjitstressregsStr = 'jitstressregs 1'
1509                                 }
1510                                 else if (scenario == 'r2r_jitstressregs2') {
1511                                     runjitstressregsStr = 'jitstressregs 2'
1512                                 }
1513                                 else if (scenario == 'r2r_jitstressregs3') {
1514                                     runjitstressregsStr = 'jitstressregs 3'
1515                                 }
1516                                 else if (scenario == 'r2r_jitstressregs4') {
1517                                     runjitstressregsStr = 'jitstressregs 4'
1518                                 }
1519                                 else if (scenario == 'r2r_jitstressregs8') {
1520                                     runjitstressregsStr = 'jitstressregs 8'
1521                                 }
1522                                 else if (scenario == 'r2r_jitstressregs0x10') {
1523                                     runjitstressregsStr = 'jitstressregs 0x10'
1524                                 }
1525                                 else if (scenario == 'r2r_jitstressregs0x80') {
1526                                     runjitstressregsStr = 'jitstressregs 0x80'
1527                                 }
1528                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1529                                     runjitstressregsStr = 'jitstressregs 0x1000'
1530                                 }
1531                                 else if (scenario == 'r2r_jitminopts') {
1532                                     runjitmioptsStr = 'jitminopts'
1533                                 }
1534                                 else if (scenario == 'r2r_jitforcerelocs') {
1535                                     runjitforcerelocsStr = 'jitforcerelocs'
1536                                 }
1537                         }
1538                         if (scenario == 'gcstress15_pri1r2r')
1539                         {
1540                             gcstressStr = 'gcstresslevel 0xF'
1541                         }
1542
1543                         if (scenario == 'jitdiff')
1544                         {
1545                             runjitdisasmStr = 'jitdisasm crossgen'
1546                         }
1547
1548                         if (scenario == 'ilrt')
1549                         {
1550                             runilasmroundtripStr = 'ilasmroundtrip'
1551                         }
1552
1553                         if (isLongGc(scenario)) {
1554                             gcTestArguments = "${scenario} sequential"
1555                         }
1556
1557                         if (scenario == 'illink')
1558                         {
1559                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1560                         }
1561
1562                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1563
1564                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1565                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1566                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1567
1568                             if (enableCorefxTesting) {
1569                                 def workspaceRelativeFxRoot = "_/fx"
1570                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1571
1572                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1573
1574                                 setTestJobTimeOut(newJob, scenario)
1575
1576                                 // Archive and process (only) the test results
1577                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1578                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1579                             }
1580                             else {
1581                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1582                             }
1583                         }
1584                         else if (isGcReliabilityFramework(scenario)) {
1585                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1586                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1587                         }
1588                         else if (architecture == 'x64' || architecture == 'x86') {
1589                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1590                         }
1591                         else if (architecture == 'x86lb') {
1592                             buildCommands += "tests\\runtest.cmd ${runtestArguments} TestEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd"
1593                         }
1594                     }
1595
1596                     if (!enableCorefxTesting) {
1597                         // Run the rest of the build
1598                         // Build the mscorlib for the other OS's
1599                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1600                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1601                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1602
1603                         if (arch == "x64") {
1604                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1605                         }
1606
1607                         // Zip up the tests directory so that we don't use so much space/time copying
1608                         // 10s of thousands of files around.
1609                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1610
1611                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1612                             // For windows, pull full test results and test drops for x86/x64.
1613                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1614                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1615                         }
1616
1617                         if (scenario == 'jitdiff') {
1618                             // retrive jit-dasm output for base commit, and run jit-diff
1619                             if (!isBuildOnly) {
1620                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1621                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1622                             }
1623                         }
1624
1625                         if (!isBuildOnly) {
1626                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1627                             setTestJobTimeOut(newJob, scenario)
1628                         }
1629                     }
1630                     break
1631                 case 'arm':
1632                     def validArmWindowsScenarios = [ "default",
1633                                                      "pri1r2r",
1634                                                      "zapdisable",
1635                                                      "minopts",
1636                                                      "tailcallstress",
1637                                                      "jitstress1",
1638                                                      "jitstress2",
1639                                                      "gcstress0x3",
1640                                                      "gcstress0xc",
1641                                                      "jitstressregs1",
1642                                                      "jitstressregs2",
1643                                                      "gcstress0xc_jitstress1",
1644                                                      "gcstress0xc_jitstress2"]
1645
1646                     assert validArmWindowsScenarios.contains(scenario)
1647
1648                     // Set time out
1649                     setTestJobTimeOut(newJob, scenario)
1650
1651                     if ( lowerConfiguration == "debug" ) {
1652                         // For Debug builds, we will do a P1 test build
1653                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1654                     }
1655                     else if (lowerConfiguration == "checked") {
1656
1657                         if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1658                         {
1659                            // Up the timeout for arm checked testing only.
1660                            // Keep the longer timeout for gcstress.
1661                            Utilities.setJobTimeout(newJob, 240)
1662                         }
1663
1664                         def machineAffinityOptions = ['use_arm64_build_machine' : true]
1665                         setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1666                         // For checked runs we will also run testing.
1667                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1668                         buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1669                     }
1670                     else if (lowerConfiguration == "release") {
1671                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1672                     }
1673                     // Add archival.
1674                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1675                     break
1676                 case 'arm64':
1677                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1678
1679                     // Set time out
1680                     setTestJobTimeOut(newJob, scenario)
1681
1682                     // Debug runs take too long to run. So build job only.
1683                     if (lowerConfiguration == "debug") {
1684                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1685                     }
1686                     else {
1687                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1688                        {
1689                            // Up the timeout for arm64 checked testing only.
1690                            // Keep the longer timeout for gcstress.
1691                            Utilities.setJobTimeout(newJob, 240)
1692                        }
1693
1694                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=1"
1695                        // Test build and run are launched together.
1696                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1697                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1698                     }
1699
1700                     // Add archival.
1701                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1702                     break
1703                 default:
1704                     println("Unknown architecture: ${architecture}");
1705                     assert false
1706                     break
1707             }
1708             break
1709         // editor brace matching: }
1710         case 'Ubuntu':
1711         case 'Ubuntu16.04':
1712         case 'Ubuntu16.10':
1713         case 'Debian8.4':
1714         case 'OSX10.12':
1715         case 'FreeBSD':
1716         case 'CentOS7.1':
1717         case 'RHEL7.2':
1718         case 'OpenSUSE42.1':
1719         case 'Tizen':
1720         case 'Fedora24': // editor brace matching: {
1721             switch (architecture) {
1722                 case 'x64':
1723                 case 'x86':
1724                     if (architecture == 'x86' && os == 'Ubuntu') {
1725                         // build and PAL test
1726                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1727                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1728                         break;
1729                     }
1730
1731                     if (scenario == 'formatting') {
1732                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1733                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1734                         break
1735                     }
1736
1737                     if (scenario == 'illink') {
1738                         assert(os == 'Ubuntu')
1739                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1740                     }
1741
1742                     def standaloneGc = ''
1743                     if (scenario == 'standalone_gc') {
1744                         standaloneGc = 'buildstandalonegc'
1745                     }
1746
1747                     if (!enableCorefxTesting) {
1748                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1749                         // only on supported OS platforms.
1750                         if (os == 'FreeBSD')
1751                         {
1752                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1753                         }
1754                         else
1755                         {
1756                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1757                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1758                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1759                         }
1760                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1761
1762                         // Set time out
1763                         setTestJobTimeOut(newJob, scenario)
1764                         // Basic archiving of the build
1765                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1766                         // And pal tests
1767                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1768                     }
1769                     else {
1770                         // Corefx stress testing
1771                         assert os == 'Ubuntu'
1772                         assert architecture == 'x64'
1773                         assert lowerConfiguration == 'checked'
1774                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1775
1776                         // Build coreclr
1777                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1778
1779                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1780                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1781
1782                         // Build and text corefx
1783                         def workspaceRelativeFxRoot = "_/fx"
1784                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1785
1786                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1787
1788                         setTestJobTimeOut(newJob, scenario)
1789
1790                         // Archive and process (only) the test results
1791                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1792                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1793                     }
1794                     break
1795                 case 'arm64':
1796                     def standaloneGc = ''
1797                     if (scenario == 'standalone_gc') {
1798                         standaloneGc = 'buildstandalonegc'
1799                     }
1800
1801                     if (!enableCorefxTesting) {
1802                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8 ${standaloneGc}"
1803
1804                         // HACK -- Arm64 does not have corefx jobs yet.
1805                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1806                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1807                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1808                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1809
1810                         // Set time out
1811                         setTestJobTimeOut(newJob, scenario)
1812                         // Basic archiving of the build
1813                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1814                     }
1815                     break
1816                 case 'arm':
1817                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1818                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1819
1820                     // default values for Ubuntu
1821                     def arm_abi="arm"
1822                     def linuxCodeName="trusty"
1823                     if (os == 'Ubuntu16.04') {
1824                         linuxCodeName="xenial"
1825                     }
1826                     else if (os == 'Tizen') {
1827                         arm_abi="armel"
1828                         linuxCodeName="tizen"
1829                     }
1830
1831                     // Unzip the Windows test binaries first. Exit with 0
1832                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1833
1834                     // Unpack the corefx binaries
1835                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1836                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1837                     if (os != 'Tizen') {
1838                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1839                     }
1840                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1841                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1842                     if (os == 'Ubuntu') {
1843                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1844                     }
1845
1846                     // Call the ARM CI script to cross build and test using docker
1847                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1848                     --mode=docker \\
1849                     --${arm_abi} \\
1850                     --linuxCodeName=${linuxCodeName} \\
1851                     --buildConfig=${lowerConfiguration} \\
1852                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1853                     --coreFxBinDir=./bin/CoreFxBinDir \\
1854                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1855
1856                     // Basic archiving of the build, no pal tests
1857                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1858                     break
1859                 default:
1860                     println("Unknown architecture: ${architecture}");
1861                     assert false
1862                     break
1863             }
1864             break
1865         // editor brace matching: }
1866         default:
1867             println("Unknown os: ${os}");
1868             assert false
1869             break
1870     } // os
1871
1872     return buildCommands
1873 }
1874
1875 // Additional scenario which can alter behavior
1876
1877 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1878 combinedScenarios.each { scenario ->
1879     [true, false].each { isPR ->
1880         Constants.architectureList.each { architecture ->
1881             Constants.configurationList.each { configuration ->
1882                 Constants.osList.each { os ->
1883                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1884                     // and reset the os to Windows_NT
1885                     def isBuildOnly = false
1886                     if (os == 'Windows_NT_BuildOnly') {
1887                         isBuildOnly = true
1888                         os = 'Windows_NT'
1889                     }
1890
1891                     // Tizen is only supported for arm architecture
1892                     if (os == 'Tizen' && architecture != 'arm') {
1893                         return
1894                     }
1895
1896                     // Skip totally unimplemented (in CI) configurations.
1897                     switch (architecture) {
1898                         case 'arm64':
1899                             if (os == 'Ubuntu16.04') {
1900                                 os = 'Ubuntu'
1901                             }
1902
1903                             // Windows and Ubuntu only
1904                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1905                                 return
1906                             }
1907                             break
1908                         case 'arm':
1909                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1910                                 return
1911                             }
1912                             break
1913                         case 'x86':
1914                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1915                                 return
1916                             }
1917                             break
1918                         case 'x86lb':
1919                             if (os != 'Windows_NT') {
1920                                 return
1921                             }
1922                             break
1923                         case 'x64':
1924                             // Everything implemented
1925                             break
1926                         default:
1927                             println("Unknown architecture: ${architecture}")
1928                             assert false
1929                             break
1930                     }
1931
1932                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1933                     // with checked builds
1934                     def enableCorefxTesting = false
1935                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1936                         if (configuration != 'Checked') {
1937                             return
1938                         }
1939
1940                         enableCorefxTesting = isCorefxTesting(scenario)
1941
1942                         // Since these are just execution time differences,
1943                         // skip platforms that don't execute the tests here (Windows_NT only)
1944                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1945                         if (!isEnabledOS || isBuildOnly) {
1946                             return
1947                         }
1948
1949                         switch (architecture) {
1950                             case 'arm':
1951                                 if ((scenario != 'gcstress0x3') &&
1952                                     (scenario != 'gcstress0xc') &&
1953                                     (scenario != 'jitstress1') &&
1954                                     (scenario != 'jitstress2') &&
1955                                     (scenario != 'jitstressregs1') &&
1956                                     (scenario != 'jitstressregs2') &&
1957                                     (scenario != 'gcstress0xc_jitstress1') &&
1958                                     (scenario != 'gcstress0xc_jitstress2') &&
1959                                     (scenario != 'minopts') &&
1960                                     (scenario != 'tailcallstress') &&
1961                                     (scenario != 'zapdisable')) {
1962                                         return
1963                                     }
1964                                     break
1965                             case 'arm64':
1966                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1967                                     return
1968                                 }
1969                                 break
1970                             case 'x64':
1971                             case 'x86':
1972                                 // x86 ubuntu: default only
1973                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1974                                     return
1975                                 }
1976                                 // Windows: Everything implemented
1977                                 break
1978                             case 'x86lb':
1979                                 // No stress modes for legacy jit.
1980                                 // (There's no technical reason we couldn't allow these.)
1981                                 return
1982                             default:
1983                                 return
1984                         }
1985                     }
1986                     else {
1987                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
1988                         // and configuration is not Checked, bail out.
1989                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1990                             return;
1991                         }
1992
1993                         // Skip scenarios
1994                         switch (scenario) {
1995                             case 'pri1':
1996                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1997                                 // the default scenario build
1998                                 if (os != 'Windows_NT') {
1999                                     return
2000                                 }
2001                                 // Only x64 for now
2002                                 if (architecture != 'x64') {
2003                                     return
2004                                 }
2005                                 break
2006                             case 'ilrt':
2007                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
2008                                 // the default scenario build
2009                                 if (os != 'Windows_NT') {
2010                                     return
2011                                 }
2012                                 // Only x64 for now
2013                                 if (architecture != 'x64') {
2014                                     return
2015                                 }
2016                                 // Release only
2017                                 if (configuration != 'Release') {
2018                                     return
2019                                 }
2020                                 break
2021                             case 'jitdiff':
2022                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2023                                     return
2024                                 }
2025                                 if (architecture != 'x64') {
2026                                     return
2027                                 }
2028                                 if (configuration != 'Checked') {
2029                                     return
2030                                 }
2031                                 break
2032                             case 'r2r':
2033                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2034                                 // the default scenario build
2035                                 if (os != 'Windows_NT') {
2036                                     return
2037                                 }
2038                                 if (architecture != 'x64') {
2039                                     return
2040                                 }
2041                                 break
2042                             case 'pri1r2r':
2043                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
2044                                 // the default scenario build
2045                                 if (os != 'Windows_NT') {
2046                                     return
2047                                 }
2048                                 if (architecture != 'x64') {
2049                                     if ((architecture != 'arm64' && architecture != 'arm') || (configuration == 'Debug')) {
2050                                         return
2051                                     }
2052                                 }
2053                                 break
2054                             case 'gcstress15_pri1r2r':
2055                             case 'r2r_jitstress1':
2056                             case 'r2r_jitstress2':
2057                             case 'r2r_jitstressregs1':
2058                             case 'r2r_jitstressregs2':
2059                             case 'r2r_jitstressregs3':
2060                             case 'r2r_jitstressregs4':
2061                             case 'r2r_jitstressregs8':
2062                             case 'r2r_jitstressregs0x10':
2063                             case 'r2r_jitstressregs0x80':
2064                             case 'r2r_jitstressregs0x1000':
2065                             case 'r2r_jitminopts':
2066                             case 'r2r_jitforcerelocs':
2067                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
2068                                 // the default scenario build
2069                                 if (os != 'Windows_NT') {
2070                                     return
2071                                 }
2072                                 if (architecture != 'x64' && architecture != 'x86') {
2073                                     return
2074                                 }
2075                                 break
2076                             case 'longgc':
2077                             case 'gcsimulator':
2078                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2079                                     return
2080                                 }
2081                                 if (architecture != 'x64') {
2082                                     return
2083                                 }
2084                                 if (configuration != 'Release') {
2085                                     return
2086                                 }
2087                                 break
2088                             case 'gc_reliability_framework':
2089                             case 'standalone_gc':
2090                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2091                                     return
2092                                 }
2093
2094                                 if (architecture != 'x64') {
2095                                     return
2096                                 }
2097
2098                                 if (configuration != 'Release' && configuration != 'Checked') {
2099                                     return
2100                                 }
2101                                 break
2102                             // We need Windows x64 Release bits for the code coverage build
2103                             case 'coverage':
2104                                 if (os != 'Windows_NT') {
2105                                     return
2106                                 }
2107                                 if (architecture != 'x64') {
2108                                     return
2109                                 }
2110                                 if (configuration != 'Release') {
2111                                     return
2112                                 }
2113                                 break
2114                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2115                             case 'formatting':
2116                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2117                                     return
2118                                 }
2119                                 if (architecture != 'x64') {
2120                                     return
2121                                 }
2122                                 if (configuration != 'Checked') {
2123                                     return
2124                                 }
2125                                 if (isBuildOnly) {
2126                                     return
2127                                 }
2128                                 break
2129                             case 'illink':
2130                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2131                                     return
2132                                 }
2133                                 if (architecture != 'x64' && architecture != 'x86') {
2134                                     return
2135                                 }
2136                                 if (isBuildOnly) {
2137                                     return
2138                                 }
2139                                 break
2140                             case 'default':
2141                                 // Nothing skipped
2142                                 break
2143                             default:
2144                                 println("Unknown scenario: ${scenario}")
2145                                 assert false
2146                                 break
2147                         }
2148                     }
2149
2150                     // Calculate names
2151                     def lowerConfiguration = configuration.toLowerCase()
2152                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2153                     def folderName = getJobFolder(scenario)
2154
2155                     // Create the new job
2156                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2157
2158                     def machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2159                     machineAffinityOptions = architecture == 'arm' ? ['use_arm64_build_machine': false] : machineAffinityOptions
2160                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2161
2162                     // Add all the standard options
2163                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2164                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2165
2166                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
2167                     def osGroup = getOSGroup(os)
2168
2169                     newJob.with {
2170                         steps {
2171                             if (os == 'Windows_NT') {
2172                                 buildCommands.each { buildCommand ->
2173                                     batchFile(buildCommand)
2174                                 }
2175                             }
2176                             else {
2177                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2178                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2179                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2180                                     // Define the Windows Tests and Corefx build job names
2181                                     def WindowTestsName = projectFolder + '/' +
2182                                                           Utilities.getFullJobName(project,
2183                                                                                    getJobName(lowerConfiguration,
2184                                                                                               'x64' ,
2185                                                                                               'windows_nt',
2186                                                                                               'default',
2187                                                                                               true),
2188                                                                                    false)
2189                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2190                                                        Utilities.getFolderName(branch)
2191
2192                                     // Copy the Windows test binaries and the Corefx build binaries
2193                                     copyArtifacts(WindowTestsName) {
2194                                         includePatterns('bin/tests/tests.zip')
2195                                         buildSelector {
2196                                             latestSuccessful(true)
2197                                         }
2198                                     }
2199
2200                                     def arm_abi = 'arm'
2201                                     def corefx_os = 'linux'
2202                                     if (os == 'Tizen') {
2203                                         arm_abi = 'armel'
2204                                         corefx_os = 'tizen'
2205                                     }
2206
2207                                     // Let's use release CoreFX to test checked CoreCLR,
2208                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2209                                     def corefx_lowerConfiguration = lowerConfiguration
2210                                     if ( lowerConfiguration == 'checked' ) {
2211                                         corefx_lowerConfiguration='release'
2212                                     }
2213
2214                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2215                                         includePatterns('bin/build.tar.gz')
2216                                         buildSelector {
2217                                             latestSuccessful(true)
2218                                         }
2219                                     }
2220                                 }
2221
2222                                 buildCommands.each { buildCommand ->
2223                                     shell(buildCommand)
2224                                 }
2225                             }
2226                         }
2227                     } // newJob.with
2228
2229                 } // os
2230             } // configuration
2231         } // architecture
2232     } // isPR
2233 } // scenario
2234
2235
2236 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2237 combinedScenarios.each { scenario ->
2238     [true, false].each { isPR ->
2239         // Architectures.  x64 only at this point
2240         ['x64', 'arm64'].each { architecture ->
2241             // Put the OS's supported for coreclr cross testing here
2242             Constants.crossList.each { os ->
2243                 if (architecture == 'arm64') {
2244                     if (os != "Ubuntu") {
2245                         return
2246                     }
2247                 }
2248
2249                 Constants.configurationList.each { configuration ->
2250
2251                     if (architecture == 'arm64') {
2252                         if (scenario != 'default' && scenario != 'pri1r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2253                             return
2254                         }
2255                     }
2256
2257                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2258                         if (configuration != 'Checked') {
2259                             return
2260                         }
2261                         if (isCorefxTesting(scenario)) {
2262                             return
2263                         }
2264                         //Skip stress modes for these scenarios
2265                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2266                             return
2267                         }
2268                     }
2269                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2270                     // and configuration is not Checked, bail out.
2271                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2272                         return;
2273                     }
2274                     // For CentOS, we only want Checked/Release pri1 builds.
2275                     else if (os == 'CentOS7.1') {
2276                         if (scenario != 'pri1' &&
2277                             scenario != 'r2r' &&
2278                             scenario != 'pri1r2r' &&
2279                             scenario != 'gcstress15_pri1r2r' &&
2280                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2281                             return
2282                         }
2283                         if (configuration != 'Checked' && configuration != 'Release') {
2284                             return
2285                         }
2286                     }
2287                     // For RedHat and Debian, we only do Release pri1 builds.
2288                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2289                         if (scenario != 'pri1') {
2290                             return
2291                         }
2292                         if (configuration != 'Release') {
2293                             return
2294                         }
2295                     }
2296                     else {
2297                         // Skip scenarios
2298                         switch (scenario) {
2299                             case 'pri1':
2300                                 // Nothing skipped
2301                                 break
2302                             case 'ilrt':
2303                                 // Release only
2304                                 if (configuration != 'Release') {
2305                                     return
2306                                 }
2307                                 break
2308                             case 'jitdiff':
2309                                 if (configuration != 'Checked') {
2310                                     return;
2311                                 }
2312                             case 'r2r':
2313                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2314                                 if (configuration != 'Checked' && configuration != 'Release') {
2315                                     return
2316                                 }
2317                                 break
2318                             case 'pri1r2r':
2319                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2320                                 if (configuration != 'Checked' && configuration != 'Release') {
2321                                     return
2322                                 }
2323                                 break
2324                             case 'gcstress15_pri1r2r':
2325                             case 'r2r_jitstress1':
2326                             case 'r2r_jitstress2':
2327                             case 'r2r_jitstressregs1':
2328                             case 'r2r_jitstressregs2':
2329                             case 'r2r_jitstressregs3':
2330                             case 'r2r_jitstressregs4':
2331                             case 'r2r_jitstressregs8':
2332                             case 'r2r_jitstressregs0x10':
2333                             case 'r2r_jitstressregs0x80':
2334                             case 'r2r_jitstressregs0x1000':
2335                             case 'r2r_jitminopts':
2336                             case 'r2r_jitforcerelocs':
2337                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2338                                 if (configuration != 'Checked' && configuration != 'Release') {
2339                                     return
2340                                 }
2341                                 break
2342                             case 'longgc':
2343                             case 'gcsimulator':
2344                                 // Long GC tests take a long time on non-Release builds
2345                                 if (configuration != 'Release') {
2346                                     return
2347                                 }
2348                                 break
2349                             case 'gc_reliability_framework':
2350                             case 'standalone_gc':
2351                                 if (configuration != 'Release' && configuration != 'Checked') {
2352                                     return
2353                                 }
2354                                 break
2355                             case 'coverage':
2356                                 //We only want Ubuntu Release for coverage
2357                                 if (os != 'Ubuntu') {
2358                                     return
2359                                 }
2360                                 if (configuration != 'Release') {
2361                                     return
2362                                 }
2363                             case 'formatting':
2364                                 return
2365                             case 'illink':
2366                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2367                                     return
2368                                 }
2369                                 break
2370                             case 'default':
2371                                 // Nothing skipped
2372                                 break
2373                             default:
2374                                 println("Unknown scenario: ${scenario}")
2375                                 assert false
2376                                 break
2377                         }
2378                     }
2379
2380                     def lowerConfiguration = configuration.toLowerCase()
2381                     def osGroup = getOSGroup(os)
2382                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2383
2384                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2385                     def inputScenario = 'default'
2386                     if (scenario == 'coverage') {
2387                         inputScenario = 'coverage'
2388                     }
2389                     def inputCoreCLRBuildName = projectFolder + '/' +
2390                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2391                     // If this is a stress scenario, there isn't any difference in the build job
2392                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2393                     // from the default scenario
2394                     def testBuildScenario = scenario
2395                     if (scenarioNeedsPri1Build(scenario)) {
2396                         testBuildScenario = 'pri1'
2397                     }
2398                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2399                         testBuildScenario = 'default'
2400                     }
2401                     def inputWindowTestsBuildName = ''
2402                     def inputWindowsTestBuildArch = architecture
2403                     if (architecture == "arm64") {
2404                         // Use the x64 test build for arm64 unix
2405                         inputWindowsTestBuildArch = "x64"
2406                     }
2407                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2408                         inputWindowTestsBuildName = projectFolder + '/' +
2409                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', 'default', true), isPR)
2410                     }
2411                     else {
2412                         inputWindowTestsBuildName = projectFolder + '/' +
2413                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2414                     }
2415                     // Enable Server GC for Ubuntu PR builds
2416                     def serverGCString = ''
2417
2418                     // Whether or not this test run should be run sequentially instead
2419                     // of in parallel. Only used for long GC tests.
2420                     def sequentialString = ''
2421
2422                     // Whether or not this test run should run a specific playlist.
2423                     // Only used for long GC tests.
2424
2425                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2426                     // for running long GC and GCSimulator tests, respectively. We don't use them
2427                     // here because using a playlist file produces much more readable output on the CI machines
2428                     // and reduces running time.
2429                     def playlistString = ''
2430
2431                     if (os == 'Ubuntu' && isPR){
2432                         serverGCString = '--useServerGC'
2433                     }
2434
2435                     // pass --crossgen to runtest.sh for crossgen builds
2436                     def crossgenStr = ''
2437                     def runcrossgentestsStr = ''
2438                     def runjitstressStr = ''
2439                     def runjitstressregsStr = ''
2440                     def runjitmioptsStr = ''
2441                     def runjitforcerelocsStr = ''
2442                     def runjitdisasmStr = ''
2443                     def runilasmroundtripStr = ''
2444                     def gcstressStr = ''
2445                     def illinkStr = ''
2446                     def layoutOnlyStr =''
2447
2448                     if (scenario == 'r2r' ||
2449                         scenario == 'pri1r2r' ||
2450                         scenario == 'gcstress15_pri1r2r' ||
2451                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2452                             crossgenStr = '--crossgen'
2453                             runcrossgentestsStr = '--runcrossgentests'
2454
2455                             if (scenario == 'r2r_jitstress1'){
2456                                 runjitstressStr = '--jitstress=1'
2457                             }
2458                             else if (scenario == 'r2r_jitstress2') {
2459                                 runjitstressStr = '--jitstress=2'
2460                             }
2461                             else if (scenario == 'r2r_jitstressregs1'){
2462                                 runjitstressregsStr = '--jitstressregs=1'
2463                             }
2464                             else if (scenario == 'r2r_jitstressregs2') {
2465                                 runjitstressregsStr = '--jitstressregs=2'
2466                             }
2467                             else if (scenario == 'r2r_jitstressregs3') {
2468                                 runjitstressregsStr = '--jitstressregs=3'
2469                             }
2470                             else if (scenario == 'r2r_jitstressregs4') {
2471                                 runjitstressregsStr = '--jitstressregs=4'
2472                             }
2473                             else if (scenario == 'r2r_jitstressregs8') {
2474                                 runjitstressregsStr = '--jitstressregs=8'
2475                             }
2476                             else if (scenario == 'r2r_jitstressregs0x10') {
2477                                 runjitstressregsStr = '--jitstressregs=0x10'
2478                             }
2479                             else if (scenario == 'r2r_jitstressregs0x80') {
2480                                 runjitstressregsStr = '--jitstressregs=0x80'
2481                             }
2482                             else if (scenario == 'r2r_jitstressregs0x1000') {
2483                                 runjitstressregsStr = '--jitstressregs=0x1000'
2484                             }
2485                             else if (scenario == 'r2r_jitminopts') {
2486                                 runjitmioptsStr = '--jitminopts'
2487                             }
2488                             else if (scenario == 'r2r_jitforcerelocs') {
2489                                 runjitforcerelocsStr = '--jitforcerelocs'
2490                             }
2491                     }
2492                     if  (scenario == 'gcstress15_pri1r2r')
2493                     {
2494                         gcstressStr = '--gcstresslevel=0xF'
2495                     }
2496
2497                     if (scenario == 'jitdiff')
2498                     {
2499                         runjitdisasmStr = '--jitdisasm --crossgen'
2500                     }
2501
2502                     if (scenario == 'illink')
2503                     {
2504                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2505                     }
2506
2507                     if (isLongGc(scenario)) {
2508                         // Long GC tests behave very poorly when they are not
2509                         // the only test running (many of them allocate until OOM).
2510                         sequentialString = '--sequential'
2511
2512                         // The Long GC playlist contains all of the tests that are
2513                         // going to be run. The GCSimulator playlist contains all of
2514                         // the GC simulator tests.
2515                         if (scenario == 'longgc') {
2516                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2517                         }
2518                         else if (scenario == 'gcsimulator') {
2519                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2520                         }
2521                     }
2522
2523                     if (isGcReliabilityFramework(scenario)) {
2524                         layoutOnlyStr = '--build-overlay-only'
2525                     }
2526
2527                     def folder = getJobFolder(scenario)
2528                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2529                         // Add parameters for the inputs
2530
2531                         parameters {
2532                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2533                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2534                         }
2535
2536                         steps {
2537                             // Set up the copies
2538
2539                             // Coreclr build containing the tests and mscorlib
2540
2541                             copyArtifacts(inputWindowTestsBuildName) {
2542                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2543                                 buildSelector {
2544                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2545                                 }
2546                             }
2547
2548                             if (scenario == 'coverage') {
2549
2550                                 // Move coreclr to clr directory
2551                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2552
2553                                 // Build coreclr
2554                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2555
2556                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2557                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2558                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2559                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2560                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2561                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2562                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2563                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2564
2565                                 // Run PAL tests
2566                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2567
2568                                 // Remove obj files for PAL tests so they're not included in coverage results
2569                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2570
2571                                 // Unzip the tests first.  Exit with 0
2572                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2573
2574                                 // Get corefx
2575                                 shell("git clone https://github.com/dotnet/corefx fx")
2576
2577                                 // Build Linux corefx
2578                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2579                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2580
2581                                 def testEnvOpt = ""
2582                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2583                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2584                                 shell("${createScriptCmds}")
2585                                 testEnvOpt = "--test-env=" + scriptFileName
2586
2587                                 // Run corefx tests
2588                                 shell("""./fx/run-test.sh \\
2589                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2590                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2591                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2592                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2593                 --configurationGroup Release""")
2594
2595
2596                                 // Run coreclr tests w/ workstation GC
2597                                 shell("""./clr/tests/runtest.sh \\
2598                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2599                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2600                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2601                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2602                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2603                 --crossgen --runcrossgentests""")
2604
2605                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2606                                 shell("""./clr/tests/runtest.sh \\
2607                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2608                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2609                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2610                 --useServerGC ${testEnvOpt}""")
2611
2612                                  // Run long-running coreclr GC tests & produce coverage reports
2613                                 shell("""./clr/tests/runtest.sh \\
2614                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2615                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2616                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2617                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2618                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2619                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2620                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2621
2622                             }
2623                             else {
2624
2625                                 // Coreclr build we are trying to test
2626
2627                                 copyArtifacts(inputCoreCLRBuildName) {
2628                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2629                                     buildSelector {
2630                                         buildNumber('${CORECLR_BUILD}')
2631                                     }
2632                                 }
2633
2634                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2635
2636                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2637                                 def osJobName
2638                                 if (os == 'Ubuntu') {
2639                                     osJobName = 'ubuntu14.04'
2640                                 }
2641                                 else {
2642                                     osJobName = os.toLowerCase()
2643                                 }
2644                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2645                                     includePatterns('bin/build.tar.gz')
2646                                     buildSelector {
2647                                         latestSuccessful(true)
2648                                     }
2649                                 }
2650
2651                                 shell ("mkdir ./bin/CoreFxBinDir")
2652                                 // Unpack the corefx binaries
2653                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2654
2655                                 // HACK -- Arm64 does not have corefx jobs yet.
2656                                 // Clone corefx and build the native packages overwriting the x64 packages.
2657                                 if (architecture == 'arm64') {
2658                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2659                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2660                                 }
2661
2662                                 // Unzip the tests first.  Exit with 0
2663                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2664
2665                                 // Execute the tests
2666                                 // If we are running a stress mode, we'll set those variables first
2667                                 def testEnvOpt = ""
2668                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2669                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2670                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2671                                     shell("${createScriptCmds}")
2672                                     testEnvOpt = "--test-env=" + scriptFileName
2673                                 }
2674
2675                                 if (isGCStressRelatedTesting(scenario)) {
2676                                     shell('./init-tools.sh')
2677                                 }
2678
2679                                 shell("""./tests/runtest.sh \\
2680                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2681                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2682                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2683                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2684                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2685                 --limitedDumpGeneration \\
2686                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} \\
2687                 ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} \\
2688                 ${illinkStr} ${sequentialString} ${playlistString} ${layoutOnlyStr}""")
2689
2690                                 if (isGcReliabilityFramework(scenario)) {
2691                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2692                                     if (serverGCString != '') {
2693                                         shell("export COMPlus_gcServer=1")
2694                                     }
2695
2696                                     shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2697                                 }
2698                             }
2699                         }
2700                     }
2701
2702                     if (scenario == 'coverage') {
2703                         // Publish coverage reports
2704                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2705                         // TODO: Add once external email sending is available again
2706                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2707                     }
2708
2709                     if (scenario == 'jitdiff') {
2710                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2711                     }
2712
2713                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2714                     if (os in ['Ubuntu']) {
2715                         SummaryBuilder summaries = new SummaryBuilder()
2716                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2717                         summaries.emit(newJob)
2718                     }
2719
2720                     setMachineAffinity(newJob, os, architecture)
2721                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2722                     // Set timeouts to 240.
2723                     setTestJobTimeOut(newJob, scenario)
2724
2725                     if (architecture == 'arm64') {
2726                         Utilities.setJobTimeout(newJob, 480)
2727                     }
2728
2729                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2730
2731                     // Create a build flow to join together the build and tests required to run this
2732                     // test.
2733                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2734                     // Linux CoreCLR test
2735                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2736                     def fullTestJobName = projectFolder + '/' + newJob.name
2737                     // Add a reference to the input jobs for report purposes
2738                     JobReport.Report.addReference(inputCoreCLRBuildName)
2739                     JobReport.Report.addReference(inputWindowTestsBuildName)
2740                     JobReport.Report.addReference(fullTestJobName)
2741                     def newFlowJob;
2742
2743                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2744                     // so that coverage data can be preserved.
2745                     if (scenario == 'coverage') {
2746                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2747                         buildFlow("""
2748 // Build the input Windows job
2749 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2750
2751 // And then build the test build
2752 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2753 """)
2754                         }
2755                     // Normal jobs copy a Windows build & a non-Windows build
2756                     } else {
2757                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2758                         buildFlow("""
2759 // Build the input jobs in parallel
2760 parallel (
2761     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2762     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2763 )
2764
2765 // And then build the test build
2766 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2767                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2768 """)
2769                         }
2770                     }
2771
2772                     setMachineAffinity(newFlowJob, os, architecture)
2773                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2774                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2775                 } // configuration
2776             } // os
2777         } // architecture
2778     } // isPR
2779 } // scenario
2780
2781 JobReport.Report.generateJobReport(out)
2782
2783 // Make the call to generate the help job
2784 Utilities.createHelperJob(this, project, branch,
2785     "Welcome to the ${project} Repository",  // This is prepended to the help message
2786     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2787
2788 Utilities.addCROSSCheck(this, project, branch)