netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'FreeBSD':'FreeBSD',
  29         'CentOS7.1': 'Linux',
  30         'OpenSUSE42.1': 'Linux',
  31         'Tizen': 'Linux']
  32     def osGroup = osGroupMap.get(os, null)
  33     assert osGroup != null : "Could not find os group for ${os}"
  34     return osGroupMap[os]
  35 }
  36
  37 // We use this class (vs variables) so that the static functions can access data here.
  38 class Constants {
  39
  40     // Innerloop build OS's
  41     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  42     // test execution in the build flow runs.  It generates the exact same build
  43     // as Windows_NT but without the tests.
  44     def static osList = [
  45                'Ubuntu',
  46                'Debian8.4',
  47                'OSX10.12',
  48                'Windows_NT',
  49                'Windows_NT_BuildOnly',
  50                'FreeBSD',
  51                'CentOS7.1',
  52                'OpenSUSE42.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  67                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  68                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  69                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  70                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  71                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  72                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  73                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  74                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  75                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  76                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  77                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  78                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  79                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  80                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  81                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  82                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  83                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  84                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  85                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  86                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  87                'corefx_baseline'                : [ : ], // corefx baseline
  88                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  89                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  90                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  91                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  92                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  93                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  94                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  95                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  96                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  97                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  98                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  99                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 100                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 101                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 102                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 103                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 104                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 105                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 107                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 109                ]
 110
 111     // This is a set of r2r jit stress scenarios
 112     def static r2rJitStressScenarios = [
 113                'r2r_jitstress1',
 114                'r2r_jitstress2',
 115                'r2r_jitstressregs1',
 116                'r2r_jitstressregs2',
 117                'r2r_jitstressregs3',
 118                'r2r_jitstressregs4',
 119                'r2r_jitstressregs8',
 120                'r2r_jitstressregs0x10',
 121                'r2r_jitstressregs0x80',
 122                'r2r_jitstressregs0x1000',
 123                'r2r_jitminopts',
 124                'r2r_jitforcerelocs']
 125
 126     // This is the basic set of scenarios
 127     def static basicScenarios = [
 128                'default',
 129                'pri1',
 130                'ilrt',
 131                'r2r',
 132                'pri1r2r',
 133                'gcstress15_pri1r2r',
 134                'longgc',
 135                'coverage',
 136                'formatting',
 137                'gcsimulator',
 138                'jitdiff',
 139                'standalone_gc',
 140                'illink'] + r2rJitStressScenarios
 141
 142     def static configurationList = ['Debug', 'Checked', 'Release']
 143
 144     // This is the set of architectures
 145     def static architectureList = ['arm', 'arm64', 'x64', 'x86']
 146 }
 147
 148 def static setMachineAffinity(def job, def os, def architecture) {
 149     if (architecture == 'arm64' && os == 'Windows_NT') {
 150         Utilities.setMachineAffinity(job, os, 'latest-arm64');
 151     } else if (architecture == 'arm64' && os == 'Ubuntu') {
 152         Utilities.setMachineAffinity(job, os, 'arm-cross-latest');
 153     } else if ((architecture == 'arm') && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
 154         Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest');
 155     } else {
 156         Utilities.setMachineAffinity(job, os, 'latest-or-auto');
 157     }
 158 }
 159
 160 def static isJITStressJob(def scenario) {
 161     return Constants.jitStressModeScenarios.containsKey(scenario) ||
 162            (Constants.r2rJitStressScenarios.indexOf(scenario) != -1)
 163 }
 164
 165 def static isGCStressRelatedTesting(def scenario) {
 166     // The 'gcstress15_pri1r2r' scenario is a basic scenario.
 167     // Detect it and make it a GCStress related.
 168     if (scenario == 'gcstress15_pri1r2r')
 169     {
 170         return true;
 171     }
 172
 173     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 174     def scenarioName = scenario.toLowerCase()
 175     def isGCStressTesting = false
 176     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 177         if (k in gcStressTestEnvVars) {
 178             isGCStressTesting = true;
 179         }
 180     }
 181     return isGCStressTesting
 182 }
 183
 184 def static isCorefxTesting(def scenario) {
 185     def corefx_prefix = 'corefx_'
 186     if (scenario.length() < corefx_prefix.length()) {
 187         return false
 188     }
 189     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 190 }
 191
 192 def static isR2R(def scenario) {
 193     return (scenario == 'r2r' || scenario == 'pri1r2r')
 194 }
 195
 196 def static isCoverage(def scenario) {
 197     return (scenario == 'coverage')
 198 }
 199
 200 def static isLongGc(def scenario) {
 201     return (scenario == 'longgc' || scenario == 'gcsimulator')
 202 }
 203
 204 def static isJitDiff(def scenario) {
 205     return (scenario == 'jitdiff')
 206 }
 207
 208 def static setTestJobTimeOut(newJob, scenario) {
 209     if (isGCStressRelatedTesting(scenario)) {
 210         Utilities.setJobTimeout(newJob, 4320)
 211     }
 212     else if (isCorefxTesting(scenario)) {
 213         Utilities.setJobTimeout(newJob, 360)
 214     }
 215     else if (Constants.jitStressModeScenarios.containsKey(scenario)) {
 216         Utilities.setJobTimeout(newJob, 240)
 217     }
 218     else if (isR2R(scenario)) {
 219         Utilities.setJobTimeout(newJob, 240)
 220     }
 221     else if (isCoverage(scenario)) {
 222         Utilities.setJobTimeout(newJob, 1440)
 223     }
 224     else if (isLongGc(scenario)) {
 225         Utilities.setJobTimeout(newJob, 1440)
 226     }
 227     else if (isJitDiff(scenario)) {
 228         Utilities.setJobTimeout(newJob, 240)
 229     }
 230     // Non-test jobs use the default timeout value.
 231 }
 232
 233 def static getJobFolder(def scenario) {
 234     if (isJITStressJob(scenario)) {
 235         return 'jitstress'
 236     }
 237     if (scenario == 'illink') {
 238         return 'illink'
 239     }
 240     return ''
 241 }
 242
 243 def static getStressModeDisplayName(def scenario) {
 244     def displayStr = ''
 245     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 246         def prefixLength = 'COMPlus_'.length()
 247         if (k.length() >= prefixLength) {
 248             def modeName = k.substring(prefixLength, k.length())
 249             displayStr += ' ' + modeName + '=' + v
 250         }
 251     }
 252     return displayStr
 253 }
 254
 255 def static getR2RStressModeDisplayName(def scenario) {
 256     // Assume the scenario name is one from the r2rJitStressScenarios list, and remove its
 257     // "r2r_" prefix.
 258     def displayStr = scenario
 259     def prefixLength = 'r2r_'.length()
 260     if (displayStr.length() >= prefixLength) {
 261         displayStr = displayStr.substring(prefixLength, displayStr.length())
 262     }
 263     return displayStr
 264 }
 265
 266 // Generates the string for creating a file that sets environment variables
 267 // that makes it possible to run stress modes.  Writes the script to the file
 268 // specified by the stepScriptLocation parameter.
 269 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 270     def stepScript = ''
 271     if (os == 'Windows_NT') {
 272         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 273         stepScript += "del ${stepScriptLocation}\r\n"
 274
 275         // Timeout in ms, default is 10 minutes. For stress
 276         // modes up this to 30 minutes
 277         def timeout = 1800000
 278
 279         // Set the Timeout
 280         stepScript += "set __TestTimeout=${timeout}\r\n"
 281         stepScript += "echo. > ${stepScriptLocation}\r\n"
 282         stressModeVars.each{ k, v ->
 283             // Write out what we are writing to the script file
 284             stepScript += "echo Setting ${k}=${v}\r\n"
 285             // Write out the set itself to the script file`
 286             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 287         }
 288     }
 289     else {
 290         stepScript += "echo Setting variables for ${stressModeName}\n"
 291         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 292         stressModeVars.each{ k, v ->
 293             // Write out what we are writing to the script file
 294             stepScript += "echo Setting ${k}=${v}\n"
 295             // Write out the set itself to the script file`
 296             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 297         }
 298         stepScript += "chmod +x ${stepScriptLocation}\n"
 299     }
 300     return stepScript
 301 }
 302
 303 // Calculates the name of the build job based on some typical parameters.
 304 //
 305 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 306     // If the architecture is x64, do not add that info into the build name.
 307     // Need to change around some systems and other builds to pick up the right builds
 308     // to do that.
 309
 310     def suffix = scenario != 'default' ? "_${scenario}" : '';
 311     if (isBuildOnly) {
 312         suffix += '_bld'
 313     }
 314     def baseName = ''
 315     switch (architecture) {
 316         case 'x64':
 317             if (scenario == 'default') {
 318                 // For now we leave x64 off of the name for compatibility with other jobs
 319                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 320             }
 321             else if (scenario == 'formatting') {
 322                 // we don't care about the configuration for the formatting job. It runs all configs
 323                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 324             }
 325             else {
 326                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 327             }
 328             break
 329         case 'arm64':
 330             // These are cross builds
 331             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 332             break
 333         case 'arm':
 334             // These are cross builds
 335             if (os == 'Tizen') {
 336                 // ABI: softfp
 337                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 338             }
 339             else {
 340                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 341             }
 342             break
 343         case 'x86':
 344             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 345             break
 346         default:
 347             println("Unknown architecture: ${architecture}");
 348             assert false
 349             break
 350     }
 351
 352     return baseName + suffix
 353 }
 354
 355 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 356     // Check scenario.
 357     switch (scenario) {
 358         case 'default':
 359             switch (architecture) {
 360                 case 'x64':
 361                 case 'x86':
 362                     if (architecture == 'x86' && os == 'Ubuntu') {
 363                         Utilities.addPeriodicTrigger(job, '@daily')
 364                     }
 365                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 366                         Utilities.addGithubPushTrigger(job)
 367                     }
 368                     break
 369                 case 'arm':
 370                     Utilities.addGithubPushTrigger(job)
 371                     break
 372                 case 'arm64':
 373                     if (os == 'Windows_NT') {
 374                         Utilities.addGithubPushTrigger(job)
 375                         // TODO: Add once external email sending is available again
 376                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 377                     }
 378                     break
 379                 default:
 380                     println("Unknown architecture: ${architecture}");
 381                     assert false
 382                     break
 383             }
 384             break
 385         case 'pri1':
 386             // Pri one gets a push trigger, and only for release
 387             if (architecture == 'x64') {
 388                 if (configuration == 'Release') {
 389                     // We expect release jobs to be Windows, or in the cross list
 390                     assert (os == 'Windows_NT') || (os in Constants.crossList)
 391                     if (!os in bidailyCrossList) {
 392                         if (isFlowJob || os == 'Windows_NT') {
 393                             Utilities.addGithubPushTrigger(job)
 394                         }
 395                     }
 396                     else {
 397                         if (isFlowJob) {
 398                             Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 399                         }
 400                     }
 401                 }
 402             }
 403             break
 404         case 'r2r':
 405             //r2r jobs that aren't pri1 can only be triggered by phrase
 406             break
 407         case 'pri1r2r':
 408             assert !(os in bidailyCrossList)
 409             //pri1 r2r gets a push trigger for checked/release
 410             if (configuration == 'Checked' || configuration == 'Release') {
 411                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 412                 if (architecture == 'x64' && os != 'OSX10.12') {
 413                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 414                     if (isFlowJob || os == 'Windows_NT') {
 415                         Utilities.addGithubPushTrigger(job)
 416                     }
 417                 // OSX10.12 pri1r2r jobs should only run every 12 hours, not daily.
 418                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 419                     if (isFlowJob) {
 420                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 421                     }
 422                 }
 423                 // For x86, only add per-commit jobs for Windows
 424                 else if (architecture == 'x86') {
 425                     if (os == 'Windows_NT') {
 426                         Utilities.addGithubPushTrigger(job)
 427                     }
 428                 }
 429                 // arm64 pri1r2r jobs should only run every 12 hours.
 430                 else if (architecture == 'arm64') {
 431                     if (os == 'Windows_NT') {
 432                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 433                         // TODO: Add once external email sending is available again
 434                         // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 435                     }
 436                 }
 437             }
 438             break
 439         case 'r2r_jitstress1':
 440         case 'r2r_jitstress2':
 441         case 'r2r_jitstressregs1':
 442         case 'r2r_jitstressregs2':
 443         case 'r2r_jitstressregs3':
 444         case 'r2r_jitstressregs4':
 445         case 'r2r_jitstressregs8':
 446         case 'r2r_jitstressregs0x10':
 447         case 'r2r_jitstressregs0x80':
 448         case 'r2r_jitstressregs0x1000':
 449         case 'r2r_jitminopts':
 450         case 'r2r_jitforcerelocs':
 451         case 'gcstress15_pri1r2r':
 452             assert !(os in bidailyCrossList)
 453
 454             // GCStress=C is currently not supported on OS X
 455             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 456                 break
 457             }
 458
 459             //GC Stress 15 pri1 r2r gets a push trigger for checked/release
 460             if (configuration == 'Checked' || configuration == 'Release') {
 461                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 462                 if (architecture == 'x64') {
 463                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 464                     if (isFlowJob || os == 'Windows_NT') {
 465                         // Add a weekly periodic trigger
 466                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 467                     }
 468                 }
 469                 // For x86, only add per-commit jobs for Windows
 470                 else if (architecture == 'x86') {
 471                     if (os == 'Windows_NT') {
 472                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 473                     }
 474                 }
 475             }
 476             break
 477         case 'longgc':
 478             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 479             assert configuration == 'Release'
 480             assert architecture == 'x64'
 481             Utilities.addPeriodicTrigger(job, '@daily')
 482             // TODO: Add once external email sending is available again
 483             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 484             break
 485         case 'gcsimulator':
 486             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 487             assert configuration == 'Release'
 488             assert architecture == 'x64'
 489             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 490             // TODO: Add once external email sending is available again
 491             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 492             break
 493         case 'standalone_gc':
 494             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 495             assert (configuration == 'Release' || configuration == 'Checked')
 496             // TODO: Add once external email sending is available again
 497             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 498             Utilities.addPeriodicTrigger(job, '@weekly')
 499             break
 500         case 'ilrt':
 501             assert !(os in bidailyCrossList)
 502             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 503             if (architecture == 'x64' && configuration == 'Release') {
 504                 // We don't expect to see a job generated except in these scenarios
 505                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 506                 if (isFlowJob || os == 'Windows_NT') {
 507                     Utilities.addPeriodicTrigger(job, '@daily')
 508                 }
 509             }
 510             break
 511         case 'jitdiff':
 512             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 513             assert configuration == 'Checked'
 514             assert (architecture == 'x64' || architecture == 'x86')
 515             Utilities.addGithubPushTrigger(job)
 516             break
 517         case 'coverage':
 518             assert (os == 'Ubuntu' || os == 'Windows_NT')
 519             assert configuration == 'Release'
 520             assert architecture == 'x64'
 521             Utilities.addPeriodicTrigger(job, '@weekly')
 522             break
 523         case 'formatting':
 524             assert (os == 'Windows_NT' || os == "Ubuntu")
 525             assert architecture == 'x64'
 526             Utilities.addGithubPushTrigger(job)
 527             break
 528         case 'jitstressregs1':
 529         case 'jitstressregs2':
 530         case 'jitstressregs3':
 531         case 'jitstressregs4':
 532         case 'jitstressregs8':
 533         case 'jitstressregs0x10':
 534         case 'jitstressregs0x80':
 535         case 'jitstressregs0x1000':
 536         case 'minopts':
 537         case 'forcerelocs':
 538         case 'jitstress1':
 539         case 'jitstress2':
 540         case 'jitstress2_jitstressregs1':
 541         case 'jitstress2_jitstressregs2':
 542         case 'jitstress2_jitstressregs3':
 543         case 'jitstress2_jitstressregs4':
 544         case 'jitstress2_jitstressregs8':
 545         case 'jitstress2_jitstressregs0x10':
 546         case 'jitstress2_jitstressregs0x80':
 547         case 'jitstress2_jitstressregs0x1000':
 548         case 'tailcallstress':
 549         case 'jitsse2only':
 550         case 'corefx_baseline':
 551         case 'corefx_minopts':
 552         case 'corefx_jitstress1':
 553         case 'corefx_jitstress2':
 554         case 'corefx_jitstressregs1':
 555         case 'corefx_jitstressregs2':
 556         case 'corefx_jitstressregs3':
 557         case 'corefx_jitstressregs4':
 558         case 'corefx_jitstressregs8':
 559         case 'corefx_jitstressregs0x10':
 560         case 'corefx_jitstressregs0x80':
 561         case 'corefx_jitstressregs0x1000':
 562         case 'zapdisable':
 563             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 564             assert (os == 'Windows_NT') || (os in Constants.crossList)
 565             Utilities.addPeriodicTrigger(job, '@daily')
 566         }
 567         break
 568         case 'heapverify1':
 569         case 'gcstress0x3':
 570             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 571                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 572                 if (architecture == 'arm64') {
 573                     assert (os == 'Windows_NT')
 574                     Utilities.addPeriodicTrigger(job, '@daily')
 575                     // TODO: Add once external email sending is available again
 576                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 577                 }
 578                 else {
 579                     Utilities.addPeriodicTrigger(job, '@weekly')
 580                 }
 581             }
 582             break
 583         case 'gcstress0xc':
 584         case 'gcstress0xc_zapdisable':
 585         case 'gcstress0xc_zapdisable_jitstress2':
 586         case 'gcstress0xc_zapdisable_heapverify1':
 587         case 'gcstress0xc_jitstress1':
 588         case 'gcstress0xc_jitstress2':
 589         case 'gcstress0xc_minopts_heapverify1':
 590             // GCStress=C is currently not supported on OS X
 591             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 592                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 593                 if (architecture == 'arm64') {
 594                     assert (os == 'Windows_NT')
 595                     // TODO: Enable a periodic trigger after tests are updated.
 596                     // Utilities.addPeriodicTrigger(job, '@daily')
 597                     // TODO: Add once external email sending is available again
 598                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 599                 }
 600                 else {
 601                     Utilities.addPeriodicTrigger(job, '@weekly')
 602                 }
 603             }
 604             break
 605
 606         case 'illink':
 607             // Testing on other operating systems TBD
 608             assert (os == 'Windows_NT' || os == 'Ubuntu')
 609             if (architecture == 'x64' || architecture == 'x86') {
 610                 if (configuration == 'Checked') {
 611                     Utilities.addPeriodicTrigger(job, '@daily')
 612                 }
 613             }
 614             break
 615
 616         default:
 617             println("Unknown scenario: ${scenario}");
 618             assert false
 619             break
 620     }
 621     return
 622 }
 623
 624 // **************************
 625 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 626 // of coreclr builds over linux/osx 10.12/freebsd/windows and debug/release/checked.  In addition, the windows
 627 // builds will do a couple extra steps.
 628 // **************************
 629
 630 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 631 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 632 // it's just used for internal builds
 633 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 634 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 635     if (isWindowsBuildOnlyJob) {
 636         return
 637     }
 638
 639     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 640     // Non pull request builds.
 641     if (!isPR) {
 642         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 643         return
 644     }
 645     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 646     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 647     def osGroup = getOSGroup(os)
 648     switch (architecture) {
 649         case 'x64': // editor brace matching: {
 650             if (scenario == 'coverage') {
 651                 assert configuration == 'Release'
 652                 if (os == 'Ubuntu') {
 653                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Coverage Build & Test", "(?i).*test\\W+coverage.*")
 654                 }
 655                 break
 656             }
 657
 658             if (scenario == 'formatting') {
 659                 assert configuration == 'Checked'
 660                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 661                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting")
 662                 }
 663                 break
 664             }
 665
 666             switch (os) {
 667                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 668                 case 'Debian8.4':
 669                 case 'RHEL7.2':
 670                     if (scenario == 'default') {
 671                         assert !isFlowJob
 672                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 673                     }
 674                     else if (scenario == 'pri1' && isFlowJob) {
 675                         assert (configuration == 'Release')
 676                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Pri 1 Build & Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 677                     }
 678                     break
 679                 case 'Ubuntu16.04':
 680                     assert !isFlowJob
 681                     assert scenario == 'default'
 682                     // Distinguish with the other architectures (arm and x86)
 683                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 684                     break
 685                 case 'Fedora24':
 686                 case 'Ubuntu16.10':
 687                 case 'OpenSUSE42.1':
 688                     assert !isFlowJob
 689                     assert scenario == 'default'
 690                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 691                     break
 692                 case 'Ubuntu':
 693                     if (scenario == 'illink') {
 694                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 695                         break
 696                     }
 697                     // fall through
 698                 case 'OSX10.12':
 699                     // Triggers on the non-flow jobs aren't necessary here
 700                     // Corefx testing uses non-flow jobs.
 701                     if (!isFlowJob && !isCorefxTesting(scenario)) {
 702                         break
 703                     }
 704                     switch (scenario) {
 705                         case 'default':
 706                             // Ubuntu uses checked for default PR tests
 707                             if (configuration == 'Checked') {
 708                                 // Default trigger
 709                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 710                             }
 711                             break
 712                         case 'pri1':
 713                             if (configuration == 'Release') {
 714                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 715                             }
 716                             break
 717                         case 'jitdiff':
 718                             if (configuration == 'Checked') {
 719                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 720                             }
 721                             break
 722                         case 'ilrt':
 723                             if (configuration == 'Release') {
 724                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 725                             }
 726                             break
 727                         case 'r2r':
 728                             if (configuration == 'Release' || configuration == 'Checked') {
 729                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 730                             }
 731                             break
 732                         case 'pri1r2r':
 733                             if (configuration == 'Release' || configuration == 'Checked') {
 734                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 735                             }
 736                             break
 737                         case 'gcstress15_pri1r2r':
 738                             if (configuration == 'Release' || configuration == 'Checked') {
 739                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 740                             }
 741                             break
 742                         case 'r2r_jitstress1':
 743                         case 'r2r_jitstress2':
 744                         case 'r2r_jitstressregs1':
 745                         case 'r2r_jitstressregs2':
 746                         case 'r2r_jitstressregs3':
 747                         case 'r2r_jitstressregs4':
 748                         case 'r2r_jitstressregs8':
 749                         case 'r2r_jitstressregs0x10':
 750                         case 'r2r_jitstressregs0x80':
 751                         case 'r2r_jitstressregs0x1000':
 752                         case 'r2r_jitminopts':
 753                         case 'r2r_jitforcerelocs':
 754                             if (configuration == 'Release' || configuration == 'Checked') {
 755                                 def displayStr = getR2RStressModeDisplayName(scenario)
 756                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 757                             }
 758                             break
 759                         case 'longgc':
 760                             if (configuration == 'Release') {
 761                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 762                             }
 763                             break
 764                         case 'gcsimulator':
 765                             if (configuration == 'Release') {
 766                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 767                             }
 768                             break
 769                         case 'minopts':
 770                         case 'forcerelocs':
 771                         case 'jitstress1':
 772                         case 'jitstress2':
 773                         case 'jitstressregs1':
 774                         case 'jitstressregs2':
 775                         case 'jitstressregs3':
 776                         case 'jitstressregs4':
 777                         case 'jitstressregs8':
 778                         case 'jitstressregs0x10':
 779                         case 'jitstressregs0x80':
 780                         case 'jitstressregs0x1000':
 781                         case 'jitstress2_jitstressregs1':
 782                         case 'jitstress2_jitstressregs2':
 783                         case 'jitstress2_jitstressregs3':
 784                         case 'jitstress2_jitstressregs4':
 785                         case 'jitstress2_jitstressregs8':
 786                         case 'jitstress2_jitstressregs0x10':
 787                         case 'jitstress2_jitstressregs0x80':
 788                         case 'jitstress2_jitstressregs0x1000':
 789                         case 'tailcallstress':
 790                         case 'jitsse2only':
 791                         case 'gcstress0x3':
 792                         case 'gcstress0xc':
 793                         case 'zapdisable':
 794                         case 'heapverify1':
 795                         case 'gcstress0xc_zapdisable':
 796                         case 'gcstress0xc_zapdisable_jitstress2':
 797                         case 'gcstress0xc_zapdisable_heapverify1':
 798                         case 'gcstress0xc_jitstress1':
 799                         case 'gcstress0xc_jitstress2':
 800                         case 'gcstress0xc_minopts_heapverify1':
 801                             def displayStr = getStressModeDisplayName(scenario)
 802                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 803                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 804                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 805                             break
 806                         case 'corefx_baseline':
 807                         case 'corefx_minopts':
 808                         case 'corefx_jitstress1':
 809                         case 'corefx_jitstress2':
 810                         case 'corefx_jitstressregs1':
 811                         case 'corefx_jitstressregs2':
 812                         case 'corefx_jitstressregs3':
 813                         case 'corefx_jitstressregs4':
 814                         case 'corefx_jitstressregs8':
 815                         case 'corefx_jitstressregs0x10':
 816                         case 'corefx_jitstressregs0x80':
 817                         case 'corefx_jitstressregs0x1000':
 818                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 819                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 820                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 821                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 822                             break
 823                         default:
 824                             println("Unknown scenario: ${scenario}");
 825                             assert false
 826                             break
 827                     }
 828                     break
 829                 case 'CentOS7.1':
 830                     switch (scenario) {
 831                         case 'pri1':
 832                             if (configuration == 'Release') {
 833                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 834                             }
 835                             break
 836                         case 'r2r':
 837                             if (configuration == 'Checked' || configuration == 'Release') {
 838                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 839                             }
 840                             break
 841                         case 'pri1r2r':
 842                             if (configuration == 'Checked' || configuration == 'Release') {
 843                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 844                             }
 845                             break
 846                         case 'gcstress15_pri1r2r':
 847                             if (configuration == 'Release' || configuration == 'Checked') {
 848                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 849                             }
 850                             break
 851                         case 'r2r_jitstress1':
 852                         case 'r2r_jitstress2':
 853                         case 'r2r_jitstressregs1':
 854                         case 'r2r_jitstressregs2':
 855                         case 'r2r_jitstressregs3':
 856                         case 'r2r_jitstressregs4':
 857                         case 'r2r_jitstressregs8':
 858                         case 'r2r_jitstressregs0x10':
 859                         case 'r2r_jitstressregs0x80':
 860                         case 'r2r_jitstressregs0x1000':
 861                         case 'r2r_jitminopts':
 862                         case 'r2r_jitforcerelocs':
 863                             if (configuration == 'Release' || configuration == 'Checked') {
 864                                 def displayStr = getR2RStressModeDisplayName(scenario)
 865                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 866                             }
 867                             break
 868                         default:
 869                             break
 870                     }
 871                 case 'Windows_NT':
 872                     switch (scenario) {
 873                         case 'default':
 874                             // Default trigger
 875                             if (configuration == 'Debug') {
 876                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
 877                             }
 878                             break
 879                         case 'pri1':
 880                             // Default trigger
 881                             if (configuration == 'Release') {
 882                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
 883                             }
 884                             break
 885                         case 'jitdiff':
 886                             if (configuration == 'Checked') {
 887                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 888                             }
 889                             break
 890                         case 'ilrt':
 891                             if (configuration == 'Release') {
 892                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 893                             }
 894                             break
 895                         case 'r2r':
 896                             if (configuration == 'Checked' || configuration == 'Release') {
 897                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 898                             }
 899                             break
 900                         case 'pri1r2r':
 901                             if (configuration == 'Checked' || configuration == 'Release') {
 902                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 903                             }
 904                             break
 905                         case 'gcstress15_pri1r2r':
 906                             if (configuration == 'Release' || configuration == 'Checked') {
 907                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 908                             }
 909                             break
 910                         case 'r2r_jitstress1':
 911                         case 'r2r_jitstress2':
 912                         case 'r2r_jitstressregs1':
 913                         case 'r2r_jitstressregs2':
 914                         case 'r2r_jitstressregs3':
 915                         case 'r2r_jitstressregs4':
 916                         case 'r2r_jitstressregs8':
 917                         case 'r2r_jitstressregs0x10':
 918                         case 'r2r_jitstressregs0x80':
 919                         case 'r2r_jitstressregs0x1000':
 920                         case 'r2r_jitminopts':
 921                         case 'r2r_jitforcerelocs':
 922                             if (configuration == 'Release' || configuration == 'Checked') {
 923                                 def displayStr = getR2RStressModeDisplayName(scenario)
 924                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 925                             }
 926                             break
 927                         case 'longgc':
 928                             if (configuration == 'Release') {
 929                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 930                             }
 931                             break
 932                         case 'gcsimulator':
 933                             if (configuration == 'Release') {
 934                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 935                             }
 936                             break
 937                         case 'standalone_gc':
 938                             if (configuration == 'Release' || configuration == 'Checked') {
 939                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 940                             }
 941                             break
 942                         case 'minopts':
 943                         case 'forcerelocs':
 944                         case 'jitstress1':
 945                         case 'jitstress2':
 946                         case 'jitstressregs1':
 947                         case 'jitstressregs2':
 948                         case 'jitstressregs3':
 949                         case 'jitstressregs4':
 950                         case 'jitstressregs8':
 951                         case 'jitstressregs0x10':
 952                         case 'jitstressregs0x80':
 953                         case 'jitstressregs0x1000':
 954                         case 'jitstress2_jitstressregs1':
 955                         case 'jitstress2_jitstressregs2':
 956                         case 'jitstress2_jitstressregs3':
 957                         case 'jitstress2_jitstressregs4':
 958                         case 'jitstress2_jitstressregs8':
 959                         case 'jitstress2_jitstressregs0x10':
 960                         case 'jitstress2_jitstressregs0x80':
 961                         case 'jitstress2_jitstressregs0x1000':
 962                         case 'tailcallstress':
 963                         case 'jitsse2only':
 964                         case 'gcstress0x3':
 965                         case 'gcstress0xc':
 966                         case 'zapdisable':
 967                         case 'heapverify1':
 968                         case 'gcstress0xc_zapdisable':
 969                         case 'gcstress0xc_zapdisable_jitstress2':
 970                         case 'gcstress0xc_zapdisable_heapverify1':
 971                         case 'gcstress0xc_jitstress1':
 972                         case 'gcstress0xc_jitstress2':
 973                         case 'gcstress0xc_minopts_heapverify1':
 974                             def displayStr = getStressModeDisplayName(scenario)
 975                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 976                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 977                                "(?i).*test\\W+${os}\\W+${scenario}.*")
 978                             break
 979                         case 'corefx_baseline':
 980                         case 'corefx_minopts':
 981                         case 'corefx_jitstress1':
 982                         case 'corefx_jitstress2':
 983                         case 'corefx_jitstressregs1':
 984                         case 'corefx_jitstressregs2':
 985                         case 'corefx_jitstressregs3':
 986                         case 'corefx_jitstressregs4':
 987                         case 'corefx_jitstressregs8':
 988                         case 'corefx_jitstressregs0x10':
 989                         case 'corefx_jitstressregs0x80':
 990                         case 'corefx_jitstressregs0x1000':
 991                             def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
 992                             assert (os == 'Windows_NT') || (os in Constants.crossList)
 993                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
 994                                "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
 995                             break
 996                         case 'illink':
 997                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 998                                     break
 999                         default:
1000                             println("Unknown scenario: ${scenario}");
1001                             assert false
1002                             break
1003                     }
1004                     break
1005                 case 'FreeBSD':
1006                     assert scenario == 'default'
1007                     if (configuration == 'Checked') {
1008                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build")
1009                     }
1010                     break
1011                 default:
1012                     println("Unknown os: ${os}");
1013                     assert false
1014                     break
1015             }
1016             break
1017         // editor brace matching: }
1018         case 'arm': // editor brace matching: {
1019             assert scenario == 'default'
1020             switch (os) {
1021                 case 'Ubuntu':
1022                 case 'Ubuntu16.04':
1023                     if ((os == 'Ubuntu' && configuration == 'Release') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1024                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1025                     }
1026                     else {
1027                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1028                     }
1029                     break;
1030                 case 'Tizen':
1031                     architecture='armel'
1032                     // Removing the regex will cause this to run on each PR.
1033                     if (configuration == 'Release' || configuration == 'Debug') {
1034                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1035                     }
1036                     else {
1037                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1038                     }
1039                     break;
1040                 case 'Windows_NT':
1041                     if (configuration == 'Debug' || configuration == 'Release')
1042                     {
1043                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build")
1044                     }
1045                     break
1046                 default:
1047                     println("NYI os: ${os}");
1048                     assert false
1049                     break
1050             }
1051             break
1052         // editor brace matching: }
1053         case 'arm64': // editor brace matching: {
1054             assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1055
1056             // Set up a private trigger
1057             def contextString = "${os} ${architecture} Cross ${configuration}"
1058             if (scenario != 'default')
1059                 contextString += " ${scenario}"
1060             contextString += " Build"
1061             // Debug builds only.
1062             if (configuration != 'Debug') {
1063                contextString += " and Test"
1064             }
1065
1066             def arm64Users = [
1067                 'adiaaida',
1068                 'AndyAyersMS',
1069                 'briansull',
1070                 'BruceForstall',
1071                 'CarolEidt',
1072                 'cmckinsey',
1073                 'erozenfeld',
1074                 'jashook',
1075                 'JosephTremoulet',
1076                 'pgavlin',
1077                 'pkukol',
1078                 'russellhadley',
1079                 'RussKeldorph',
1080                 'sandreenko',
1081                 'sivarv',
1082                 'swaroop-sridhar',
1083                 'gkhanna79',
1084                 'jkotas',
1085                 'markwilkie',
1086                 'rahku',
1087                 'ramarag',
1088                 'tzwlai',
1089                 'weshaggard'
1090             ]
1091
1092             switch (os) {
1093                 case 'Windows_NT':
1094                     switch (scenario) {
1095                         case 'default':
1096                             // For now only run Debug jobs on PR Trigger.
1097                             if (configuration != 'Debug') {
1098                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1099                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*", null, arm64Users)
1100                             }
1101                             else {
1102                                 // Add "Checked Build And Test" and "Debug Build" to the above users' PRs since many of them
1103                                 // are at higher risk of ARM64-breaking changes.
1104                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1105                             }
1106                             break
1107                         case 'pri1r2r':
1108                         case 'gcstress0x3':
1109                         case 'gcstress0xc':
1110                             Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString,
1111                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*", null, arm64Users)
1112                             break
1113                     }
1114                     break
1115                 default:
1116                     println("NYI os: ${os}");
1117                     assert false
1118                     break
1119             }
1120             break
1121         // editor brace matching: }
1122         case 'x86': // editor brace matching: {
1123             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1124             if (os == 'Ubuntu') {
1125                 // on-demand only for ubuntu x86
1126                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1127                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1128                 break
1129             }
1130             switch (scenario) {
1131                 case 'default':
1132                     if (configuration == 'Checked') {
1133                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test")
1134                     }
1135                     else if (configuration == 'Release') {
1136                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1137                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1138                     }
1139                     break
1140                 case 'pri1':
1141                     if (configuration == 'Release') {
1142                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Priority 1 Build and Test")
1143                     }
1144                     break
1145                 case 'ilrt':
1146                     if (configuration == 'Release') {
1147                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1148                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1149                     }
1150                     break
1151                 case 'r2r':
1152                     if (configuration == 'Checked' || configuration == 'Release') {
1153                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri0 Build & Test",
1154                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1155                     }
1156                     break
1157                 case 'pri1r2r':
1158                     if (configuration == 'Checked' || configuration == 'Release') {
1159                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} R2R pri1 Build & Test",
1160                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1161                     }
1162                     break
1163                 case 'gcstress15_pri1r2r':
1164                     if (configuration == 'Release' || configuration == 'Checked') {
1165                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test",
1166                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1167                     }
1168                     break
1169                 case 'r2r_jitstress1':
1170                 case 'r2r_jitstress2':
1171                 case 'r2r_jitstressregs1':
1172                 case 'r2r_jitstressregs2':
1173                 case 'r2r_jitstressregs3':
1174                 case 'r2r_jitstressregs4':
1175                 case 'r2r_jitstressregs8':
1176                 case 'r2r_jitstressregs0x10':
1177                 case 'r2r_jitstressregs0x80':
1178                 case 'r2r_jitstressregs0x1000':
1179                 case 'r2r_jitminopts':
1180                 case 'r2r_jitforcerelocs':
1181                     if (configuration == 'Release' || configuration == 'Checked') {
1182                         def displayStr = getR2RStressModeDisplayName(scenario)
1183                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} R2R Build & Test",
1184                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1185                     }
1186                     break
1187                 case 'longgc':
1188                     if (configuration == 'Release') {
1189                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1190                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1191                     }
1192                     break
1193                 case 'gcsimulator':
1194                     if (configuration == 'Release') {
1195                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1196                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1197                     }
1198                     break
1199                 case 'standalone_gc':
1200                     if (configuration == 'Release' || configuration == 'Checked') {
1201                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1202                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1203                     }
1204                     break
1205                 case 'minopts':
1206                 case 'forcerelocs':
1207                 case 'jitstress1':
1208                 case 'jitstress2':
1209                 case 'jitstressregs1':
1210                 case 'jitstressregs2':
1211                 case 'jitstressregs3':
1212                 case 'jitstressregs4':
1213                 case 'jitstressregs8':
1214                 case 'jitstressregs0x10':
1215                 case 'jitstressregs0x80':
1216                 case 'jitstressregs0x1000':
1217                 case 'jitstress2_jitstressregs1':
1218                 case 'jitstress2_jitstressregs2':
1219                 case 'jitstress2_jitstressregs3':
1220                 case 'jitstress2_jitstressregs4':
1221                 case 'jitstress2_jitstressregs8':
1222                 case 'jitstress2_jitstressregs0x10':
1223                 case 'jitstress2_jitstressregs0x80':
1224                 case 'jitstress2_jitstressregs0x1000':
1225                 case 'tailcallstress':
1226                 case 'jitsse2only':
1227                 case 'gcstress0x3':
1228                 case 'gcstress0xc':
1229                 case 'zapdisable':
1230                 case 'heapverify1':
1231                 case 'gcstress0xc_zapdisable':
1232                 case 'gcstress0xc_zapdisable_jitstress2':
1233                 case 'gcstress0xc_zapdisable_heapverify1':
1234                 case 'gcstress0xc_jitstress1':
1235                 case 'gcstress0xc_jitstress2':
1236                 case 'gcstress0xc_minopts_heapverify1':
1237                     def displayStr = getStressModeDisplayName(scenario)
1238                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1239                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1240                     break
1241                 case 'corefx_baseline':
1242                 case 'corefx_minopts':
1243                 case 'corefx_jitstress1':
1244                 case 'corefx_jitstress2':
1245                 case 'corefx_jitstressregs1':
1246                 case 'corefx_jitstressregs2':
1247                 case 'corefx_jitstressregs3':
1248                 case 'corefx_jitstressregs4':
1249                 case 'corefx_jitstressregs8':
1250                 case 'corefx_jitstressregs0x10':
1251                 case 'corefx_jitstressregs0x80':
1252                 case 'corefx_jitstressregs0x1000':
1253                     def displayName = ('CoreFx ' + getStressModeDisplayName(scenario)).trim()
1254                     assert (os == 'Windows_NT') || (os in Constants.crossList)
1255                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayName})",
1256                        "(?i).*test\\W+${os}\\W+${architecture}\\W+${scenario}.*")
1257                     break
1258                 case 'illink':
1259                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1260                     break
1261                 default:
1262                     println("Unknown scenario: ${os} ${architecture} ${scenario}");
1263                     assert false
1264                     break
1265             }
1266             break
1267         // editor brace matching: }
1268         default:
1269             println("Unknown architecture: ${architecture}");
1270             assert false
1271             break
1272     }
1273 }
1274
1275 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def enableCorefxTesting, def isBuildOnly) {
1276     def buildCommands = [];
1277     def osGroup = getOSGroup(os)
1278     def lowerConfiguration = configuration.toLowerCase()
1279
1280     // Calculate the build steps, archival, and xunit results
1281     switch (os) {
1282         case 'Windows_NT': // editor brace matching: {
1283             switch (architecture) {
1284                 case 'x64':
1285                 case 'x86':
1286                     def arch = architecture
1287                     def buildOpts = ''
1288
1289                     if (scenario == 'illink') {
1290                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1291                     }
1292
1293                     if (Constants.jitStressModeScenarios.containsKey(scenario) ||
1294                             scenario == 'default' ||
1295                             scenario == 'r2r' ||
1296                             scenario == 'jitdiff' ||
1297                             scenario == 'ilrt' ||
1298                             scenario == 'illink' ||
1299                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1300                         buildOpts += enableCorefxTesting ? ' skiptests' : ''
1301                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1302                     }
1303
1304                     // For Pri 1 tests, we must shorten the output test binary path names.
1305                     // if __TestIntermediateDir is already set, build-test.cmd will
1306                     // output test binaries to that directory. If it is not set, the
1307                     // binaries are sent to a default directory whose name is about
1308                     // 35 characters long.
1309
1310                     else if (scenario == 'pri1' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'|| scenario == 'coverage') {
1311                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts} -priority=1"
1312                     }
1313                     else if (isLongGc(scenario)) {
1314                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} skiptests"
1315                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
1316                     }
1317                     else if (scenario == 'standalone_gc') {
1318                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} ${buildOpts} buildstandalonegc"
1319                     }
1320                     else if (scenario == 'formatting') {
1321                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1322                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1323                         break
1324                     }
1325                     else {
1326                         println("Unknown scenario: ${scenario}")
1327                         assert false
1328                     }
1329
1330                     // If we are running a stress mode, we should write out the set of key
1331                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1332
1333                     if (!isBuildOnly) {
1334                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
1335                         def crossgenStr = ''
1336                         def runcrossgentestsStr = ''
1337                         def runjitstressStr = ''
1338                         def runjitstressregsStr = ''
1339                         def runjitmioptsStr = ''
1340                         def runjitforcerelocsStr = ''
1341                         def runjitdisasmStr = ''
1342                         def runilasmroundtripStr = ''
1343                         def gcstressStr = ''
1344                         def runtestArguments = ''
1345                         def gcTestArguments = ''
1346                         def illinkArguments = ''
1347
1348                         if (scenario == 'r2r' ||
1349                             scenario == 'pri1r2r' ||
1350                             scenario == 'gcstress15_pri1r2r' ||
1351                             Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1352                                 crossgenStr = 'crossgen'
1353                                 runcrossgentestsStr = 'runcrossgentests'
1354
1355                                 if (scenario == 'r2r_jitstress1'){
1356                                     runjitstressStr = 'jitstress 1'
1357                                 }
1358                                 else if (scenario == 'r2r_jitstress2') {
1359                                     runjitstressStr = 'jitstress 2'
1360                                 }
1361                                 else if (scenario == 'r2r_jitstressregs1'){
1362                                     runjitstressregsStr = 'jitstressregs 1'
1363                                 }
1364                                 else if (scenario == 'r2r_jitstressregs2') {
1365                                     runjitstressregsStr = 'jitstressregs 2'
1366                                 }
1367                                 else if (scenario == 'r2r_jitstressregs3') {
1368                                     runjitstressregsStr = 'jitstressregs 3'
1369                                 }
1370                                 else if (scenario == 'r2r_jitstressregs4') {
1371                                     runjitstressregsStr = 'jitstressregs 4'
1372                                 }
1373                                 else if (scenario == 'r2r_jitstressregs8') {
1374                                     runjitstressregsStr = 'jitstressregs 8'
1375                                 }
1376                                 else if (scenario == 'r2r_jitstressregs0x10') {
1377                                     runjitstressregsStr = 'jitstressregs 0x10'
1378                                 }
1379                                 else if (scenario == 'r2r_jitstressregs0x80') {
1380                                     runjitstressregsStr = 'jitstressregs 0x80'
1381                                 }
1382                                 else if (scenario == 'r2r_jitstressregs0x1000') {
1383                                     runjitstressregsStr = 'jitstressregs 0x1000'
1384                                 }
1385                                 else if (scenario == 'r2r_jitminopts') {
1386                                     runjitmioptsStr = 'jitminopts'
1387                                 }
1388                                 else if (scenario == 'r2r_jitforcerelocs') {
1389                                     runjitforcerelocsStr = 'jitforcerelocs'
1390                                 }
1391                         }
1392                         if (scenario == 'gcstress15_pri1r2r')
1393                         {
1394                             gcstressStr = 'gcstresslevel 0xF'
1395                         }
1396
1397                         if (scenario == 'jitdiff')
1398                         {
1399                             runjitdisasmStr = 'jitdisasm crossgen'
1400                         }
1401
1402                         if (scenario == 'ilrt')
1403                         {
1404                             runilasmroundtripStr = 'ilasmroundtrip'
1405                         }
1406
1407                         if (isLongGc(scenario)) {
1408                             gcTestArguments = "${scenario} sequential"
1409                         }
1410
1411                         if (scenario == 'illink')
1412                         {
1413                             illinkArguments = "link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1414                         }
1415
1416                         runtestArguments = "${lowerConfiguration} ${arch} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${gcTestArguments} ${illinkArguments} collectdumps"
1417
1418                         if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1419                             def stepScriptLocation = "%WORKSPACE%\\SetStressModes.bat"
1420                             buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], stepScriptLocation)
1421
1422                             if (enableCorefxTesting) {
1423                                 def workspaceRelativeFxRoot = "_/fx"
1424                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1425
1426                                 buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${stepScriptLocation}"
1427
1428                                 setTestJobTimeOut(newJob, scenario)
1429
1430                                 // Archive and process (only) the test results
1431                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1432                                 Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1433                             }
1434                             else {
1435                                 buildCommands += "%WORKSPACE%\\tests\\runtest.cmd ${runtestArguments} TestEnv ${stepScriptLocation}"
1436                             }
1437                         }
1438                         else if (architecture == 'x64' || architecture == 'x86') {
1439                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1440                         }
1441                     }
1442
1443                     if (!enableCorefxTesting) {
1444                         // Run the rest of the build
1445                         // Build the mscorlib for the other OS's
1446                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1447                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} freebsdmscorlib"
1448                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1449
1450                         // Zip up the tests directory so that we don't use so much space/time copying
1451                         // 10s of thousands of files around.
1452                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1453
1454                         if (!Constants.jitStressModeScenarios.containsKey(scenario)) {
1455                             // For windows, pull full test results and test drops for x86/x64.
1456                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1457                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1458                         }
1459
1460                         if (scenario == 'jitdiff') {
1461                             // retrive jit-dasm output for base commit, and run jit-diff
1462                             if (!isBuildOnly) {
1463                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1464                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1465                             }
1466                         }
1467
1468                         if (!isBuildOnly) {
1469                             if (architecture == 'x64' || !isPR) {
1470                                 Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml')
1471                             }
1472                             setTestJobTimeOut(newJob, scenario)
1473                         }
1474                     }
1475                     break
1476                 case 'arm':
1477                     assert (scenario == 'default')
1478
1479                     // Set time out
1480                     setTestJobTimeOut(newJob, scenario)
1481
1482                     if ( lowerConfiguration == "debug" ) {
1483                         // For Debug builds, we will do a P1 test build
1484                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} -priority=1"
1485                     }
1486                     else {
1487                         buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture}"
1488                     }
1489                     // Add archival.
1490                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1491                     break
1492                 case 'arm64':
1493                     assert (scenario == 'default') || (scenario == 'pri1r2r') || (scenario == 'gcstress0x3') || (scenario == 'gcstress0xc')
1494
1495                     // Set time out
1496                     setTestJobTimeOut(newJob, scenario)
1497
1498                     // Debug runs take too long to run. So build job only.
1499                     if (lowerConfiguration == "debug") {
1500                        buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1501                     }
1502                     else {
1503                        if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1504                        {
1505                            // Up the timeout for arm64 checked testing only.
1506                            // Keep the longer timeout for gcstress.
1507                            Utilities.setJobTimeout(newJob, 240)
1508                        }
1509
1510                        buildCommands += "set __TestIntermediateDir=int&&build.cmd skiptests ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2"
1511                        // Test build and run are launched together.
1512                        buildCommands += "python tests\\scripts\\arm64_post_build.py -repo_root %WORKSPACE% -arch ${architecture} -build_type ${lowerConfiguration} -scenario ${scenario} -key_location C:\\tools\\key.txt"
1513                        //Utilities.addXUnitDotNETResults(newJob, 'bin/tests/testResults.xml')
1514                     }
1515
1516                     // Add archival.
1517                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1518                     break
1519                 default:
1520                     println("Unknown architecture: ${architecture}");
1521                     assert false
1522                     break
1523             }
1524             break
1525         // editor brace matching: }
1526         case 'Ubuntu':
1527         case 'Ubuntu16.04':
1528         case 'Ubuntu16.10':
1529         case 'Debian8.4':
1530         case 'OSX10.12':
1531         case 'FreeBSD':
1532         case 'CentOS7.1':
1533         case 'RHEL7.2':
1534         case 'OpenSUSE42.1':
1535         case 'Tizen':
1536         case 'Fedora24': // editor brace matching: {
1537             switch (architecture) {
1538                 case 'x64':
1539                 case 'x86':
1540                     if (architecture == 'x86' && os == 'Ubuntu') {
1541                         // build and PAL test
1542                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1543                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1544                         break;
1545                     }
1546
1547                     if (scenario == 'formatting') {
1548                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1549                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1550                         break
1551                     }
1552
1553                     if (scenario == 'illink') {
1554                         assert(os == 'Ubuntu')
1555                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1556                     }
1557
1558                     def standaloneGc = ''
1559                     if (scenario == 'standalone_gc') {
1560                         standaloneGc = 'buildstandalonegc'
1561                     }
1562
1563                     if (!enableCorefxTesting) {
1564                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1565                         // only on supported OS platforms.
1566                         if (os == 'FreeBSD')
1567                         {
1568                             buildCommands += "./build.sh skipmscorlib verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1569                         }
1570                         else
1571                         {
1572                             def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1573                             def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1574                             buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture} ${standaloneGc}"
1575                         }
1576                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1577
1578                         // Set time out
1579                         setTestJobTimeOut(newJob, scenario)
1580                         // Basic archiving of the build
1581                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1582                         // And pal tests
1583                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1584                     }
1585                     else {
1586                         // Corefx stress testing
1587                         assert os == 'Ubuntu'
1588                         assert architecture == 'x64'
1589                         assert lowerConfiguration == 'checked'
1590                         assert Constants.jitStressModeScenarios.containsKey(scenario)
1591
1592                         // Build coreclr
1593                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1594
1595                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1596                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1597
1598                         // Build and text corefx
1599                         def workspaceRelativeFxRoot = "_/fx"
1600                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1601
1602                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1603
1604                         setTestJobTimeOut(newJob, scenario)
1605
1606                         // Archive and process (only) the test results
1607                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1608                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1609                     }
1610                     break
1611                 case 'arm64':
1612                     // We don't run the cross build except on Ubuntu
1613                     assert os == 'Ubuntu'
1614
1615                     buildCommands += """echo \"Using rootfs in /opt/aarch64-linux-gnu-root\"
1616                         ROOTFS_DIR=/opt/aarch64-linux-gnu-root ./build.sh skipmscorlib arm64 cross verbose ${lowerConfiguration}"""
1617
1618                     // Basic archiving of the build, no pal tests
1619                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1620                     break
1621                 case 'arm':
1622                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1623                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1624
1625                     // default values for Ubuntu
1626                     def arm_abi="arm"
1627                     def linuxCodeName="trusty"
1628                     if (os == 'Ubuntu16.04') {
1629                         linuxCodeName="xenial"
1630                     }
1631                     else if (os == 'Tizen') {
1632                         arm_abi="armel"
1633                         linuxCodeName="tizen"
1634                     }
1635
1636                     // Unzip the Windows test binaries first. Exit with 0
1637                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1638
1639                     // Unpack the corefx binaries
1640                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1641                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1642                     if (os != 'Tizen') {
1643                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1644                     }
1645                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1646                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1647                     if (os == 'Ubuntu') {
1648                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1649                     }
1650
1651                     // Call the ARM CI script to cross build and test using docker
1652                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1653                     --mode=docker \\
1654                     --${arm_abi} \\
1655                     --linuxCodeName=${linuxCodeName} \\
1656                     --buildConfig=${lowerConfiguration} \\
1657                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1658                     --coreFxBinDir=./bin/CoreFxBinDir \\
1659                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1660
1661                     // Basic archiving of the build, no pal tests
1662                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1663                     break
1664                 default:
1665                     println("Unknown architecture: ${architecture}");
1666                     assert false
1667                     break
1668             }
1669             break
1670         // editor brace matching: }
1671         default:
1672             println("Unknown os: ${os}");
1673             assert false
1674             break
1675     } // os
1676
1677     return buildCommands
1678 }
1679
1680 // Additional scenario which can alter behavior
1681
1682 def combinedScenarios = Constants.basicScenarios + Constants.jitStressModeScenarios.keySet()
1683 combinedScenarios.each { scenario ->
1684     [true, false].each { isPR ->
1685         Constants.architectureList.each { architecture ->
1686             Constants.configurationList.each { configuration ->
1687                 Constants.osList.each { os ->
1688                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1689                     // and reset the os to Windows_NT
1690                     def isBuildOnly = false
1691                     if (os == 'Windows_NT_BuildOnly') {
1692                         isBuildOnly = true
1693                         os = 'Windows_NT'
1694                     }
1695
1696                     // WinArm32 is only built for Debug and Release
1697                     if (os == 'Windows_NT' && architecture == 'arm')
1698                     {
1699                         if (configuration == 'Checked')
1700                         {
1701                             return
1702                         }
1703                     }
1704                     // Tizen is only supported for arm architecture
1705                     if (os == 'Tizen' && architecture != 'arm') {
1706                         return
1707                     }
1708
1709                     // Skip totally unimplemented (in CI) configurations.
1710                     switch (architecture) {
1711                         case 'arm64':
1712                             // Windows only
1713                             if (os != 'Windows_NT' || isBuildOnly) {
1714                                 return
1715                             }
1716                             break
1717                         case 'arm':
1718                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1719                                 return
1720                             }
1721                             break
1722                         case 'x86':
1723                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1724                                 return
1725                             }
1726                             break
1727                         case 'x64':
1728                             // Everything implemented
1729                             break
1730                         default:
1731                             println("Unknown architecture: ${architecture}")
1732                             assert false
1733                             break
1734                     }
1735
1736                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1737                     // with checked builds
1738                     def enableCorefxTesting = false
1739                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
1740                         if (configuration != 'Checked') {
1741                             return
1742                         }
1743
1744                         enableCorefxTesting = isCorefxTesting(scenario)
1745
1746                         // Since these are just execution time differences,
1747                         // skip platforms that don't execute the tests here (Windows_NT only)
1748                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && enableCorefxTesting)
1749                         if (!isEnabledOS || isBuildOnly) {
1750                             return
1751                         }
1752
1753                         switch (architecture) {
1754                             case 'arm64':
1755                                 if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc')) {
1756                                     return
1757                                 }
1758                                 break
1759                             case 'x64':
1760                             case 'x86':
1761                                 // x86 ubuntu: default only
1762                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1763                                     return
1764                                 }
1765                                 // Windows: Everything implemented
1766                                 break
1767                             default:
1768                                 return
1769                         }
1770                     }
1771                     else {
1772                         // If this is a r2r jitstress, jitstressregs, jitminopts, or forcerelocs scenario
1773                         // and configuration is not Checked, bail out.
1774                         if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
1775                             return;
1776                         }
1777
1778                         // Skip scenarios
1779                         switch (scenario) {
1780                             case 'pri1':
1781                                 // The pri1 build isn't necessary except for Windows_NT.  Non-Windows NT uses
1782                                 // the default scenario build
1783                                 if (os != 'Windows_NT') {
1784                                     return
1785                                 }
1786                                 // Only x64 for now
1787                                 if (architecture != 'x64') {
1788                                     return
1789                                 }
1790                                 break
1791                             case 'ilrt':
1792                                 // The ilrt build isn't necessary except for Windows_NT.  Non-Windows NT uses
1793                                 // the default scenario build
1794                                 if (os != 'Windows_NT') {
1795                                     return
1796                                 }
1797                                 // Only x64 for now
1798                                 if (architecture != 'x64') {
1799                                     return
1800                                 }
1801                                 // Release only
1802                                 if (configuration != 'Release') {
1803                                     return
1804                                 }
1805                                 break
1806                             case 'jitdiff':
1807                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1808                                     return
1809                                 }
1810                                 if (architecture != 'x64') {
1811                                     return
1812                                 }
1813                                 if (configuration != 'Checked') {
1814                                     return
1815                                 }
1816                                 break
1817                             case 'r2r':
1818                                 // The r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1819                                 // the default scenario build
1820                                 if (os != 'Windows_NT') {
1821                                     return
1822                                 }
1823                                 if (architecture != 'x64') {
1824                                     return
1825                                 }
1826                                 break
1827                             case 'pri1r2r':
1828                                 // The pri1r2r build isn't necessary except for Windows_NT.  Non-Windows NT uses
1829                                 // the default scenario build
1830                                 if (os != 'Windows_NT') {
1831                                     return
1832                                 }
1833                                 if (architecture != 'x64') {
1834                                     if (architecture != 'arm64' || configuration == 'Debug') {
1835                                         return
1836                                     }
1837                                 }
1838                                 break
1839                             case 'gcstress15_pri1r2r':
1840                             case 'r2r_jitstress1':
1841                             case 'r2r_jitstress2':
1842                             case 'r2r_jitstressregs1':
1843                             case 'r2r_jitstressregs2':
1844                             case 'r2r_jitstressregs3':
1845                             case 'r2r_jitstressregs4':
1846                             case 'r2r_jitstressregs8':
1847                             case 'r2r_jitstressregs0x10':
1848                             case 'r2r_jitstressregs0x80':
1849                             case 'r2r_jitstressregs0x1000':
1850                             case 'r2r_jitminopts':
1851                             case 'r2r_jitforcerelocs':
1852                                 // The above builds are not necessary except for Windows_NT.  Non-Windows NT uses
1853                                 // the default scenario build
1854                                 if (os != 'Windows_NT') {
1855                                     return
1856                                 }
1857                                 if (architecture != 'x64' && architecture != 'x86') {
1858                                     return
1859                                 }
1860                                 break
1861                             case 'longgc':
1862                             case 'gcsimulator':
1863                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1864                                     return
1865                                 }
1866                                 if (architecture != 'x64') {
1867                                     return
1868                                 }
1869                                 if (configuration != 'Release') {
1870                                     return
1871                                 }
1872                                 break
1873                             case 'standalone_gc':
1874                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1875                                     return
1876                                 }
1877
1878                                 if (architecture != 'x64') {
1879                                     return
1880                                 }
1881
1882                                 if (configuration != 'Release' && configuration != 'Checked') {
1883                                     return
1884                                 }
1885                                 break
1886                             // We need Windows x64 Release bits for the code coverage build
1887                             case 'coverage':
1888                                 if (os != 'Windows_NT') {
1889                                     return
1890                                 }
1891                                 if (architecture != 'x64') {
1892                                     return
1893                                 }
1894                                 if (configuration != 'Release') {
1895                                     return
1896                                 }
1897                                 break
1898                             // We only run Windows and Ubuntu x64 Checked for formatting right now
1899                             case 'formatting':
1900                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
1901                                     return
1902                                 }
1903                                 if (architecture != 'x64') {
1904                                     return
1905                                 }
1906                                 if (configuration != 'Checked') {
1907                                     return
1908                                 }
1909                                 if (isBuildOnly) {
1910                                     return
1911                                 }
1912                                 break
1913                             case 'illink':
1914                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
1915                                     return
1916                                 }
1917                                 if (architecture != 'x64' && architecture != 'x86') {
1918                                     return
1919                                 }
1920                                 if (isBuildOnly) {
1921                                     return
1922                                 }
1923                                 break
1924                             case 'default':
1925                                 // Nothing skipped
1926                                 break
1927                             default:
1928                                 println("Unknown scenario: ${scenario}")
1929                                 assert false
1930                                 break
1931                         }
1932                     }
1933
1934                     // Calculate names
1935                     def lowerConfiguration = configuration.toLowerCase()
1936                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
1937                     def folderName = getJobFolder(scenario)
1938
1939                     // Create the new job
1940                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
1941
1942                     setMachineAffinity(newJob, os, architecture)
1943
1944                     // Add all the standard options
1945                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
1946                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
1947
1948                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, enableCorefxTesting, isBuildOnly)
1949                     def osGroup = getOSGroup(os)
1950
1951                     newJob.with {
1952                         steps {
1953                             if (os == 'Windows_NT') {
1954                                 buildCommands.each { buildCommand ->
1955                                     batchFile(buildCommand)
1956                                 }
1957                             }
1958                             else {
1959                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
1960                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
1961                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
1962                                     // Define the Windows Tests and Corefx build job names
1963                                     def WindowTestsName = projectFolder + '/' +
1964                                                           Utilities.getFullJobName(project,
1965                                                                                    getJobName(lowerConfiguration,
1966                                                                                               'x64' ,
1967                                                                                               'windows_nt',
1968                                                                                               'default',
1969                                                                                               true),
1970                                                                                    false)
1971                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
1972                                                        Utilities.getFolderName(branch)
1973
1974                                     // Copy the Windows test binaries and the Corefx build binaries
1975                                     copyArtifacts(WindowTestsName) {
1976                                         excludePatterns('**/testResults.xml', '**/*.ni.dll')
1977                                         buildSelector {
1978                                             latestSuccessful(true)
1979                                         }
1980                                     }
1981
1982                                     // Defaults for Ubuntu
1983                                     def arm_abi = 'arm'
1984                                     def corefx_os = 'ubuntu14.04'
1985                                     if (os == 'Ubuntu16.04') {
1986                                         arm_abi = 'arm'
1987                                         corefx_os = 'ubuntu16.04'
1988                                     }
1989                                     else if (os == 'Tizen') {
1990                                         arm_abi = 'armel'
1991                                         corefx_os = 'tizen'
1992                                     }
1993
1994                                     // Let's use release CoreFX to test checked CoreCLR,
1995                                     // because we do not generate checked CoreFX in CoreFX CI yet.
1996                                     def corefx_lowerConfiguration = lowerConfiguration
1997                                     if ( lowerConfiguration == 'checked' ) {
1998                                         corefx_lowerConfiguration='release'
1999                                     }
2000                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2001                                         includePatterns('bin/build.tar.gz')
2002                                         buildSelector {
2003                                             latestSuccessful(true)
2004                                         }
2005                                     }
2006                                 }
2007
2008                                 buildCommands.each { buildCommand ->
2009                                     shell(buildCommand)
2010                                 }
2011                             }
2012                         }
2013                     } // newJob.with
2014
2015                 } // os
2016             } // configuration
2017         } // architecture
2018     } // isPR
2019 } // scenario
2020
2021
2022 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
2023 combinedScenarios.each { scenario ->
2024     [true, false].each { isPR ->
2025         // Architectures.  x64 only at this point
2026         ['x64'].each { architecture ->
2027             // Put the OS's supported for coreclr cross testing here
2028             Constants.crossList.each { os ->
2029                 Constants.configurationList.each { configuration ->
2030
2031                     if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2032                         if (configuration != 'Checked') {
2033                             return
2034                         }
2035                         if (isCorefxTesting(scenario)) {
2036                             return
2037                         }
2038                         //Skip stress modes for these scenarios
2039                         if (os == 'RHEL7.2' || os == 'Debian8.4') {
2040                             return
2041                         }
2042                     }
2043                     // If this is a r2r jitstress, jitstressregs, jitminopts or forcerelocs scenario
2044                     // and configuration is not Checked, bail out.
2045                     else if (configuration != 'Checked' && Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2046                         return;
2047                     }
2048                     // For CentOS, we only want Checked/Release pri1 builds.
2049                     else if (os == 'CentOS7.1') {
2050                         if (scenario != 'pri1' &&
2051                             scenario != 'r2r' &&
2052                             scenario != 'pri1r2r' &&
2053                             scenario != 'gcstress15_pri1r2r' &&
2054                             Constants.r2rJitStressScenarios.indexOf(scenario) == -1) {
2055                             return
2056                         }
2057                         if (configuration != 'Checked' && configuration != 'Release') {
2058                             return
2059                         }
2060                     }
2061                     // For RedHat and Debian, we only do Release pri1 builds.
2062                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2063                         if (scenario != 'pri1') {
2064                             return
2065                         }
2066                         if (configuration != 'Release') {
2067                             return
2068                         }
2069                     }
2070                     else {
2071                         // Skip scenarios
2072                         switch (scenario) {
2073                             case 'pri1':
2074                                 // Nothing skipped
2075                                 break
2076                             case 'ilrt':
2077                                 // Release only
2078                                 if (configuration != 'Release') {
2079                                     return
2080                                 }
2081                                 break
2082                             case 'jitdiff':
2083                                 if (configuration != 'Checked') {
2084                                     return;
2085                                 }
2086                             case 'r2r':
2087                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2088                                 if (configuration != 'Checked' && configuration != 'Release') {
2089                                     return
2090                                 }
2091                                 break
2092                             case 'pri1r2r':
2093                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2094                                 if (configuration != 'Checked' && configuration != 'Release') {
2095                                     return
2096                                 }
2097                                 break
2098                             case 'gcstress15_pri1r2r':
2099                             case 'r2r_jitstress1':
2100                             case 'r2r_jitstress2':
2101                             case 'r2r_jitstressregs1':
2102                             case 'r2r_jitstressregs2':
2103                             case 'r2r_jitstressregs3':
2104                             case 'r2r_jitstressregs4':
2105                             case 'r2r_jitstressregs8':
2106                             case 'r2r_jitstressregs0x10':
2107                             case 'r2r_jitstressregs0x80':
2108                             case 'r2r_jitstressregs0x1000':
2109                             case 'r2r_jitminopts':
2110                             case 'r2r_jitforcerelocs':
2111                                 //Skip configs that aren't Checked or Release (so just Debug, for now)
2112                                 if (configuration != 'Checked' && configuration != 'Release') {
2113                                     return
2114                                 }
2115                                 break
2116                             case 'longgc':
2117                             case 'gcsimulator':
2118                                 // Long GC tests take a long time on non-Release builds
2119                                 if (configuration != 'Release') {
2120                                     return
2121                                 }
2122                                 break
2123                             case 'standalone_gc':
2124                                 if (configuration != 'Release' && configuration != 'Checked') {
2125                                     return
2126                                 }
2127                             case 'coverage':
2128                                 //We only want Ubuntu Release for coverage
2129                                 if (os != 'Ubuntu') {
2130                                     return
2131                                 }
2132                                 if (configuration != 'Release') {
2133                                     return
2134                                 }
2135                             case 'formatting':
2136                                 return
2137                             case 'illink':
2138                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2139                                     return
2140                                 }
2141                                 break
2142                             case 'default':
2143                                 // Nothing skipped
2144                                 break
2145                             default:
2146                                 println("Unknown scenario: ${scenario}")
2147                                 assert false
2148                                 break
2149                         }
2150                     }
2151
2152                     def lowerConfiguration = configuration.toLowerCase()
2153                     def osGroup = getOSGroup(os)
2154                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2155
2156                     // Unless this is a coverage test run, we want to copy over the default build of coreclr.
2157                     def inputScenario = 'default'
2158                     if (scenario == 'coverage') {
2159                         inputScenario = 'coverage'
2160                     }
2161                     def inputCoreCLRBuildName = projectFolder + '/' +
2162                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputScenario, false), isPR)
2163                     // If this is a stress scenario, there isn't any difference in the build job
2164                     // so we didn't create a build only job for windows_nt specific to that stress mode.  Just copy
2165                     // from the default scenario
2166                     def testBuildScenario = scenario
2167                     if (testBuildScenario == 'coverage' || testBuildScenario == 'pri1r2r'|| testBuildScenario == 'gcstress15_pri1r2r') {
2168                         testBuildScenario = 'pri1'
2169                     }
2170                     else if ( testBuildScenario == 'r2r' || Constants.r2rJitStressScenarios.indexOf(testBuildScenario) != -1 || isLongGc(testBuildScenario)) {
2171                         testBuildScenario = 'default'
2172                     }
2173                     def inputWindowTestsBuildName = ''
2174                     if (Constants.jitStressModeScenarios.containsKey(testBuildScenario)) {
2175                         inputWindowTestsBuildName = projectFolder + '/' +
2176                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', 'default', true), isPR)
2177                     }
2178                     else {
2179                         inputWindowTestsBuildName = projectFolder + '/' +
2180                             Utilities.getFullJobName(project, getJobName(configuration, architecture, 'windows_nt', testBuildScenario, true), isPR)
2181                     }
2182                     // Enable Server GC for Ubuntu PR builds
2183                     def serverGCString = ''
2184
2185                     // Whether or not this test run should be run sequentially instead
2186                     // of in parallel. Only used for long GC tests.
2187                     def sequentialString = ''
2188
2189                     // Whether or not this test run should run a specific playlist.
2190                     // Only used for long GC tests.
2191
2192                     // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2193                     // for running long GC and GCSimulator tests, respectively. We don't use them
2194                     // here because using a playlist file produces much more readable output on the CI machines
2195                     // and reduces running time.
2196                     def playlistString = ''
2197
2198                     if (os == 'Ubuntu' && isPR){
2199                         serverGCString = '--useServerGC'
2200                     }
2201
2202                     // pass --crossgen to runtest.sh for crossgen builds
2203                     def crossgenStr = ''
2204                     def runcrossgentestsStr = ''
2205                     def runjitstressStr = ''
2206                     def runjitstressregsStr = ''
2207                     def runjitmioptsStr = ''
2208                     def runjitforcerelocsStr = ''
2209                     def runjitdisasmStr = ''
2210                     def runilasmroundtripStr = ''
2211                     def gcstressStr = ''
2212                     def illinkStr = ''
2213
2214                     if (scenario == 'r2r' ||
2215                         scenario == 'pri1r2r' ||
2216                         scenario == 'gcstress15_pri1r2r' ||
2217                         Constants.r2rJitStressScenarios.indexOf(scenario) != -1) {
2218                             crossgenStr = '--crossgen'
2219                             runcrossgentestsStr = '--runcrossgentests'
2220
2221                             if (scenario == 'r2r_jitstress1'){
2222                                 runjitstressStr = '--jitstress=1'
2223                             }
2224                             else if (scenario == 'r2r_jitstress2') {
2225                                 runjitstressStr = '--jitstress=2'
2226                             }
2227                             else if (scenario == 'r2r_jitstressregs1'){
2228                                 runjitstressregsStr = '--jitstressregs=1'
2229                             }
2230                             else if (scenario == 'r2r_jitstressregs2') {
2231                                 runjitstressregsStr = '--jitstressregs=2'
2232                             }
2233                             else if (scenario == 'r2r_jitstressregs3') {
2234                                 runjitstressregsStr = '--jitstressregs=3'
2235                             }
2236                             else if (scenario == 'r2r_jitstressregs4') {
2237                                 runjitstressregsStr = '--jitstressregs=4'
2238                             }
2239                             else if (scenario == 'r2r_jitstressregs8') {
2240                                 runjitstressregsStr = '--jitstressregs=8'
2241                             }
2242                             else if (scenario == 'r2r_jitstressregs0x10') {
2243                                 runjitstressregsStr = '--jitstressregs=0x10'
2244                             }
2245                             else if (scenario == 'r2r_jitstressregs0x80') {
2246                                 runjitstressregsStr = '--jitstressregs=0x80'
2247                             }
2248                             else if (scenario == 'r2r_jitstressregs0x1000') {
2249                                 runjitstressregsStr = '--jitstressregs=0x1000'
2250                             }
2251                             else if (scenario == 'r2r_jitminopts') {
2252                                 runjitmioptsStr = '--jitminopts'
2253                             }
2254                             else if (scenario == 'r2r_jitforcerelocs') {
2255                                 runjitforcerelocsStr = '--jitforcerelocs'
2256                             }
2257                     }
2258                     if  (scenario == 'gcstress15_pri1r2r')
2259                     {
2260                         gcstressStr = '--gcstresslevel=0xF'
2261                     }
2262
2263                     if (scenario == 'jitdiff')
2264                     {
2265                         runjitdisasmStr = '--jitdisasm --crossgen'
2266                     }
2267
2268                     if (scenario == 'illink')
2269                     {
2270                         illinkStr = '--link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2271                     }
2272
2273                     if (isLongGc(scenario)) {
2274                         // Long GC tests behave very poorly when they are not
2275                         // the only test running (many of them allocate until OOM).
2276                         sequentialString = '--sequential'
2277
2278                         // The Long GC playlist contains all of the tests that are
2279                         // going to be run. The GCSimulator playlist contains all of
2280                         // the GC simulator tests.
2281                         if (scenario == 'longgc') {
2282                             playlistString = '--long-gc --playlist=./tests/longRunningGcTests.txt'
2283                         }
2284                         else if (scenario == 'gcsimulator') {
2285                             playlistString = '--gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2286                         }
2287                     }
2288
2289                     def folder = getJobFolder(scenario)
2290                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2291                         // Add parameters for the inputs
2292
2293                         parameters {
2294                             stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2295                             stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2296                         }
2297
2298                         steps {
2299                             // Set up the copies
2300
2301                             // Coreclr build containing the tests and mscorlib
2302
2303                             copyArtifacts(inputWindowTestsBuildName) {
2304                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2305                                 buildSelector {
2306                                     buildNumber('${CORECLR_WINDOWS_BUILD}')
2307                                 }
2308                             }
2309
2310                             if (scenario == 'coverage') {
2311
2312                                 // Move coreclr to clr directory
2313                                 shell("rm -rf .clr; mkdir .clr; mv * .clr; mv .git .clr; mv .clr clr")
2314
2315                                 // Build coreclr
2316                                 shell("./clr/build.sh coverage verbose ${lowerConfiguration} ${architecture}")
2317
2318                                 // Remove folders from obj that we don't expect to be covered. May update this later.
2319                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ToolBox")
2320                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/debug")
2321                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ilasm")
2322                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/ildasm")
2323                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/dbgshim")
2324                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordac")
2325                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/dlls/mscordbi")
2326
2327                                 // Run PAL tests
2328                                 shell("./clr/src/pal/tests/palsuite/runpaltests.sh \$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration} \$(pwd)/clr/bin/paltestout")
2329
2330                                 // Remove obj files for PAL tests so they're not included in coverage results
2331                                 shell("rm -rf ./clr/bin/obj/Linux.x64.Release/src/pal/tests")
2332
2333                                 // Unzip the tests first.  Exit with 0
2334                                 shell("unzip -q -o ./clr/bin/tests/tests.zip -d ./clr/bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2335
2336                                 // Get corefx
2337                                 shell("git clone https://github.com/dotnet/corefx fx")
2338
2339                                 // Build Linux corefx
2340                                 shell("./fx/build-native.sh -release -buildArch=x64 -os=Linux")
2341                                 shell("./fx/build-managed.sh -release -buildArch=x64 -osgroup=Linux -skiptests")
2342
2343                                 def testEnvOpt = ""
2344                                 def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2345                                 def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios['heapverify1'], scriptFileName)
2346                                 shell("${createScriptCmds}")
2347                                 testEnvOpt = "--test-env=" + scriptFileName
2348
2349                                 // Run corefx tests
2350                                 shell("""./fx/run-test.sh \\
2351                 --coreclr-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2352                 --mscorlib-bins \$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration} \\
2353                 --corefx-tests \$(pwd)/fx/bin/tests/${osGroup}.AnyCPU.${configuration} \\
2354                 --corefx-native-bins \$(pwd)/fx/bin/${osGroup}.${architecture}.${configuration} \\
2355                 --configurationGroup Release""")
2356
2357
2358                                 // Run coreclr tests w/ workstation GC
2359                                 shell("""./clr/tests/runtest.sh \\
2360                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2361                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2362                 --coreClrBinDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2363                 --mscorlibDir=\"\$(pwd)/clr/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2364                 --coreFxBinDir=\"\$(pwd)/fx/bin/runtime/netcoreapp-${osGroup}-Release-${architecture}\" \\
2365                 --crossgen --runcrossgentests""")
2366
2367                                 // Run coreclr tests w/ server GC & HeapVerify enabled
2368                                 shell("""./clr/tests/runtest.sh \\
2369                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2370                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2371                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2372                 --useServerGC ${testEnvOpt}""")
2373
2374                                  // Run long-running coreclr GC tests & produce coverage reports
2375                                 shell("""./clr/tests/runtest.sh \\
2376                 --testRootDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2377                 --testNativeBinDir=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2378                 --coreOverlayDir=\"\$(pwd)/clr/bin/tests/Windows_NT.${architecture}.${configuration}/Tests/coreoverlay\" \\
2379                 --long-gc --playlist=\"\$(pwd)/clr/tests/longRunningGcTests.txt\" --coreclr-coverage\\
2380                 --coreclr-objs=\"\$(pwd)/clr/bin/obj/${osGroup}.${architecture}.${configuration}\" \\
2381                 --coreclr-src=\"\$(pwd)/clr/src\" \\
2382                 --coverage-output-dir=\"\${WORKSPACE}/coverage\" """)
2383
2384                             }
2385                             else {
2386
2387                                 // Coreclr build we are trying to test
2388
2389                                 copyArtifacts(inputCoreCLRBuildName) {
2390                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2391                                     buildSelector {
2392                                         buildNumber('${CORECLR_BUILD}')
2393                                     }
2394                                 }
2395
2396                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2397
2398                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2399                                 def osJobName
2400                                 if (os == 'Ubuntu') {
2401                                     osJobName = 'ubuntu14.04'
2402                                 }
2403                                 else {
2404                                     osJobName = os.toLowerCase()
2405                                 }
2406                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2407                                     includePatterns('bin/build.tar.gz')
2408                                     buildSelector {
2409                                         latestSuccessful(true)
2410                                     }
2411                                 }
2412
2413                                 shell ("mkdir ./bin/CoreFxBinDir")
2414                                 // Unpack the corefx binaries
2415                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2416
2417                                 // Unzip the tests first.  Exit with 0
2418                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2419
2420                                 // Execute the tests
2421                                 // If we are running a stress mode, we'll set those variables first
2422                                 def testEnvOpt = ""
2423                                 if (Constants.jitStressModeScenarios.containsKey(scenario)) {
2424                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2425                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2426                                     shell("${createScriptCmds}")
2427                                     testEnvOpt = "--test-env=" + scriptFileName
2428                                 }
2429
2430                                 if (isGCStressRelatedTesting(scenario)) {
2431                                     shell('./init-tools.sh')
2432                                 }
2433
2434                                 shell("""./tests/runtest.sh \\
2435                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2436                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2437                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2438                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2439                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2440                 --limitedDumpGeneration \\
2441                 ${testEnvOpt} ${serverGCString} ${gcstressStr} ${crossgenStr} ${runcrossgentestsStr} ${runjitstressStr} ${runjitstressregsStr} ${runjitmioptsStr} ${runjitforcerelocsStr} ${runjitdisasmStr} ${runilasmroundtripStr} ${illinkStr} ${sequentialString} ${playlistString}""")
2442                             }
2443                         }
2444                     }
2445
2446                     if (scenario == 'coverage') {
2447                         // Publish coverage reports
2448                         Utilities.addHtmlPublisher(newJob, '${WORKSPACE}/coverage/Coverage/reports', 'Code Coverage Report', 'coreclr.html')
2449                         // TODO: Add once external email sending is available again
2450                         // addEmailPublisher(newJob, 'clrcoverage@microsoft.com')
2451                     }
2452
2453                     if (scenario == 'jitdiff') {
2454                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2455                     }
2456
2457                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2458                     if (os in ['Ubuntu']) {
2459                         SummaryBuilder summaries = new SummaryBuilder()
2460                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2461                         summaries.emit(newJob)
2462                     }
2463
2464                     setMachineAffinity(newJob, os, architecture)
2465                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2466                     // Set timeouts to 240.
2467                     setTestJobTimeOut(newJob, scenario)
2468                     Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2469
2470                     // Create a build flow to join together the build and tests required to run this
2471                     // test.
2472                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2473                     // Linux CoreCLR test
2474                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2475                     def fullTestJobName = projectFolder + '/' + newJob.name
2476                     // Add a reference to the input jobs for report purposes
2477                     JobReport.Report.addReference(inputCoreCLRBuildName)
2478                     JobReport.Report.addReference(inputWindowTestsBuildName)
2479                     JobReport.Report.addReference(fullTestJobName)
2480                     def newFlowJob;
2481
2482                     // If this is a coverage job, we don't copy any input coreCLR build - instead, we build it as part of the flow job,
2483                     // so that coverage data can be preserved.
2484                     if (scenario == 'coverage') {
2485                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2486                         buildFlow("""
2487 // Build the input Windows job
2488 windowsBuildJob = build(params, '${inputWindowTestsBuildName}')
2489
2490 // And then build the test build
2491 build(params + [CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2492 """)
2493                         }
2494                     // Normal jobs copy a Windows build & a non-Windows build
2495                     } else {
2496                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2497                         buildFlow("""
2498 // Build the input jobs in parallel
2499 parallel (
2500     { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2501     { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2502 )
2503
2504 // And then build the test build
2505 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2506                 CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2507 """)
2508                         }
2509                     }
2510
2511                     setMachineAffinity(newFlowJob, os, architecture)
2512                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2513                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2514                 } // configuration
2515             } // os
2516         } // architecture
2517     } // isPR
2518 } // scenario
2519
2520 JobReport.Report.generateJobReport(out)
2521
2522 // Make the call to generate the help job
2523 Utilities.createHelperJob(this, project, branch,
2524     "Welcome to the ${project} Repository",  // This is prepended to the help message
2525     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2526
2527 Utilities.addCROSSCheck(this, project, branch)