netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'CentOS7.1',
  49                'RHEL7.2',
  50                'Ubuntu16.04',
  51                'Ubuntu16.10',
  52                'Fedora24',
  53                'Tizen']
  54
  55     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  56
  57     // This is a set of JIT stress modes combined with the set of variables that
  58     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  59     // the values are the environment variables
  60     def static jitStressModeScenarios = [
  61                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  62                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  63                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  64                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  65                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  66                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  67                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  68                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  69                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  70                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  71                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  72                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  73                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  74                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  75                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  76                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  77                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  78                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  79                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  80                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  81                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  82                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  83                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  84                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  85                'corefx_baseline'                : [ : ], // corefx baseline
  86                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  87                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  88                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  89                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  90                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  91                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  92                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  93                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  94                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  95                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  96                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  97                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  98                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  99                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 100                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 101                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 102                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 103                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 104                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 105                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 106                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 107                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 108     ]
 109
 110     // This is a set of ReadyToRun stress scenarios
 111     def static r2rStressScenarios = [
 112                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 113                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 114                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 115                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 116                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 117                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 118                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 119                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 120                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 121                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 122                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 123                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 124                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 125     ]
 126
 127     // This is the basic set of scenarios
 128     def static basicScenarios = [
 129                'default',
 130                'ilrt',
 131                'r2r',
 132                'longgc',
 133                'formatting',
 134                'gcsimulator',
 135                'jitdiff',
 136                'standalone_gc',
 137                'gc_reliability_framework',
 138                'illink'] + r2rStressScenarios.keySet()
 139
 140     def static allScenarios = basicScenarios + jitStressModeScenarios.keySet()
 141
 142     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 143     // to Tests.lst file categories to exclude.
 144     //
 145     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 146     // and with the same values, with some commented out, for easier maintenance.
 147     //
 148     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 149     //
 150     def static validArmWindowsScenarios = [
 151                'default':                                [],
 152                // 'ilrt'
 153                // 'r2r':                                    ["R2R_FAIL"],
 154                // 'longgc'
 155                // 'formatting'
 156                // 'gcsimulator'
 157                // 'jitdiff'
 158                // 'standalone_gc'
 159                // 'gc_reliability_framework'
 160                // 'illink'
 161                // 'r2r_jitstress1'
 162                // 'r2r_jitstress2'
 163                // 'r2r_jitstressregs1'
 164                // 'r2r_jitstressregs2'
 165                // 'r2r_jitstressregs3'
 166                // 'r2r_jitstressregs4'
 167                // 'r2r_jitstressregs8'
 168                // 'r2r_jitstressregs0x10'
 169                // 'r2r_jitstressregs0x80'
 170                // 'r2r_jitstressregs0x1000'
 171                // 'r2r_jitminopts'
 172                // 'r2r_jitforcerelocs'
 173                // 'r2r_gcstress15'
 174                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 175                // 'tieredcompilation'
 176                // 'forcerelocs'
 177                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 179                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 180                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 181                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 196                // 'jitsse2only'                          // Only relevant to xarch
 197                // 'jitnosimd'
 198                // 'corefx_baseline'
 199                // 'corefx_minopts'
 200                // 'corefx_tieredcompilation'
 201                // 'corefx_jitstress1'
 202                // 'corefx_jitstress2'
 203                // 'corefx_jitstressregs1'
 204                // 'corefx_jitstressregs2'
 205                // 'corefx_jitstressregs3'
 206                // 'corefx_jitstressregs4'
 207                // 'corefx_jitstressregs8'
 208                // 'corefx_jitstressregs0x10'
 209                // 'corefx_jitstressregs0x80'
 210                // 'corefx_jitstressregs0x1000'
 211                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 212                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 213                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 214                // 'heapverify1'
 215                // 'gcstress0xc_zapdisable'
 216                // 'gcstress0xc_zapdisable_jitstress2'
 217                // 'gcstress0xc_zapdisable_heapverify1'
 218                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 219                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 220                // 'gcstress0xc_minopts_heapverify1'
 221
 222                //
 223                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 224                //
 225
 226                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 227                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 228                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 245     ]
 246
 247     def static configurationList = ['Debug', 'Checked', 'Release']
 248
 249     // This is the set of architectures
 250     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86', 'x86lb']
 251 }
 252
 253 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 254     assert os instanceof String
 255     assert architecture instanceof String
 256
 257     def armArches = ['arm', 'armlb', 'arm64']
 258     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 259
 260     if (!(architecture in armArches)) {
 261         assert options == null
 262         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 263
 264         return
 265     }
 266
 267     // This is an arm(64) job.
 268     //
 269     // There are several options.
 270     //
 271     // Windows_NT
 272     //
 273     // Arm32 (Build) -> latest-arm64
 274     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 275     // Arm32 (Test)  -> arm64-windows_nt
 276     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 277     //
 278     // Arm64 (Build) -> latest-arm64
 279     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 280     // Arm64 (Test)  -> arm64-windows_nt
 281     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 282     //
 283     // Ubuntu
 284     //
 285     // Arm32 (Build) -> arm-cross-latest
 286     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 287     // Arm32 (Test)  -> NYI Arch not supported
 288     //       |->
 289     //
 290     // Arm64 (Build) -> arm64-cross-latest
 291     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 292     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 293     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 294     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 295     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 296
 297     // This has to be a arm arch
 298     assert architecture in armArches
 299     if (os == "Windows_NT") {
 300         // Arm(64) windows jobs share the same machines for now
 301         def isBuild = options['use_arm64_build_machine'] == true
 302
 303         if (isBuild == true) {
 304             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 305         } else {
 306             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 307         }
 308     } else {
 309         assert os != 'Windows_NT'
 310         assert os in supportedArmLinuxOs
 311
 312         if (architecture == 'arm' || architecture == 'armlb') {
 313             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 314         } else {
 315             // Arm64 Linux
 316             if (options['is_build_only'] == true) {
 317                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 318             } else {
 319                 // Arm64 Test Machines
 320                 if (options['large_pages'] == false) {
 321                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 322                 } else {
 323                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 324                 }
 325             }
 326         }
 327     }
 328 }
 329
 330 def static isGCStressRelatedTesting(def scenario) {
 331     // The 'r2r_gcstress15' scenario is a basic scenario.
 332     // Detect it and make it a GCStress related.
 333     if (scenario == 'r2r_gcstress15')
 334     {
 335         return true;
 336     }
 337
 338     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 339     def scenarioName = scenario.toLowerCase()
 340     def isGCStressTesting = false
 341     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 342         if (k in gcStressTestEnvVars) {
 343             isGCStressTesting = true;
 344         }
 345     }
 346     return isGCStressTesting
 347 }
 348
 349 def static isCoreFxScenario(def scenario) {
 350     def corefx_prefix = 'corefx_'
 351     if (scenario.length() < corefx_prefix.length()) {
 352         return false
 353     }
 354     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 355 }
 356
 357 def static isR2RBaselineScenario(def scenario) {
 358     return (scenario == 'r2r')
 359 }
 360
 361 def static isR2RStressScenario(def scenario) {
 362     return Constants.r2rStressScenarios.containsKey(scenario)
 363 }
 364
 365 def static isR2RScenario(def scenario) {
 366     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 367 }
 368
 369 def static isJitStressScenario(def scenario) {
 370     return Constants.jitStressModeScenarios.containsKey(scenario)
 371 }
 372
 373 def static isLongGc(def scenario) {
 374     return (scenario == 'longgc' || scenario == 'gcsimulator')
 375 }
 376
 377 def static isJitDiff(def scenario) {
 378     return (scenario == 'jitdiff')
 379 }
 380
 381 def static isGcReliabilityFramework(def scenario) {
 382     return (scenario == 'gc_reliability_framework')
 383 }
 384
 385 def static isArmWindowsScenario(def scenario) {
 386     return Constants.validArmWindowsScenarios.containsKey(scenario)
 387 }
 388
 389 def static setTestJobTimeOut(newJob, scenario) {
 390     if (isGCStressRelatedTesting(scenario)) {
 391         Utilities.setJobTimeout(newJob, 4320)
 392     }
 393     else if (isCoreFxScenario(scenario)) {
 394         Utilities.setJobTimeout(newJob, 360)
 395     }
 396     else if (isJitStressScenario(scenario)) {
 397         Utilities.setJobTimeout(newJob, 240)
 398     }
 399     else if (isR2RBaselineScenario(scenario)) {
 400         Utilities.setJobTimeout(newJob, 240)
 401     }
 402     else if (isLongGc(scenario)) {
 403         Utilities.setJobTimeout(newJob, 1440)
 404     }
 405     else if (isJitDiff(scenario)) {
 406         Utilities.setJobTimeout(newJob, 240)
 407     }
 408     else if (isGcReliabilityFramework(scenario)) {
 409         Utilities.setJobTimeout(newJob, 1440)
 410     }
 411     // Non-test jobs use the default timeout value.
 412 }
 413
 414 def static getJobFolder(def scenario) {
 415     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 416         return 'jitstress'
 417     }
 418     if (scenario == 'illink') {
 419         return 'illink'
 420     }
 421     return ''
 422 }
 423
 424 def static getStressModeDisplayName(def scenario) {
 425     def displayStr = ''
 426     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 427         def prefixLength = 'COMPlus_'.length()
 428         if (k.length() >= prefixLength) {
 429             def modeName = k.substring(prefixLength, k.length())
 430             displayStr += ' ' + modeName + '=' + v
 431         }
 432     }
 433
 434     if (isCoreFxScenario(scenario)) {
 435         displayStr = ('CoreFx ' + displayStr).trim()
 436     }
 437
 438     return displayStr
 439 }
 440
 441 def static getR2RDisplayName(def scenario) {
 442     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 443     def displayStr = scenario
 444     def prefixLength = 'r2r_'.length()
 445     if (displayStr.length() >= prefixLength) {
 446         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 447     } else if (scenario == 'r2r') {
 448         displayStr = "R2R"
 449     }
 450     return displayStr
 451 }
 452
 453 // Generates the string for creating a file that sets environment variables
 454 // that makes it possible to run stress modes.  Writes the script to the file
 455 // specified by the stepScriptLocation parameter.
 456 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 457     def stepScript = ''
 458     if (os == 'Windows_NT') {
 459         // Timeout in ms, default is 10 minutes. For stress modes up this to 30 minutes.
 460         // BUG?: it seems this is ignored, as this script will be run in an environment where
 461         //       environment variables will be discarded before this is used.
 462         def timeout = 1800000
 463         stepScript += "set __TestTimeout=${timeout}\r\n"
 464
 465         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 466         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 467
 468         // Create at least an empty script.
 469         stepScript += "echo. > ${stepScriptLocation}\r\n"
 470
 471         stressModeVars.each{ k, v ->
 472             // Write out what we are writing to the script file
 473             stepScript += "echo Setting ${k}=${v}\r\n"
 474             // Write out the set itself to the script file`
 475             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 476         }
 477     }
 478     else {
 479         stepScript += "echo Setting variables for ${stressModeName}\n"
 480         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 481         stressModeVars.each{ k, v ->
 482             // Write out what we are writing to the script file
 483             stepScript += "echo Setting ${k}=${v}\n"
 484             // Write out the set itself to the script file`
 485             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 486         }
 487         stepScript += "chmod +x ${stepScriptLocation}\n"
 488     }
 489
 490     return stepScript
 491 }
 492
 493 // Append an existing script to a stress mode script already created by genStressModeScriptStep().
 494 // Returns string of commands to do this.
 495 def static appendStressModeScriptStep(def os, def appendScript, def stepScriptLocation) {
 496     assert (os == 'Windows_NT')
 497     def stepScript = ''
 498     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 499     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 500
 501     // Display the resulting script. This is useful when looking at the output log file.
 502     stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 503     stepScript += "type ${stepScriptLocation}\r\n"
 504
 505     return stepScript
 506 }
 507
 508 // Calculates the name of the build job based on some typical parameters.
 509 //
 510 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 511     // If the architecture is x64, do not add that info into the build name.
 512     // Need to change around some systems and other builds to pick up the right builds
 513     // to do that.
 514
 515     def suffix = scenario != 'default' ? "_${scenario}" : '';
 516     if (isBuildOnly) {
 517         suffix += '_bld'
 518     }
 519     def baseName = ''
 520     switch (architecture) {
 521         case 'x64':
 522             if (scenario == 'default') {
 523                 // For now we leave x64 off of the name for compatibility with other jobs
 524                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 525             }
 526             else if (scenario == 'formatting') {
 527                 // we don't care about the configuration for the formatting job. It runs all configs
 528                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 529             }
 530             else {
 531                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 532             }
 533             break
 534         case 'arm64':
 535             if (os.toLowerCase() == "windows_nt") {
 536                 // These are cross builds
 537                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 538             }
 539             else {
 540                 // Defaults to a small page size set of machines.
 541                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 542             }
 543             break
 544         case 'arm':
 545             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 546             break
 547         case 'armlb':
 548             // These are cross builds
 549             if (os == 'Tizen') {
 550                 // ABI: softfp
 551                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 552             }
 553             else {
 554                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 555             }
 556             break
 557         case 'x86':
 558         case 'x86lb':
 559         case 'x86_arm_altjit':
 560         case 'x64_arm64_altjit':
 561             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 562             break
 563         default:
 564             println("Unknown architecture: ${architecture}");
 565             assert false
 566             break
 567     }
 568
 569     return baseName + suffix
 570 }
 571
 572 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 573
 574     // Check scenario.
 575     switch (scenario) {
 576         case 'default':
 577             switch (architecture) {
 578                 case 'x64':
 579                 case 'x86':
 580                 case 'x86lb':
 581                     if (architecture == 'x86' && os == 'Ubuntu') {
 582                         Utilities.addPeriodicTrigger(job, '@daily')
 583                     }
 584                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 585                         Utilities.addGithubPushTrigger(job)
 586                     }
 587                     break
 588                 case 'arm':
 589                 case 'armlb':
 590                 case 'x86_arm_altjit':
 591                 case 'x64_arm64_altjit':
 592                     Utilities.addGithubPushTrigger(job)
 593                     break
 594                 case 'arm64':
 595                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 596                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 597                     break
 598                 default:
 599                     println("Unknown architecture: ${architecture}");
 600                     assert false
 601                     break
 602             }
 603             break
 604         case 'r2r':
 605             assert !(os in bidailyCrossList)
 606             // r2r gets a push trigger for checked/release
 607             if (configuration == 'Checked' || configuration == 'Release') {
 608                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 609                 if (architecture == 'x64' && os != 'OSX10.12') {
 610                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 611                     if (isFlowJob || os == 'Windows_NT') {
 612                         Utilities.addGithubPushTrigger(job)
 613                     }
 614                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 615                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 616                     if (isFlowJob) {
 617                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 618                     }
 619                 }
 620                 // For x86, only add per-commit jobs for Windows
 621                 else if (architecture == 'x86' || architecture == 'x86lb') {
 622                     if (os == 'Windows_NT') {
 623                         Utilities.addGithubPushTrigger(job)
 624                     }
 625                 }
 626                 // arm64 r2r jobs should only run daily.
 627                 else if (architecture == 'arm64') {
 628                     if (os == 'Windows_NT') {
 629                         Utilities.addPeriodicTrigger(job, '@daily')
 630                     }
 631                 }
 632             }
 633             break
 634         case 'r2r_jitstress1':
 635         case 'r2r_jitstress2':
 636         case 'r2r_jitstressregs1':
 637         case 'r2r_jitstressregs2':
 638         case 'r2r_jitstressregs3':
 639         case 'r2r_jitstressregs4':
 640         case 'r2r_jitstressregs8':
 641         case 'r2r_jitstressregs0x10':
 642         case 'r2r_jitstressregs0x80':
 643         case 'r2r_jitstressregs0x1000':
 644         case 'r2r_jitminopts':
 645         case 'r2r_jitforcerelocs':
 646         case 'r2r_gcstress15':
 647             assert !(os in bidailyCrossList)
 648
 649             // GCStress=C is currently not supported on OS X
 650             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 651                 break
 652             }
 653
 654             // GC Stress 15 r2r gets a push trigger for checked/release
 655             if (configuration == 'Checked' || configuration == 'Release') {
 656                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 657                 if (architecture == 'x64') {
 658                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 659                     if (isFlowJob || os == 'Windows_NT') {
 660                         // Add a weekly periodic trigger
 661                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 662                     }
 663                 }
 664                 // For x86, only add per-commit jobs for Windows
 665                 else if (architecture == 'x86') {
 666                     if (os == 'Windows_NT') {
 667                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 668                     }
 669                 }
 670             }
 671             break
 672         case 'longgc':
 673             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 674             assert configuration == 'Release'
 675             assert architecture == 'x64'
 676             Utilities.addPeriodicTrigger(job, '@daily')
 677             // TODO: Add once external email sending is available again
 678             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 679             break
 680         case 'gcsimulator':
 681             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 682             assert configuration == 'Release'
 683             assert architecture == 'x64'
 684             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 685             // TODO: Add once external email sending is available again
 686             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 687             break
 688         case 'standalone_gc':
 689             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 690             assert (configuration == 'Release' || configuration == 'Checked')
 691             // TODO: Add once external email sending is available again
 692             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 693             Utilities.addPeriodicTrigger(job, '@daily')
 694             break
 695         case 'gc_reliability_framework':
 696             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 697             assert (configuration == 'Release' || configuration == 'Checked')
 698             // Only triggered by phrase.
 699             break
 700         case 'ilrt':
 701             assert !(os in bidailyCrossList)
 702             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 703             if (architecture == 'x64' && configuration == 'Release') {
 704                 // We don't expect to see a job generated except in these scenarios
 705                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 706                 if (isFlowJob || os == 'Windows_NT') {
 707                     Utilities.addPeriodicTrigger(job, '@daily')
 708                 }
 709             }
 710             break
 711         case 'jitdiff':
 712             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 713             assert configuration == 'Checked'
 714             assert (architecture == 'x64' || architecture == 'x86')
 715             Utilities.addGithubPushTrigger(job)
 716             break
 717         case 'formatting':
 718             assert (os == 'Windows_NT' || os == "Ubuntu")
 719             assert architecture == 'x64'
 720             Utilities.addGithubPushTrigger(job)
 721             break
 722         case 'jitstressregs1':
 723         case 'jitstressregs2':
 724         case 'jitstressregs3':
 725         case 'jitstressregs4':
 726         case 'jitstressregs8':
 727         case 'jitstressregs0x10':
 728         case 'jitstressregs0x80':
 729         case 'jitstressregs0x1000':
 730         case 'minopts':
 731         case 'forcerelocs':
 732         case 'jitstress1':
 733         case 'jitstress2':
 734         case 'jitstress2_jitstressregs1':
 735         case 'jitstress2_jitstressregs2':
 736         case 'jitstress2_jitstressregs3':
 737         case 'jitstress2_jitstressregs4':
 738         case 'jitstress2_jitstressregs8':
 739         case 'jitstress2_jitstressregs0x10':
 740         case 'jitstress2_jitstressregs0x80':
 741         case 'jitstress2_jitstressregs0x1000':
 742         case 'tailcallstress':
 743         case 'jitsse2only':
 744         case 'jitnosimd':
 745         case 'corefx_baseline':
 746         case 'corefx_minopts':
 747         case 'corefx_jitstress1':
 748         case 'corefx_jitstress2':
 749         case 'corefx_jitstressregs1':
 750         case 'corefx_jitstressregs2':
 751         case 'corefx_jitstressregs3':
 752         case 'corefx_jitstressregs4':
 753         case 'corefx_jitstressregs8':
 754         case 'corefx_jitstressregs0x10':
 755         case 'corefx_jitstressregs0x80':
 756         case 'corefx_jitstressregs0x1000':
 757         case 'zapdisable':
 758             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 759                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 760                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 761                     if (os == 'Windows_NT') {
 762                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 763                         Utilities.addPeriodicTrigger(job, '@weekly')
 764                     }
 765                 }
 766                 else {
 767                     Utilities.addPeriodicTrigger(job, '@daily')
 768                 }
 769             }
 770             break
 771         case 'heapverify1':
 772         case 'gcstress0x3':
 773             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 774                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 775                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 776                     if (os == 'Windows_NT') {
 777                         Utilities.addPeriodicTrigger(job, '@daily')
 778                     }
 779                     // TODO: Add once external email sending is available again
 780                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 781                 }
 782                 else {
 783                     Utilities.addPeriodicTrigger(job, '@weekly')
 784                 }
 785             }
 786             break
 787         case 'gcstress0xc':
 788         case 'gcstress0xc_zapdisable':
 789         case 'gcstress0xc_zapdisable_jitstress2':
 790         case 'gcstress0xc_zapdisable_heapverify1':
 791         case 'gcstress0xc_jitstress1':
 792         case 'gcstress0xc_jitstress2':
 793         case 'gcstress0xc_minopts_heapverify1':
 794             // GCStress=C is currently not supported on OS X
 795             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 796                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 797                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 798                     // TODO: Enable a periodic trigger after tests are updated.
 799                     // Utilities.addPeriodicTrigger(job, '@daily')
 800                     // TODO: Add once external email sending is available again
 801                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 802                 }
 803                 else {
 804                     Utilities.addPeriodicTrigger(job, '@weekly')
 805                 }
 806             }
 807             break
 808
 809         case 'illink':
 810             // Testing on other operating systems TBD
 811             assert (os == 'Windows_NT' || os == 'Ubuntu')
 812             if (architecture == 'x64' || architecture == 'x86') {
 813                 if (configuration == 'Checked') {
 814                     Utilities.addPeriodicTrigger(job, '@daily')
 815                 }
 816             }
 817             break
 818
 819         case 'tieredcompilation':
 820         case 'corefx_tieredcompilation':
 821             // No periodic jobs just yet, still testing
 822             break
 823
 824         default:
 825             println("Unknown scenario: ${scenario}");
 826             assert false
 827             break
 828     }
 829     return
 830 }
 831
 832 // **************************
 833 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 834 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 835 // builds will do a couple extra steps.
 836 // **************************
 837
 838 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 839 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 840 // it's just used for internal builds
 841 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 842 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 843     if (isWindowsBuildOnlyJob) {
 844         return
 845     }
 846
 847     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 848     // Non pull request builds.
 849     if (!isPR) {
 850         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 851         return
 852     }
 853
 854      def arm64Users = [
 855         'AndyAyersMS',
 856         'briansull',
 857         'BruceForstall',
 858         'CarolEidt',
 859         'cmckinsey',
 860         'erozenfeld',
 861         'janvorli',
 862         'jashook',
 863         'JosephTremoulet',
 864         'pgodeq',
 865         'pgavlin',
 866         'rartemev',
 867         'russellhadley',
 868         'RussKeldorph',
 869         'sandreenko',
 870         'sdmaclea',
 871         'swaroop-sridhar',
 872         'jkotas',
 873         'markwilkie',
 874         'weshaggard'
 875     ]
 876
 877     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 878     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 879     def osGroup = getOSGroup(os)
 880     switch (architecture) {
 881         case 'x64': // editor brace matching: {
 882             if (scenario == 'formatting') {
 883                 assert configuration == 'Checked'
 884                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 885                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
 886                 }
 887                 break
 888             }
 889
 890             switch (os) {
 891                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 892                 case 'Debian8.4':
 893                 case 'RHEL7.2':
 894                     if (scenario == 'default') {
 895                         assert !isFlowJob
 896                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 897                     }
 898                     break
 899                 case 'Ubuntu16.04':
 900                     assert !isFlowJob
 901                     assert scenario == 'default'
 902                     // Distinguish with the other architectures (arm and x86)
 903                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 904                     break
 905                 case 'Fedora24':
 906                 case 'Ubuntu16.10':
 907                     assert !isFlowJob
 908                     assert scenario == 'default'
 909                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 910                     break
 911                 case 'Ubuntu':
 912                     if (scenario == 'illink') {
 913                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 914                         break
 915                     }
 916                     // fall through
 917                 case 'OSX10.12':
 918                     // Triggers on the non-flow jobs aren't necessary here
 919                     // Corefx testing uses non-flow jobs.
 920                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
 921                         break
 922                     }
 923                     switch (scenario) {
 924                         case 'default':
 925                             // OSX uses checked for default PR tests
 926                             if (configuration == 'Checked') {
 927                                 // Default trigger
 928                                 assert !job.name.contains("centos")
 929                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
 930                             }
 931                             break
 932                         case 'jitdiff':
 933                             if (configuration == 'Checked') {
 934                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 935                             }
 936                             break
 937                         case 'ilrt':
 938                             if (configuration == 'Release') {
 939                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 940                             }
 941                             break
 942                         case 'longgc':
 943                             if (configuration == 'Release') {
 944                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 945                             }
 946                             break
 947                         case 'gcsimulator':
 948                             if (configuration == 'Release') {
 949                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 950                             }
 951                             break
 952                         case 'standalone_gc':
 953                             if (configuration == 'Release' || configuration == 'Checked') {
 954                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 955                             }
 956                             break
 957                         case 'gc_reliability_framework':
 958                             if (configuration == 'Release' || configuration == 'Checked') {
 959                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 960                             }
 961                             break
 962                         default:
 963                             if (isJitStressScenario(scenario)) {
 964                                 def displayStr = getStressModeDisplayName(scenario)
 965                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 966                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
 967                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 968                             }
 969                             else if (isR2RScenario(scenario)) {
 970                                 if (configuration == 'Release' || configuration == 'Checked') {
 971                                     def displayStr = getR2RDisplayName(scenario)
 972                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
 973                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 974                                 }
 975                             }
 976                             else {
 977                                 println("Unknown scenario: ${scenario}");
 978                                 assert false
 979                             }
 980                             break
 981                     }
 982                     break
 983
 984                 case 'CentOS7.1':
 985                     switch (scenario) {
 986                         case 'default':
 987                             // CentOS uses checked for default PR tests while debug is build only
 988                             if (configuration == 'Debug') {
 989                                 // Default trigger
 990                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
 991                             }
 992
 993                             // Make sure this is a flow job to get build and test.
 994                             if (configuration == 'Checked' && isFlowJob) {
 995                                 assert job.name.contains("flow")
 996                                 // Default trigger
 997                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
 998                             }
 999                             break
1000                         default:
1001                             if (isR2RScenario(scenario)) {
1002                                 if (configuration == 'Release' || configuration == 'Checked') {
1003                                     def displayStr = getR2RDisplayName(scenario)
1004                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1005                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1006                                 }
1007                             }
1008                             break
1009                     }
1010                     break
1011
1012                 case 'Windows_NT':
1013                     switch (scenario) {
1014                         case 'default':
1015                             // Default trigger
1016                             if (configuration == 'Checked') {
1017                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1018                             }
1019                             break
1020                         case 'jitdiff':
1021                             if (configuration == 'Checked') {
1022                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1023                             }
1024                             break
1025                         case 'ilrt':
1026                             if (configuration == 'Release') {
1027                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1028                             }
1029                             break
1030                         case 'longgc':
1031                             if (configuration == 'Release') {
1032                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1033                             }
1034                             break
1035                         case 'gcsimulator':
1036                             if (configuration == 'Release') {
1037                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1038                             }
1039                             break
1040                         case 'standalone_gc':
1041                             if (configuration == 'Release' || configuration == 'Checked') {
1042                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1043                             }
1044                             break
1045                         case 'gc_reliability_framework':
1046                             if (configuration == 'Release' || configuration == 'Checked') {
1047                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1048                             }
1049                             break
1050                         case 'illink':
1051                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1052                             break
1053                         default:
1054                             if (isJitStressScenario(scenario)) {
1055                                 def displayStr = getStressModeDisplayName(scenario)
1056                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1057                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1058                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1059                             }
1060                             else if (isR2RScenario(scenario)) {
1061                                 if (configuration == 'Release' || configuration == 'Checked') {
1062                                     def displayStr = getR2RDisplayName(scenario)
1063                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1064                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1065                                 }
1066                             }
1067                             else {
1068                                 println("Unknown scenario: ${scenario}");
1069                                 assert false
1070                             }
1071                             break
1072                     }
1073                     break
1074                 default:
1075                     println("Unknown os: ${os}");
1076                     assert false
1077                     break
1078             }
1079             break
1080         // editor brace matching: }
1081         case 'armlb':
1082         case 'arm': // editor brace matching: {
1083             switch (os) {
1084                 case 'Ubuntu':
1085                 case 'Ubuntu16.04':
1086                     if (architecture == 'armlb') { // Ubuntu arm is only for armlb currently
1087                         assert scenario == 'default'
1088                         job.with {
1089                             publishers {
1090                                 azureVMAgentPostBuildAction {
1091                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1092                                 }
1093                             }
1094                         }
1095                         if ((os == 'Ubuntu' && configuration == 'Debug') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1096                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1097                         }
1098                         else {
1099                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1100                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1101                         }
1102                     }
1103                     break
1104                 case 'Tizen':
1105                     if (architecture == 'armlb') { // Tizen armel is only for armlb currently
1106                         architecture='armel'
1107                         job.with {
1108                             publishers {
1109                                 azureVMAgentPostBuildAction {
1110                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1111                                 }
1112                             }
1113                         }
1114                         // Removing the regex will cause this to run on each PR.
1115                         if (configuration == 'Checked') {
1116                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1117                         }
1118                         else {
1119                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1120                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1121                         }
1122                     }
1123                     break
1124                 case 'Windows_NT':
1125                     // Set up a private trigger
1126                     def contextString = "${os} ${architecture} Cross ${configuration}"
1127                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1128                     if (scenario == 'default') {
1129                         contextString += " Innerloop"
1130                         triggerString += "\\W+Innerloop"
1131                     }
1132                     else {
1133                         contextString += " ${scenario}"
1134                         triggerString += "\\W+${scenario}"
1135                     }
1136
1137                     if (configuration == 'Debug') {
1138                         contextString += " Build"
1139                         triggerString += "\\W+Build"
1140                     } else {
1141                         contextString += " Build and Test"
1142                         triggerString += "\\W+Build and Test"
1143                     }
1144
1145                     triggerString += ".*"
1146
1147                     switch (scenario) {
1148                         case 'default':
1149                             // Only Checked is a default trigger.
1150                             if (configuration == 'Checked')
1151                             {
1152                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1153                             }
1154                             else
1155                             {
1156                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1157                             }
1158                             break
1159                         default:
1160                             // Stress jobs will use this code path.
1161                             if (isArmWindowsScenario(scenario)) {
1162                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1163                             }
1164                             break
1165                     }
1166                     break
1167                 default:
1168                     println("NYI os: ${os}");
1169                     assert false
1170                     break
1171             }
1172             break
1173         // editor brace matching: }
1174         case 'arm64': // editor brace matching: {
1175             // Set up a private trigger
1176             def contextString = "${os} ${architecture} Cross ${configuration}"
1177             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1178             if (scenario == 'default') {
1179                 contextString += " Innerloop"
1180                 triggerString += "\\W+Innerloop"
1181             }
1182             else {
1183                 contextString += " ${scenario}"
1184                 triggerString += "\\W+${scenario}"
1185             }
1186
1187             if (configuration == 'Debug') {
1188                 contextString += " Build"
1189                 triggerString += "\\W+Build"
1190             } else {
1191                 contextString += " Build and Test"
1192                 triggerString += "\\W+Build and Test"
1193             }
1194
1195             triggerString += ".*"
1196
1197             switch (os) {
1198                 case 'Ubuntu':
1199                 case 'Ubuntu16.04':
1200                     switch (scenario) {
1201                         case 'default':
1202                             if (configuration == 'Debug' && !isFlowJob) {
1203                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1204                             }
1205                             else {
1206                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1207                             }
1208
1209                             break
1210                         default:
1211                             if (isR2RScenario(scenario)) {
1212                                 if (configuration == 'Checked' || configuration == 'Release') {
1213                                     def displayStr = getR2RDisplayName(scenario)
1214                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1215                                 }
1216                             }
1217                             break
1218                     }
1219                     break
1220                 case 'Windows_NT':
1221                     assert isArmWindowsScenario(scenario)
1222                     switch (scenario) {
1223                         case 'default':
1224                             if (configuration == 'Debug' || configuration == 'Checked') {
1225                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1226                             }
1227                             else {
1228                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1229                             }
1230                             break
1231                         default:
1232                             // Stress jobs will use this code path.
1233                             if (isArmWindowsScenario(scenario)) {
1234                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1235                             }
1236                             break
1237                     }
1238                     break
1239                 default:
1240                     println("NYI os: ${os}");
1241                     assert false
1242                     break
1243             }
1244             break
1245         // editor brace matching: }
1246         case 'x86': // editor brace matching: {
1247             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1248             if (os == 'Ubuntu') {
1249                 // on-demand only for ubuntu x86
1250                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1251                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1252                 break
1253             }
1254             switch (scenario) {
1255                 case 'default':
1256                     if (configuration == 'Checked') {
1257                         assert !job.name.contains("centos")
1258                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1259                     }
1260                     else {
1261                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1262                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1263                     }
1264                     break
1265                 case 'ilrt':
1266                     if (configuration == 'Release') {
1267                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1268                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1269                     }
1270                     break
1271                 case 'longgc':
1272                     if (configuration == 'Release') {
1273                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1274                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1275                     }
1276                     break
1277                 case 'gcsimulator':
1278                     if (configuration == 'Release') {
1279                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1280                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1281                     }
1282                     break
1283                 case 'standalone_gc':
1284                     if (configuration == 'Release' || configuration == 'Checked') {
1285                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1286                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1287                     }
1288                     break
1289                 case 'illink':
1290                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1291                     break
1292                 default:
1293                     if (isJitStressScenario(scenario)) {
1294                         def displayStr = getStressModeDisplayName(scenario)
1295                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1296                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1297                     }
1298                     else if (isR2RScenario(scenario)) {
1299                         if (configuration == 'Release' || configuration == 'Checked') {
1300                             def displayStr = getR2RDisplayName(scenario)
1301                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1302                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1303                         }
1304                     }
1305                     else {
1306                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1307                         assert false
1308                     }
1309                     break
1310             }
1311             break
1312          // editor brace matching: }
1313         case 'x86lb': // editor brace matching: {
1314             assert (os == 'Windows_NT')
1315
1316             def arch = 'x86'
1317             def jit = 'legacy_backend'
1318             switch (scenario) {
1319                 case 'default':
1320                     if (configuration == 'Checked') {
1321                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${arch} ${jit} ${configuration} Build and Test",
1322                             "(?i).*test\\W+${os}\\W+${arch}\\W+${jit}\\W+${configuration}\\W+Build and Test.*")
1323                     }
1324                     break
1325                 default:
1326                     println("Unknown scenario: ${os} ${arch} ${jit} ${scenario}");
1327                     assert false
1328                     break
1329             }
1330             break
1331         // editor brace matching: }
1332         case 'x64_arm64_altjit':
1333         case 'x86_arm_altjit': // editor brace matching: {
1334             assert (os == 'Windows_NT')
1335             switch (scenario) {
1336                 case 'default':
1337                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1338                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1339                     break
1340                 default:
1341                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1342                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1343                     break
1344             }
1345             break
1346         // editor brace matching: }
1347         default:
1348             println("Unknown architecture: ${architecture}");
1349             assert false
1350             break
1351     }
1352 }
1353
1354 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1355     def buildCommands = [];
1356     def osGroup = getOSGroup(os)
1357     def lowerConfiguration = configuration.toLowerCase()
1358
1359     def priority = '1'
1360     if (scenario == 'default' && isPR == true) {
1361         priority = '0'
1362     }
1363
1364     def enableCorefxTesting = isCoreFxScenario(scenario)
1365
1366     // Calculate the build steps, archival, and xunit results
1367     switch (os) {
1368         case 'Windows_NT': // editor brace matching: {
1369             switch (architecture) {
1370                 case 'x64':
1371                 case 'x86':
1372                 case 'x86lb':
1373                 case 'x86_arm_altjit':
1374                 case 'x64_arm64_altjit':
1375                     def arch = architecture
1376                     def buildOpts = ''
1377                     if ((architecture == 'x86lb') || (architecture == 'x86_arm_altjit')) {
1378                         arch = 'x86'
1379                     }
1380                     else if (architecture == 'x64_arm64_altjit') {
1381                         arch = 'x64'
1382                     }
1383
1384                     if (scenario == 'formatting') {
1385                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1386                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1387                         break
1388                     }
1389
1390                     if (scenario == 'illink') {
1391                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1392                     }
1393
1394                     // If it is a release build for windows, ensure PGO is used, else fail the build
1395                     if ((lowerConfiguration == 'release') &&
1396                         (scenario in Constants.basicScenarios) &&
1397                         (architecture != 'x86lb') &&
1398                         (architecture != 'x86_arm_altjit') &&
1399                         (architecture != 'x64_arm64_altjit')) {
1400
1401                         buildOpts += ' -enforcepgo'
1402                     }
1403
1404                     if (enableCorefxTesting) {
1405                         buildOpts += ' skiptests';
1406                     } else {
1407                         buildOpts += " -priority=${priority}"
1408                     }
1409
1410                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1411                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1412                     // 35 characters long.
1413
1414                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1415
1416                     if (!isBuildOnly) {
1417                         def runtestArguments = ''
1418                         def testOpts = 'collectdumps'
1419
1420                         if (isR2RScenario(scenario)) {
1421
1422                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1423                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1424                             // to cause the tests to be crossgen'ed.
1425
1426                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1427                                 testOpts += ' crossgenaltjit protononjit.dll'
1428                             } else {
1429                                 testOpts += ' crossgen'
1430                             }
1431
1432                             testOpts += ' runcrossgentests'
1433
1434                             if (scenario == 'r2r_jitstress1') {
1435                                 testOpts += ' jitstress 1'
1436                             }
1437                             else if (scenario == 'r2r_jitstress2') {
1438                                 testOpts += ' jitstress 2'
1439                             }
1440                             else if (scenario == 'r2r_jitstressregs1') {
1441                                 testOpts += ' jitstressregs 1'
1442                             }
1443                             else if (scenario == 'r2r_jitstressregs2') {
1444                                 testOpts += ' jitstressregs 2'
1445                             }
1446                             else if (scenario == 'r2r_jitstressregs3') {
1447                                 testOpts += ' jitstressregs 3'
1448                             }
1449                             else if (scenario == 'r2r_jitstressregs4') {
1450                                 testOpts += ' jitstressregs 4'
1451                             }
1452                             else if (scenario == 'r2r_jitstressregs8') {
1453                                 testOpts += ' jitstressregs 8'
1454                             }
1455                             else if (scenario == 'r2r_jitstressregs0x10') {
1456                                 testOpts += ' jitstressregs 0x10'
1457                             }
1458                             else if (scenario == 'r2r_jitstressregs0x80') {
1459                                 testOpts += ' jitstressregs 0x80'
1460                             }
1461                             else if (scenario == 'r2r_jitstressregs0x1000') {
1462                                 testOpts += ' jitstressregs 0x1000'
1463                             }
1464                             else if (scenario == 'r2r_jitminopts') {
1465                                 testOpts += ' jitminopts'
1466                             }
1467                             else if (scenario == 'r2r_jitforcerelocs') {
1468                                 testOpts += ' jitforcerelocs'
1469                             }
1470                             else if (scenario == 'r2r_gcstress15') {
1471                                 testOpts += ' gcstresslevel 0xF'
1472                             }
1473                         }
1474                         else if (scenario == 'jitdiff') {
1475                             testOpts += ' jitdisasm crossgen'
1476                         }
1477                         else if (scenario == 'ilrt') {
1478                             testOpts += ' ilasmroundtrip'
1479                         }
1480                         else if (isLongGc(scenario)) {
1481                             testOpts += " ${scenario} sequential"
1482                         }
1483                         else if (scenario == 'standalone_gc') {
1484                             testOpts += ' gcname clrgc.dll'
1485                         }
1486                         else if (scenario == 'illink') {
1487                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1488                         }
1489
1490                         // If we are running a stress mode, we should write out the set of key
1491                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1492
1493                         def envScriptPath = ''
1494                         if (isJitStressScenario(scenario)) {
1495                             def buildCommandsStr = ''
1496                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1497                             buildCommandsStr += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], envScriptPath)
1498                             if (architecture == 'x86lb') {
1499                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd", envScriptPath)
1500                             }
1501                             else if (architecture == 'x86_arm_altjit') {
1502                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1503                             }
1504                             else if (architecture == 'x64_arm64_altjit') {
1505                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1506                             }
1507
1508                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1509                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1510                             // into a single script.
1511                             buildCommands += buildCommandsStr
1512                         }
1513                         else if (architecture == 'x86lb') {
1514                             envScriptPath = "%WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd"
1515                         }
1516                         else if (architecture == 'x86_arm_altjit') {
1517                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1518                         }
1519                         else if (architecture == 'x64_arm64_altjit') {
1520                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1521                         }
1522                         if (envScriptPath != '') {
1523                             testOpts += " TestEnv ${envScriptPath}"
1524                         }
1525
1526                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1527
1528                         if (enableCorefxTesting) {
1529                             def workspaceRelativeFxRoot = "_/fx"
1530                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1531
1532                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1533
1534                             setTestJobTimeOut(newJob, scenario)
1535
1536                             // Archive and process (only) the test results
1537                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1538                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1539
1540                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1541                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1542                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1543                         }
1544                         else if (isGcReliabilityFramework(scenario)) {
1545                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1546                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1547                         }
1548                         else {
1549                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1550                         }
1551                     }
1552
1553                     if (!enableCorefxTesting) {
1554                         // Run the rest of the build
1555                         // Build the mscorlib for the other OS's
1556                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1557                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1558
1559                         if (arch == "x64") {
1560                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1561                         }
1562
1563                         // Zip up the tests directory so that we don't use so much space/time copying
1564                         // 10s of thousands of files around.
1565                         buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1566
1567                         if (!isJitStressScenario(scenario)) {
1568                             // For windows, pull full test results and test drops for x86/x64.
1569                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1570                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1571                         }
1572
1573                         if (scenario == 'jitdiff') {
1574                             // retrive jit-dasm output for base commit, and run jit-diff
1575                             if (!isBuildOnly) {
1576                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1577                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1578                             }
1579                         }
1580
1581                         if (!isBuildOnly) {
1582                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1583                             setTestJobTimeOut(newJob, scenario)
1584                         }
1585                     }
1586                     break
1587                 case 'armlb':
1588                 case 'arm':
1589                     assert isArmWindowsScenario(scenario)
1590
1591                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1592                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1593
1594                     // Set time out
1595                     setTestJobTimeOut(newJob, scenario)
1596
1597                     if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1598                     {
1599                         // Up the timeout for arm checked testing only.
1600                         // Keep the longer timeout for gcstress.
1601                         Utilities.setJobTimeout(newJob, 240)
1602                     }
1603
1604                     def buildArchitecture = 'arm'
1605
1606                     // For 'arm' (the RyuJIT/arm32 architecture), tell build.cmd to use RyuJIT/arm32 for crossgen compilation.
1607                     // RyuJIT/arm32 is currently not the default JIT; it is an aljit. So, this is a special case.
1608                     def armCrossgenOpt = ''
1609                     if (architecture == 'arm') {
1610                         armCrossgenOpt = '-altjitcrossgen'
1611                     }
1612
1613                     // This is now a build only job. Do not run tests. Use the flow job.
1614                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1615
1616                     // Zip up the tests directory so that we don't use so much space/time copying
1617                     // 10s of thousands of files around.
1618                     buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1619
1620                     // Add archival.
1621                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1622                     break
1623                 case 'arm64':
1624                     assert isArmWindowsScenario(scenario)
1625
1626                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1627                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1628
1629                     // Set time out
1630                     setTestJobTimeOut(newJob, scenario)
1631                     if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1632                     {
1633                         // Up the timeout for arm checked testing only.
1634                         // Keep the longer timeout for gcstress.
1635                         Utilities.setJobTimeout(newJob, 240)
1636                     }
1637
1638                     // This is now a build only job. Do not run tests. Use the flow job.
1639                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1640
1641                     // Zip up the tests directory so that we don't use so much space/time copying
1642                     // 10s of thousands of files around.
1643                     buildCommands += "powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1644
1645                     // Add archival.
1646                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1647                     break
1648                 default:
1649                     println("Unknown architecture: ${architecture}");
1650                     assert false
1651                     break
1652             }
1653             break
1654         // editor brace matching: }
1655         case 'Ubuntu':
1656         case 'Ubuntu16.04':
1657         case 'Ubuntu16.10':
1658         case 'Debian8.4':
1659         case 'OSX10.12':
1660         case 'CentOS7.1':
1661         case 'RHEL7.2':
1662         case 'Tizen':
1663         case 'Fedora24': // editor brace matching: {
1664             switch (architecture) {
1665                 case 'x64':
1666                 case 'x86':
1667                     if (architecture == 'x86' && os == 'Ubuntu') {
1668                         // build and PAL test
1669                         buildCommands += "./tests/scripts/x86_ci_script.sh --buildConfig=${lowerConfiguration}"
1670                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1671                         break
1672                     }
1673
1674                     if (scenario == 'formatting') {
1675                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1676                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1677                         break
1678                     }
1679
1680                     if (scenario == 'illink') {
1681                         assert(os == 'Ubuntu')
1682                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1683                     }
1684
1685                     if (!enableCorefxTesting) {
1686                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1687                         // only on supported OS platforms.
1688                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1689                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1690                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1691                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1692
1693                         // Set time out
1694                         setTestJobTimeOut(newJob, scenario)
1695                         // Basic archiving of the build
1696                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1697                         // And pal tests
1698                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1699                     }
1700                     else {
1701                         // Corefx stress testing
1702                         assert os == 'Ubuntu'
1703                         assert architecture == 'x64'
1704                         assert lowerConfiguration == 'checked'
1705                         assert isJitStressScenario(scenario)
1706
1707                         // Build coreclr
1708                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1709
1710                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1711                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1712
1713                         // Build and text corefx
1714                         def workspaceRelativeFxRoot = "_/fx"
1715                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1716
1717                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1718
1719                         setTestJobTimeOut(newJob, scenario)
1720
1721                         // Archive and process (only) the test results
1722                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1723                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1724                     }
1725                     break
1726                 case 'arm64':
1727                     if (!enableCorefxTesting) {
1728                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1729
1730                         // HACK -- Arm64 does not have corefx jobs yet.
1731                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1732                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1733                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1734                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1735
1736                         setTestJobTimeOut(newJob, scenario)
1737                         // Basic archiving of the build
1738                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1739                     }
1740                     break
1741                 case 'armlb':
1742                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1743                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1744
1745                     // default values for Ubuntu
1746                     def arm_abi="arm"
1747                     def linuxCodeName="trusty"
1748                     if (os == 'Ubuntu16.04') {
1749                         linuxCodeName="xenial"
1750                     }
1751                     else if (os == 'Tizen') {
1752                         arm_abi="armel"
1753                         linuxCodeName="tizen"
1754                     }
1755
1756                     // Unzip the Windows test binaries first. Exit with 0
1757                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1758
1759                     // Unpack the corefx binaries
1760                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1761                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1762                     if (os != 'Tizen') {
1763                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1764                     }
1765                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1766                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1767                     if (os == 'Ubuntu') {
1768                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1769                     }
1770
1771                     // Call the ARM CI script to cross build and test using docker
1772                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1773                     --mode=docker \\
1774                     --${arm_abi} \\
1775                     --linuxCodeName=${linuxCodeName} \\
1776                     --buildConfig=${lowerConfiguration} \\
1777                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1778                     --coreFxBinDir=./bin/CoreFxBinDir \\
1779                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1780
1781                     // Basic archiving of the build, no pal tests
1782                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1783                     break
1784                 default:
1785                     println("Unknown architecture: ${architecture}");
1786                     assert false
1787                     break
1788             }
1789             break
1790         // editor brace matching: }
1791         default:
1792             println("Unknown os: ${os}");
1793             assert false
1794             break
1795     } // os
1796
1797     return buildCommands
1798 }
1799
1800 // Additional scenario which can alter behavior
1801
1802 Constants.allScenarios.each { scenario ->
1803     [true, false].each { isPR ->
1804         Constants.architectureList.each { architecture ->
1805             Constants.configurationList.each { configuration ->
1806                 Constants.osList.each { os ->
1807                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1808                     // and reset the os to Windows_NT
1809                     def isBuildOnly = false
1810                     if (os == 'Windows_NT_BuildOnly') {
1811                         isBuildOnly = true
1812                         os = 'Windows_NT'
1813                     }
1814
1815                     // Tizen is only supported for arm legacy_backend architecture
1816                     if (os == 'Tizen' && architecture != 'armlb') {
1817                         return
1818                     }
1819
1820                     // Skip totally unimplemented (in CI) configurations.
1821                     switch (architecture) {
1822                         case 'arm64':
1823                             if (os == 'Ubuntu16.04') {
1824                                 os = 'Ubuntu'
1825                             }
1826
1827                             // Windows and Ubuntu only
1828                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1829                                 return
1830                             }
1831                             break
1832                         case 'armlb':
1833                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1834                                 return
1835                             }
1836                             break
1837                         case 'arm':
1838                             if (os != 'Windows_NT') {
1839                                 return
1840                             }
1841                             break
1842                         case 'x86':
1843                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1844                                 return
1845                             }
1846                             break
1847                         case 'x86lb':
1848                         case 'x86_arm_altjit':
1849                         case 'x64_arm64_altjit':
1850                             if (os != 'Windows_NT') {
1851                                 return
1852                             }
1853                             break
1854                         case 'x64':
1855                             // Everything implemented
1856                             break
1857                         default:
1858                             println("Unknown architecture: ${architecture}")
1859                             assert false
1860                             break
1861                     }
1862
1863                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1864                     // with checked builds)
1865                     if (isJitStressScenario(scenario)) {
1866                         if (configuration != 'Checked') {
1867                             return
1868                         }
1869
1870                         // Since these are just execution time differences,
1871                         // skip platforms that don't execute the tests here (Windows_NT only)
1872                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
1873                         if (!isEnabledOS || isBuildOnly) {
1874                             return
1875                         }
1876
1877                         switch (architecture) {
1878                             case 'x64':
1879                             case 'x86':
1880                             case 'x86_arm_altjit':
1881                             case 'x64_arm64_altjit':
1882                                 // x86 ubuntu: default only
1883                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1884                                     return
1885                                 }
1886                                 // Windows: Everything implemented
1887                                 break
1888
1889                             default:
1890                                 // arm, arm64, armlb: stress is handled through flow jobs.
1891                                 // x86lb: No stress modes for legacy jit. (There's no technical reason we couldn't allow these.)
1892                                 return
1893                         }
1894                     }
1895                     else if (isR2RScenario(scenario)) {
1896                         if (os != 'Windows_NT') {
1897                             return
1898                         }
1899                         // R2R test runs are not implemented for arm/armlb/arm64.
1900                         if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
1901                             return
1902                         }
1903                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
1904                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
1905                             return
1906                         }
1907                     }
1908                     else {
1909                         // Skip scenarios
1910                         switch (scenario) {
1911                             case 'ilrt':
1912                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1913                                 // the default scenario build
1914                                 if (os != 'Windows_NT') {
1915                                     return
1916                                 }
1917                                 // Only x64 for now
1918                                 if (architecture != 'x64') {
1919                                     return
1920                                 }
1921                                 // Release only
1922                                 if (configuration != 'Release') {
1923                                     return
1924                                 }
1925                                 break
1926                             case 'jitdiff':
1927                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1928                                     return
1929                                 }
1930                                 if (architecture != 'x64') {
1931                                     return
1932                                 }
1933                                 if (configuration != 'Checked') {
1934                                     return
1935                                 }
1936                                 break
1937                             case 'longgc':
1938                             case 'gcsimulator':
1939                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1940                                     return
1941                                 }
1942                                 if (architecture != 'x64') {
1943                                     return
1944                                 }
1945                                 if (configuration != 'Release') {
1946                                     return
1947                                 }
1948                                 break
1949                             case 'gc_reliability_framework':
1950                             case 'standalone_gc':
1951                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1952                                     return
1953                                 }
1954
1955                                 if (architecture != 'x64') {
1956                                     return
1957                                 }
1958
1959                                 if (configuration != 'Release' && configuration != 'Checked') {
1960                                     return
1961                                 }
1962                                 break
1963                             // We only run Windows and Ubuntu x64 Checked for formatting right now
1964                             case 'formatting':
1965                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
1966                                     return
1967                                 }
1968                                 if (architecture != 'x64') {
1969                                     return
1970                                 }
1971                                 if (configuration != 'Checked') {
1972                                     return
1973                                 }
1974                                 if (isBuildOnly) {
1975                                     return
1976                                 }
1977                                 break
1978                             case 'illink':
1979                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
1980                                     return
1981                                 }
1982                                 if (architecture != 'x64' && architecture != 'x86') {
1983                                     return
1984                                 }
1985                                 if (isBuildOnly) {
1986                                     return
1987                                 }
1988                                 break
1989                             case 'default':
1990                                 // Nothing skipped
1991                                 break
1992                             default:
1993                                 println("Unknown scenario: ${scenario}")
1994                                 assert false
1995                                 break
1996                         }
1997                     }
1998
1999                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2000                     // runtime behavior, not compile-time behavior, are not interesting.
2001                     switch (architecture) {
2002                         case 'x86_arm_altjit':
2003                         case 'x64_arm64_altjit':
2004                             if (isGCStressRelatedTesting(scenario)) {
2005                                 return
2006                             }
2007                             break
2008                         default:
2009                             break
2010                     }
2011
2012                     // Only implement 'default' for x86lb.
2013                     if ((architecture == 'x86lb') && (scenario != 'default')) {
2014                         return
2015                     }
2016
2017                     // Calculate names
2018                     def lowerConfiguration = configuration.toLowerCase()
2019                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2020                     def folderName = getJobFolder(scenario)
2021
2022                     // Create the new job
2023                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2024
2025                     def machineAffinityOptions = null
2026
2027                     if (os != 'Windows_NT') {
2028                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2029                     }
2030                     else {
2031                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2032                     }
2033
2034                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2035
2036                     // Add all the standard options
2037                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2038                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly)
2039
2040                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2041                     def osGroup = getOSGroup(os)
2042
2043                     newJob.with {
2044                         steps {
2045                             if (os == 'Windows_NT') {
2046                                 buildCommands.each { buildCommand ->
2047                                     batchFile(buildCommand)
2048                                 }
2049                             }
2050                             else {
2051                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2052                                 if ( architecture == 'armlb' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2053                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2054                                     // Define the Windows Tests and Corefx build job names
2055                                     def WindowTestsName = projectFolder + '/' +
2056                                                           Utilities.getFullJobName(project,
2057                                                                                    getJobName(lowerConfiguration,
2058                                                                                               'x64' ,
2059                                                                                               'windows_nt',
2060                                                                                               'default',
2061                                                                                               true),
2062                                                                                    false)
2063                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2064                                                        Utilities.getFolderName(branch)
2065
2066                                     // Copy the Windows test binaries and the Corefx build binaries
2067                                     copyArtifacts(WindowTestsName) {
2068                                         includePatterns('bin/tests/tests.zip')
2069                                         buildSelector {
2070                                             latestSuccessful(true)
2071                                         }
2072                                     }
2073
2074                                     def arm_abi = 'arm'
2075                                     def corefx_os = 'linux'
2076                                     if (os == 'Tizen') {
2077                                         arm_abi = 'armel'
2078                                         corefx_os = 'tizen'
2079                                     }
2080
2081                                     // Let's use release CoreFX to test checked CoreCLR,
2082                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2083                                     def corefx_lowerConfiguration = lowerConfiguration
2084                                     if ( lowerConfiguration == 'checked' ) {
2085                                         corefx_lowerConfiguration='release'
2086                                     }
2087
2088                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2089                                         includePatterns('bin/build.tar.gz')
2090                                         buildSelector {
2091                                             latestSuccessful(true)
2092                                         }
2093                                     }
2094                                 }
2095
2096                                 buildCommands.each { buildCommand ->
2097                                     shell(buildCommand)
2098                                 }
2099                             }
2100                         }
2101                     } // newJob.with
2102
2103                 } // os
2104             } // configuration
2105         } // architecture
2106     } // isPR
2107 } // scenario
2108
2109
2110 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2111 Constants.allScenarios.each { scenario ->
2112     [true, false].each { isPR ->
2113         ['arm', 'armlb', 'x64', 'arm64'].each { architecture ->
2114             Constants.crossList.each { os ->
2115                 if (architecture == 'arm64') {
2116                     if (os != "Ubuntu" && os != "Windows_NT") {
2117                         return
2118                     }
2119                 } else if (architecture == 'arm' || architecture == 'armlb') {
2120                     if (os != 'Windows_NT') {
2121                         return
2122                     }
2123                 }
2124
2125                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2126
2127                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2128                     return
2129                 }
2130
2131                 Constants.configurationList.each { configuration ->
2132
2133                     // First, filter based on OS.
2134
2135                     if (os == 'Windows_NT') {
2136                         if (!isArmWindowsScenario(scenario)) {
2137                             return
2138                         }
2139                     }
2140                     else {
2141                         if (architecture == 'arm64') {
2142                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2143                                 return
2144                             }
2145                         }
2146                     }
2147
2148                     // For CentOS, we only want Checked/Release builds.
2149                     if (os == 'CentOS7.1') {
2150                         if (configuration != 'Checked' && configuration != 'Release') {
2151                             return
2152                         }
2153                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2154                             return
2155                         }
2156                     }
2157
2158                     // For RedHat and Debian, we only do Release builds.
2159                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2160                         if (configuration != 'Release') {
2161                             return
2162                         }
2163                         if (scenario != 'default') {
2164                             return
2165                         }
2166                     }
2167
2168                     // Next, filter based on scenario.
2169
2170                     if (isJitStressScenario(scenario)) {
2171                         if (configuration != 'Checked') {
2172                             return
2173                         }
2174                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2175                         if (isCoreFxScenario(scenario)) {
2176                             return
2177                         }
2178                     }
2179                     else if (isR2RBaselineScenario(scenario)) {
2180                         if (configuration != 'Checked' && configuration != 'Release') {
2181                             return
2182                         }
2183                     }
2184                     else if (isR2RStressScenario(scenario)) {
2185                         if (configuration != 'Checked') {
2186                             return
2187                         }
2188                     }
2189                     else {
2190                         // Skip scenarios
2191                         switch (scenario) {
2192                             case 'ilrt':
2193                             case 'longgc':
2194                             case 'gcsimulator':
2195                                 // Long GC tests take a long time on non-Release builds
2196                                 // ilrt is also Release only
2197                                 if (configuration != 'Release') {
2198                                     return
2199                                 }
2200                                 break
2201                             case 'jitdiff':
2202                                 if (configuration != 'Checked') {
2203                                     return;
2204                                 }
2205                                 break
2206                             case 'gc_reliability_framework':
2207                             case 'standalone_gc':
2208                                 if (configuration != 'Release' && configuration != 'Checked') {
2209                                     return
2210                                 }
2211                                 break
2212                             case 'formatting':
2213                                 return
2214                             case 'illink':
2215                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2216                                     return
2217                                 }
2218                                 break
2219                             case 'default':
2220                                 // Nothing skipped
2221                                 break
2222                             default:
2223                                 println("Unknown scenario: ${scenario}")
2224                                 assert false
2225                                 break
2226                         }
2227                     }
2228
2229                     // Done filtering. Now, create the jobs.
2230
2231                     def lowerConfiguration = configuration.toLowerCase()
2232                     def osGroup = getOSGroup(os)
2233                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2234
2235                     def inputCoreCLRBuildName = projectFolder + '/' +
2236                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2237
2238                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2239                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2240                     def testBuildScenario = scenario
2241                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2242                         testBuildScenario = 'default'
2243                     }
2244
2245                     def inputWindowsTestBuildArch = architecture
2246                     if (architecture == "arm64" && os != "Windows_NT") {
2247                         // Use the x64 test build for arm64 unix
2248                         inputWindowsTestBuildArch = "x64"
2249                     }
2250
2251                     def inputWindowTestsBuildName = projectFolder + '/' +
2252                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2253
2254                     // Enable Server GC for Ubuntu PR builds
2255                     def serverGCString = ''
2256                     if (os == 'Ubuntu' && isPR) {
2257                         serverGCString = '--useServerGC'
2258                     }
2259
2260                     def testOpts = ''
2261
2262                     if (isR2RScenario(scenario)) {
2263
2264                         testOpts += ' --crossgen --runcrossgentests'
2265
2266                         if (scenario == 'r2r_jitstress1') {
2267                             testOpts += ' --jitstress=1'
2268                         }
2269                         else if (scenario == 'r2r_jitstress2') {
2270                             testOpts += ' --jitstress=2'
2271                         }
2272                         else if (scenario == 'r2r_jitstressregs1') {
2273                             testOpts += ' --jitstressregs=1'
2274                         }
2275                         else if (scenario == 'r2r_jitstressregs2') {
2276                             testOpts += ' --jitstressregs=2'
2277                         }
2278                         else if (scenario == 'r2r_jitstressregs3') {
2279                             testOpts += ' --jitstressregs=3'
2280                         }
2281                         else if (scenario == 'r2r_jitstressregs4') {
2282                             testOpts += ' --jitstressregs=4'
2283                         }
2284                         else if (scenario == 'r2r_jitstressregs8') {
2285                             testOpts += ' --jitstressregs=8'
2286                         }
2287                         else if (scenario == 'r2r_jitstressregs0x10') {
2288                             testOpts += ' --jitstressregs=0x10'
2289                         }
2290                         else if (scenario == 'r2r_jitstressregs0x80') {
2291                             testOpts += ' --jitstressregs=0x80'
2292                         }
2293                         else if (scenario == 'r2r_jitstressregs0x1000') {
2294                             testOpts += ' --jitstressregs=0x1000'
2295                         }
2296                         else if (scenario == 'r2r_jitminopts') {
2297                             testOpts += ' --jitminopts'
2298                         }
2299                         else if (scenario == 'r2r_jitforcerelocs') {
2300                             testOpts += ' --jitforcerelocs'
2301                         }
2302                         else if (scenario == 'r2r_gcstress15') {
2303                             testOpts += ' --gcstresslevel=0xF'
2304                         }
2305                     }
2306                     else if (scenario == 'jitdiff') {
2307                         testOpts += ' --jitdisasm --crossgen'
2308                     }
2309                     else if (scenario == 'illink') {
2310                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2311                     }
2312                     else if (isLongGc(scenario)) {
2313                         // Long GC tests behave very poorly when they are not
2314                         // the only test running (many of them allocate until OOM).
2315                         testOpts += ' --sequential'
2316
2317                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2318                         // for running long GC and GCSimulator tests, respectively. We don't use them
2319                         // here because using a playlist file produces much more readable output on the CI machines
2320                         // and reduces running time.
2321                         //
2322                         // The Long GC playlist contains all of the tests that are
2323                         // going to be run. The GCSimulator playlist contains all of
2324                         // the GC simulator tests.
2325                         if (scenario == 'longgc') {
2326                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2327                         }
2328                         else if (scenario == 'gcsimulator') {
2329                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2330                         }
2331                     }
2332                     else if (isGcReliabilityFramework(scenario)) {
2333                         testOpts += ' --build-overlay-only'
2334                     }
2335                     else if (scenario == 'standalone_gc') {
2336                         if (osGroup == 'OSX') {
2337                             testOpts += ' --gcname=libclrgc.dylib'
2338                         }
2339                         else if (osGroup == 'Linux') {
2340                             testOpts += ' --gcname=libclrgc.so'
2341                         }
2342                         else {
2343                             println("Unexpected OS group: ${osGroup} for os ${os}")
2344                             assert false
2345                         }
2346                     }
2347
2348                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2349
2350                     def folder = getJobFolder(scenario)
2351                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2352                         // Add parameters for the inputs
2353
2354                         if (windowsArmJob == true) {
2355                             parameters {
2356                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2357                             }
2358                         }
2359                         else {
2360                             parameters {
2361                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2362                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2363                             }
2364                         }
2365
2366                         steps {
2367                             // Set up the copies
2368
2369                             // Coreclr build containing the tests and mscorlib
2370
2371                             if (windowsArmJob != true) {
2372                                 copyArtifacts(inputWindowTestsBuildName) {
2373                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2374                                     buildSelector {
2375                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2376                                     }
2377                                 }
2378                             }
2379
2380                             // Coreclr build we are trying to test
2381
2382                             copyArtifacts(inputCoreCLRBuildName) {
2383                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2384                                 buildSelector {
2385                                     buildNumber('${CORECLR_BUILD}')
2386                                 }
2387                             }
2388
2389                             // Windows CoreCLR Arm(64) will restore corefx
2390                             // packages correctly.
2391                             //
2392                             // In addition, test steps are entirely different
2393                             // because we do not have a unified runner
2394                             if (windowsArmJob != true) {
2395                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2396
2397                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2398                                 def osJobName
2399                                 if (os == 'Ubuntu') {
2400                                     osJobName = 'ubuntu14.04'
2401                                 }
2402                                 else {
2403                                     osJobName = os.toLowerCase()
2404                                 }
2405                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2406                                     includePatterns('bin/build.tar.gz')
2407                                     buildSelector {
2408                                         latestSuccessful(true)
2409                                     }
2410                                 }
2411
2412                                 shell ("mkdir ./bin/CoreFxBinDir")
2413                                 // Unpack the corefx binaries
2414                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2415
2416                                 // HACK -- Arm64 does not have corefx jobs yet.
2417                                 // Clone corefx and build the native packages overwriting the x64 packages.
2418                                 if (architecture == 'arm64') {
2419                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2420                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2421                                 }
2422
2423                                 // Unzip the tests first.  Exit with 0
2424                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2425
2426                                 // Execute the tests
2427                                 // If we are running a stress mode, we'll set those variables first
2428                                 def testEnvOpt = ""
2429                                 if (isJitStressScenario(scenario)) {
2430                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2431                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2432                                     shell("${createScriptCmds}")
2433                                     testEnvOpt = "--test-env=" + scriptFileName
2434                                 }
2435
2436                                 if (isGCStressRelatedTesting(scenario)) {
2437                                     shell('./init-tools.sh')
2438                                 }
2439
2440                                 shell("""./tests/runtest.sh \\
2441                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2442                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2443                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2444                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2445                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2446                 --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2447
2448                                 if (isGcReliabilityFramework(scenario)) {
2449                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2450                                     if (serverGCString != '') {
2451                                         shell("export COMPlus_gcServer=1")
2452                                     }
2453
2454                                     shell("./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2455                                 }
2456                             }
2457
2458                             else { // windowsArmJob == true
2459                                 // Unzip tests.
2460                                 batchFile("powershell -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2461
2462                                 // Build the build commands
2463                                 def buildCommands = ""
2464
2465                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2466                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2467                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2468
2469                                 // For all jobs
2470                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2471
2472                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2473                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2474
2475                                 // Arm(32) ryujit
2476                                 if (architecture == "arm") {
2477                                     // **This is an AltJit**
2478
2479                                     addEnvVariable("COMPlus_AltJit", "*")
2480                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2481                                     addEnvVariable("COMPlus_AltJitName", "protojit.dll")
2482                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2483                                 }
2484
2485                                 // If we are running a stress mode, we'll set those variables as well
2486                                 def stressValues = null
2487                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2488                                     if (isJitStressScenario(scenario)) {
2489                                         stressValues = Constants.jitStressModeScenarios[scenario]
2490                                     }
2491                                     else {
2492                                         stressValues = Constants.r2rStressScenarios[scenario]
2493                                     }
2494
2495                                     stressValues.each { key, value ->
2496                                         addEnvVariable(key, value)
2497                                     }
2498                                 }
2499
2500                                 // TODO: do whatever is necessary to support enabling R2R testing. Environment variables,
2501                                 // crossgen the framework assemblies, etc.
2502
2503                                 // Create the smarty command
2504                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2505                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2506                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2507
2508                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "arm") { addExclude("PROTOJIT_" + exclude) } else { addExclude(exclude) } }
2509
2510                                 if (architecture == "arm") {
2511                                     addExclude("PROTOJIT_FAIL")
2512                                 }
2513
2514                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2515                                     def failTag = "JITSTRESS_FAIL"
2516                                     def excludeTag = "JITSTRESS_EXCLUDE"
2517
2518                                     if (scenario.contains('gc')) {
2519                                         failTag = "GCSTRESS_FAIL"
2520                                         excludeTag = "GCSTRESS_EXCLUDE"
2521                                     }
2522
2523                                     addArchSpecificExclude(architecture, failTag)
2524                                     addArchSpecificExclude(architecture, excludeTag)
2525                                 }
2526
2527                                 else {
2528                                     addExclude("pri1")
2529                                 }
2530
2531                                 smartyCommand += "/lstFile Tests.lst"
2532
2533                                 def testListArch = [
2534                                     'arm64': 'arm64',
2535                                     'arm': 'arm',
2536                                     'armlb': 'arm'
2537                                 ]
2538
2539                                 def archLocation = testListArch[architecture]
2540
2541                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2542                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2543                                 addCommand("${smartyCommand}")
2544
2545                                 batchFile(buildCommands)
2546                             }
2547                         }
2548                     }
2549
2550                     if (scenario == 'jitdiff') {
2551                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2552                     }
2553
2554                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2555                     if (os in ['Ubuntu']) {
2556                         SummaryBuilder summaries = new SummaryBuilder()
2557                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2558                         summaries.emit(newJob)
2559                     }
2560
2561                     def affinityOptions = null
2562
2563                     if (windowsArmJob == true) {
2564                         affinityOptions = [
2565                             "use_arm64_build_machine" : false
2566                         ]
2567                     }
2568
2569                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2570                         affinityOptions = [
2571                             "large_pages" : false
2572                         ]
2573                     }
2574
2575                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2576                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2577
2578                     // REVIEW: Should the scenario-base timeout be set in the "else" clause of the "architecture==arm64"?
2579                     //         Should the "architecture==arm64" include arm/armlb?
2580                     //         Should we be overriding possibly higher timeouts that would be set by setTestJobTimeOut()?
2581                     setTestJobTimeOut(newJob, scenario)
2582                     if (architecture == 'arm64') {
2583                         Utilities.setJobTimeout(newJob, 240)
2584                     }
2585
2586                     if (windowsArmJob != true) {
2587                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2588                     }
2589                     else {
2590                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2591                     }
2592
2593                     // Create a build flow to join together the build and tests required to run this test.
2594                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2595                     // Linux CoreCLR test
2596                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2597                     def fullTestJobName = projectFolder + '/' + newJob.name
2598                     // Add a reference to the input jobs for report purposes
2599                     JobReport.Report.addReference(inputCoreCLRBuildName)
2600                     JobReport.Report.addReference(inputWindowTestsBuildName)
2601                     JobReport.Report.addReference(fullTestJobName)
2602                     def newFlowJob = null
2603
2604                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2605                         // Do not create the flow job for RHEL jobs.
2606                         return
2607                     }
2608
2609                     if (windowsArmJob == true) {
2610                         // For Windows arm jobs there is no reason to build a parallel test job.
2611                         // The product build supports building and archiving the tests.
2612
2613                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2614                         buildFlow("""
2615 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2616
2617 // And then build the test build
2618 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2619 """)
2620                         }
2621                     }
2622                     else {
2623                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2624                         buildFlow("""
2625 // Build the input jobs in parallel
2626 parallel (
2627 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2628 { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2629 )
2630
2631 // And then build the test build
2632 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2633             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2634 """)
2635                         }
2636                     }
2637
2638                     // For the flow jobs set the machine affinity as x64 if an armarch.
2639                     def flowArch = architecture
2640
2641                     if (flowArch in validWindowsNTCrossArches) {
2642                         flowArch = 'x64'
2643                         affinityOptions = null
2644                     }
2645
2646                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2647                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2648                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false)
2649                 } // configuration
2650             } // os
2651         } // architecture
2652     } // isPR
2653 } // scenario
2654
2655 JobReport.Report.generateJobReport(out)
2656
2657 // Make the call to generate the help job
2658 Utilities.createHelperJob(this, project, branch,
2659     "Welcome to the ${project} Repository",  // This is prepended to the help message
2660     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2661
2662 Utilities.addCROSSCheck(this, project, branch)