netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'corefx_baseline'                : [ : ], // corefx baseline
  90                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  91                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  92                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  93                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  94                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  95                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  96                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  97                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  98                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  99                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 100                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 101                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 102                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 103                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 104                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 105                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 107                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 109                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 110                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 111                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 112     ]
 113
 114     // This is a set of ReadyToRun stress scenarios
 115     def static r2rStressScenarios = [
 116                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 117                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 118                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 119                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 120                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 121                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 122                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 123                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 124                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 125                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 126                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 127                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 128                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 129     ]
 130
 131     // This is the basic set of scenarios
 132     def static basicScenarios = [
 133                'default',
 134                'ilrt',
 135                'r2r',
 136                'longgc',
 137                'formatting',
 138                'gcsimulator',
 139                'jitdiff',
 140                'standalone_gc',
 141                'gc_reliability_framework',
 142                'illink'] + r2rStressScenarios.keySet()
 143
 144     def static allScenarios = basicScenarios + jitStressModeScenarios.keySet()
 145
 146     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 147     // to Tests.lst file categories to exclude.
 148     //
 149     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 150     // and with the same values, with some commented out, for easier maintenance.
 151     //
 152     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 153     //
 154     def static validArmWindowsScenarios = [
 155                'default':                                [],
 156                // 'ilrt'
 157                // 'r2r':                                    ["R2R_FAIL"],
 158                // 'longgc'
 159                // 'formatting'
 160                // 'gcsimulator'
 161                // 'jitdiff'
 162                // 'standalone_gc'
 163                // 'gc_reliability_framework'
 164                // 'illink'
 165                // 'r2r_jitstress1'
 166                // 'r2r_jitstress2'
 167                // 'r2r_jitstressregs1'
 168                // 'r2r_jitstressregs2'
 169                // 'r2r_jitstressregs3'
 170                // 'r2r_jitstressregs4'
 171                // 'r2r_jitstressregs8'
 172                // 'r2r_jitstressregs0x10'
 173                // 'r2r_jitstressregs0x80'
 174                // 'r2r_jitstressregs0x1000'
 175                // 'r2r_jitminopts'
 176                // 'r2r_jitforcerelocs'
 177                // 'r2r_gcstress15'
 178                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 179                'tieredcompilation':                      [],
 180                'forcerelocs':                            [],
 181                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 196                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 197                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 198                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 199                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 200                // 'jitsse2only'                          // Only relevant to xarch
 201                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 202                // 'corefx_baseline'
 203                // 'corefx_minopts'
 204                // 'corefx_tieredcompilation'
 205                // 'corefx_jitstress1'
 206                // 'corefx_jitstress2'
 207                // 'corefx_jitstressregs1'
 208                // 'corefx_jitstressregs2'
 209                // 'corefx_jitstressregs3'
 210                // 'corefx_jitstressregs4'
 211                // 'corefx_jitstressregs8'
 212                // 'corefx_jitstressregs0x10'
 213                // 'corefx_jitstressregs0x80'
 214                // 'corefx_jitstressregs0x1000'
 215                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 216                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 217                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'heapverify1':                            [],
 219                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 220                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 221                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 222                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 223                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 224                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 225
 226                //
 227                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 228                //
 229
 230                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 231                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 249     ]
 250
 251     def static configurationList = ['Debug', 'Checked', 'Release']
 252
 253     // This is the set of architectures
 254     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 255 }
 256
 257 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 258     assert os instanceof String
 259     assert architecture instanceof String
 260
 261     def armArches = ['arm', 'armlb', 'arm64']
 262     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 263
 264     if (!(architecture in armArches)) {
 265         assert options == null
 266         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 267
 268         return
 269     }
 270
 271     // This is an arm(64) job.
 272     //
 273     // There are several options.
 274     //
 275     // Windows_NT
 276     //
 277     // Arm32 (Build) -> latest-arm64
 278     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 279     // Arm32 (Test)  -> arm64-windows_nt
 280     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 281     //
 282     // Arm64 (Build) -> latest-arm64
 283     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 284     // Arm64 (Test)  -> arm64-windows_nt
 285     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 286     //
 287     // Ubuntu
 288     //
 289     // Arm32 (Build) -> arm-cross-latest
 290     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 291     // Arm32 (Test)  -> NYI Arch not supported
 292     //       |->
 293     //
 294     // Arm64 (Build) -> arm64-cross-latest
 295     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 296     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 297     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 298     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 299     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 300
 301     // This has to be a arm arch
 302     assert architecture in armArches
 303     if (os == "Windows_NT") {
 304         // Arm(64) windows jobs share the same machines for now
 305         def isBuild = options['use_arm64_build_machine'] == true
 306
 307         if (isBuild == true) {
 308             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 309         } else {
 310             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 311         }
 312     } else {
 313         assert os != 'Windows_NT'
 314         assert os in supportedArmLinuxOs
 315
 316         if (architecture == 'arm' || architecture == 'armlb') {
 317             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 318         } else {
 319             // Arm64 Linux
 320             if (options['is_build_only'] == true) {
 321                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 322             } else {
 323                 // Arm64 Test Machines
 324                 if (options['large_pages'] == false) {
 325                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 326                 } else {
 327                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 328                 }
 329             }
 330         }
 331     }
 332 }
 333
 334 def static isGCStressRelatedTesting(def scenario) {
 335     // The 'r2r_gcstress15' scenario is a basic scenario.
 336     // Detect it and make it a GCStress related.
 337     if (scenario == 'r2r_gcstress15')
 338     {
 339         return true;
 340     }
 341
 342     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 343     def scenarioName = scenario.toLowerCase()
 344     def isGCStressTesting = false
 345     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 346         if (k in gcStressTestEnvVars) {
 347             isGCStressTesting = true;
 348         }
 349     }
 350     return isGCStressTesting
 351 }
 352
 353 def static isCoreFxScenario(def scenario) {
 354     def corefx_prefix = 'corefx_'
 355     if (scenario.length() < corefx_prefix.length()) {
 356         return false
 357     }
 358     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 359 }
 360
 361 def static isR2RBaselineScenario(def scenario) {
 362     return (scenario == 'r2r')
 363 }
 364
 365 def static isR2RStressScenario(def scenario) {
 366     return Constants.r2rStressScenarios.containsKey(scenario)
 367 }
 368
 369 def static isR2RScenario(def scenario) {
 370     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 371 }
 372
 373 def static isJitStressScenario(def scenario) {
 374     return Constants.jitStressModeScenarios.containsKey(scenario)
 375 }
 376
 377 def static isLongGc(def scenario) {
 378     return (scenario == 'longgc' || scenario == 'gcsimulator')
 379 }
 380
 381 def static isJitDiff(def scenario) {
 382     return (scenario == 'jitdiff')
 383 }
 384
 385 def static isGcReliabilityFramework(def scenario) {
 386     return (scenario == 'gc_reliability_framework')
 387 }
 388
 389 def static isArmWindowsScenario(def scenario) {
 390     return Constants.validArmWindowsScenarios.containsKey(scenario)
 391 }
 392
 393 def static setTestJobTimeOut(newJob, scenario) {
 394     if (isGCStressRelatedTesting(scenario)) {
 395         Utilities.setJobTimeout(newJob, 4320)
 396     }
 397     else if (isCoreFxScenario(scenario)) {
 398         Utilities.setJobTimeout(newJob, 360)
 399     }
 400     else if (isJitStressScenario(scenario)) {
 401         Utilities.setJobTimeout(newJob, 240)
 402     }
 403     else if (isR2RBaselineScenario(scenario)) {
 404         Utilities.setJobTimeout(newJob, 240)
 405     }
 406     else if (isLongGc(scenario)) {
 407         Utilities.setJobTimeout(newJob, 1440)
 408     }
 409     else if (isJitDiff(scenario)) {
 410         Utilities.setJobTimeout(newJob, 240)
 411     }
 412     else if (isGcReliabilityFramework(scenario)) {
 413         Utilities.setJobTimeout(newJob, 1440)
 414     }
 415     // Non-test jobs use the default timeout value.
 416 }
 417
 418 def static getJobFolder(def scenario) {
 419     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 420         return 'jitstress'
 421     }
 422     if (scenario == 'illink') {
 423         return 'illink'
 424     }
 425     return ''
 426 }
 427
 428 def static getStressModeDisplayName(def scenario) {
 429     def displayStr = ''
 430     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 431         def prefixLength = 'COMPlus_'.length()
 432         if (k.length() >= prefixLength) {
 433             def modeName = k.substring(prefixLength, k.length())
 434             displayStr += ' ' + modeName + '=' + v
 435         }
 436     }
 437
 438     if (isCoreFxScenario(scenario)) {
 439         displayStr = ('CoreFx ' + displayStr).trim()
 440     }
 441
 442     return displayStr
 443 }
 444
 445 def static getR2RDisplayName(def scenario) {
 446     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 447     def displayStr = scenario
 448     def prefixLength = 'r2r_'.length()
 449     if (displayStr.length() >= prefixLength) {
 450         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 451     } else if (scenario == 'r2r') {
 452         displayStr = "R2R"
 453     }
 454     return displayStr
 455 }
 456
 457 // Generates the string for creating a file that sets environment variables
 458 // that makes it possible to run stress modes.  Writes the script to the file
 459 // specified by the stepScriptLocation parameter.
 460 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 461     def stepScript = ''
 462     if (os == 'Windows_NT') {
 463         // Timeout in ms, default is 10 minutes. For stress modes up this to 30 minutes.
 464         // BUG?: it seems this is ignored, as this script will be run in an environment where
 465         //       environment variables will be discarded before this is used.
 466         def timeout = 1800000
 467         stepScript += "set __TestTimeout=${timeout}\r\n"
 468
 469         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 470         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 471
 472         // Create at least an empty script.
 473         stepScript += "echo. > ${stepScriptLocation}\r\n"
 474
 475         stressModeVars.each{ k, v ->
 476             // Write out what we are writing to the script file
 477             stepScript += "echo Setting ${k}=${v}\r\n"
 478             // Write out the set itself to the script file`
 479             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 480         }
 481     }
 482     else {
 483         stepScript += "echo Setting variables for ${stressModeName}\n"
 484         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 485         stressModeVars.each{ k, v ->
 486             // Write out what we are writing to the script file
 487             stepScript += "echo Setting ${k}=${v}\n"
 488             // Write out the set itself to the script file`
 489             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 490         }
 491         stepScript += "chmod +x ${stepScriptLocation}\n"
 492     }
 493
 494     return stepScript
 495 }
 496
 497 // Append an existing script to a stress mode script already created by genStressModeScriptStep().
 498 // Returns string of commands to do this.
 499 def static appendStressModeScriptStep(def os, def appendScript, def stepScriptLocation) {
 500     assert (os == 'Windows_NT')
 501     def stepScript = ''
 502     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 503     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 504
 505     // Display the resulting script. This is useful when looking at the output log file.
 506     stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 507     stepScript += "type ${stepScriptLocation}\r\n"
 508
 509     return stepScript
 510 }
 511
 512 def static isNeedDocker(def architecture, def os, def isBuild) {
 513     if (isBuild) {
 514         if (architecture == 'x86' && os == 'Ubuntu') {
 515             return true
 516         }
 517         else if (architecture == 'arm') {
 518             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 519                 return true
 520             }
 521         }
 522     }
 523     else {
 524         if (architecture == 'x86' && os == 'Ubuntu') {
 525             return true
 526         }
 527     }
 528     return false
 529 }
 530
 531 def static getDockerImageName(def architecture, def os, def isBuild) {
 532     // We must change some docker private images to official later
 533     if (isBuild) {
 534         if (architecture == 'x86' && os == 'Ubuntu') {
 535             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 536         }
 537         else if (architecture == 'arm') {
 538             if (os == 'Ubuntu') {
 539                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 540             }
 541             else if (os == 'Ubuntu16.04') {
 542                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 543             }
 544             else if (os == 'Tizen') {
 545                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 546             }
 547         }
 548     }
 549     else {
 550         if (architecture == 'x86' && os == 'Ubuntu') {
 551             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 552         }
 553     }
 554     println("Unknown architecture to use docker: ${architecture} ${os}");
 555     assert false
 556 }
 557
 558 // Calculates the name of the build job based on some typical parameters.
 559 //
 560 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 561     // If the architecture is x64, do not add that info into the build name.
 562     // Need to change around some systems and other builds to pick up the right builds
 563     // to do that.
 564
 565     def suffix = scenario != 'default' ? "_${scenario}" : '';
 566     if (isBuildOnly) {
 567         suffix += '_bld'
 568     }
 569     def baseName = ''
 570     switch (architecture) {
 571         case 'x64':
 572             if (scenario == 'default') {
 573                 // For now we leave x64 off of the name for compatibility with other jobs
 574                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 575             }
 576             else if (scenario == 'formatting') {
 577                 // we don't care about the configuration for the formatting job. It runs all configs
 578                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 579             }
 580             else {
 581                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 582             }
 583             break
 584         case 'arm64':
 585             if (os.toLowerCase() == "windows_nt") {
 586                 // These are cross builds
 587                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 588             }
 589             else {
 590                 // Defaults to a small page size set of machines.
 591                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 592             }
 593             break
 594         case 'arm':
 595             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 596             break
 597         case 'armlb':
 598             // These are cross builds
 599             if (os == 'Tizen') {
 600                 // ABI: softfp
 601                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 602             }
 603             else {
 604                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 605             }
 606             break
 607         case 'x86':
 608         case 'x86_arm_altjit':
 609         case 'x64_arm64_altjit':
 610             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 611             break
 612         default:
 613             println("Unknown architecture: ${architecture}");
 614             assert false
 615             break
 616     }
 617
 618     return baseName + suffix
 619 }
 620
 621 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 622
 623     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 624     if (os == 'Windows_NT') {
 625         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 626             if (!(branch in Constants.WindowsArm64Branches)) {
 627                 return
 628             }
 629         }
 630     }
 631
 632     // Check scenario.
 633     switch (scenario) {
 634         case 'default':
 635             switch (architecture) {
 636                 case 'x64':
 637                 case 'x86':
 638                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 639                         Utilities.addPeriodicTrigger(job, '@daily')
 640                     }
 641                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 642                         Utilities.addGithubPushTrigger(job)
 643                     }
 644                     break
 645                 case 'arm':
 646                 case 'armlb':
 647                 case 'x86_arm_altjit':
 648                 case 'x64_arm64_altjit':
 649                     Utilities.addGithubPushTrigger(job)
 650                     break
 651                 case 'arm64':
 652                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 653                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 654                     break
 655                 default:
 656                     println("Unknown architecture: ${architecture}");
 657                     assert false
 658                     break
 659             }
 660             break
 661         case 'r2r':
 662             assert !(os in bidailyCrossList)
 663             // r2r gets a push trigger for checked/release
 664             if (configuration == 'Checked' || configuration == 'Release') {
 665                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 666                 if (architecture == 'x64' && os != 'OSX10.12') {
 667                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 668                     if (isFlowJob || os == 'Windows_NT') {
 669                         Utilities.addGithubPushTrigger(job)
 670                     }
 671                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 672                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 673                     if (isFlowJob) {
 674                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 675                     }
 676                 }
 677                 // For x86, only add per-commit jobs for Windows
 678                 else if (architecture == 'x86') {
 679                     if (os == 'Windows_NT') {
 680                         Utilities.addGithubPushTrigger(job)
 681                     }
 682                 }
 683                 // arm64 r2r jobs should only run daily.
 684                 else if (architecture == 'arm64') {
 685                     if (os == 'Windows_NT') {
 686                         Utilities.addPeriodicTrigger(job, '@daily')
 687                     }
 688                 }
 689             }
 690             break
 691         case 'r2r_jitstress1':
 692         case 'r2r_jitstress2':
 693         case 'r2r_jitstressregs1':
 694         case 'r2r_jitstressregs2':
 695         case 'r2r_jitstressregs3':
 696         case 'r2r_jitstressregs4':
 697         case 'r2r_jitstressregs8':
 698         case 'r2r_jitstressregs0x10':
 699         case 'r2r_jitstressregs0x80':
 700         case 'r2r_jitstressregs0x1000':
 701         case 'r2r_jitminopts':
 702         case 'r2r_jitforcerelocs':
 703         case 'r2r_gcstress15':
 704             assert !(os in bidailyCrossList)
 705
 706             // GCStress=C is currently not supported on OS X
 707             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 708                 break
 709             }
 710
 711             // GC Stress 15 r2r gets a push trigger for checked/release
 712             if (configuration == 'Checked' || configuration == 'Release') {
 713                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 714                 if (architecture == 'x64') {
 715                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 716                     if (isFlowJob || os == 'Windows_NT') {
 717                         // Add a weekly periodic trigger
 718                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 719                     }
 720                 }
 721                 // For x86, only add per-commit jobs for Windows
 722                 else if (architecture == 'x86') {
 723                     if (os == 'Windows_NT') {
 724                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 725                     }
 726                 }
 727             }
 728             break
 729         case 'longgc':
 730             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 731             assert configuration == 'Release'
 732             assert architecture == 'x64'
 733             Utilities.addPeriodicTrigger(job, '@daily')
 734             // TODO: Add once external email sending is available again
 735             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 736             break
 737         case 'gcsimulator':
 738             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 739             assert configuration == 'Release'
 740             assert architecture == 'x64'
 741             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 742             // TODO: Add once external email sending is available again
 743             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 744             break
 745         case 'standalone_gc':
 746             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 747             assert (configuration == 'Release' || configuration == 'Checked')
 748             // TODO: Add once external email sending is available again
 749             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 750             Utilities.addPeriodicTrigger(job, '@daily')
 751             break
 752         case 'gc_reliability_framework':
 753             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 754             assert (configuration == 'Release' || configuration == 'Checked')
 755             // Only triggered by phrase.
 756             break
 757         case 'ilrt':
 758             assert !(os in bidailyCrossList)
 759             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 760             if (architecture == 'x64' && configuration == 'Release') {
 761                 // We don't expect to see a job generated except in these scenarios
 762                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 763                 if (isFlowJob || os == 'Windows_NT') {
 764                     Utilities.addPeriodicTrigger(job, '@daily')
 765                 }
 766             }
 767             break
 768         case 'jitdiff':
 769             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 770             assert configuration == 'Checked'
 771             assert (architecture == 'x64' || architecture == 'x86')
 772             Utilities.addGithubPushTrigger(job)
 773             break
 774         case 'formatting':
 775             assert (os == 'Windows_NT' || os == "Ubuntu")
 776             assert architecture == 'x64'
 777             Utilities.addGithubPushTrigger(job)
 778             break
 779         case 'jitstressregs1':
 780         case 'jitstressregs2':
 781         case 'jitstressregs3':
 782         case 'jitstressregs4':
 783         case 'jitstressregs8':
 784         case 'jitstressregs0x10':
 785         case 'jitstressregs0x80':
 786         case 'jitstressregs0x1000':
 787         case 'minopts':
 788         case 'forcerelocs':
 789         case 'jitstress1':
 790         case 'jitstress2':
 791         case 'jitstress2_jitstressregs1':
 792         case 'jitstress2_jitstressregs2':
 793         case 'jitstress2_jitstressregs3':
 794         case 'jitstress2_jitstressregs4':
 795         case 'jitstress2_jitstressregs8':
 796         case 'jitstress2_jitstressregs0x10':
 797         case 'jitstress2_jitstressregs0x80':
 798         case 'jitstress2_jitstressregs0x1000':
 799         case 'tailcallstress':
 800         case 'jitsse2only':
 801         case 'jitnosimd':
 802         case 'corefx_baseline':
 803         case 'corefx_minopts':
 804         case 'corefx_jitstress1':
 805         case 'corefx_jitstress2':
 806         case 'corefx_jitstressregs1':
 807         case 'corefx_jitstressregs2':
 808         case 'corefx_jitstressregs3':
 809         case 'corefx_jitstressregs4':
 810         case 'corefx_jitstressregs8':
 811         case 'corefx_jitstressregs0x10':
 812         case 'corefx_jitstressregs0x80':
 813         case 'corefx_jitstressregs0x1000':
 814         case 'zapdisable':
 815             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 816                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 817                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 818                     if (os == 'Windows_NT') {
 819                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 820                         Utilities.addPeriodicTrigger(job, '@weekly')
 821                     }
 822                 }
 823                 else {
 824                     Utilities.addPeriodicTrigger(job, '@daily')
 825                 }
 826             }
 827             break
 828         case 'heapverify1':
 829         case 'gcstress0x3':
 830             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 831                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 832                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 833                     if (os == 'Windows_NT') {
 834                         Utilities.addPeriodicTrigger(job, '@daily')
 835                     }
 836                     // TODO: Add once external email sending is available again
 837                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 838                 }
 839                 else {
 840                     Utilities.addPeriodicTrigger(job, '@weekly')
 841                 }
 842             }
 843             break
 844         case 'gcstress0xc':
 845         case 'gcstress0xc_zapdisable':
 846         case 'gcstress0xc_zapdisable_jitstress2':
 847         case 'gcstress0xc_zapdisable_heapverify1':
 848         case 'gcstress0xc_jitstress1':
 849         case 'gcstress0xc_jitstress2':
 850         case 'gcstress0xc_minopts_heapverify1':
 851             // GCStress=C is currently not supported on OS X
 852             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 853                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 854                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 855                     // TODO: Enable a periodic trigger after tests are updated.
 856                     // Utilities.addPeriodicTrigger(job, '@daily')
 857                     // TODO: Add once external email sending is available again
 858                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 859                 }
 860                 else {
 861                     Utilities.addPeriodicTrigger(job, '@weekly')
 862                 }
 863             }
 864             break
 865
 866         case 'illink':
 867             // Testing on other operating systems TBD
 868             assert (os == 'Windows_NT' || os == 'Ubuntu')
 869             if (architecture == 'x64' || architecture == 'x86') {
 870                 if (configuration == 'Checked') {
 871                     Utilities.addPeriodicTrigger(job, '@daily')
 872                 }
 873             }
 874             break
 875
 876         case 'tieredcompilation':
 877         case 'corefx_tieredcompilation':
 878             // No periodic jobs just yet, still testing
 879             break
 880
 881         default:
 882             println("Unknown scenario: ${scenario}");
 883             assert false
 884             break
 885     }
 886     return
 887 }
 888
 889 // **************************
 890 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 891 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 892 // builds will do a couple extra steps.
 893 // **************************
 894
 895 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 896 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 897 // it's just used for internal builds
 898 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 899 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 900     if (isWindowsBuildOnlyJob) {
 901         return
 902     }
 903
 904     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 905     // Non pull request builds.
 906     if (!isPR) {
 907         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 908         return
 909     }
 910
 911      def arm64Users = [
 912         'AndyAyersMS',
 913         'briansull',
 914         'BruceForstall',
 915         'CarolEidt',
 916         'cmckinsey',
 917         'erozenfeld',
 918         'janvorli',
 919         'jashook',
 920         'JosephTremoulet',
 921         'pgodeq',
 922         'pgavlin',
 923         'rartemev',
 924         'russellhadley',
 925         'RussKeldorph',
 926         'sandreenko',
 927         'sdmaclea',
 928         'swaroop-sridhar',
 929         'jkotas',
 930         'markwilkie',
 931         'weshaggard'
 932     ]
 933
 934     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 935     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 936     def osGroup = getOSGroup(os)
 937     switch (architecture) {
 938         case 'x64': // editor brace matching: {
 939             if (scenario == 'formatting') {
 940                 assert configuration == 'Checked'
 941                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 942                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
 943                 }
 944                 break
 945             }
 946
 947             switch (os) {
 948                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 949                 case 'Debian8.4':
 950                 case 'RHEL7.2':
 951                     if (scenario == 'default') {
 952                         assert !isFlowJob
 953                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 954                     }
 955                     break
 956                 case 'Ubuntu16.04':
 957                     assert !isFlowJob
 958                     assert scenario == 'default'
 959                     // Distinguish with the other architectures (arm and x86)
 960                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 961                     break
 962                 case 'Fedora24':
 963                 case 'Ubuntu16.10':
 964                     assert !isFlowJob
 965                     assert scenario == 'default'
 966                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 967                     break
 968                 case 'Ubuntu':
 969                     if (scenario == 'illink') {
 970                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 971                         break
 972                     }
 973                     // fall through
 974                 case 'OSX10.12':
 975                     // Triggers on the non-flow jobs aren't necessary here
 976                     // Corefx testing uses non-flow jobs.
 977                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
 978                         break
 979                     }
 980                     switch (scenario) {
 981                         case 'default':
 982                             // OSX uses checked for default PR tests
 983                             if (configuration == 'Checked') {
 984                                 // Default trigger
 985                                 assert !job.name.contains("centos")
 986                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
 987                             }
 988                             break
 989                         case 'jitdiff':
 990                             if (configuration == 'Checked') {
 991                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 992                             }
 993                             break
 994                         case 'ilrt':
 995                             if (configuration == 'Release') {
 996                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 997                             }
 998                             break
 999                         case 'longgc':
1000                             if (configuration == 'Release') {
1001                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1002                             }
1003                             break
1004                         case 'gcsimulator':
1005                             if (configuration == 'Release') {
1006                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1007                             }
1008                             break
1009                         case 'standalone_gc':
1010                             if (configuration == 'Release' || configuration == 'Checked') {
1011                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1012                             }
1013                             break
1014                         case 'gc_reliability_framework':
1015                             if (configuration == 'Release' || configuration == 'Checked') {
1016                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1017                             }
1018                             break
1019                         default:
1020                             if (isJitStressScenario(scenario)) {
1021                                 def displayStr = getStressModeDisplayName(scenario)
1022                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1023                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1024                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1025                             }
1026                             else if (isR2RScenario(scenario)) {
1027                                 if (configuration == 'Release' || configuration == 'Checked') {
1028                                     def displayStr = getR2RDisplayName(scenario)
1029                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1030                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1031                                 }
1032                             }
1033                             else {
1034                                 println("Unknown scenario: ${scenario}");
1035                                 assert false
1036                             }
1037                             break
1038                     }
1039                     break
1040
1041                 case 'CentOS7.1':
1042                     switch (scenario) {
1043                         case 'default':
1044                             // CentOS uses checked for default PR tests while debug is build only
1045                             if (configuration == 'Debug') {
1046                                 // Default trigger
1047                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1048                             }
1049
1050                             // Make sure this is a flow job to get build and test.
1051                             if (configuration == 'Checked' && isFlowJob) {
1052                                 assert job.name.contains("flow")
1053                                 // Default trigger
1054                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1055                             }
1056                             break
1057                         default:
1058                             if (isR2RScenario(scenario)) {
1059                                 if (configuration == 'Release' || configuration == 'Checked') {
1060                                     def displayStr = getR2RDisplayName(scenario)
1061                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1062                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1063                                 }
1064                             }
1065                             break
1066                     }
1067                     break
1068
1069                 case 'Windows_NT':
1070                     switch (scenario) {
1071                         case 'default':
1072                             // Default trigger
1073                             if (configuration == 'Checked') {
1074                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1075                             }
1076                             break
1077                         case 'jitdiff':
1078                             if (configuration == 'Checked') {
1079                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1080                             }
1081                             break
1082                         case 'ilrt':
1083                             if (configuration == 'Release') {
1084                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1085                             }
1086                             break
1087                         case 'longgc':
1088                             if (configuration == 'Release') {
1089                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1090                             }
1091                             break
1092                         case 'gcsimulator':
1093                             if (configuration == 'Release') {
1094                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1095                             }
1096                             break
1097                         case 'standalone_gc':
1098                             if (configuration == 'Release' || configuration == 'Checked') {
1099                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1100                             }
1101                             break
1102                         case 'gc_reliability_framework':
1103                             if (configuration == 'Release' || configuration == 'Checked') {
1104                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1105                             }
1106                             break
1107                         case 'illink':
1108                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1109                             break
1110                         default:
1111                             if (isJitStressScenario(scenario)) {
1112                                 def displayStr = getStressModeDisplayName(scenario)
1113                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1114                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1115                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1116                             }
1117                             else if (isR2RScenario(scenario)) {
1118                                 if (configuration == 'Release' || configuration == 'Checked') {
1119                                     def displayStr = getR2RDisplayName(scenario)
1120                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1121                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1122                                 }
1123                             }
1124                             else {
1125                                 println("Unknown scenario: ${scenario}");
1126                                 assert false
1127                             }
1128                             break
1129                     }
1130                     break
1131                 default:
1132                     println("Unknown os: ${os}");
1133                     assert false
1134                     break
1135             }
1136             break
1137         // editor brace matching: }
1138         case 'armlb':
1139         case 'arm': // editor brace matching: {
1140             switch (os) {
1141                 case 'Ubuntu':
1142                 case 'Ubuntu16.04':
1143                     if (architecture == 'armlb') { // Ubuntu arm is only for armlb currently
1144                         assert scenario == 'default'
1145                         job.with {
1146                             publishers {
1147                                 azureVMAgentPostBuildAction {
1148                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1149                                 }
1150                             }
1151                         }
1152                         if ((os == 'Ubuntu' && configuration == 'Debug') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1153                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1154                         }
1155                         else {
1156                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1157                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1158                         }
1159                     }
1160                     break
1161                 case 'Tizen':
1162                     if (architecture == 'armlb') { // Tizen armel is only for armlb currently
1163                         architecture='armel'
1164                         job.with {
1165                             publishers {
1166                                 azureVMAgentPostBuildAction {
1167                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1168                                 }
1169                             }
1170                         }
1171                         // Removing the regex will cause this to run on each PR.
1172                         if (configuration == 'Checked') {
1173                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1174                         }
1175                         else {
1176                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1177                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1178                         }
1179                     }
1180                     break
1181                 case 'Windows_NT':
1182                     // Triggers on the non-flow jobs aren't necessary here
1183                     if (!isFlowJob) {
1184                         break
1185                     }
1186
1187                     // Set up a private trigger
1188                     def contextString = "${os} ${architecture} Cross ${configuration}"
1189                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1190                     if (scenario == 'default') {
1191                         contextString += " Innerloop"
1192                         triggerString += "\\W+Innerloop"
1193                     }
1194                     else {
1195                         contextString += " ${scenario}"
1196                         triggerString += "\\W+${scenario}"
1197                     }
1198
1199                     if (configuration == 'Debug') {
1200                         contextString += " Build"
1201                         triggerString += "\\W+Build"
1202                     } else {
1203                         contextString += " Build and Test"
1204                         triggerString += "\\W+Build and Test"
1205                     }
1206
1207                     triggerString += ".*"
1208
1209                     switch (scenario) {
1210                         case 'default':
1211                             // Only Checked is a default trigger.
1212                             if (configuration == 'Checked')
1213                             {
1214                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1215                             }
1216                             else
1217                             {
1218                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1219                             }
1220                             break
1221                         default:
1222                             // Stress jobs will use this code path.
1223                             if (isArmWindowsScenario(scenario)) {
1224                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1225                             }
1226                             break
1227                     }
1228                     break
1229                 default:
1230                     println("NYI os: ${os}");
1231                     assert false
1232                     break
1233             }
1234             break
1235         // editor brace matching: }
1236         case 'arm64': // editor brace matching: {
1237             // Set up a private trigger
1238             def contextString = "${os} ${architecture} Cross ${configuration}"
1239             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1240             if (scenario == 'default') {
1241                 contextString += " Innerloop"
1242                 triggerString += "\\W+Innerloop"
1243             }
1244             else {
1245                 contextString += " ${scenario}"
1246                 triggerString += "\\W+${scenario}"
1247             }
1248
1249             if (configuration == 'Debug') {
1250                 contextString += " Build"
1251                 triggerString += "\\W+Build"
1252             } else {
1253                 contextString += " Build and Test"
1254                 triggerString += "\\W+Build and Test"
1255             }
1256
1257             triggerString += ".*"
1258
1259             switch (os) {
1260                 case 'Ubuntu':
1261                 case 'Ubuntu16.04':
1262                     switch (scenario) {
1263                         case 'default':
1264                             if (configuration == 'Debug' && !isFlowJob) {
1265                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1266                             }
1267                             else {
1268                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1269                             }
1270
1271                             break
1272                         default:
1273                             if (isR2RScenario(scenario)) {
1274                                 if (configuration == 'Checked' || configuration == 'Release') {
1275                                     def displayStr = getR2RDisplayName(scenario)
1276                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1277                                 }
1278                             }
1279                             break
1280                     }
1281                     break
1282                 case 'Windows_NT':
1283                     // Triggers on the non-flow jobs aren't necessary here
1284                     if (!isFlowJob) {
1285                         break
1286                     }
1287
1288                     assert isArmWindowsScenario(scenario)
1289                     switch (scenario) {
1290                         case 'default':
1291                             if (configuration == 'Checked') {
1292                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1293                             }
1294                             else {
1295                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1296                             }
1297                             break
1298                         default:
1299                             // Stress jobs will use this code path.
1300                             if (isArmWindowsScenario(scenario)) {
1301                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1302                             }
1303                             break
1304                     }
1305                     break
1306                 default:
1307                     println("NYI os: ${os}");
1308                     assert false
1309                     break
1310             }
1311             break
1312         // editor brace matching: }
1313         case 'x86': // editor brace matching: {
1314             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1315             if (os == 'Ubuntu') {
1316                 // Triggers on the non-flow jobs aren't necessary here
1317                 if (!isFlowJob) {
1318                     break
1319                 }
1320                 // on-demand only for ubuntu x86
1321                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1322                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1323                 break
1324             }
1325             switch (scenario) {
1326                 case 'default':
1327                     if (configuration == 'Checked') {
1328                         assert !job.name.contains("centos")
1329                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1330                     }
1331                     else {
1332                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1333                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1334                     }
1335                     break
1336                 case 'ilrt':
1337                     if (configuration == 'Release') {
1338                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1339                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1340                     }
1341                     break
1342                 case 'longgc':
1343                     if (configuration == 'Release') {
1344                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1345                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1346                     }
1347                     break
1348                 case 'gcsimulator':
1349                     if (configuration == 'Release') {
1350                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1351                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1352                     }
1353                     break
1354                 case 'standalone_gc':
1355                     if (configuration == 'Release' || configuration == 'Checked') {
1356                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1357                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1358                     }
1359                     break
1360                 case 'illink':
1361                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1362                     break
1363                 default:
1364                     if (isJitStressScenario(scenario)) {
1365                         def displayStr = getStressModeDisplayName(scenario)
1366                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1367                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1368                     }
1369                     else if (isR2RScenario(scenario)) {
1370                         if (configuration == 'Release' || configuration == 'Checked') {
1371                             def displayStr = getR2RDisplayName(scenario)
1372                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1373                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1374                         }
1375                     }
1376                     else {
1377                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1378                         assert false
1379                     }
1380                     break
1381             }
1382             break
1383          // editor brace matching: }
1384         case 'x64_arm64_altjit':
1385         case 'x86_arm_altjit': // editor brace matching: {
1386             assert (os == 'Windows_NT')
1387             switch (scenario) {
1388                 case 'default':
1389                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1390                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1391                     break
1392                 default:
1393                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1394                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1395                     break
1396             }
1397             break
1398         // editor brace matching: }
1399         default:
1400             println("Unknown architecture: ${architecture}");
1401             assert false
1402             break
1403     }
1404 }
1405
1406 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1407     def buildCommands = [];
1408     def osGroup = getOSGroup(os)
1409     def lowerConfiguration = configuration.toLowerCase()
1410
1411     def priority = '1'
1412     if (scenario == 'default' && isPR == true) {
1413         priority = '0'
1414     }
1415
1416     def enableCorefxTesting = isCoreFxScenario(scenario)
1417
1418     // Calculate the build steps, archival, and xunit results
1419     switch (os) {
1420         case 'Windows_NT': // editor brace matching: {
1421             switch (architecture) {
1422                 case 'x64':
1423                 case 'x86':
1424                 case 'x86_arm_altjit':
1425                 case 'x64_arm64_altjit':
1426                     def arch = architecture
1427                     def buildOpts = ''
1428                     if (architecture == 'x86_arm_altjit') {
1429                         arch = 'x86'
1430                     }
1431                     else if (architecture == 'x64_arm64_altjit') {
1432                         arch = 'x64'
1433                     }
1434
1435                     if (scenario == 'formatting') {
1436                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1437                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1438                         break
1439                     }
1440
1441                     if (scenario == 'illink') {
1442                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1443                     }
1444
1445                     // If it is a release build for windows, ensure PGO is used, else fail the build
1446                     if ((lowerConfiguration == 'release') &&
1447                         (scenario in Constants.basicScenarios) &&
1448                         (architecture != 'x86_arm_altjit') &&
1449                         (architecture != 'x64_arm64_altjit')) {
1450
1451                         buildOpts += ' -enforcepgo'
1452                     }
1453
1454                     if (enableCorefxTesting) {
1455                         buildOpts += ' skiptests';
1456                     } else {
1457                         buildOpts += " -priority=${priority}"
1458                     }
1459
1460                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1461                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1462                     // 35 characters long.
1463
1464                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1465
1466                     if (!isBuildOnly) {
1467                         def runtestArguments = ''
1468                         def testOpts = 'collectdumps'
1469
1470                         if (isR2RScenario(scenario)) {
1471
1472                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1473                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1474                             // to cause the tests to be crossgen'ed.
1475
1476                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1477                                 testOpts += ' crossgenaltjit protononjit.dll'
1478                             } else {
1479                                 testOpts += ' crossgen'
1480                             }
1481
1482                             testOpts += ' runcrossgentests'
1483
1484                             if (scenario == 'r2r_jitstress1') {
1485                                 testOpts += ' jitstress 1'
1486                             }
1487                             else if (scenario == 'r2r_jitstress2') {
1488                                 testOpts += ' jitstress 2'
1489                             }
1490                             else if (scenario == 'r2r_jitstressregs1') {
1491                                 testOpts += ' jitstressregs 1'
1492                             }
1493                             else if (scenario == 'r2r_jitstressregs2') {
1494                                 testOpts += ' jitstressregs 2'
1495                             }
1496                             else if (scenario == 'r2r_jitstressregs3') {
1497                                 testOpts += ' jitstressregs 3'
1498                             }
1499                             else if (scenario == 'r2r_jitstressregs4') {
1500                                 testOpts += ' jitstressregs 4'
1501                             }
1502                             else if (scenario == 'r2r_jitstressregs8') {
1503                                 testOpts += ' jitstressregs 8'
1504                             }
1505                             else if (scenario == 'r2r_jitstressregs0x10') {
1506                                 testOpts += ' jitstressregs 0x10'
1507                             }
1508                             else if (scenario == 'r2r_jitstressregs0x80') {
1509                                 testOpts += ' jitstressregs 0x80'
1510                             }
1511                             else if (scenario == 'r2r_jitstressregs0x1000') {
1512                                 testOpts += ' jitstressregs 0x1000'
1513                             }
1514                             else if (scenario == 'r2r_jitminopts') {
1515                                 testOpts += ' jitminopts'
1516                             }
1517                             else if (scenario == 'r2r_jitforcerelocs') {
1518                                 testOpts += ' jitforcerelocs'
1519                             }
1520                             else if (scenario == 'r2r_gcstress15') {
1521                                 testOpts += ' gcstresslevel 0xF'
1522                             }
1523                         }
1524                         else if (scenario == 'jitdiff') {
1525                             testOpts += ' jitdisasm crossgen'
1526                         }
1527                         else if (scenario == 'ilrt') {
1528                             testOpts += ' ilasmroundtrip'
1529                         }
1530                         else if (isLongGc(scenario)) {
1531                             testOpts += " ${scenario} sequential"
1532                         }
1533                         else if (scenario == 'standalone_gc') {
1534                             testOpts += ' gcname clrgc.dll'
1535                         }
1536                         else if (scenario == 'illink') {
1537                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1538                         }
1539
1540                         // If we are running a stress mode, we should write out the set of key
1541                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1542
1543                         def envScriptPath = ''
1544                         if (isJitStressScenario(scenario)) {
1545                             def buildCommandsStr = ''
1546                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1547                             buildCommandsStr += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], envScriptPath)
1548                             if (architecture == 'x86_arm_altjit') {
1549                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1550                             }
1551                             else if (architecture == 'x64_arm64_altjit') {
1552                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1553                             }
1554
1555                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1556                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1557                             // into a single script.
1558                             buildCommands += buildCommandsStr
1559                         }
1560                         else if (architecture == 'x86_arm_altjit') {
1561                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1562                         }
1563                         else if (architecture == 'x64_arm64_altjit') {
1564                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1565                         }
1566                         if (envScriptPath != '') {
1567                             testOpts += " TestEnv ${envScriptPath}"
1568                         }
1569
1570                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1571
1572                         if (enableCorefxTesting) {
1573                             def workspaceRelativeFxRoot = "_/fx"
1574                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1575
1576                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1577
1578                             setTestJobTimeOut(newJob, scenario)
1579
1580                             // Archive and process (only) the test results
1581                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1582                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1583
1584                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1585                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1586                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1587                         }
1588                         else if (isGcReliabilityFramework(scenario)) {
1589                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1590                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1591                         }
1592                         else {
1593                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1594                         }
1595                     }
1596
1597                     if (!enableCorefxTesting) {
1598                         // Run the rest of the build
1599                         // Build the mscorlib for the other OS's
1600                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1601                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1602
1603                         if (arch == "x64") {
1604                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1605                         }
1606
1607                         // Zip up the tests directory so that we don't use so much space/time copying
1608                         // 10s of thousands of files around.
1609                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1610
1611                         if (!isJitStressScenario(scenario)) {
1612                             // For windows, pull full test results and test drops for x86/x64.
1613                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1614                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1615                         }
1616
1617                         if (scenario == 'jitdiff') {
1618                             // retrive jit-dasm output for base commit, and run jit-diff
1619                             if (!isBuildOnly) {
1620                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1621                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1622                             }
1623                         }
1624
1625                         if (!isBuildOnly) {
1626                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1627                             setTestJobTimeOut(newJob, scenario)
1628                         }
1629                     }
1630                     break
1631                 case 'armlb':
1632                 case 'arm':
1633                     assert isArmWindowsScenario(scenario)
1634
1635                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1636                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1637
1638                     // Set time out
1639                     setTestJobTimeOut(newJob, scenario)
1640
1641                     if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1642                     {
1643                         // Up the timeout for arm checked testing only.
1644                         // Keep the longer timeout for gcstress.
1645                         Utilities.setJobTimeout(newJob, 240)
1646                     }
1647
1648                     def buildArchitecture = 'arm'
1649
1650                     // For 'arm' (the RyuJIT/arm32 architecture), tell build.cmd to use RyuJIT/arm32 for crossgen compilation.
1651                     // RyuJIT/arm32 is currently not the default JIT; it is an aljit. So, this is a special case.
1652                     def armCrossgenOpt = ''
1653                     if (architecture == 'arm') {
1654                         armCrossgenOpt = '-altjitcrossgen'
1655                     }
1656
1657                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1658                     priority = '1'
1659
1660                     // This is now a build only job. Do not run tests. Use the flow job.
1661                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1662
1663                     // Zip up the tests directory so that we don't use so much space/time copying
1664                     // 10s of thousands of files around.
1665                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1666
1667                     // Add archival.
1668                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1669                     break
1670                 case 'arm64':
1671                     assert isArmWindowsScenario(scenario)
1672
1673                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1674                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1675
1676                     // Set time out
1677                     setTestJobTimeOut(newJob, scenario)
1678                     if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1679                     {
1680                         // Up the timeout for arm checked testing only.
1681                         // Keep the longer timeout for gcstress.
1682                         Utilities.setJobTimeout(newJob, 240)
1683                     }
1684
1685                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1686                     priority = '1'
1687
1688                     // This is now a build only job. Do not run tests. Use the flow job.
1689                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1690
1691                     // Zip up the tests directory so that we don't use so much space/time copying
1692                     // 10s of thousands of files around.
1693                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1694
1695                     // Add archival.
1696                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1697                     break
1698                 default:
1699                     println("Unknown architecture: ${architecture}");
1700                     assert false
1701                     break
1702             }
1703             break
1704         // editor brace matching: }
1705         case 'Ubuntu':
1706         case 'Ubuntu16.04':
1707         case 'Ubuntu16.10':
1708         case 'Debian8.4':
1709         case 'OSX10.12':
1710         case 'CentOS7.1':
1711         case 'RHEL7.2':
1712         case 'Tizen':
1713         case 'Fedora24': // editor brace matching: {
1714             switch (architecture) {
1715                 case 'x64':
1716                 case 'x86':
1717                     if (architecture == 'x86' && os == 'Ubuntu') {
1718                         // build and PAL test
1719                         def dockerImage = getDockerImageName(architecture, os, true)
1720                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1721                         dockerImage = getDockerImageName(architecture, os, false)
1722                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1723                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1724                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1725                         break
1726                     }
1727
1728                     if (scenario == 'formatting') {
1729                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1730                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1731                         break
1732                     }
1733
1734                     if (scenario == 'illink') {
1735                         assert(os == 'Ubuntu')
1736                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1737                     }
1738
1739                     if (!enableCorefxTesting) {
1740                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1741                         // only on supported OS platforms.
1742                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1743                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1744                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1745                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1746
1747                         // Set time out
1748                         setTestJobTimeOut(newJob, scenario)
1749                         // Basic archiving of the build
1750                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1751                         // And pal tests
1752                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1753                     }
1754                     else {
1755                         // Corefx stress testing
1756                         assert os == 'Ubuntu'
1757                         assert architecture == 'x64'
1758                         assert lowerConfiguration == 'checked'
1759                         assert isJitStressScenario(scenario)
1760
1761                         // Build coreclr
1762                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1763
1764                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1765                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1766
1767                         // Build and text corefx
1768                         def workspaceRelativeFxRoot = "_/fx"
1769                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1770
1771                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1772
1773                         setTestJobTimeOut(newJob, scenario)
1774
1775                         // Archive and process (only) the test results
1776                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1777                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1778                     }
1779                     break
1780                 case 'arm64':
1781                     if (!enableCorefxTesting) {
1782                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1783
1784                         // HACK -- Arm64 does not have corefx jobs yet.
1785                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1786                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1787                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1788                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1789
1790                         setTestJobTimeOut(newJob, scenario)
1791                         // Basic archiving of the build
1792                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1793                     }
1794                     break
1795                 case 'armlb':
1796                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1797                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1798
1799                     // default values for Ubuntu
1800                     def arm_abi="arm"
1801                     def linuxCodeName="trusty"
1802                     if (os == 'Ubuntu16.04') {
1803                         linuxCodeName="xenial"
1804                     }
1805                     else if (os == 'Tizen') {
1806                         arm_abi="armel"
1807                         linuxCodeName="tizen"
1808                     }
1809
1810                     // Unzip the Windows test binaries first. Exit with 0
1811                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1812
1813                     // Unpack the corefx binaries
1814                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1815                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1816                     if (os != 'Tizen') {
1817                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1818                     }
1819                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1820                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1821                     if (os == 'Ubuntu') {
1822                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1823                     }
1824
1825                     // Call the ARM CI script to cross build and test using docker
1826                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1827                     --mode=docker \\
1828                     --${arm_abi} \\
1829                     --linuxCodeName=${linuxCodeName} \\
1830                     --buildConfig=${lowerConfiguration} \\
1831                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1832                     --coreFxBinDir=./bin/CoreFxBinDir \\
1833                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1834
1835                     // Basic archiving of the build, no pal tests
1836                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1837                     break
1838                 default:
1839                     println("Unknown architecture: ${architecture}");
1840                     assert false
1841                     break
1842             }
1843             break
1844         // editor brace matching: }
1845         default:
1846             println("Unknown os: ${os}");
1847             assert false
1848             break
1849     } // os
1850
1851     return buildCommands
1852 }
1853
1854 Constants.allScenarios.each { scenario ->
1855     [true, false].each { isPR ->
1856         Constants.architectureList.each { architecture ->
1857             Constants.configurationList.each { configuration ->
1858                 Constants.osList.each { os ->
1859                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1860                     // and reset the os to Windows_NT
1861                     def isBuildOnly = false
1862                     if (os == 'Windows_NT_BuildOnly') {
1863                         isBuildOnly = true
1864                         os = 'Windows_NT'
1865                     }
1866
1867                     // Tizen is only supported for arm legacy_backend architecture
1868                     if (os == 'Tizen' && architecture != 'armlb') {
1869                         return
1870                     }
1871
1872                     // Skip totally unimplemented (in CI) configurations.
1873                     switch (architecture) {
1874                         case 'arm64':
1875                             if (os == 'Ubuntu16.04') {
1876                                 os = 'Ubuntu'
1877                             }
1878
1879                             // Windows and Ubuntu only
1880                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1881                                 return
1882                             }
1883                             break
1884                         case 'armlb':
1885                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1886                                 return
1887                             }
1888                             break
1889                         case 'arm':
1890                             if (os != 'Windows_NT') {
1891                                 return
1892                             }
1893                             break
1894                         case 'x86':
1895                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1896                                 return
1897                             }
1898                             break
1899                         case 'x86_arm_altjit':
1900                         case 'x64_arm64_altjit':
1901                             if (os != 'Windows_NT') {
1902                                 return
1903                             }
1904                             break
1905                         case 'x64':
1906                             // Everything implemented
1907                             break
1908                         default:
1909                             println("Unknown architecture: ${architecture}")
1910                             assert false
1911                             break
1912                     }
1913
1914                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1915                     // with checked builds)
1916                     if (isJitStressScenario(scenario)) {
1917                         if (configuration != 'Checked') {
1918                             return
1919                         }
1920
1921                         // Since these are just execution time differences,
1922                         // skip platforms that don't execute the tests here (Windows_NT only)
1923                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
1924                         if (!isEnabledOS || isBuildOnly) {
1925                             return
1926                         }
1927
1928                         switch (architecture) {
1929                             case 'x64':
1930                             case 'x86':
1931                             case 'x86_arm_altjit':
1932                             case 'x64_arm64_altjit':
1933                                 // x86 ubuntu: default only
1934                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1935                                     return
1936                                 }
1937                                 // Windows: Everything implemented
1938                                 break
1939
1940                             default:
1941                                 // arm, arm64, armlb: stress is handled through flow jobs.
1942                                 return
1943                         }
1944                     }
1945                     else if (isR2RScenario(scenario)) {
1946                         if (os != 'Windows_NT') {
1947                             return
1948                         }
1949                         // R2R test runs are not implemented for arm/armlb/arm64.
1950                         if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
1951                             return
1952                         }
1953                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
1954                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
1955                             return
1956                         }
1957                     }
1958                     else {
1959                         // Skip scenarios
1960                         switch (scenario) {
1961                             case 'ilrt':
1962                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1963                                 // the default scenario build
1964                                 if (os != 'Windows_NT') {
1965                                     return
1966                                 }
1967                                 // Only x64 for now
1968                                 if (architecture != 'x64') {
1969                                     return
1970                                 }
1971                                 // Release only
1972                                 if (configuration != 'Release') {
1973                                     return
1974                                 }
1975                                 break
1976                             case 'jitdiff':
1977                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1978                                     return
1979                                 }
1980                                 if (architecture != 'x64') {
1981                                     return
1982                                 }
1983                                 if (configuration != 'Checked') {
1984                                     return
1985                                 }
1986                                 break
1987                             case 'longgc':
1988                             case 'gcsimulator':
1989                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1990                                     return
1991                                 }
1992                                 if (architecture != 'x64') {
1993                                     return
1994                                 }
1995                                 if (configuration != 'Release') {
1996                                     return
1997                                 }
1998                                 break
1999                             case 'gc_reliability_framework':
2000                             case 'standalone_gc':
2001                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2002                                     return
2003                                 }
2004
2005                                 if (architecture != 'x64') {
2006                                     return
2007                                 }
2008
2009                                 if (configuration != 'Release' && configuration != 'Checked') {
2010                                     return
2011                                 }
2012                                 break
2013                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2014                             case 'formatting':
2015                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2016                                     return
2017                                 }
2018                                 if (architecture != 'x64') {
2019                                     return
2020                                 }
2021                                 if (configuration != 'Checked') {
2022                                     return
2023                                 }
2024                                 if (isBuildOnly) {
2025                                     return
2026                                 }
2027                                 break
2028                             case 'illink':
2029                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2030                                     return
2031                                 }
2032                                 if (architecture != 'x64' && architecture != 'x86') {
2033                                     return
2034                                 }
2035                                 if (isBuildOnly) {
2036                                     return
2037                                 }
2038                                 break
2039                             case 'default':
2040                                 // Nothing skipped
2041                                 break
2042                             default:
2043                                 println("Unknown scenario: ${scenario}")
2044                                 assert false
2045                                 break
2046                         }
2047                     }
2048
2049                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2050                     // runtime behavior, not compile-time behavior, are not interesting.
2051                     switch (architecture) {
2052                         case 'x86_arm_altjit':
2053                         case 'x64_arm64_altjit':
2054                             if (isGCStressRelatedTesting(scenario)) {
2055                                 return
2056                             }
2057                             break
2058                         default:
2059                             break
2060                     }
2061
2062                     // Calculate names
2063                     def lowerConfiguration = configuration.toLowerCase()
2064                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2065                     def folderName = getJobFolder(scenario)
2066
2067                     // Create the new job
2068                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2069
2070                     def machineAffinityOptions = null
2071
2072                     if (os != 'Windows_NT') {
2073                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2074                     }
2075                     else {
2076                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2077                     }
2078
2079                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2080
2081                     // Add all the standard options
2082                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2083                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2084
2085                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2086                     def osGroup = getOSGroup(os)
2087
2088                     newJob.with {
2089                         steps {
2090                             if (os == 'Windows_NT') {
2091                                 buildCommands.each { buildCommand ->
2092                                     batchFile(buildCommand)
2093                                 }
2094                             }
2095                             else {
2096                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2097                                 if ( architecture == 'armlb' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2098                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2099                                     // Define the Windows Tests and Corefx build job names
2100                                     def WindowTestsName = projectFolder + '/' +
2101                                                           Utilities.getFullJobName(project,
2102                                                                                    getJobName(lowerConfiguration,
2103                                                                                               'x64' ,
2104                                                                                               'windows_nt',
2105                                                                                               'default',
2106                                                                                               true),
2107                                                                                    false)
2108                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2109                                                        Utilities.getFolderName(branch)
2110
2111                                     // Copy the Windows test binaries and the Corefx build binaries
2112                                     copyArtifacts(WindowTestsName) {
2113                                         includePatterns('bin/tests/tests.zip')
2114                                         buildSelector {
2115                                             latestSuccessful(true)
2116                                         }
2117                                     }
2118
2119                                     def arm_abi = 'arm'
2120                                     def corefx_os = 'linux'
2121                                     if (os == 'Tizen') {
2122                                         arm_abi = 'armel'
2123                                         corefx_os = 'tizen'
2124                                     }
2125
2126                                     // Let's use release CoreFX to test checked CoreCLR,
2127                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2128                                     def corefx_lowerConfiguration = lowerConfiguration
2129                                     if ( lowerConfiguration == 'checked' ) {
2130                                         corefx_lowerConfiguration='release'
2131                                     }
2132
2133                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2134                                         includePatterns('bin/build.tar.gz')
2135                                         buildSelector {
2136                                             latestSuccessful(true)
2137                                         }
2138                                     }
2139                                 }
2140
2141                                 buildCommands.each { buildCommand ->
2142                                     shell(buildCommand)
2143                                 }
2144                             }
2145                         }
2146                     } // newJob.with
2147
2148                 } // os
2149             } // configuration
2150         } // architecture
2151     } // isPR
2152 } // scenario
2153
2154
2155 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2156 Constants.allScenarios.each { scenario ->
2157     [true, false].each { isPR ->
2158         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2159             Constants.crossList.each { os ->
2160                 if (architecture == 'arm64') {
2161                     if (os != "Ubuntu" && os != "Windows_NT") {
2162                         return
2163                     }
2164                 } else if (architecture == 'arm' || architecture == 'armlb') {
2165                     if (os != 'Windows_NT') {
2166                         return
2167                     }
2168                 }
2169                 else if (architecture == 'x86') {
2170                     if (os != "Ubuntu") {
2171                         return
2172                     }
2173                 }
2174
2175                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2176
2177                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2178                     return
2179                 }
2180
2181                 Constants.configurationList.each { configuration ->
2182
2183                     // First, filter based on OS.
2184
2185                     if (os == 'Windows_NT') {
2186                         if (!isArmWindowsScenario(scenario)) {
2187                             return
2188                         }
2189                     }
2190                     else {
2191                         // Non-Windows
2192                         if (architecture == 'arm64') {
2193                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2194                                 return
2195                             }
2196                         }
2197                         else if (architecture == 'x86') {
2198                             // Linux/x86 only want default test
2199                             if (scenario != 'default') {
2200                                 return
2201                             }
2202                         }
2203                     }
2204
2205                     // For CentOS, we only want Checked/Release builds.
2206                     if (os == 'CentOS7.1') {
2207                         if (configuration != 'Checked' && configuration != 'Release') {
2208                             return
2209                         }
2210                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2211                             return
2212                         }
2213                     }
2214
2215                     // For RedHat and Debian, we only do Release builds.
2216                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2217                         if (configuration != 'Release') {
2218                             return
2219                         }
2220                         if (scenario != 'default') {
2221                             return
2222                         }
2223                     }
2224
2225                     // Next, filter based on scenario.
2226
2227                     if (isJitStressScenario(scenario)) {
2228                         if (configuration != 'Checked') {
2229                             return
2230                         }
2231                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2232                         if (isCoreFxScenario(scenario)) {
2233                             return
2234                         }
2235                     }
2236                     else if (isR2RBaselineScenario(scenario)) {
2237                         if (configuration != 'Checked' && configuration != 'Release') {
2238                             return
2239                         }
2240                     }
2241                     else if (isR2RStressScenario(scenario)) {
2242                         if (configuration != 'Checked') {
2243                             return
2244                         }
2245                     }
2246                     else {
2247                         // Skip scenarios
2248                         switch (scenario) {
2249                             case 'ilrt':
2250                             case 'longgc':
2251                             case 'gcsimulator':
2252                                 // Long GC tests take a long time on non-Release builds
2253                                 // ilrt is also Release only
2254                                 if (configuration != 'Release') {
2255                                     return
2256                                 }
2257                                 break
2258                             case 'jitdiff':
2259                                 if (configuration != 'Checked') {
2260                                     return;
2261                                 }
2262                                 break
2263                             case 'gc_reliability_framework':
2264                             case 'standalone_gc':
2265                                 if (configuration != 'Release' && configuration != 'Checked') {
2266                                     return
2267                                 }
2268                                 break
2269                             case 'formatting':
2270                                 return
2271                             case 'illink':
2272                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2273                                     return
2274                                 }
2275                                 break
2276                             case 'default':
2277                                 // Nothing skipped
2278                                 break
2279                             default:
2280                                 println("Unknown scenario: ${scenario}")
2281                                 assert false
2282                                 break
2283                         }
2284                     }
2285
2286                     // Done filtering. Now, create the jobs.
2287
2288                     def lowerConfiguration = configuration.toLowerCase()
2289                     def osGroup = getOSGroup(os)
2290                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2291
2292                     def inputCoreCLRBuildName = projectFolder + '/' +
2293                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2294
2295                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2296                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2297                     def testBuildScenario = scenario
2298                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2299                         testBuildScenario = 'default'
2300                     }
2301
2302                     def inputWindowsTestBuildArch = architecture
2303                     if (architecture == "arm64" && os != "Windows_NT") {
2304                         // Use the x64 test build for arm64 unix
2305                         inputWindowsTestBuildArch = "x64"
2306                     }
2307
2308                     def inputWindowTestsBuildName = projectFolder + '/' +
2309                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2310
2311                     // Enable Server GC for Ubuntu PR builds
2312                     def serverGCString = ''
2313                     if (os == 'Ubuntu' && isPR) {
2314                         serverGCString = '--useServerGC'
2315                     }
2316
2317                     def testOpts = ''
2318
2319                     if (isR2RScenario(scenario)) {
2320
2321                         testOpts += ' --crossgen --runcrossgentests'
2322
2323                         if (scenario == 'r2r_jitstress1') {
2324                             testOpts += ' --jitstress=1'
2325                         }
2326                         else if (scenario == 'r2r_jitstress2') {
2327                             testOpts += ' --jitstress=2'
2328                         }
2329                         else if (scenario == 'r2r_jitstressregs1') {
2330                             testOpts += ' --jitstressregs=1'
2331                         }
2332                         else if (scenario == 'r2r_jitstressregs2') {
2333                             testOpts += ' --jitstressregs=2'
2334                         }
2335                         else if (scenario == 'r2r_jitstressregs3') {
2336                             testOpts += ' --jitstressregs=3'
2337                         }
2338                         else if (scenario == 'r2r_jitstressregs4') {
2339                             testOpts += ' --jitstressregs=4'
2340                         }
2341                         else if (scenario == 'r2r_jitstressregs8') {
2342                             testOpts += ' --jitstressregs=8'
2343                         }
2344                         else if (scenario == 'r2r_jitstressregs0x10') {
2345                             testOpts += ' --jitstressregs=0x10'
2346                         }
2347                         else if (scenario == 'r2r_jitstressregs0x80') {
2348                             testOpts += ' --jitstressregs=0x80'
2349                         }
2350                         else if (scenario == 'r2r_jitstressregs0x1000') {
2351                             testOpts += ' --jitstressregs=0x1000'
2352                         }
2353                         else if (scenario == 'r2r_jitminopts') {
2354                             testOpts += ' --jitminopts'
2355                         }
2356                         else if (scenario == 'r2r_jitforcerelocs') {
2357                             testOpts += ' --jitforcerelocs'
2358                         }
2359                         else if (scenario == 'r2r_gcstress15') {
2360                             testOpts += ' --gcstresslevel=0xF'
2361                         }
2362                     }
2363                     else if (scenario == 'jitdiff') {
2364                         testOpts += ' --jitdisasm --crossgen'
2365                     }
2366                     else if (scenario == 'illink') {
2367                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2368                     }
2369                     else if (isLongGc(scenario)) {
2370                         // Long GC tests behave very poorly when they are not
2371                         // the only test running (many of them allocate until OOM).
2372                         testOpts += ' --sequential'
2373
2374                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2375                         // for running long GC and GCSimulator tests, respectively. We don't use them
2376                         // here because using a playlist file produces much more readable output on the CI machines
2377                         // and reduces running time.
2378                         //
2379                         // The Long GC playlist contains all of the tests that are
2380                         // going to be run. The GCSimulator playlist contains all of
2381                         // the GC simulator tests.
2382                         if (scenario == 'longgc') {
2383                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2384                         }
2385                         else if (scenario == 'gcsimulator') {
2386                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2387                         }
2388                     }
2389                     else if (isGcReliabilityFramework(scenario)) {
2390                         testOpts += ' --build-overlay-only'
2391                     }
2392                     else if (scenario == 'standalone_gc') {
2393                         if (osGroup == 'OSX') {
2394                             testOpts += ' --gcname=libclrgc.dylib'
2395                         }
2396                         else if (osGroup == 'Linux') {
2397                             testOpts += ' --gcname=libclrgc.so'
2398                         }
2399                         else {
2400                             println("Unexpected OS group: ${osGroup} for os ${os}")
2401                             assert false
2402                         }
2403                     }
2404
2405                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2406
2407                     def folder = getJobFolder(scenario)
2408                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2409                         // Add parameters for the inputs
2410
2411                         if (windowsArmJob == true) {
2412                             parameters {
2413                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2414                             }
2415                         }
2416                         else {
2417                             parameters {
2418                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2419                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2420                             }
2421                         }
2422
2423                         steps {
2424                             // Set up the copies
2425
2426                             // Coreclr build containing the tests and mscorlib
2427
2428                             if (windowsArmJob != true) {
2429                                 copyArtifacts(inputWindowTestsBuildName) {
2430                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2431                                     buildSelector {
2432                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2433                                     }
2434                                 }
2435                             }
2436
2437                             // Coreclr build we are trying to test
2438
2439                             copyArtifacts(inputCoreCLRBuildName) {
2440                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2441                                 buildSelector {
2442                                     buildNumber('${CORECLR_BUILD}')
2443                                 }
2444                             }
2445
2446                             // Windows CoreCLR Arm(64) will restore corefx
2447                             // packages correctly.
2448                             //
2449                             // In addition, test steps are entirely different
2450                             // because we do not have a unified runner
2451                             if (windowsArmJob != true) {
2452                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2453
2454                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2455                                 def osJobName
2456                                 if (os == 'Ubuntu') {
2457                                     osJobName = 'ubuntu14.04'
2458                                 }
2459                                 else if (architecture == 'x86') {
2460                                     if (os == 'Ubuntu') {
2461                                         // Linux/x86 corefx jobs does not build managed yet
2462                                         // Clone linux/arm corefx managed packages and overwrite linux/x86 native
2463                                         osJobName = "linux_arm_cross"
2464                                     }
2465                                 }
2466                                 else {
2467                                     osJobName = os.toLowerCase()
2468                                 }
2469                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2470                                     includePatterns('bin/build.tar.gz')
2471                                     buildSelector {
2472                                         latestSuccessful(true)
2473                                     }
2474                                 }
2475
2476                                 shell("mkdir ./bin/CoreFxBinDir")
2477                                 // Unpack the corefx binaries
2478                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2479
2480                                 // HACK -- Arm64 does not have corefx jobs yet.
2481                                 // Clone corefx and build the native packages overwriting the x64 packages.
2482                                 if (architecture == 'arm64') {
2483                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2484                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2485                                 }
2486                                 else if (architecture == 'x86') {
2487                                     shell("mkdir ./bin/CoreFxNative")
2488
2489                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2490                                         includePatterns('bin/build.tar.gz')
2491                                         targetDirectory('bin/CoreFxNative')
2492                                         buildSelector {
2493                                             latestSuccessful(true)
2494                                         }
2495                                     }
2496
2497                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2498                                 }
2499
2500                                 // Unzip the tests first.  Exit with 0
2501                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2502
2503                                 // Execute the tests
2504                                 def runDocker = isNeedDocker(architecture, os, false)
2505                                 def dockerPrefix = ""
2506                                 def dockerCmd = ""
2507                                 if (runDocker) {
2508                                     def dockerImage = getDockerImageName(architecture, os, false)
2509                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2510                                     dockerCmd = dockerPrefix + "${dockerImage} "
2511                                 }
2512
2513                                 // If we are running a stress mode, we'll set those variables first
2514                                 def testEnvOpt = ""
2515                                 if (isJitStressScenario(scenario)) {
2516                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2517                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2518                                     shell("${createScriptCmds}")
2519                                     testEnvOpt = "--test-env=" + scriptFileName
2520                                 }
2521
2522                                 if (isGCStressRelatedTesting(scenario)) {
2523                                     shell('./init-tools.sh')
2524                                 }
2525
2526                                 shell("""${dockerCmd}./tests/runtest.sh \\
2527                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2528                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2529                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2530                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2531                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2532                 --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2533
2534                                 if (isGcReliabilityFramework(scenario)) {
2535                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2536                                     if (serverGCString != '') {
2537                                         if (runDocker) {
2538                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2539                                         }
2540                                         else {
2541                                             shell("export COMPlus_gcServer=1")
2542                                         }
2543                                     }
2544
2545                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2546                                 }
2547                             }
2548
2549                             else { // windowsArmJob == true
2550                                 // Unzip tests.
2551                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2552
2553                                 // Build the build commands
2554                                 def buildCommands = ""
2555
2556                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2557                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2558                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2559
2560                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2561                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2562
2563                                 // For all jobs
2564                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2565
2566                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2567                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2568
2569                                 // Arm(32) ryujit
2570                                 if (architecture == "arm") {
2571                                     // **This is an AltJit**
2572
2573                                     addEnvVariable("COMPlus_AltJit", "*")
2574                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2575                                     addEnvVariable("COMPlus_AltJitName", "protojit.dll")
2576                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2577                                 }
2578
2579                                 // If we are running a stress mode, we'll set those variables as well
2580                                 def stressValues = null
2581                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2582                                     if (isJitStressScenario(scenario)) {
2583                                         stressValues = Constants.jitStressModeScenarios[scenario]
2584                                     }
2585                                     else {
2586                                         stressValues = Constants.r2rStressScenarios[scenario]
2587                                     }
2588
2589                                     stressValues.each { key, value ->
2590                                         addEnvVariable(key, value)
2591                                     }
2592                                 }
2593
2594                                 // TODO: do whatever is necessary to support enabling R2R testing. Environment variables,
2595                                 // crossgen the framework assemblies, etc.
2596
2597                                 // Create the smarty command
2598                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2599                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2600                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2601
2602                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "arm") { addExclude("PROTOJIT_" + exclude) } else { addExclude(exclude) } }
2603
2604                                 if (architecture == "arm") {
2605                                     addExclude("PROTOJIT_FAIL")
2606                                 }
2607
2608                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2609                                     def failTag = "JITSTRESS_FAIL"
2610                                     def excludeTag = "JITSTRESS_EXCLUDE"
2611
2612                                     if (scenario.contains('gc')) {
2613                                         failTag = "GCSTRESS_FAIL"
2614                                         excludeTag = "GCSTRESS_EXCLUDE"
2615                                     }
2616
2617                                     addArchSpecificExclude(architecture, failTag)
2618                                     addArchSpecificExclude(architecture, excludeTag)
2619                                 }
2620
2621                                 else {
2622                                     addExclude("pri1")
2623                                 }
2624
2625                                 smartyCommand += "/lstFile Tests.lst"
2626
2627                                 def testListArch = [
2628                                     'arm64': 'arm64',
2629                                     'arm': 'arm',
2630                                     'armlb': 'arm'
2631                                 ]
2632
2633                                 def archLocation = testListArch[architecture]
2634
2635                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2636                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2637                                 addCommand("${smartyCommand}")
2638
2639                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2640                                 // However, we also need to remove all the variables that were set during this batch file, so we
2641                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2642                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2643                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2644                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2645                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2646                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2647
2648                                 // ZIP up the smarty output, no matter what the smarty result.
2649                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2650
2651                                 addCommand("echo %errorlevel%")
2652                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2653
2654                                 // Use the smarty errorlevel as the script errorlevel.
2655                                 addCommand("exit /b %__save_smarty_errorlevel%")
2656
2657                                 batchFile(buildCommands)
2658                             }
2659                         }
2660                     }
2661
2662                     if (scenario == 'jitdiff') {
2663                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2664                     }
2665
2666                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2667                     if (os in ['Ubuntu']) {
2668                         SummaryBuilder summaries = new SummaryBuilder()
2669                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2670                         summaries.emit(newJob)
2671                     }
2672
2673                     def affinityOptions = null
2674
2675                     if (windowsArmJob == true) {
2676                         affinityOptions = [
2677                             "use_arm64_build_machine" : false
2678                         ]
2679                     }
2680
2681                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2682                         affinityOptions = [
2683                             "large_pages" : false
2684                         ]
2685                     }
2686
2687                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2688                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2689
2690                     // REVIEW: Should the scenario-base timeout be set in the "else" clause of the "architecture==arm64"?
2691                     //         Should the "architecture==arm64" include arm/armlb?
2692                     //         Should we be overriding possibly higher timeouts that would be set by setTestJobTimeOut()?
2693                     setTestJobTimeOut(newJob, scenario)
2694                     if (architecture == 'arm64') {
2695                         Utilities.setJobTimeout(newJob, 240)
2696                     }
2697
2698                     if (windowsArmJob != true) {
2699                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2700                     }
2701                     else {
2702                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2703
2704                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2705                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2706                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2707                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2708                     }
2709
2710                     // Create a build flow to join together the build and tests required to run this test.
2711                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2712                     // Linux CoreCLR test
2713                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2714                     def fullTestJobName = projectFolder + '/' + newJob.name
2715                     // Add a reference to the input jobs for report purposes
2716                     JobReport.Report.addReference(inputCoreCLRBuildName)
2717                     JobReport.Report.addReference(inputWindowTestsBuildName)
2718                     JobReport.Report.addReference(fullTestJobName)
2719                     def newFlowJob = null
2720
2721                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2722                         // Do not create the flow job for RHEL jobs.
2723                         return
2724                     }
2725
2726                     if (windowsArmJob == true) {
2727                         // For Windows arm jobs there is no reason to build a parallel test job.
2728                         // The product build supports building and archiving the tests.
2729
2730                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2731                         buildFlow("""
2732 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2733
2734 // And then build the test build
2735 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2736 """)
2737                         }
2738                     }
2739                     else {
2740                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2741                         buildFlow("""
2742 // Build the input jobs in parallel
2743 parallel (
2744 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2745 { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2746 )
2747
2748 // And then build the test build
2749 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2750             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2751 """)
2752                         }
2753                     }
2754
2755                     // For the flow jobs set the machine affinity as x64 if an armarch.
2756                     def flowArch = architecture
2757
2758                     if (flowArch in validWindowsNTCrossArches) {
2759                         flowArch = 'x64'
2760                         affinityOptions = null
2761                     }
2762
2763                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2764                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2765                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2766                 } // configuration
2767             } // os
2768         } // architecture
2769     } // isPR
2770 } // scenario
2771
2772 JobReport.Report.generateJobReport(out)
2773
2774 // Make the call to generate the help job
2775 Utilities.createHelperJob(this, project, branch,
2776     "Welcome to the ${project} Repository",  // This is prepended to the help message
2777     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2778
2779 Utilities.addCROSSCheck(this, project, branch)