netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'CentOS7.1',
  49                'RHEL7.2',
  50                'Ubuntu16.04',
  51                'Ubuntu16.10',
  52                'Fedora24',
  53                'Tizen']
  54
  55     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  56
  57     // This is a set of JIT stress modes combined with the set of variables that
  58     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  59     // the values are the environment variables
  60     def static jitStressModeScenarios = [
  61                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  62                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  63                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  64                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  65                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  66                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  67                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  68                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  69                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  70                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  71                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  72                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  73                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  74                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  75                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  76                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  77                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  78                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  79                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  80                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  81                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  82                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  83                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  84                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  85                'corefx_baseline'                : [ : ], // corefx baseline
  86                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  87                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  88                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  89                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  90                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  91                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  92                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  93                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  94                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  95                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  96                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  97                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  98                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  99                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 100                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 101                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 102                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 103                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 104                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 105                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 106                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 107                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 108     ]
 109
 110     // This is a set of ReadyToRun stress scenarios
 111     def static r2rStressScenarios = [
 112                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 113                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 114                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 115                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 116                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 117                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 118                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 119                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 120                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 121                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 122                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 123                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 124                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 125     ]
 126
 127     // This is the basic set of scenarios
 128     def static basicScenarios = [
 129                'default',
 130                'ilrt',
 131                'r2r',
 132                'longgc',
 133                'formatting',
 134                'gcsimulator',
 135                'jitdiff',
 136                'standalone_gc',
 137                'gc_reliability_framework',
 138                'illink'] + r2rStressScenarios.keySet()
 139
 140     def static allScenarios = basicScenarios + jitStressModeScenarios.keySet()
 141
 142     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 143     // to Tests.lst file categories to exclude.
 144     //
 145     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 146     // and with the same values, with some commented out, for easier maintenance.
 147     //
 148     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 149     //
 150     def static validArmWindowsScenarios = [
 151                'default':                                [],
 152                // 'ilrt'
 153                // 'r2r':                                    ["R2R_FAIL"],
 154                // 'longgc'
 155                // 'formatting'
 156                // 'gcsimulator'
 157                // 'jitdiff'
 158                // 'standalone_gc'
 159                // 'gc_reliability_framework'
 160                // 'illink'
 161                // 'r2r_jitstress1'
 162                // 'r2r_jitstress2'
 163                // 'r2r_jitstressregs1'
 164                // 'r2r_jitstressregs2'
 165                // 'r2r_jitstressregs3'
 166                // 'r2r_jitstressregs4'
 167                // 'r2r_jitstressregs8'
 168                // 'r2r_jitstressregs0x10'
 169                // 'r2r_jitstressregs0x80'
 170                // 'r2r_jitstressregs0x1000'
 171                // 'r2r_jitminopts'
 172                // 'r2r_jitforcerelocs'
 173                // 'r2r_gcstress15'
 174                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 175                'tieredcompilation':                      [],
 176                'forcerelocs':                            [],
 177                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 179                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 180                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 181                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 196                // 'jitsse2only'                          // Only relevant to xarch
 197                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 198                // 'corefx_baseline'
 199                // 'corefx_minopts'
 200                // 'corefx_tieredcompilation'
 201                // 'corefx_jitstress1'
 202                // 'corefx_jitstress2'
 203                // 'corefx_jitstressregs1'
 204                // 'corefx_jitstressregs2'
 205                // 'corefx_jitstressregs3'
 206                // 'corefx_jitstressregs4'
 207                // 'corefx_jitstressregs8'
 208                // 'corefx_jitstressregs0x10'
 209                // 'corefx_jitstressregs0x80'
 210                // 'corefx_jitstressregs0x1000'
 211                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 212                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 213                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 214                'heapverify1':                            [],
 215                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 216                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 217                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 219                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 220                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 221
 222                //
 223                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 224                //
 225
 226                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 227                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 228                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 245     ]
 246
 247     def static configurationList = ['Debug', 'Checked', 'Release']
 248
 249     // This is the set of architectures
 250     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 251 }
 252
 253 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 254     assert os instanceof String
 255     assert architecture instanceof String
 256
 257     def armArches = ['arm', 'armlb', 'arm64']
 258     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 259
 260     if (!(architecture in armArches)) {
 261         assert options == null
 262         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 263
 264         return
 265     }
 266
 267     // This is an arm(64) job.
 268     //
 269     // There are several options.
 270     //
 271     // Windows_NT
 272     //
 273     // Arm32 (Build) -> latest-arm64
 274     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 275     // Arm32 (Test)  -> arm64-windows_nt
 276     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 277     //
 278     // Arm64 (Build) -> latest-arm64
 279     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 280     // Arm64 (Test)  -> arm64-windows_nt
 281     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 282     //
 283     // Ubuntu
 284     //
 285     // Arm32 (Build) -> arm-cross-latest
 286     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 287     // Arm32 (Test)  -> NYI Arch not supported
 288     //       |->
 289     //
 290     // Arm64 (Build) -> arm64-cross-latest
 291     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 292     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 293     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 294     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 295     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 296
 297     // This has to be a arm arch
 298     assert architecture in armArches
 299     if (os == "Windows_NT") {
 300         // Arm(64) windows jobs share the same machines for now
 301         def isBuild = options['use_arm64_build_machine'] == true
 302
 303         if (isBuild == true) {
 304             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 305         } else {
 306             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 307         }
 308     } else {
 309         assert os != 'Windows_NT'
 310         assert os in supportedArmLinuxOs
 311
 312         if (architecture == 'arm' || architecture == 'armlb') {
 313             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 314         } else {
 315             // Arm64 Linux
 316             if (options['is_build_only'] == true) {
 317                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 318             } else {
 319                 // Arm64 Test Machines
 320                 if (options['large_pages'] == false) {
 321                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 322                 } else {
 323                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 324                 }
 325             }
 326         }
 327     }
 328 }
 329
 330 def static isGCStressRelatedTesting(def scenario) {
 331     // The 'r2r_gcstress15' scenario is a basic scenario.
 332     // Detect it and make it a GCStress related.
 333     if (scenario == 'r2r_gcstress15')
 334     {
 335         return true;
 336     }
 337
 338     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 339     def scenarioName = scenario.toLowerCase()
 340     def isGCStressTesting = false
 341     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 342         if (k in gcStressTestEnvVars) {
 343             isGCStressTesting = true;
 344         }
 345     }
 346     return isGCStressTesting
 347 }
 348
 349 def static isCoreFxScenario(def scenario) {
 350     def corefx_prefix = 'corefx_'
 351     if (scenario.length() < corefx_prefix.length()) {
 352         return false
 353     }
 354     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 355 }
 356
 357 def static isR2RBaselineScenario(def scenario) {
 358     return (scenario == 'r2r')
 359 }
 360
 361 def static isR2RStressScenario(def scenario) {
 362     return Constants.r2rStressScenarios.containsKey(scenario)
 363 }
 364
 365 def static isR2RScenario(def scenario) {
 366     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 367 }
 368
 369 def static isJitStressScenario(def scenario) {
 370     return Constants.jitStressModeScenarios.containsKey(scenario)
 371 }
 372
 373 def static isLongGc(def scenario) {
 374     return (scenario == 'longgc' || scenario == 'gcsimulator')
 375 }
 376
 377 def static isJitDiff(def scenario) {
 378     return (scenario == 'jitdiff')
 379 }
 380
 381 def static isGcReliabilityFramework(def scenario) {
 382     return (scenario == 'gc_reliability_framework')
 383 }
 384
 385 def static isArmWindowsScenario(def scenario) {
 386     return Constants.validArmWindowsScenarios.containsKey(scenario)
 387 }
 388
 389 def static setTestJobTimeOut(newJob, isPR, architecture, scenario) {
 390     if (isGCStressRelatedTesting(scenario)) {
 391         Utilities.setJobTimeout(newJob, 4320)
 392     }
 393     else if (isCoreFxScenario(scenario)) {
 394         Utilities.setJobTimeout(newJob, 360)
 395     }
 396     else if (isJitStressScenario(scenario)) {
 397         Utilities.setJobTimeout(newJob, 240)
 398     }
 399     else if (isR2RBaselineScenario(scenario)) {
 400         Utilities.setJobTimeout(newJob, 240)
 401     }
 402     else if (isLongGc(scenario)) {
 403         Utilities.setJobTimeout(newJob, 1440)
 404     }
 405     else if (isJitDiff(scenario)) {
 406         Utilities.setJobTimeout(newJob, 240)
 407     }
 408     else if (isGcReliabilityFramework(scenario)) {
 409         Utilities.setJobTimeout(newJob, 1440)
 410     }
 411     else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 412         Utilities.setJobTimeout(newJob, 240)
 413     }
 414     else if (!(scenario == 'default' && isPR == true)) {
 415         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 416         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 417         Utilities.setJobTimeout(newJob, 240)
 418     }
 419     // Non-test jobs use the default timeout value.
 420 }
 421
 422 def static getJobFolder(def scenario) {
 423     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 424         return 'jitstress'
 425     }
 426     if (scenario == 'illink') {
 427         return 'illink'
 428     }
 429     return ''
 430 }
 431
 432 def static getStressModeDisplayName(def scenario) {
 433     def displayStr = ''
 434     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 435         def prefixLength = 'COMPlus_'.length()
 436         if (k.length() >= prefixLength) {
 437             def modeName = k.substring(prefixLength, k.length())
 438             displayStr += ' ' + modeName + '=' + v
 439         }
 440     }
 441
 442     if (isCoreFxScenario(scenario)) {
 443         displayStr = ('CoreFx ' + displayStr).trim()
 444     }
 445
 446     return displayStr
 447 }
 448
 449 def static getR2RDisplayName(def scenario) {
 450     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 451     def displayStr = scenario
 452     def prefixLength = 'r2r_'.length()
 453     if (displayStr.length() >= prefixLength) {
 454         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 455     } else if (scenario == 'r2r') {
 456         displayStr = "R2R"
 457     }
 458     return displayStr
 459 }
 460
 461 // Generates the string for creating a file that sets environment variables
 462 // that makes it possible to run stress modes.  Writes the script to the file
 463 // specified by the stepScriptLocation parameter.
 464 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 465     def stepScript = ''
 466     if (os == 'Windows_NT') {
 467         // Timeout in ms, default is 10 minutes. For stress modes up this to 30 minutes.
 468         // BUG?: it seems this is ignored, as this script will be run in an environment where
 469         //       environment variables will be discarded before this is used.
 470         def timeout = 1800000
 471         stepScript += "set __TestTimeout=${timeout}\r\n"
 472
 473         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 474         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 475
 476         // Create at least an empty script.
 477         stepScript += "echo. > ${stepScriptLocation}\r\n"
 478
 479         stressModeVars.each{ k, v ->
 480             // Write out what we are writing to the script file
 481             stepScript += "echo Setting ${k}=${v}\r\n"
 482             // Write out the set itself to the script file`
 483             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 484         }
 485     }
 486     else {
 487         stepScript += "echo Setting variables for ${stressModeName}\n"
 488         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 489         stressModeVars.each{ k, v ->
 490             // Write out what we are writing to the script file
 491             stepScript += "echo Setting ${k}=${v}\n"
 492             // Write out the set itself to the script file`
 493             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 494         }
 495         stepScript += "chmod +x ${stepScriptLocation}\n"
 496     }
 497
 498     return stepScript
 499 }
 500
 501 // Append an existing script to a stress mode script already created by genStressModeScriptStep().
 502 // Returns string of commands to do this.
 503 def static appendStressModeScriptStep(def os, def appendScript, def stepScriptLocation) {
 504     assert (os == 'Windows_NT')
 505     def stepScript = ''
 506     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 507     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 508
 509     // Display the resulting script. This is useful when looking at the output log file.
 510     stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 511     stepScript += "type ${stepScriptLocation}\r\n"
 512
 513     return stepScript
 514 }
 515
 516 def static isNeedDocker(def architecture, def os, def isBuild) {
 517     if (isBuild) {
 518         if (architecture == 'x86' && os == 'Ubuntu') {
 519             return true
 520         }
 521         else if (architecture == 'arm') {
 522             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 523                 return true
 524             }
 525         }
 526     }
 527     else {
 528         if (architecture == 'x86' && os == 'Ubuntu') {
 529             return true
 530         }
 531     }
 532     return false
 533 }
 534
 535 def static getDockerImageName(def architecture, def os, def isBuild) {
 536     // We must change some docker private images to official later
 537     if (isBuild) {
 538         if (architecture == 'x86' && os == 'Ubuntu') {
 539             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 540         }
 541         else if (architecture == 'arm') {
 542             if (os == 'Ubuntu') {
 543                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 544             }
 545             else if (os == 'Ubuntu16.04') {
 546                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 547             }
 548             else if (os == 'Tizen') {
 549                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 550             }
 551         }
 552     }
 553     else {
 554         if (architecture == 'x86' && os == 'Ubuntu') {
 555             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 556         }
 557     }
 558     println("Unknown architecture to use docker: ${architecture} ${os}");
 559     assert false
 560 }
 561
 562 // Calculates the name of the build job based on some typical parameters.
 563 //
 564 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 565     // If the architecture is x64, do not add that info into the build name.
 566     // Need to change around some systems and other builds to pick up the right builds
 567     // to do that.
 568
 569     def suffix = scenario != 'default' ? "_${scenario}" : '';
 570     if (isBuildOnly) {
 571         suffix += '_bld'
 572     }
 573     def baseName = ''
 574     switch (architecture) {
 575         case 'x64':
 576             if (scenario == 'default') {
 577                 // For now we leave x64 off of the name for compatibility with other jobs
 578                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 579             }
 580             else if (scenario == 'formatting') {
 581                 // we don't care about the configuration for the formatting job. It runs all configs
 582                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 583             }
 584             else {
 585                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 586             }
 587             break
 588         case 'arm64':
 589             if (os.toLowerCase() == "windows_nt") {
 590                 // These are cross builds
 591                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 592             }
 593             else {
 594                 // Defaults to a small page size set of machines.
 595                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 596             }
 597             break
 598         case 'arm':
 599             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 600             break
 601         case 'armlb':
 602             // These are cross builds
 603             if (os == 'Tizen') {
 604                 // ABI: softfp
 605                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 606             }
 607             else {
 608                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 609             }
 610             break
 611         case 'x86':
 612         case 'x86_arm_altjit':
 613         case 'x64_arm64_altjit':
 614             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 615             break
 616         default:
 617             println("Unknown architecture: ${architecture}");
 618             assert false
 619             break
 620     }
 621
 622     return baseName + suffix
 623 }
 624
 625 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 626
 627     // Check scenario.
 628     switch (scenario) {
 629         case 'default':
 630             switch (architecture) {
 631                 case 'x64':
 632                 case 'x86':
 633                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 634                         Utilities.addPeriodicTrigger(job, '@daily')
 635                     }
 636                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 637                         Utilities.addGithubPushTrigger(job)
 638                     }
 639                     break
 640                 case 'arm':
 641                 case 'armlb':
 642                 case 'x86_arm_altjit':
 643                 case 'x64_arm64_altjit':
 644                     Utilities.addGithubPushTrigger(job)
 645                     break
 646                 case 'arm64':
 647                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 648                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 649                     break
 650                 default:
 651                     println("Unknown architecture: ${architecture}");
 652                     assert false
 653                     break
 654             }
 655             break
 656         case 'r2r':
 657             assert !(os in bidailyCrossList)
 658             // r2r gets a push trigger for checked/release
 659             if (configuration == 'Checked' || configuration == 'Release') {
 660                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 661                 if (architecture == 'x64' && os != 'OSX10.12') {
 662                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 663                     if (isFlowJob || os == 'Windows_NT') {
 664                         Utilities.addGithubPushTrigger(job)
 665                     }
 666                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 667                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 668                     if (isFlowJob) {
 669                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 670                     }
 671                 }
 672                 // For x86, only add per-commit jobs for Windows
 673                 else if (architecture == 'x86') {
 674                     if (os == 'Windows_NT') {
 675                         Utilities.addGithubPushTrigger(job)
 676                     }
 677                 }
 678                 // arm64 r2r jobs should only run daily.
 679                 else if (architecture == 'arm64') {
 680                     if (os == 'Windows_NT') {
 681                         Utilities.addPeriodicTrigger(job, '@daily')
 682                     }
 683                 }
 684             }
 685             break
 686         case 'r2r_jitstress1':
 687         case 'r2r_jitstress2':
 688         case 'r2r_jitstressregs1':
 689         case 'r2r_jitstressregs2':
 690         case 'r2r_jitstressregs3':
 691         case 'r2r_jitstressregs4':
 692         case 'r2r_jitstressregs8':
 693         case 'r2r_jitstressregs0x10':
 694         case 'r2r_jitstressregs0x80':
 695         case 'r2r_jitstressregs0x1000':
 696         case 'r2r_jitminopts':
 697         case 'r2r_jitforcerelocs':
 698         case 'r2r_gcstress15':
 699             assert !(os in bidailyCrossList)
 700
 701             // GCStress=C is currently not supported on OS X
 702             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 703                 break
 704             }
 705
 706             // GC Stress 15 r2r gets a push trigger for checked/release
 707             if (configuration == 'Checked' || configuration == 'Release') {
 708                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 709                 if (architecture == 'x64') {
 710                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 711                     if (isFlowJob || os == 'Windows_NT') {
 712                         // Add a weekly periodic trigger
 713                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 714                     }
 715                 }
 716                 // For x86, only add per-commit jobs for Windows
 717                 else if (architecture == 'x86') {
 718                     if (os == 'Windows_NT') {
 719                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 720                     }
 721                 }
 722             }
 723             break
 724         case 'longgc':
 725             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 726             assert configuration == 'Release'
 727             assert architecture == 'x64'
 728             Utilities.addPeriodicTrigger(job, '@daily')
 729             // TODO: Add once external email sending is available again
 730             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 731             break
 732         case 'gcsimulator':
 733             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 734             assert configuration == 'Release'
 735             assert architecture == 'x64'
 736             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 737             // TODO: Add once external email sending is available again
 738             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 739             break
 740         case 'standalone_gc':
 741             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 742             assert (configuration == 'Release' || configuration == 'Checked')
 743             // TODO: Add once external email sending is available again
 744             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 745             Utilities.addPeriodicTrigger(job, '@daily')
 746             break
 747         case 'gc_reliability_framework':
 748             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 749             assert (configuration == 'Release' || configuration == 'Checked')
 750             // Only triggered by phrase.
 751             break
 752         case 'ilrt':
 753             assert !(os in bidailyCrossList)
 754             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 755             if (architecture == 'x64' && configuration == 'Release') {
 756                 // We don't expect to see a job generated except in these scenarios
 757                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 758                 if (isFlowJob || os == 'Windows_NT') {
 759                     Utilities.addPeriodicTrigger(job, '@daily')
 760                 }
 761             }
 762             break
 763         case 'jitdiff':
 764             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 765             assert configuration == 'Checked'
 766             assert (architecture == 'x64' || architecture == 'x86')
 767             Utilities.addGithubPushTrigger(job)
 768             break
 769         case 'formatting':
 770             assert (os == 'Windows_NT' || os == "Ubuntu")
 771             assert architecture == 'x64'
 772             Utilities.addGithubPushTrigger(job)
 773             break
 774         case 'jitstressregs1':
 775         case 'jitstressregs2':
 776         case 'jitstressregs3':
 777         case 'jitstressregs4':
 778         case 'jitstressregs8':
 779         case 'jitstressregs0x10':
 780         case 'jitstressregs0x80':
 781         case 'jitstressregs0x1000':
 782         case 'minopts':
 783         case 'forcerelocs':
 784         case 'jitstress1':
 785         case 'jitstress2':
 786         case 'jitstress2_jitstressregs1':
 787         case 'jitstress2_jitstressregs2':
 788         case 'jitstress2_jitstressregs3':
 789         case 'jitstress2_jitstressregs4':
 790         case 'jitstress2_jitstressregs8':
 791         case 'jitstress2_jitstressregs0x10':
 792         case 'jitstress2_jitstressregs0x80':
 793         case 'jitstress2_jitstressregs0x1000':
 794         case 'tailcallstress':
 795         case 'jitsse2only':
 796         case 'jitnosimd':
 797         case 'corefx_baseline':
 798         case 'corefx_minopts':
 799         case 'corefx_jitstress1':
 800         case 'corefx_jitstress2':
 801         case 'corefx_jitstressregs1':
 802         case 'corefx_jitstressregs2':
 803         case 'corefx_jitstressregs3':
 804         case 'corefx_jitstressregs4':
 805         case 'corefx_jitstressregs8':
 806         case 'corefx_jitstressregs0x10':
 807         case 'corefx_jitstressregs0x80':
 808         case 'corefx_jitstressregs0x1000':
 809         case 'zapdisable':
 810             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 811                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 812                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 813                     if (os == 'Windows_NT') {
 814                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 815                         Utilities.addPeriodicTrigger(job, '@weekly')
 816                     }
 817                 }
 818                 else {
 819                     Utilities.addPeriodicTrigger(job, '@daily')
 820                 }
 821             }
 822             break
 823         case 'heapverify1':
 824         case 'gcstress0x3':
 825             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 826                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 827                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 828                     if (os == 'Windows_NT') {
 829                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 830                         Utilities.addPeriodicTrigger(job, '@weekly')
 831                     }
 832                     // TODO: Add once external email sending is available again
 833                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 834                 }
 835                 else {
 836                     Utilities.addPeriodicTrigger(job, '@weekly')
 837                 }
 838             }
 839             break
 840         case 'gcstress0xc':
 841         case 'gcstress0xc_zapdisable':
 842         case 'gcstress0xc_zapdisable_jitstress2':
 843         case 'gcstress0xc_zapdisable_heapverify1':
 844         case 'gcstress0xc_jitstress1':
 845         case 'gcstress0xc_jitstress2':
 846         case 'gcstress0xc_minopts_heapverify1':
 847             // GCStress=C is currently not supported on OS X
 848             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 849                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 850                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 851                     if (os == 'Windows_NT') {
 852                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 853                         Utilities.addPeriodicTrigger(job, '@weekly')
 854                     }
 855                     // TODO: Add once external email sending is available again
 856                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 857                 }
 858                 else {
 859                     Utilities.addPeriodicTrigger(job, '@weekly')
 860                 }
 861             }
 862             break
 863
 864         case 'illink':
 865             // Testing on other operating systems TBD
 866             assert (os == 'Windows_NT' || os == 'Ubuntu')
 867             if (architecture == 'x64' || architecture == 'x86') {
 868                 if (configuration == 'Checked') {
 869                     Utilities.addPeriodicTrigger(job, '@daily')
 870                 }
 871             }
 872             break
 873
 874         case 'tieredcompilation':
 875         case 'corefx_tieredcompilation':
 876             // No periodic jobs just yet, still testing
 877             break
 878
 879         default:
 880             println("Unknown scenario: ${scenario}");
 881             assert false
 882             break
 883     }
 884     return
 885 }
 886
 887 // **************************
 888 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 889 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 890 // builds will do a couple extra steps.
 891 // **************************
 892
 893 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 894 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 895 // it's just used for internal builds
 896 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 897 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 898     if (isWindowsBuildOnlyJob) {
 899         return
 900     }
 901
 902     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 903     // Non pull request builds.
 904     if (!isPR) {
 905         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 906         return
 907     }
 908
 909      def arm64Users = [
 910         'AndyAyersMS',
 911         'briansull',
 912         'BruceForstall',
 913         'CarolEidt',
 914         'cmckinsey',
 915         'erozenfeld',
 916         'janvorli',
 917         'jashook',
 918         'JosephTremoulet',
 919         'pgodeq',
 920         'pgavlin',
 921         'rartemev',
 922         'russellhadley',
 923         'RussKeldorph',
 924         'sandreenko',
 925         'sdmaclea',
 926         'swaroop-sridhar',
 927         'jkotas',
 928         'markwilkie',
 929         'weshaggard'
 930     ]
 931
 932     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 933     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 934     def osGroup = getOSGroup(os)
 935     switch (architecture) {
 936         case 'x64': // editor brace matching: {
 937             if (scenario == 'formatting') {
 938                 assert configuration == 'Checked'
 939                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 940                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
 941                 }
 942                 break
 943             }
 944
 945             switch (os) {
 946                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 947                 case 'Debian8.4':
 948                 case 'RHEL7.2':
 949                     if (scenario == 'default') {
 950                         assert !isFlowJob
 951                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 952                     }
 953                     break
 954                 case 'Ubuntu16.04':
 955                     assert !isFlowJob
 956                     assert scenario == 'default'
 957                     // Distinguish with the other architectures (arm and x86)
 958                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 959                     break
 960                 case 'Fedora24':
 961                 case 'Ubuntu16.10':
 962                     assert !isFlowJob
 963                     assert scenario == 'default'
 964                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 965                     break
 966                 case 'Ubuntu':
 967                     if (scenario == 'illink') {
 968                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 969                         break
 970                     }
 971                     // fall through
 972                 case 'OSX10.12':
 973                     // Triggers on the non-flow jobs aren't necessary here
 974                     // Corefx testing uses non-flow jobs.
 975                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
 976                         break
 977                     }
 978                     switch (scenario) {
 979                         case 'default':
 980                             // OSX uses checked for default PR tests
 981                             if (configuration == 'Checked') {
 982                                 // Default trigger
 983                                 assert !job.name.contains("centos")
 984                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
 985                             }
 986                             break
 987                         case 'jitdiff':
 988                             if (configuration == 'Checked') {
 989                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 990                             }
 991                             break
 992                         case 'ilrt':
 993                             if (configuration == 'Release') {
 994                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 995                             }
 996                             break
 997                         case 'longgc':
 998                             if (configuration == 'Release') {
 999                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1000                             }
1001                             break
1002                         case 'gcsimulator':
1003                             if (configuration == 'Release') {
1004                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1005                             }
1006                             break
1007                         case 'standalone_gc':
1008                             if (configuration == 'Release' || configuration == 'Checked') {
1009                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1010                             }
1011                             break
1012                         case 'gc_reliability_framework':
1013                             if (configuration == 'Release' || configuration == 'Checked') {
1014                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1015                             }
1016                             break
1017                         default:
1018                             if (isJitStressScenario(scenario)) {
1019                                 def displayStr = getStressModeDisplayName(scenario)
1020                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1021                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1022                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1023                             }
1024                             else if (isR2RScenario(scenario)) {
1025                                 if (configuration == 'Release' || configuration == 'Checked') {
1026                                     def displayStr = getR2RDisplayName(scenario)
1027                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1028                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1029                                 }
1030                             }
1031                             else {
1032                                 println("Unknown scenario: ${scenario}");
1033                                 assert false
1034                             }
1035                             break
1036                     }
1037                     break
1038
1039                 case 'CentOS7.1':
1040                     switch (scenario) {
1041                         case 'default':
1042                             // CentOS uses checked for default PR tests while debug is build only
1043                             if (configuration == 'Debug') {
1044                                 // Default trigger
1045                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1046                             }
1047
1048                             // Make sure this is a flow job to get build and test.
1049                             if (configuration == 'Checked' && isFlowJob) {
1050                                 assert job.name.contains("flow")
1051                                 // Default trigger
1052                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1053                             }
1054                             break
1055                         default:
1056                             if (isR2RScenario(scenario)) {
1057                                 if (configuration == 'Release' || configuration == 'Checked') {
1058                                     def displayStr = getR2RDisplayName(scenario)
1059                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1060                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1061                                 }
1062                             }
1063                             break
1064                     }
1065                     break
1066
1067                 case 'Windows_NT':
1068                     switch (scenario) {
1069                         case 'default':
1070                             // Default trigger
1071                             if (configuration == 'Checked') {
1072                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1073                             }
1074                             break
1075                         case 'jitdiff':
1076                             if (configuration == 'Checked') {
1077                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1078                             }
1079                             break
1080                         case 'ilrt':
1081                             if (configuration == 'Release') {
1082                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1083                             }
1084                             break
1085                         case 'longgc':
1086                             if (configuration == 'Release') {
1087                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1088                             }
1089                             break
1090                         case 'gcsimulator':
1091                             if (configuration == 'Release') {
1092                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1093                             }
1094                             break
1095                         case 'standalone_gc':
1096                             if (configuration == 'Release' || configuration == 'Checked') {
1097                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1098                             }
1099                             break
1100                         case 'gc_reliability_framework':
1101                             if (configuration == 'Release' || configuration == 'Checked') {
1102                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1103                             }
1104                             break
1105                         case 'illink':
1106                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1107                             break
1108                         default:
1109                             if (isJitStressScenario(scenario)) {
1110                                 def displayStr = getStressModeDisplayName(scenario)
1111                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1112                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1113                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1114                             }
1115                             else if (isR2RScenario(scenario)) {
1116                                 if (configuration == 'Release' || configuration == 'Checked') {
1117                                     def displayStr = getR2RDisplayName(scenario)
1118                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1119                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1120                                 }
1121                             }
1122                             else {
1123                                 println("Unknown scenario: ${scenario}");
1124                                 assert false
1125                             }
1126                             break
1127                     }
1128                     break
1129                 default:
1130                     println("Unknown os: ${os}");
1131                     assert false
1132                     break
1133             }
1134             break
1135         // editor brace matching: }
1136         case 'armlb':
1137         case 'arm': // editor brace matching: {
1138             switch (os) {
1139                 case 'Ubuntu':
1140                 case 'Ubuntu16.04':
1141                     if (architecture == 'armlb') { // Ubuntu arm is only for armlb currently
1142                         assert scenario == 'default'
1143                         job.with {
1144                             publishers {
1145                                 azureVMAgentPostBuildAction {
1146                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1147                                 }
1148                             }
1149                         }
1150                         if ((os == 'Ubuntu' && configuration == 'Debug') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1151                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1152                         }
1153                         else {
1154                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1155                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1156                         }
1157                     }
1158                     break
1159                 case 'Tizen':
1160                     if (architecture == 'armlb') { // Tizen armel is only for armlb currently
1161                         architecture='armel'
1162                         job.with {
1163                             publishers {
1164                                 azureVMAgentPostBuildAction {
1165                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1166                                 }
1167                             }
1168                         }
1169                         // Removing the regex will cause this to run on each PR.
1170                         if (configuration == 'Checked') {
1171                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1172                         }
1173                         else {
1174                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1175                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1176                         }
1177                     }
1178                     break
1179                 case 'Windows_NT':
1180                     // Triggers on the non-flow jobs aren't necessary here
1181                     if (!isFlowJob) {
1182                         break
1183                     }
1184
1185                     // Set up a private trigger
1186                     def contextString = "${os} ${architecture} Cross ${configuration}"
1187                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1188                     if (scenario == 'default') {
1189                         contextString += " Innerloop"
1190                         triggerString += "\\W+Innerloop"
1191                     }
1192                     else {
1193                         contextString += " ${scenario}"
1194                         triggerString += "\\W+${scenario}"
1195                     }
1196
1197                     if (configuration == 'Debug') {
1198                         contextString += " Build"
1199                         triggerString += "\\W+Build"
1200                     } else {
1201                         contextString += " Build and Test"
1202                         triggerString += "\\W+Build and Test"
1203                     }
1204
1205                     triggerString += ".*"
1206
1207                     switch (scenario) {
1208                         case 'default':
1209                             // Only Checked is a default trigger.
1210                             if (configuration == 'Checked')
1211                             {
1212                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1213                             }
1214                             else
1215                             {
1216                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1217                             }
1218                             break
1219                         default:
1220                             // Stress jobs will use this code path.
1221                             if (isArmWindowsScenario(scenario)) {
1222                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1223                             }
1224                             break
1225                     }
1226                     break
1227                 default:
1228                     println("NYI os: ${os}");
1229                     assert false
1230                     break
1231             }
1232             break
1233         // editor brace matching: }
1234         case 'arm64': // editor brace matching: {
1235             // Set up a private trigger
1236             def contextString = "${os} ${architecture} Cross ${configuration}"
1237             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1238             if (scenario == 'default') {
1239                 contextString += " Innerloop"
1240                 triggerString += "\\W+Innerloop"
1241             }
1242             else {
1243                 contextString += " ${scenario}"
1244                 triggerString += "\\W+${scenario}"
1245             }
1246
1247             if (configuration == 'Debug') {
1248                 contextString += " Build"
1249                 triggerString += "\\W+Build"
1250             } else {
1251                 contextString += " Build and Test"
1252                 triggerString += "\\W+Build and Test"
1253             }
1254
1255             triggerString += ".*"
1256
1257             switch (os) {
1258                 case 'Ubuntu':
1259                 case 'Ubuntu16.04':
1260                     switch (scenario) {
1261                         case 'default':
1262                             if (configuration == 'Debug' && !isFlowJob) {
1263                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1264                             }
1265                             else {
1266                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1267                             }
1268
1269                             break
1270                         default:
1271                             if (isR2RScenario(scenario)) {
1272                                 if (configuration == 'Checked' || configuration == 'Release') {
1273                                     def displayStr = getR2RDisplayName(scenario)
1274                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1275                                 }
1276                             }
1277                             break
1278                     }
1279                     break
1280                 case 'Windows_NT':
1281                     // Triggers on the non-flow jobs aren't necessary here
1282                     if (!isFlowJob) {
1283                         break
1284                     }
1285
1286                     assert isArmWindowsScenario(scenario)
1287                     switch (scenario) {
1288                         case 'default':
1289                             if (configuration == 'Checked') {
1290                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1291                             }
1292                             else {
1293                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1294                             }
1295                             break
1296                         default:
1297                             // Stress jobs will use this code path.
1298                             if (isArmWindowsScenario(scenario)) {
1299                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1300                             }
1301                             break
1302                     }
1303                     break
1304                 default:
1305                     println("NYI os: ${os}");
1306                     assert false
1307                     break
1308             }
1309             break
1310         // editor brace matching: }
1311         case 'x86': // editor brace matching: {
1312             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1313             if (os == 'Ubuntu') {
1314                 // Triggers on the non-flow jobs aren't necessary here
1315                 if (!isFlowJob) {
1316                     break
1317                 }
1318                 // on-demand only for ubuntu x86
1319                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1320                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1321                 break
1322             }
1323             switch (scenario) {
1324                 case 'default':
1325                     if (configuration == 'Checked') {
1326                         assert !job.name.contains("centos")
1327                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1328                     }
1329                     else {
1330                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1331                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1332                     }
1333                     break
1334                 case 'ilrt':
1335                     if (configuration == 'Release') {
1336                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1337                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1338                     }
1339                     break
1340                 case 'longgc':
1341                     if (configuration == 'Release') {
1342                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1343                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1344                     }
1345                     break
1346                 case 'gcsimulator':
1347                     if (configuration == 'Release') {
1348                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1349                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1350                     }
1351                     break
1352                 case 'standalone_gc':
1353                     if (configuration == 'Release' || configuration == 'Checked') {
1354                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1355                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1356                     }
1357                     break
1358                 case 'illink':
1359                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1360                     break
1361                 default:
1362                     if (isJitStressScenario(scenario)) {
1363                         def displayStr = getStressModeDisplayName(scenario)
1364                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1365                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1366                     }
1367                     else if (isR2RScenario(scenario)) {
1368                         if (configuration == 'Release' || configuration == 'Checked') {
1369                             def displayStr = getR2RDisplayName(scenario)
1370                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1371                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1372                         }
1373                     }
1374                     else {
1375                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1376                         assert false
1377                     }
1378                     break
1379             }
1380             break
1381          // editor brace matching: }
1382         case 'x64_arm64_altjit':
1383         case 'x86_arm_altjit': // editor brace matching: {
1384             assert (os == 'Windows_NT')
1385             switch (scenario) {
1386                 case 'default':
1387                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1388                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1389                     break
1390                 default:
1391                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1392                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1393                     break
1394             }
1395             break
1396         // editor brace matching: }
1397         default:
1398             println("Unknown architecture: ${architecture}");
1399             assert false
1400             break
1401     }
1402 }
1403
1404 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1405     def buildCommands = [];
1406     def osGroup = getOSGroup(os)
1407     def lowerConfiguration = configuration.toLowerCase()
1408
1409     def priority = '1'
1410     if (scenario == 'default' && isPR == true) {
1411         priority = '0'
1412     }
1413
1414     def enableCorefxTesting = isCoreFxScenario(scenario)
1415
1416     // Calculate the build steps, archival, and xunit results
1417     switch (os) {
1418         case 'Windows_NT': // editor brace matching: {
1419             switch (architecture) {
1420                 case 'x64':
1421                 case 'x86':
1422                 case 'x86_arm_altjit':
1423                 case 'x64_arm64_altjit':
1424                     def arch = architecture
1425                     def buildOpts = ''
1426                     if (architecture == 'x86_arm_altjit') {
1427                         arch = 'x86'
1428                     }
1429                     else if (architecture == 'x64_arm64_altjit') {
1430                         arch = 'x64'
1431                     }
1432
1433                     if (scenario == 'formatting') {
1434                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1435                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1436                         break
1437                     }
1438
1439                     if (scenario == 'illink') {
1440                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1441                     }
1442
1443                     // If it is a release build for windows, ensure PGO is used, else fail the build
1444                     if ((lowerConfiguration == 'release') &&
1445                         (scenario in Constants.basicScenarios) &&
1446                         (architecture != 'x86_arm_altjit') &&
1447                         (architecture != 'x64_arm64_altjit')) {
1448
1449                         buildOpts += ' -enforcepgo'
1450                     }
1451
1452                     if (enableCorefxTesting) {
1453                         buildOpts += ' skiptests';
1454                     } else {
1455                         buildOpts += " -priority=${priority}"
1456                     }
1457
1458                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1459                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1460                     // 35 characters long.
1461
1462                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1463
1464                     if (!isBuildOnly) {
1465                         def runtestArguments = ''
1466                         def testOpts = 'collectdumps'
1467
1468                         if (isR2RScenario(scenario)) {
1469
1470                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1471                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1472                             // to cause the tests to be crossgen'ed.
1473
1474                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1475                                 testOpts += ' crossgenaltjit protononjit.dll'
1476                             } else {
1477                                 testOpts += ' crossgen'
1478                             }
1479
1480                             testOpts += ' runcrossgentests'
1481
1482                             if (scenario == 'r2r_jitstress1') {
1483                                 testOpts += ' jitstress 1'
1484                             }
1485                             else if (scenario == 'r2r_jitstress2') {
1486                                 testOpts += ' jitstress 2'
1487                             }
1488                             else if (scenario == 'r2r_jitstressregs1') {
1489                                 testOpts += ' jitstressregs 1'
1490                             }
1491                             else if (scenario == 'r2r_jitstressregs2') {
1492                                 testOpts += ' jitstressregs 2'
1493                             }
1494                             else if (scenario == 'r2r_jitstressregs3') {
1495                                 testOpts += ' jitstressregs 3'
1496                             }
1497                             else if (scenario == 'r2r_jitstressregs4') {
1498                                 testOpts += ' jitstressregs 4'
1499                             }
1500                             else if (scenario == 'r2r_jitstressregs8') {
1501                                 testOpts += ' jitstressregs 8'
1502                             }
1503                             else if (scenario == 'r2r_jitstressregs0x10') {
1504                                 testOpts += ' jitstressregs 0x10'
1505                             }
1506                             else if (scenario == 'r2r_jitstressregs0x80') {
1507                                 testOpts += ' jitstressregs 0x80'
1508                             }
1509                             else if (scenario == 'r2r_jitstressregs0x1000') {
1510                                 testOpts += ' jitstressregs 0x1000'
1511                             }
1512                             else if (scenario == 'r2r_jitminopts') {
1513                                 testOpts += ' jitminopts'
1514                             }
1515                             else if (scenario == 'r2r_jitforcerelocs') {
1516                                 testOpts += ' jitforcerelocs'
1517                             }
1518                             else if (scenario == 'r2r_gcstress15') {
1519                                 testOpts += ' gcstresslevel 0xF'
1520                             }
1521                         }
1522                         else if (scenario == 'jitdiff') {
1523                             testOpts += ' jitdisasm crossgen'
1524                         }
1525                         else if (scenario == 'ilrt') {
1526                             testOpts += ' ilasmroundtrip'
1527                         }
1528                         else if (isLongGc(scenario)) {
1529                             testOpts += " ${scenario} sequential"
1530                         }
1531                         else if (scenario == 'standalone_gc') {
1532                             testOpts += ' gcname clrgc.dll'
1533                         }
1534                         else if (scenario == 'illink') {
1535                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1536                         }
1537
1538                         // If we are running a stress mode, we should write out the set of key
1539                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1540
1541                         def envScriptPath = ''
1542                         if (isJitStressScenario(scenario)) {
1543                             def buildCommandsStr = ''
1544                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1545                             buildCommandsStr += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], envScriptPath)
1546                             if (architecture == 'x86_arm_altjit') {
1547                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1548                             }
1549                             else if (architecture == 'x64_arm64_altjit') {
1550                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1551                             }
1552
1553                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1554                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1555                             // into a single script.
1556                             buildCommands += buildCommandsStr
1557                         }
1558                         else if (architecture == 'x86_arm_altjit') {
1559                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1560                         }
1561                         else if (architecture == 'x64_arm64_altjit') {
1562                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1563                         }
1564                         if (envScriptPath != '') {
1565                             testOpts += " TestEnv ${envScriptPath}"
1566                         }
1567
1568                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1569
1570                         if (enableCorefxTesting) {
1571                             def workspaceRelativeFxRoot = "_/fx"
1572                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1573
1574                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1575
1576                             setTestJobTimeOut(newJob, isPR, architecture, scenario)
1577
1578                             // Archive and process (only) the test results
1579                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1580                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1581
1582                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1583                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1584                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1585                         }
1586                         else if (isGcReliabilityFramework(scenario)) {
1587                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1588                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1589                         }
1590                         else {
1591                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1592                         }
1593                     }
1594
1595                     if (!enableCorefxTesting) {
1596                         // Run the rest of the build
1597                         // Build the mscorlib for the other OS's
1598                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1599                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1600
1601                         if (arch == "x64") {
1602                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1603                         }
1604
1605                         // Zip up the tests directory so that we don't use so much space/time copying
1606                         // 10s of thousands of files around.
1607                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1608
1609                         if (!isJitStressScenario(scenario)) {
1610                             // For windows, pull full test results and test drops for x86/x64.
1611                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1612                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1613                         }
1614
1615                         if (scenario == 'jitdiff') {
1616                             // retrive jit-dasm output for base commit, and run jit-diff
1617                             if (!isBuildOnly) {
1618                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1619                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1620                             }
1621                         }
1622
1623                         if (!isBuildOnly) {
1624                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1625                             setTestJobTimeOut(newJob, isPR, architecture, scenario)
1626                         }
1627                     }
1628                     break
1629                 case 'armlb':
1630                 case 'arm':
1631                     assert isArmWindowsScenario(scenario)
1632
1633                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1634                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1635
1636                     setTestJobTimeOut(newJob, isPR, architecture, scenario)
1637
1638                     def buildArchitecture = 'arm'
1639
1640                     // For 'arm' (the RyuJIT/arm32 architecture), tell build.cmd to use RyuJIT/arm32 for crossgen compilation.
1641                     // RyuJIT/arm32 is currently not the default JIT; it is an aljit. So, this is a special case.
1642                     def armCrossgenOpt = ''
1643                     if (architecture == 'arm') {
1644                         armCrossgenOpt = '-altjitcrossgen'
1645                     }
1646
1647                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1648                     priority = '1'
1649
1650                     // This is now a build only job. Do not run tests. Use the flow job.
1651                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1652
1653                     // Zip up the tests directory so that we don't use so much space/time copying
1654                     // 10s of thousands of files around.
1655                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1656
1657                     // Add archival.
1658                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1659                     break
1660                 case 'arm64':
1661                     assert isArmWindowsScenario(scenario)
1662
1663                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1664                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1665
1666                     setTestJobTimeOut(newJob, isPR, architecture, scenario)
1667
1668                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1669                     priority = '1'
1670
1671                     // This is now a build only job. Do not run tests. Use the flow job.
1672                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1673
1674                     // Zip up the tests directory so that we don't use so much space/time copying
1675                     // 10s of thousands of files around.
1676                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1677
1678                     // Add archival.
1679                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1680                     break
1681                 default:
1682                     println("Unknown architecture: ${architecture}");
1683                     assert false
1684                     break
1685             }
1686             break
1687         // editor brace matching: }
1688         case 'Ubuntu':
1689         case 'Ubuntu16.04':
1690         case 'Ubuntu16.10':
1691         case 'Debian8.4':
1692         case 'OSX10.12':
1693         case 'CentOS7.1':
1694         case 'RHEL7.2':
1695         case 'Tizen':
1696         case 'Fedora24': // editor brace matching: {
1697             switch (architecture) {
1698                 case 'x64':
1699                 case 'x86':
1700                     if (architecture == 'x86' && os == 'Ubuntu') {
1701                         // build and PAL test
1702                         def dockerImage = getDockerImageName(architecture, os, true)
1703                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1704                         dockerImage = getDockerImageName(architecture, os, false)
1705                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1706                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1707                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1708                         break
1709                     }
1710
1711                     if (scenario == 'formatting') {
1712                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1713                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1714                         break
1715                     }
1716
1717                     if (scenario == 'illink') {
1718                         assert(os == 'Ubuntu')
1719                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1720                     }
1721
1722                     if (!enableCorefxTesting) {
1723                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1724                         // only on supported OS platforms.
1725                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1726                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1727                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1728                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1729
1730                         setTestJobTimeOut(newJob, isPR, architecture, scenario)
1731
1732                         // Basic archiving of the build
1733                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1734                         // And pal tests
1735                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1736                     }
1737                     else {
1738                         // Corefx stress testing
1739                         assert os == 'Ubuntu'
1740                         assert architecture == 'x64'
1741                         assert lowerConfiguration == 'checked'
1742                         assert isJitStressScenario(scenario)
1743
1744                         // Build coreclr
1745                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1746
1747                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1748                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1749
1750                         // Build and text corefx
1751                         def workspaceRelativeFxRoot = "_/fx"
1752                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1753
1754                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1755
1756                         setTestJobTimeOut(newJob, isPR, architecture, scenario)
1757
1758                         // Archive and process (only) the test results
1759                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1760                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1761                     }
1762                     break
1763                 case 'arm64':
1764                     if (!enableCorefxTesting) {
1765                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1766
1767                         // HACK -- Arm64 does not have corefx jobs yet.
1768                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1769                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1770                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1771                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1772
1773                         setTestJobTimeOut(newJob, isPR, architecture, scenario)
1774                         // Basic archiving of the build
1775                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1776                     }
1777                     break
1778                 case 'armlb':
1779                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1780                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1781
1782                     // default values for Ubuntu
1783                     def arm_abi="arm"
1784                     def linuxCodeName="trusty"
1785                     if (os == 'Ubuntu16.04') {
1786                         linuxCodeName="xenial"
1787                     }
1788                     else if (os == 'Tizen') {
1789                         arm_abi="armel"
1790                         linuxCodeName="tizen"
1791                     }
1792
1793                     // Unzip the Windows test binaries first. Exit with 0
1794                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1795
1796                     // Unpack the corefx binaries
1797                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1798                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1799                     if (os != 'Tizen') {
1800                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1801                     }
1802                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1803                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1804                     if (os == 'Ubuntu') {
1805                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1806                     }
1807
1808                     // Call the ARM CI script to cross build and test using docker
1809                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1810                     --mode=docker \\
1811                     --${arm_abi} \\
1812                     --linuxCodeName=${linuxCodeName} \\
1813                     --buildConfig=${lowerConfiguration} \\
1814                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1815                     --coreFxBinDir=./bin/CoreFxBinDir \\
1816                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1817
1818                     // Basic archiving of the build, no pal tests
1819                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1820                     break
1821                 default:
1822                     println("Unknown architecture: ${architecture}");
1823                     assert false
1824                     break
1825             }
1826             break
1827         // editor brace matching: }
1828         default:
1829             println("Unknown os: ${os}");
1830             assert false
1831             break
1832     } // os
1833
1834     return buildCommands
1835 }
1836
1837 Constants.allScenarios.each { scenario ->
1838     [true, false].each { isPR ->
1839         Constants.architectureList.each { architecture ->
1840             Constants.configurationList.each { configuration ->
1841                 Constants.osList.each { os ->
1842                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1843                     // and reset the os to Windows_NT
1844                     def isBuildOnly = false
1845                     if (os == 'Windows_NT_BuildOnly') {
1846                         isBuildOnly = true
1847                         os = 'Windows_NT'
1848                     }
1849
1850                     // Tizen is only supported for arm legacy_backend architecture
1851                     if (os == 'Tizen' && architecture != 'armlb') {
1852                         return
1853                     }
1854
1855                     // Skip totally unimplemented (in CI) configurations.
1856                     switch (architecture) {
1857                         case 'arm64':
1858                             if (os == 'Ubuntu16.04') {
1859                                 os = 'Ubuntu'
1860                             }
1861
1862                             // Windows and Ubuntu only
1863                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1864                                 return
1865                             }
1866                             break
1867                         case 'armlb':
1868                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1869                                 return
1870                             }
1871                             break
1872                         case 'arm':
1873                             if (os != 'Windows_NT') {
1874                                 return
1875                             }
1876                             break
1877                         case 'x86':
1878                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1879                                 return
1880                             }
1881                             break
1882                         case 'x86_arm_altjit':
1883                         case 'x64_arm64_altjit':
1884                             if (os != 'Windows_NT') {
1885                                 return
1886                             }
1887                             break
1888                         case 'x64':
1889                             // Everything implemented
1890                             break
1891                         default:
1892                             println("Unknown architecture: ${architecture}")
1893                             assert false
1894                             break
1895                     }
1896
1897                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1898                     // with checked builds)
1899                     if (isJitStressScenario(scenario)) {
1900                         if (configuration != 'Checked') {
1901                             return
1902                         }
1903
1904                         // Since these are just execution time differences,
1905                         // skip platforms that don't execute the tests here (Windows_NT only)
1906                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
1907                         if (!isEnabledOS || isBuildOnly) {
1908                             return
1909                         }
1910
1911                         switch (architecture) {
1912                             case 'x64':
1913                             case 'x86':
1914                             case 'x86_arm_altjit':
1915                             case 'x64_arm64_altjit':
1916                                 // x86 ubuntu: default only
1917                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1918                                     return
1919                                 }
1920                                 // Windows: Everything implemented
1921                                 break
1922
1923                             default:
1924                                 // arm, arm64, armlb: stress is handled through flow jobs.
1925                                 return
1926                         }
1927                     }
1928                     else if (isR2RScenario(scenario)) {
1929                         if (os != 'Windows_NT') {
1930                             return
1931                         }
1932                         // R2R test runs are not implemented for arm/armlb/arm64.
1933                         if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
1934                             return
1935                         }
1936                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
1937                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
1938                             return
1939                         }
1940                     }
1941                     else {
1942                         // Skip scenarios
1943                         switch (scenario) {
1944                             case 'ilrt':
1945                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1946                                 // the default scenario build
1947                                 if (os != 'Windows_NT') {
1948                                     return
1949                                 }
1950                                 // Only x64 for now
1951                                 if (architecture != 'x64') {
1952                                     return
1953                                 }
1954                                 // Release only
1955                                 if (configuration != 'Release') {
1956                                     return
1957                                 }
1958                                 break
1959                             case 'jitdiff':
1960                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1961                                     return
1962                                 }
1963                                 if (architecture != 'x64') {
1964                                     return
1965                                 }
1966                                 if (configuration != 'Checked') {
1967                                     return
1968                                 }
1969                                 break
1970                             case 'longgc':
1971                             case 'gcsimulator':
1972                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1973                                     return
1974                                 }
1975                                 if (architecture != 'x64') {
1976                                     return
1977                                 }
1978                                 if (configuration != 'Release') {
1979                                     return
1980                                 }
1981                                 break
1982                             case 'gc_reliability_framework':
1983                             case 'standalone_gc':
1984                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1985                                     return
1986                                 }
1987
1988                                 if (architecture != 'x64') {
1989                                     return
1990                                 }
1991
1992                                 if (configuration != 'Release' && configuration != 'Checked') {
1993                                     return
1994                                 }
1995                                 break
1996                             // We only run Windows and Ubuntu x64 Checked for formatting right now
1997                             case 'formatting':
1998                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
1999                                     return
2000                                 }
2001                                 if (architecture != 'x64') {
2002                                     return
2003                                 }
2004                                 if (configuration != 'Checked') {
2005                                     return
2006                                 }
2007                                 if (isBuildOnly) {
2008                                     return
2009                                 }
2010                                 break
2011                             case 'illink':
2012                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2013                                     return
2014                                 }
2015                                 if (architecture != 'x64' && architecture != 'x86') {
2016                                     return
2017                                 }
2018                                 if (isBuildOnly) {
2019                                     return
2020                                 }
2021                                 break
2022                             case 'default':
2023                                 // Nothing skipped
2024                                 break
2025                             default:
2026                                 println("Unknown scenario: ${scenario}")
2027                                 assert false
2028                                 break
2029                         }
2030                     }
2031
2032                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2033                     // runtime behavior, not compile-time behavior, are not interesting.
2034                     switch (architecture) {
2035                         case 'x86_arm_altjit':
2036                         case 'x64_arm64_altjit':
2037                             if (isGCStressRelatedTesting(scenario)) {
2038                                 return
2039                             }
2040                             break
2041                         default:
2042                             break
2043                     }
2044
2045                     // Calculate names
2046                     def lowerConfiguration = configuration.toLowerCase()
2047                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2048                     def folderName = getJobFolder(scenario)
2049
2050                     // Create the new job
2051                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2052
2053                     def machineAffinityOptions = null
2054
2055                     if (os != 'Windows_NT') {
2056                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2057                     }
2058                     else {
2059                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2060                     }
2061
2062                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2063
2064                     // Add all the standard options
2065                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2066                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2067
2068                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2069                     def osGroup = getOSGroup(os)
2070
2071                     newJob.with {
2072                         steps {
2073                             if (os == 'Windows_NT') {
2074                                 buildCommands.each { buildCommand ->
2075                                     batchFile(buildCommand)
2076                                 }
2077                             }
2078                             else {
2079                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2080                                 if ( architecture == 'armlb' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2081                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2082                                     // Define the Windows Tests and Corefx build job names
2083                                     def WindowTestsName = projectFolder + '/' +
2084                                                           Utilities.getFullJobName(project,
2085                                                                                    getJobName(lowerConfiguration,
2086                                                                                               'x64' ,
2087                                                                                               'windows_nt',
2088                                                                                               'default',
2089                                                                                               true),
2090                                                                                    false)
2091                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2092                                                        Utilities.getFolderName(branch)
2093
2094                                     // Copy the Windows test binaries and the Corefx build binaries
2095                                     copyArtifacts(WindowTestsName) {
2096                                         includePatterns('bin/tests/tests.zip')
2097                                         buildSelector {
2098                                             latestSuccessful(true)
2099                                         }
2100                                     }
2101
2102                                     def arm_abi = 'arm'
2103                                     def corefx_os = 'linux'
2104                                     if (os == 'Tizen') {
2105                                         arm_abi = 'armel'
2106                                         corefx_os = 'tizen'
2107                                     }
2108
2109                                     // Let's use release CoreFX to test checked CoreCLR,
2110                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2111                                     def corefx_lowerConfiguration = lowerConfiguration
2112                                     if ( lowerConfiguration == 'checked' ) {
2113                                         corefx_lowerConfiguration='release'
2114                                     }
2115
2116                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2117                                         includePatterns('bin/build.tar.gz')
2118                                         buildSelector {
2119                                             latestSuccessful(true)
2120                                         }
2121                                     }
2122                                 }
2123
2124                                 buildCommands.each { buildCommand ->
2125                                     shell(buildCommand)
2126                                 }
2127                             }
2128                         }
2129                     } // newJob.with
2130
2131                 } // os
2132             } // configuration
2133         } // architecture
2134     } // isPR
2135 } // scenario
2136
2137
2138 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2139 Constants.allScenarios.each { scenario ->
2140     [true, false].each { isPR ->
2141         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2142             Constants.crossList.each { os ->
2143                 if (architecture == 'arm64') {
2144                     if (os != "Ubuntu" && os != "Windows_NT") {
2145                         return
2146                     }
2147                 } else if (architecture == 'arm' || architecture == 'armlb') {
2148                     if (os != 'Windows_NT') {
2149                         return
2150                     }
2151                 }
2152                 else if (architecture == 'x86') {
2153                     if (os != "Ubuntu") {
2154                         return
2155                     }
2156                 }
2157
2158                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2159
2160                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2161                     return
2162                 }
2163
2164                 Constants.configurationList.each { configuration ->
2165
2166                     // First, filter based on OS.
2167
2168                     if (os == 'Windows_NT') {
2169                         if (!isArmWindowsScenario(scenario)) {
2170                             return
2171                         }
2172                     }
2173                     else {
2174                         // Non-Windows
2175                         if (architecture == 'arm64') {
2176                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2177                                 return
2178                             }
2179                         }
2180                         else if (architecture == 'x86') {
2181                             // Linux/x86 only want default test
2182                             if (scenario != 'default') {
2183                                 return
2184                             }
2185                         }
2186                     }
2187
2188                     // For CentOS, we only want Checked/Release builds.
2189                     if (os == 'CentOS7.1') {
2190                         if (configuration != 'Checked' && configuration != 'Release') {
2191                             return
2192                         }
2193                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2194                             return
2195                         }
2196                     }
2197
2198                     // For RedHat and Debian, we only do Release builds.
2199                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2200                         if (configuration != 'Release') {
2201                             return
2202                         }
2203                         if (scenario != 'default') {
2204                             return
2205                         }
2206                     }
2207
2208                     // Next, filter based on scenario.
2209
2210                     if (isJitStressScenario(scenario)) {
2211                         if (configuration != 'Checked') {
2212                             return
2213                         }
2214                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2215                         if (isCoreFxScenario(scenario)) {
2216                             return
2217                         }
2218                     }
2219                     else if (isR2RBaselineScenario(scenario)) {
2220                         if (configuration != 'Checked' && configuration != 'Release') {
2221                             return
2222                         }
2223                     }
2224                     else if (isR2RStressScenario(scenario)) {
2225                         if (configuration != 'Checked') {
2226                             return
2227                         }
2228                     }
2229                     else {
2230                         // Skip scenarios
2231                         switch (scenario) {
2232                             case 'ilrt':
2233                             case 'longgc':
2234                             case 'gcsimulator':
2235                                 // Long GC tests take a long time on non-Release builds
2236                                 // ilrt is also Release only
2237                                 if (configuration != 'Release') {
2238                                     return
2239                                 }
2240                                 break
2241                             case 'jitdiff':
2242                                 if (configuration != 'Checked') {
2243                                     return;
2244                                 }
2245                                 break
2246                             case 'gc_reliability_framework':
2247                             case 'standalone_gc':
2248                                 if (configuration != 'Release' && configuration != 'Checked') {
2249                                     return
2250                                 }
2251                                 break
2252                             case 'formatting':
2253                                 return
2254                             case 'illink':
2255                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2256                                     return
2257                                 }
2258                                 break
2259                             case 'default':
2260                                 // Nothing skipped
2261                                 break
2262                             default:
2263                                 println("Unknown scenario: ${scenario}")
2264                                 assert false
2265                                 break
2266                         }
2267                     }
2268
2269                     // Done filtering. Now, create the jobs.
2270
2271                     def lowerConfiguration = configuration.toLowerCase()
2272                     def osGroup = getOSGroup(os)
2273                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2274
2275                     def inputCoreCLRBuildName = projectFolder + '/' +
2276                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2277
2278                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2279                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2280                     def testBuildScenario = scenario
2281                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2282                         testBuildScenario = 'default'
2283                     }
2284
2285                     def inputWindowsTestBuildArch = architecture
2286                     if (architecture == "arm64" && os != "Windows_NT") {
2287                         // Use the x64 test build for arm64 unix
2288                         inputWindowsTestBuildArch = "x64"
2289                     }
2290
2291                     def inputWindowTestsBuildName = projectFolder + '/' +
2292                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2293
2294                     // Enable Server GC for Ubuntu PR builds
2295                     def serverGCString = ''
2296                     if (os == 'Ubuntu' && isPR) {
2297                         serverGCString = '--useServerGC'
2298                     }
2299
2300                     def testOpts = ''
2301
2302                     if (isR2RScenario(scenario)) {
2303
2304                         testOpts += ' --crossgen --runcrossgentests'
2305
2306                         if (scenario == 'r2r_jitstress1') {
2307                             testOpts += ' --jitstress=1'
2308                         }
2309                         else if (scenario == 'r2r_jitstress2') {
2310                             testOpts += ' --jitstress=2'
2311                         }
2312                         else if (scenario == 'r2r_jitstressregs1') {
2313                             testOpts += ' --jitstressregs=1'
2314                         }
2315                         else if (scenario == 'r2r_jitstressregs2') {
2316                             testOpts += ' --jitstressregs=2'
2317                         }
2318                         else if (scenario == 'r2r_jitstressregs3') {
2319                             testOpts += ' --jitstressregs=3'
2320                         }
2321                         else if (scenario == 'r2r_jitstressregs4') {
2322                             testOpts += ' --jitstressregs=4'
2323                         }
2324                         else if (scenario == 'r2r_jitstressregs8') {
2325                             testOpts += ' --jitstressregs=8'
2326                         }
2327                         else if (scenario == 'r2r_jitstressregs0x10') {
2328                             testOpts += ' --jitstressregs=0x10'
2329                         }
2330                         else if (scenario == 'r2r_jitstressregs0x80') {
2331                             testOpts += ' --jitstressregs=0x80'
2332                         }
2333                         else if (scenario == 'r2r_jitstressregs0x1000') {
2334                             testOpts += ' --jitstressregs=0x1000'
2335                         }
2336                         else if (scenario == 'r2r_jitminopts') {
2337                             testOpts += ' --jitminopts'
2338                         }
2339                         else if (scenario == 'r2r_jitforcerelocs') {
2340                             testOpts += ' --jitforcerelocs'
2341                         }
2342                         else if (scenario == 'r2r_gcstress15') {
2343                             testOpts += ' --gcstresslevel=0xF'
2344                         }
2345                     }
2346                     else if (scenario == 'jitdiff') {
2347                         testOpts += ' --jitdisasm --crossgen'
2348                     }
2349                     else if (scenario == 'illink') {
2350                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2351                     }
2352                     else if (isLongGc(scenario)) {
2353                         // Long GC tests behave very poorly when they are not
2354                         // the only test running (many of them allocate until OOM).
2355                         testOpts += ' --sequential'
2356
2357                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2358                         // for running long GC and GCSimulator tests, respectively. We don't use them
2359                         // here because using a playlist file produces much more readable output on the CI machines
2360                         // and reduces running time.
2361                         //
2362                         // The Long GC playlist contains all of the tests that are
2363                         // going to be run. The GCSimulator playlist contains all of
2364                         // the GC simulator tests.
2365                         if (scenario == 'longgc') {
2366                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2367                         }
2368                         else if (scenario == 'gcsimulator') {
2369                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2370                         }
2371                     }
2372                     else if (isGcReliabilityFramework(scenario)) {
2373                         testOpts += ' --build-overlay-only'
2374                     }
2375                     else if (scenario == 'standalone_gc') {
2376                         if (osGroup == 'OSX') {
2377                             testOpts += ' --gcname=libclrgc.dylib'
2378                         }
2379                         else if (osGroup == 'Linux') {
2380                             testOpts += ' --gcname=libclrgc.so'
2381                         }
2382                         else {
2383                             println("Unexpected OS group: ${osGroup} for os ${os}")
2384                             assert false
2385                         }
2386                     }
2387
2388                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2389
2390                     def folder = getJobFolder(scenario)
2391                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2392                         // Add parameters for the inputs
2393
2394                         if (windowsArmJob == true) {
2395                             parameters {
2396                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2397                             }
2398                         }
2399                         else {
2400                             parameters {
2401                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2402                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2403                             }
2404                         }
2405
2406                         steps {
2407                             // Set up the copies
2408
2409                             // Coreclr build containing the tests and mscorlib
2410
2411                             if (windowsArmJob != true) {
2412                                 copyArtifacts(inputWindowTestsBuildName) {
2413                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2414                                     buildSelector {
2415                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2416                                     }
2417                                 }
2418                             }
2419
2420                             // Coreclr build we are trying to test
2421
2422                             copyArtifacts(inputCoreCLRBuildName) {
2423                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2424                                 buildSelector {
2425                                     buildNumber('${CORECLR_BUILD}')
2426                                 }
2427                             }
2428
2429                             // Windows CoreCLR Arm(64) will restore corefx
2430                             // packages correctly.
2431                             //
2432                             // In addition, test steps are entirely different
2433                             // because we do not have a unified runner
2434                             if (windowsArmJob != true) {
2435                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2436
2437                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2438                                 def osJobName
2439                                 if (os == 'Ubuntu') {
2440                                     osJobName = 'ubuntu14.04'
2441                                 }
2442                                 else if (architecture == 'x86') {
2443                                     if (os == 'Ubuntu') {
2444                                         // Linux/x86 corefx jobs does not build managed yet
2445                                         // Clone linux/arm corefx managed packages and overwrite linux/x86 native
2446                                         osJobName = "linux_arm_cross"
2447                                     }
2448                                 }
2449                                 else {
2450                                     osJobName = os.toLowerCase()
2451                                 }
2452                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2453                                     includePatterns('bin/build.tar.gz')
2454                                     buildSelector {
2455                                         latestSuccessful(true)
2456                                     }
2457                                 }
2458
2459                                 shell("mkdir ./bin/CoreFxBinDir")
2460                                 // Unpack the corefx binaries
2461                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2462
2463                                 // HACK -- Arm64 does not have corefx jobs yet.
2464                                 // Clone corefx and build the native packages overwriting the x64 packages.
2465                                 if (architecture == 'arm64') {
2466                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2467                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2468                                 }
2469                                 else if (architecture == 'x86') {
2470                                     shell("mkdir ./bin/CoreFxNative")
2471
2472                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2473                                         includePatterns('bin/build.tar.gz')
2474                                         targetDirectory('bin/CoreFxNative')
2475                                         buildSelector {
2476                                             latestSuccessful(true)
2477                                         }
2478                                     }
2479
2480                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2481                                 }
2482
2483                                 // Unzip the tests first.  Exit with 0
2484                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2485
2486                                 // Execute the tests
2487                                 def runDocker = isNeedDocker(architecture, os, false)
2488                                 def dockerPrefix = ""
2489                                 def dockerCmd = ""
2490                                 if (runDocker) {
2491                                     def dockerImage = getDockerImageName(architecture, os, false)
2492                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2493                                     dockerCmd = dockerPrefix + "${dockerImage} "
2494                                 }
2495
2496                                 // If we are running a stress mode, we'll set those variables first
2497                                 def testEnvOpt = ""
2498                                 if (isJitStressScenario(scenario)) {
2499                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2500                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2501                                     shell("${createScriptCmds}")
2502                                     testEnvOpt = "--test-env=" + scriptFileName
2503                                 }
2504
2505                                 if (isGCStressRelatedTesting(scenario)) {
2506                                     shell('./init-tools.sh')
2507                                 }
2508
2509                                 shell("""${dockerCmd}./tests/runtest.sh \\
2510                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2511                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2512                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2513                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2514                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2515                 --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2516
2517                                 if (isGcReliabilityFramework(scenario)) {
2518                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2519                                     if (serverGCString != '') {
2520                                         if (runDocker) {
2521                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2522                                         }
2523                                         else {
2524                                             shell("export COMPlus_gcServer=1")
2525                                         }
2526                                     }
2527
2528                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2529                                 }
2530                             }
2531
2532                             else { // windowsArmJob == true
2533                                 // Unzip tests.
2534                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2535
2536                                 // Build the build commands
2537                                 def buildCommands = ""
2538
2539                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2540                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2541                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2542
2543                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2544                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2545
2546                                 // For all jobs
2547                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2548
2549                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2550                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2551
2552                                 // Arm(32) ryujit
2553                                 if (architecture == "arm") {
2554                                     // **This is an AltJit**
2555
2556                                     addEnvVariable("COMPlus_AltJit", "*")
2557                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2558                                     addEnvVariable("COMPlus_AltJitName", "protojit.dll")
2559                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2560                                 }
2561
2562                                 // If we are running a stress mode, we'll set those variables as well
2563                                 def stressValues = null
2564                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2565                                     if (isJitStressScenario(scenario)) {
2566                                         stressValues = Constants.jitStressModeScenarios[scenario]
2567                                     }
2568                                     else {
2569                                         stressValues = Constants.r2rStressScenarios[scenario]
2570                                     }
2571
2572                                     stressValues.each { key, value ->
2573                                         addEnvVariable(key, value)
2574                                     }
2575                                 }
2576
2577                                 // TODO: do whatever is necessary to support enabling R2R testing. Environment variables,
2578                                 // crossgen the framework assemblies, etc.
2579
2580                                 // Create the smarty command
2581                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2582                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2583                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2584
2585                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "arm") { addExclude("PROTOJIT_" + exclude) } else { addExclude(exclude) } }
2586
2587                                 if (architecture == "arm") {
2588                                     addExclude("PROTOJIT_FAIL")
2589                                 }
2590
2591                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2592                                     def failTag = "JITSTRESS_FAIL"
2593                                     def excludeTag = "JITSTRESS_EXCLUDE"
2594
2595                                     if (scenario.contains('gc')) {
2596                                         failTag = "GCSTRESS_FAIL"
2597                                         excludeTag = "GCSTRESS_EXCLUDE"
2598                                     }
2599
2600                                     addArchSpecificExclude(architecture, failTag)
2601                                     addArchSpecificExclude(architecture, excludeTag)
2602                                 }
2603                                 else {
2604                                     addExclude("pri1")
2605                                 }
2606
2607                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2608                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2609                                 addExclude("LONG_RUNNING")
2610
2611                                 smartyCommand += "/lstFile Tests.lst"
2612
2613                                 def testListArch = [
2614                                     'arm64': 'arm64',
2615                                     'arm': 'arm',
2616                                     'armlb': 'arm'
2617                                 ]
2618
2619                                 def archLocation = testListArch[architecture]
2620
2621                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2622                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2623                                 addCommand("${smartyCommand}")
2624
2625                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2626                                 // However, we also need to remove all the variables that were set during this batch file, so we
2627                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2628                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2629                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2630                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2631                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2632                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2633
2634                                 // ZIP up the smarty output, no matter what the smarty result.
2635                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2636
2637                                 addCommand("echo %errorlevel%")
2638                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2639
2640                                 // Use the smarty errorlevel as the script errorlevel.
2641                                 addCommand("exit /b %__save_smarty_errorlevel%")
2642
2643                                 batchFile(buildCommands)
2644                             }
2645                         }
2646                     }
2647
2648                     if (scenario == 'jitdiff') {
2649                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2650                     }
2651
2652                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2653                     if (os in ['Ubuntu']) {
2654                         SummaryBuilder summaries = new SummaryBuilder()
2655                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2656                         summaries.emit(newJob)
2657                     }
2658
2659                     def affinityOptions = null
2660
2661                     if (windowsArmJob == true) {
2662                         affinityOptions = [
2663                             "use_arm64_build_machine" : false
2664                         ]
2665                     }
2666
2667                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2668                         affinityOptions = [
2669                             "large_pages" : false
2670                         ]
2671                     }
2672
2673                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2674                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2675
2676                     setTestJobTimeOut(newJob, isPR, architecture, scenario)
2677
2678                     if (windowsArmJob != true) {
2679                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2680                     }
2681                     else {
2682                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2683
2684                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2685                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2686                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2687                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2688                     }
2689
2690                     // Create a build flow to join together the build and tests required to run this test.
2691                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2692                     // Linux CoreCLR test
2693                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2694                     def fullTestJobName = projectFolder + '/' + newJob.name
2695                     // Add a reference to the input jobs for report purposes
2696                     JobReport.Report.addReference(inputCoreCLRBuildName)
2697                     JobReport.Report.addReference(inputWindowTestsBuildName)
2698                     JobReport.Report.addReference(fullTestJobName)
2699                     def newFlowJob = null
2700
2701                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2702                         // Do not create the flow job for RHEL jobs.
2703                         return
2704                     }
2705
2706                     if (windowsArmJob == true) {
2707                         // For Windows arm jobs there is no reason to build a parallel test job.
2708                         // The product build supports building and archiving the tests.
2709
2710                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2711                         buildFlow("""
2712 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2713
2714 // And then build the test build
2715 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2716 """)
2717                         }
2718                     }
2719                     else {
2720                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2721                         buildFlow("""
2722 // Build the input jobs in parallel
2723 parallel (
2724 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2725 { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2726 )
2727
2728 // And then build the test build
2729 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2730             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2731 """)
2732                         }
2733                     }
2734
2735                     // For the flow jobs set the machine affinity as x64 if an armarch.
2736                     def flowArch = architecture
2737
2738                     if (flowArch in validWindowsNTCrossArches) {
2739                         flowArch = 'x64'
2740                         affinityOptions = null
2741                     }
2742
2743                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2744                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2745                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2746                 } // configuration
2747             } // os
2748         } // architecture
2749     } // isPR
2750 } // scenario
2751
2752 JobReport.Report.generateJobReport(out)
2753
2754 // Make the call to generate the help job
2755 Utilities.createHelperJob(this, project, branch,
2756     "Welcome to the ${project} Repository",  // This is prepended to the help message
2757     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2758
2759 Utilities.addCROSSCheck(this, project, branch)