netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'CentOS7.1',
  49                'RHEL7.2',
  50                'Ubuntu16.04',
  51                'Ubuntu16.10',
  52                'Fedora24',
  53                'Tizen']
  54
  55     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  56
  57     // This is a set of JIT stress modes combined with the set of variables that
  58     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  59     // the values are the environment variables
  60     def static jitStressModeScenarios = [
  61                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  62                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  63                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  64                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  65                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  66                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  67                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  68                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  69                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  70                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  71                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  72                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  73                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  74                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  75                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  76                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  77                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  78                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  79                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  80                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  81                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  82                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  83                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  84                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  85                'corefx_baseline'                : [ : ], // corefx baseline
  86                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  87                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  88                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  89                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  90                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  91                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  92                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  93                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  94                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  95                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  96                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  97                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  98                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  99                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 100                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 101                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 102                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 103                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 104                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 105                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 106                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 107                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 108     ]
 109
 110     // This is a set of ReadyToRun stress scenarios
 111     def static r2rStressScenarios = [
 112                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 113                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 114                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 115                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 116                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 117                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 118                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 119                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 120                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 121                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 122                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 123                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 124                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 125     ]
 126
 127     // This is the basic set of scenarios
 128     def static basicScenarios = [
 129                'default',
 130                'ilrt',
 131                'r2r',
 132                'longgc',
 133                'formatting',
 134                'gcsimulator',
 135                'jitdiff',
 136                'standalone_gc',
 137                'gc_reliability_framework',
 138                'illink'] + r2rStressScenarios.keySet()
 139
 140     def static allScenarios = basicScenarios + jitStressModeScenarios.keySet()
 141
 142     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 143     // to Tests.lst file categories to exclude.
 144     //
 145     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 146     // and with the same values, with some commented out, for easier maintenance.
 147     //
 148     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 149     //
 150     def static validArmWindowsScenarios = [
 151                'default':                                [],
 152                // 'ilrt'
 153                // 'r2r':                                    ["R2R_FAIL"],
 154                // 'longgc'
 155                // 'formatting'
 156                // 'gcsimulator'
 157                // 'jitdiff'
 158                // 'standalone_gc'
 159                // 'gc_reliability_framework'
 160                // 'illink'
 161                // 'r2r_jitstress1'
 162                // 'r2r_jitstress2'
 163                // 'r2r_jitstressregs1'
 164                // 'r2r_jitstressregs2'
 165                // 'r2r_jitstressregs3'
 166                // 'r2r_jitstressregs4'
 167                // 'r2r_jitstressregs8'
 168                // 'r2r_jitstressregs0x10'
 169                // 'r2r_jitstressregs0x80'
 170                // 'r2r_jitstressregs0x1000'
 171                // 'r2r_jitminopts'
 172                // 'r2r_jitforcerelocs'
 173                // 'r2r_gcstress15'
 174                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 175                'tieredcompilation':                      [],
 176                'forcerelocs':                            [],
 177                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 179                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 180                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 181                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 196                // 'jitsse2only'                          // Only relevant to xarch
 197                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 198                // 'corefx_baseline'
 199                // 'corefx_minopts'
 200                // 'corefx_tieredcompilation'
 201                // 'corefx_jitstress1'
 202                // 'corefx_jitstress2'
 203                // 'corefx_jitstressregs1'
 204                // 'corefx_jitstressregs2'
 205                // 'corefx_jitstressregs3'
 206                // 'corefx_jitstressregs4'
 207                // 'corefx_jitstressregs8'
 208                // 'corefx_jitstressregs0x10'
 209                // 'corefx_jitstressregs0x80'
 210                // 'corefx_jitstressregs0x1000'
 211                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 212                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 213                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 214                'heapverify1':                            [],
 215                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 216                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 217                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 219                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 220                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 221
 222                //
 223                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 224                //
 225
 226                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 227                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 228                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 245     ]
 246
 247     def static configurationList = ['Debug', 'Checked', 'Release']
 248
 249     // This is the set of architectures
 250     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 251 }
 252
 253 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 254     assert os instanceof String
 255     assert architecture instanceof String
 256
 257     def armArches = ['arm', 'armlb', 'arm64']
 258     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 259
 260     if (!(architecture in armArches)) {
 261         assert options == null
 262         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 263
 264         return
 265     }
 266
 267     // This is an arm(64) job.
 268     //
 269     // There are several options.
 270     //
 271     // Windows_NT
 272     //
 273     // Arm32 (Build) -> latest-arm64
 274     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 275     // Arm32 (Test)  -> arm64-windows_nt
 276     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 277     //
 278     // Arm64 (Build) -> latest-arm64
 279     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 280     // Arm64 (Test)  -> arm64-windows_nt
 281     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 282     //
 283     // Ubuntu
 284     //
 285     // Arm32 (Build) -> arm-cross-latest
 286     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 287     // Arm32 (Test)  -> NYI Arch not supported
 288     //       |->
 289     //
 290     // Arm64 (Build) -> arm64-cross-latest
 291     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 292     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 293     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 294     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 295     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 296
 297     // This has to be a arm arch
 298     assert architecture in armArches
 299     if (os == "Windows_NT") {
 300         // Arm(64) windows jobs share the same machines for now
 301         def isBuild = options['use_arm64_build_machine'] == true
 302
 303         if (isBuild == true) {
 304             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 305         } else {
 306             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 307         }
 308     } else {
 309         assert os != 'Windows_NT'
 310         assert os in supportedArmLinuxOs
 311
 312         if (architecture == 'arm' || architecture == 'armlb') {
 313             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 314         } else {
 315             // Arm64 Linux
 316             if (options['is_build_only'] == true) {
 317                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 318             } else {
 319                 // Arm64 Test Machines
 320                 if (options['large_pages'] == false) {
 321                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 322                 } else {
 323                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 324                 }
 325             }
 326         }
 327     }
 328 }
 329
 330 def static isGCStressRelatedTesting(def scenario) {
 331     // The 'r2r_gcstress15' scenario is a basic scenario.
 332     // Detect it and make it a GCStress related.
 333     if (scenario == 'r2r_gcstress15')
 334     {
 335         return true;
 336     }
 337
 338     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 339     def scenarioName = scenario.toLowerCase()
 340     def isGCStressTesting = false
 341     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 342         if (k in gcStressTestEnvVars) {
 343             isGCStressTesting = true;
 344         }
 345     }
 346     return isGCStressTesting
 347 }
 348
 349 def static isCoreFxScenario(def scenario) {
 350     def corefx_prefix = 'corefx_'
 351     if (scenario.length() < corefx_prefix.length()) {
 352         return false
 353     }
 354     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 355 }
 356
 357 def static isR2RBaselineScenario(def scenario) {
 358     return (scenario == 'r2r')
 359 }
 360
 361 def static isR2RStressScenario(def scenario) {
 362     return Constants.r2rStressScenarios.containsKey(scenario)
 363 }
 364
 365 def static isR2RScenario(def scenario) {
 366     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 367 }
 368
 369 def static isJitStressScenario(def scenario) {
 370     return Constants.jitStressModeScenarios.containsKey(scenario)
 371 }
 372
 373 def static isLongGc(def scenario) {
 374     return (scenario == 'longgc' || scenario == 'gcsimulator')
 375 }
 376
 377 def static isJitDiff(def scenario) {
 378     return (scenario == 'jitdiff')
 379 }
 380
 381 def static isGcReliabilityFramework(def scenario) {
 382     return (scenario == 'gc_reliability_framework')
 383 }
 384
 385 def static isArmWindowsScenario(def scenario) {
 386     return Constants.validArmWindowsScenarios.containsKey(scenario)
 387 }
 388
 389 def static setTestJobTimeOut(newJob, scenario) {
 390     if (isGCStressRelatedTesting(scenario)) {
 391         Utilities.setJobTimeout(newJob, 4320)
 392     }
 393     else if (isCoreFxScenario(scenario)) {
 394         Utilities.setJobTimeout(newJob, 360)
 395     }
 396     else if (isJitStressScenario(scenario)) {
 397         Utilities.setJobTimeout(newJob, 240)
 398     }
 399     else if (isR2RBaselineScenario(scenario)) {
 400         Utilities.setJobTimeout(newJob, 240)
 401     }
 402     else if (isLongGc(scenario)) {
 403         Utilities.setJobTimeout(newJob, 1440)
 404     }
 405     else if (isJitDiff(scenario)) {
 406         Utilities.setJobTimeout(newJob, 240)
 407     }
 408     else if (isGcReliabilityFramework(scenario)) {
 409         Utilities.setJobTimeout(newJob, 1440)
 410     }
 411     // Non-test jobs use the default timeout value.
 412 }
 413
 414 def static getJobFolder(def scenario) {
 415     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 416         return 'jitstress'
 417     }
 418     if (scenario == 'illink') {
 419         return 'illink'
 420     }
 421     return ''
 422 }
 423
 424 def static getStressModeDisplayName(def scenario) {
 425     def displayStr = ''
 426     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 427         def prefixLength = 'COMPlus_'.length()
 428         if (k.length() >= prefixLength) {
 429             def modeName = k.substring(prefixLength, k.length())
 430             displayStr += ' ' + modeName + '=' + v
 431         }
 432     }
 433
 434     if (isCoreFxScenario(scenario)) {
 435         displayStr = ('CoreFx ' + displayStr).trim()
 436     }
 437
 438     return displayStr
 439 }
 440
 441 def static getR2RDisplayName(def scenario) {
 442     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 443     def displayStr = scenario
 444     def prefixLength = 'r2r_'.length()
 445     if (displayStr.length() >= prefixLength) {
 446         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 447     } else if (scenario == 'r2r') {
 448         displayStr = "R2R"
 449     }
 450     return displayStr
 451 }
 452
 453 // Generates the string for creating a file that sets environment variables
 454 // that makes it possible to run stress modes.  Writes the script to the file
 455 // specified by the stepScriptLocation parameter.
 456 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 457     def stepScript = ''
 458     if (os == 'Windows_NT') {
 459         // Timeout in ms, default is 10 minutes. For stress modes up this to 30 minutes.
 460         // BUG?: it seems this is ignored, as this script will be run in an environment where
 461         //       environment variables will be discarded before this is used.
 462         def timeout = 1800000
 463         stepScript += "set __TestTimeout=${timeout}\r\n"
 464
 465         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 466         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 467
 468         // Create at least an empty script.
 469         stepScript += "echo. > ${stepScriptLocation}\r\n"
 470
 471         stressModeVars.each{ k, v ->
 472             // Write out what we are writing to the script file
 473             stepScript += "echo Setting ${k}=${v}\r\n"
 474             // Write out the set itself to the script file`
 475             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 476         }
 477     }
 478     else {
 479         stepScript += "echo Setting variables for ${stressModeName}\n"
 480         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 481         stressModeVars.each{ k, v ->
 482             // Write out what we are writing to the script file
 483             stepScript += "echo Setting ${k}=${v}\n"
 484             // Write out the set itself to the script file`
 485             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 486         }
 487         stepScript += "chmod +x ${stepScriptLocation}\n"
 488     }
 489
 490     return stepScript
 491 }
 492
 493 // Append an existing script to a stress mode script already created by genStressModeScriptStep().
 494 // Returns string of commands to do this.
 495 def static appendStressModeScriptStep(def os, def appendScript, def stepScriptLocation) {
 496     assert (os == 'Windows_NT')
 497     def stepScript = ''
 498     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 499     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 500
 501     // Display the resulting script. This is useful when looking at the output log file.
 502     stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 503     stepScript += "type ${stepScriptLocation}\r\n"
 504
 505     return stepScript
 506 }
 507
 508 def static isNeedDocker(def architecture, def os, def isBuild) {
 509     if (isBuild) {
 510         if (architecture == 'x86' && os == 'Ubuntu') {
 511             return true
 512         }
 513         else if (architecture == 'arm') {
 514             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 515                 return true
 516             }
 517         }
 518     }
 519     else {
 520         if (architecture == 'x86' && os == 'Ubuntu') {
 521             return true
 522         }
 523     }
 524     return false
 525 }
 526
 527 def static getDockerImageName(def architecture, def os, def isBuild) {
 528     // We must change some docker private images to official later
 529     if (isBuild) {
 530         if (architecture == 'x86' && os == 'Ubuntu') {
 531             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 532         }
 533         else if (architecture == 'arm') {
 534             if (os == 'Ubuntu') {
 535                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 536             }
 537             else if (os == 'Ubuntu16.04') {
 538                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 539             }
 540             else if (os == 'Tizen') {
 541                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 542             }
 543         }
 544     }
 545     else {
 546         if (architecture == 'x86' && os == 'Ubuntu') {
 547             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 548         }
 549     }
 550     println("Unknown architecture to use docker: ${architecture} ${os}");
 551     assert false
 552 }
 553
 554 // Calculates the name of the build job based on some typical parameters.
 555 //
 556 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 557     // If the architecture is x64, do not add that info into the build name.
 558     // Need to change around some systems and other builds to pick up the right builds
 559     // to do that.
 560
 561     def suffix = scenario != 'default' ? "_${scenario}" : '';
 562     if (isBuildOnly) {
 563         suffix += '_bld'
 564     }
 565     def baseName = ''
 566     switch (architecture) {
 567         case 'x64':
 568             if (scenario == 'default') {
 569                 // For now we leave x64 off of the name for compatibility with other jobs
 570                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 571             }
 572             else if (scenario == 'formatting') {
 573                 // we don't care about the configuration for the formatting job. It runs all configs
 574                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 575             }
 576             else {
 577                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 578             }
 579             break
 580         case 'arm64':
 581             if (os.toLowerCase() == "windows_nt") {
 582                 // These are cross builds
 583                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 584             }
 585             else {
 586                 // Defaults to a small page size set of machines.
 587                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 588             }
 589             break
 590         case 'arm':
 591             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 592             break
 593         case 'armlb':
 594             // These are cross builds
 595             if (os == 'Tizen') {
 596                 // ABI: softfp
 597                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 598             }
 599             else {
 600                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 601             }
 602             break
 603         case 'x86':
 604         case 'x86_arm_altjit':
 605         case 'x64_arm64_altjit':
 606             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 607             break
 608         default:
 609             println("Unknown architecture: ${architecture}");
 610             assert false
 611             break
 612     }
 613
 614     return baseName + suffix
 615 }
 616
 617 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 618
 619     // Check scenario.
 620     switch (scenario) {
 621         case 'default':
 622             switch (architecture) {
 623                 case 'x64':
 624                 case 'x86':
 625                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 626                         Utilities.addPeriodicTrigger(job, '@daily')
 627                     }
 628                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 629                         Utilities.addGithubPushTrigger(job)
 630                     }
 631                     break
 632                 case 'arm':
 633                 case 'armlb':
 634                 case 'x86_arm_altjit':
 635                 case 'x64_arm64_altjit':
 636                     Utilities.addGithubPushTrigger(job)
 637                     break
 638                 case 'arm64':
 639                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 640                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 641                     break
 642                 default:
 643                     println("Unknown architecture: ${architecture}");
 644                     assert false
 645                     break
 646             }
 647             break
 648         case 'r2r':
 649             assert !(os in bidailyCrossList)
 650             // r2r gets a push trigger for checked/release
 651             if (configuration == 'Checked' || configuration == 'Release') {
 652                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 653                 if (architecture == 'x64' && os != 'OSX10.12') {
 654                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 655                     if (isFlowJob || os == 'Windows_NT') {
 656                         Utilities.addGithubPushTrigger(job)
 657                     }
 658                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 659                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 660                     if (isFlowJob) {
 661                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 662                     }
 663                 }
 664                 // For x86, only add per-commit jobs for Windows
 665                 else if (architecture == 'x86') {
 666                     if (os == 'Windows_NT') {
 667                         Utilities.addGithubPushTrigger(job)
 668                     }
 669                 }
 670                 // arm64 r2r jobs should only run daily.
 671                 else if (architecture == 'arm64') {
 672                     if (os == 'Windows_NT') {
 673                         Utilities.addPeriodicTrigger(job, '@daily')
 674                     }
 675                 }
 676             }
 677             break
 678         case 'r2r_jitstress1':
 679         case 'r2r_jitstress2':
 680         case 'r2r_jitstressregs1':
 681         case 'r2r_jitstressregs2':
 682         case 'r2r_jitstressregs3':
 683         case 'r2r_jitstressregs4':
 684         case 'r2r_jitstressregs8':
 685         case 'r2r_jitstressregs0x10':
 686         case 'r2r_jitstressregs0x80':
 687         case 'r2r_jitstressregs0x1000':
 688         case 'r2r_jitminopts':
 689         case 'r2r_jitforcerelocs':
 690         case 'r2r_gcstress15':
 691             assert !(os in bidailyCrossList)
 692
 693             // GCStress=C is currently not supported on OS X
 694             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 695                 break
 696             }
 697
 698             // GC Stress 15 r2r gets a push trigger for checked/release
 699             if (configuration == 'Checked' || configuration == 'Release') {
 700                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 701                 if (architecture == 'x64') {
 702                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 703                     if (isFlowJob || os == 'Windows_NT') {
 704                         // Add a weekly periodic trigger
 705                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 706                     }
 707                 }
 708                 // For x86, only add per-commit jobs for Windows
 709                 else if (architecture == 'x86') {
 710                     if (os == 'Windows_NT') {
 711                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 712                     }
 713                 }
 714             }
 715             break
 716         case 'longgc':
 717             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 718             assert configuration == 'Release'
 719             assert architecture == 'x64'
 720             Utilities.addPeriodicTrigger(job, '@daily')
 721             // TODO: Add once external email sending is available again
 722             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 723             break
 724         case 'gcsimulator':
 725             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 726             assert configuration == 'Release'
 727             assert architecture == 'x64'
 728             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 729             // TODO: Add once external email sending is available again
 730             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 731             break
 732         case 'standalone_gc':
 733             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 734             assert (configuration == 'Release' || configuration == 'Checked')
 735             // TODO: Add once external email sending is available again
 736             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 737             Utilities.addPeriodicTrigger(job, '@daily')
 738             break
 739         case 'gc_reliability_framework':
 740             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 741             assert (configuration == 'Release' || configuration == 'Checked')
 742             // Only triggered by phrase.
 743             break
 744         case 'ilrt':
 745             assert !(os in bidailyCrossList)
 746             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 747             if (architecture == 'x64' && configuration == 'Release') {
 748                 // We don't expect to see a job generated except in these scenarios
 749                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 750                 if (isFlowJob || os == 'Windows_NT') {
 751                     Utilities.addPeriodicTrigger(job, '@daily')
 752                 }
 753             }
 754             break
 755         case 'jitdiff':
 756             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 757             assert configuration == 'Checked'
 758             assert (architecture == 'x64' || architecture == 'x86')
 759             Utilities.addGithubPushTrigger(job)
 760             break
 761         case 'formatting':
 762             assert (os == 'Windows_NT' || os == "Ubuntu")
 763             assert architecture == 'x64'
 764             Utilities.addGithubPushTrigger(job)
 765             break
 766         case 'jitstressregs1':
 767         case 'jitstressregs2':
 768         case 'jitstressregs3':
 769         case 'jitstressregs4':
 770         case 'jitstressregs8':
 771         case 'jitstressregs0x10':
 772         case 'jitstressregs0x80':
 773         case 'jitstressregs0x1000':
 774         case 'minopts':
 775         case 'forcerelocs':
 776         case 'jitstress1':
 777         case 'jitstress2':
 778         case 'jitstress2_jitstressregs1':
 779         case 'jitstress2_jitstressregs2':
 780         case 'jitstress2_jitstressregs3':
 781         case 'jitstress2_jitstressregs4':
 782         case 'jitstress2_jitstressregs8':
 783         case 'jitstress2_jitstressregs0x10':
 784         case 'jitstress2_jitstressregs0x80':
 785         case 'jitstress2_jitstressregs0x1000':
 786         case 'tailcallstress':
 787         case 'jitsse2only':
 788         case 'jitnosimd':
 789         case 'corefx_baseline':
 790         case 'corefx_minopts':
 791         case 'corefx_jitstress1':
 792         case 'corefx_jitstress2':
 793         case 'corefx_jitstressregs1':
 794         case 'corefx_jitstressregs2':
 795         case 'corefx_jitstressregs3':
 796         case 'corefx_jitstressregs4':
 797         case 'corefx_jitstressregs8':
 798         case 'corefx_jitstressregs0x10':
 799         case 'corefx_jitstressregs0x80':
 800         case 'corefx_jitstressregs0x1000':
 801         case 'zapdisable':
 802             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 803                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 804                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 805                     if (os == 'Windows_NT') {
 806                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 807                         Utilities.addPeriodicTrigger(job, '@weekly')
 808                     }
 809                 }
 810                 else {
 811                     Utilities.addPeriodicTrigger(job, '@daily')
 812                 }
 813             }
 814             break
 815         case 'heapverify1':
 816         case 'gcstress0x3':
 817             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 818                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 819                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 820                     if (os == 'Windows_NT') {
 821                         Utilities.addPeriodicTrigger(job, '@daily')
 822                     }
 823                     // TODO: Add once external email sending is available again
 824                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 825                 }
 826                 else {
 827                     Utilities.addPeriodicTrigger(job, '@weekly')
 828                 }
 829             }
 830             break
 831         case 'gcstress0xc':
 832         case 'gcstress0xc_zapdisable':
 833         case 'gcstress0xc_zapdisable_jitstress2':
 834         case 'gcstress0xc_zapdisable_heapverify1':
 835         case 'gcstress0xc_jitstress1':
 836         case 'gcstress0xc_jitstress2':
 837         case 'gcstress0xc_minopts_heapverify1':
 838             // GCStress=C is currently not supported on OS X
 839             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 840                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 841                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 842                     // TODO: Enable a periodic trigger after tests are updated.
 843                     // Utilities.addPeriodicTrigger(job, '@daily')
 844                     // TODO: Add once external email sending is available again
 845                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 846                 }
 847                 else {
 848                     Utilities.addPeriodicTrigger(job, '@weekly')
 849                 }
 850             }
 851             break
 852
 853         case 'illink':
 854             // Testing on other operating systems TBD
 855             assert (os == 'Windows_NT' || os == 'Ubuntu')
 856             if (architecture == 'x64' || architecture == 'x86') {
 857                 if (configuration == 'Checked') {
 858                     Utilities.addPeriodicTrigger(job, '@daily')
 859                 }
 860             }
 861             break
 862
 863         case 'tieredcompilation':
 864         case 'corefx_tieredcompilation':
 865             // No periodic jobs just yet, still testing
 866             break
 867
 868         default:
 869             println("Unknown scenario: ${scenario}");
 870             assert false
 871             break
 872     }
 873     return
 874 }
 875
 876 // **************************
 877 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 878 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 879 // builds will do a couple extra steps.
 880 // **************************
 881
 882 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 883 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 884 // it's just used for internal builds
 885 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 886 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 887     if (isWindowsBuildOnlyJob) {
 888         return
 889     }
 890
 891     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 892     // Non pull request builds.
 893     if (!isPR) {
 894         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 895         return
 896     }
 897
 898      def arm64Users = [
 899         'AndyAyersMS',
 900         'briansull',
 901         'BruceForstall',
 902         'CarolEidt',
 903         'cmckinsey',
 904         'erozenfeld',
 905         'janvorli',
 906         'jashook',
 907         'JosephTremoulet',
 908         'pgodeq',
 909         'pgavlin',
 910         'rartemev',
 911         'russellhadley',
 912         'RussKeldorph',
 913         'sandreenko',
 914         'sdmaclea',
 915         'swaroop-sridhar',
 916         'jkotas',
 917         'markwilkie',
 918         'weshaggard'
 919     ]
 920
 921     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 922     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 923     def osGroup = getOSGroup(os)
 924     switch (architecture) {
 925         case 'x64': // editor brace matching: {
 926             if (scenario == 'formatting') {
 927                 assert configuration == 'Checked'
 928                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 929                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
 930                 }
 931                 break
 932             }
 933
 934             switch (os) {
 935                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 936                 case 'Debian8.4':
 937                 case 'RHEL7.2':
 938                     if (scenario == 'default') {
 939                         assert !isFlowJob
 940                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 941                     }
 942                     break
 943                 case 'Ubuntu16.04':
 944                     assert !isFlowJob
 945                     assert scenario == 'default'
 946                     // Distinguish with the other architectures (arm and x86)
 947                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 948                     break
 949                 case 'Fedora24':
 950                 case 'Ubuntu16.10':
 951                     assert !isFlowJob
 952                     assert scenario == 'default'
 953                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 954                     break
 955                 case 'Ubuntu':
 956                     if (scenario == 'illink') {
 957                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 958                         break
 959                     }
 960                     // fall through
 961                 case 'OSX10.12':
 962                     // Triggers on the non-flow jobs aren't necessary here
 963                     // Corefx testing uses non-flow jobs.
 964                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
 965                         break
 966                     }
 967                     switch (scenario) {
 968                         case 'default':
 969                             // OSX uses checked for default PR tests
 970                             if (configuration == 'Checked') {
 971                                 // Default trigger
 972                                 assert !job.name.contains("centos")
 973                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
 974                             }
 975                             break
 976                         case 'jitdiff':
 977                             if (configuration == 'Checked') {
 978                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 979                             }
 980                             break
 981                         case 'ilrt':
 982                             if (configuration == 'Release') {
 983                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 984                             }
 985                             break
 986                         case 'longgc':
 987                             if (configuration == 'Release') {
 988                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 989                             }
 990                             break
 991                         case 'gcsimulator':
 992                             if (configuration == 'Release') {
 993                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 994                             }
 995                             break
 996                         case 'standalone_gc':
 997                             if (configuration == 'Release' || configuration == 'Checked') {
 998                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 999                             }
1000                             break
1001                         case 'gc_reliability_framework':
1002                             if (configuration == 'Release' || configuration == 'Checked') {
1003                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1004                             }
1005                             break
1006                         default:
1007                             if (isJitStressScenario(scenario)) {
1008                                 def displayStr = getStressModeDisplayName(scenario)
1009                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1010                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1011                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1012                             }
1013                             else if (isR2RScenario(scenario)) {
1014                                 if (configuration == 'Release' || configuration == 'Checked') {
1015                                     def displayStr = getR2RDisplayName(scenario)
1016                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1017                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1018                                 }
1019                             }
1020                             else {
1021                                 println("Unknown scenario: ${scenario}");
1022                                 assert false
1023                             }
1024                             break
1025                     }
1026                     break
1027
1028                 case 'CentOS7.1':
1029                     switch (scenario) {
1030                         case 'default':
1031                             // CentOS uses checked for default PR tests while debug is build only
1032                             if (configuration == 'Debug') {
1033                                 // Default trigger
1034                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1035                             }
1036
1037                             // Make sure this is a flow job to get build and test.
1038                             if (configuration == 'Checked' && isFlowJob) {
1039                                 assert job.name.contains("flow")
1040                                 // Default trigger
1041                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1042                             }
1043                             break
1044                         default:
1045                             if (isR2RScenario(scenario)) {
1046                                 if (configuration == 'Release' || configuration == 'Checked') {
1047                                     def displayStr = getR2RDisplayName(scenario)
1048                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1049                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1050                                 }
1051                             }
1052                             break
1053                     }
1054                     break
1055
1056                 case 'Windows_NT':
1057                     switch (scenario) {
1058                         case 'default':
1059                             // Default trigger
1060                             if (configuration == 'Checked') {
1061                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1062                             }
1063                             break
1064                         case 'jitdiff':
1065                             if (configuration == 'Checked') {
1066                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1067                             }
1068                             break
1069                         case 'ilrt':
1070                             if (configuration == 'Release') {
1071                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1072                             }
1073                             break
1074                         case 'longgc':
1075                             if (configuration == 'Release') {
1076                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1077                             }
1078                             break
1079                         case 'gcsimulator':
1080                             if (configuration == 'Release') {
1081                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1082                             }
1083                             break
1084                         case 'standalone_gc':
1085                             if (configuration == 'Release' || configuration == 'Checked') {
1086                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1087                             }
1088                             break
1089                         case 'gc_reliability_framework':
1090                             if (configuration == 'Release' || configuration == 'Checked') {
1091                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1092                             }
1093                             break
1094                         case 'illink':
1095                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1096                             break
1097                         default:
1098                             if (isJitStressScenario(scenario)) {
1099                                 def displayStr = getStressModeDisplayName(scenario)
1100                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1101                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1102                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1103                             }
1104                             else if (isR2RScenario(scenario)) {
1105                                 if (configuration == 'Release' || configuration == 'Checked') {
1106                                     def displayStr = getR2RDisplayName(scenario)
1107                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1108                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1109                                 }
1110                             }
1111                             else {
1112                                 println("Unknown scenario: ${scenario}");
1113                                 assert false
1114                             }
1115                             break
1116                     }
1117                     break
1118                 default:
1119                     println("Unknown os: ${os}");
1120                     assert false
1121                     break
1122             }
1123             break
1124         // editor brace matching: }
1125         case 'armlb':
1126         case 'arm': // editor brace matching: {
1127             switch (os) {
1128                 case 'Ubuntu':
1129                 case 'Ubuntu16.04':
1130                     if (architecture == 'armlb') { // Ubuntu arm is only for armlb currently
1131                         assert scenario == 'default'
1132                         job.with {
1133                             publishers {
1134                                 azureVMAgentPostBuildAction {
1135                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1136                                 }
1137                             }
1138                         }
1139                         if ((os == 'Ubuntu' && configuration == 'Debug') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1140                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1141                         }
1142                         else {
1143                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1144                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1145                         }
1146                     }
1147                     break
1148                 case 'Tizen':
1149                     if (architecture == 'armlb') { // Tizen armel is only for armlb currently
1150                         architecture='armel'
1151                         job.with {
1152                             publishers {
1153                                 azureVMAgentPostBuildAction {
1154                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1155                                 }
1156                             }
1157                         }
1158                         // Removing the regex will cause this to run on each PR.
1159                         if (configuration == 'Checked') {
1160                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1161                         }
1162                         else {
1163                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1164                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1165                         }
1166                     }
1167                     break
1168                 case 'Windows_NT':
1169                     // Triggers on the non-flow jobs aren't necessary here
1170                     if (!isFlowJob) {
1171                         break
1172                     }
1173
1174                     // Set up a private trigger
1175                     def contextString = "${os} ${architecture} Cross ${configuration}"
1176                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1177                     if (scenario == 'default') {
1178                         contextString += " Innerloop"
1179                         triggerString += "\\W+Innerloop"
1180                     }
1181                     else {
1182                         contextString += " ${scenario}"
1183                         triggerString += "\\W+${scenario}"
1184                     }
1185
1186                     if (configuration == 'Debug') {
1187                         contextString += " Build"
1188                         triggerString += "\\W+Build"
1189                     } else {
1190                         contextString += " Build and Test"
1191                         triggerString += "\\W+Build and Test"
1192                     }
1193
1194                     triggerString += ".*"
1195
1196                     switch (scenario) {
1197                         case 'default':
1198                             // Only Checked is a default trigger.
1199                             if (configuration == 'Checked')
1200                             {
1201                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1202                             }
1203                             else
1204                             {
1205                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1206                             }
1207                             break
1208                         default:
1209                             // Stress jobs will use this code path.
1210                             if (isArmWindowsScenario(scenario)) {
1211                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1212                             }
1213                             break
1214                     }
1215                     break
1216                 default:
1217                     println("NYI os: ${os}");
1218                     assert false
1219                     break
1220             }
1221             break
1222         // editor brace matching: }
1223         case 'arm64': // editor brace matching: {
1224             // Set up a private trigger
1225             def contextString = "${os} ${architecture} Cross ${configuration}"
1226             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1227             if (scenario == 'default') {
1228                 contextString += " Innerloop"
1229                 triggerString += "\\W+Innerloop"
1230             }
1231             else {
1232                 contextString += " ${scenario}"
1233                 triggerString += "\\W+${scenario}"
1234             }
1235
1236             if (configuration == 'Debug') {
1237                 contextString += " Build"
1238                 triggerString += "\\W+Build"
1239             } else {
1240                 contextString += " Build and Test"
1241                 triggerString += "\\W+Build and Test"
1242             }
1243
1244             triggerString += ".*"
1245
1246             switch (os) {
1247                 case 'Ubuntu':
1248                 case 'Ubuntu16.04':
1249                     switch (scenario) {
1250                         case 'default':
1251                             if (configuration == 'Debug' && !isFlowJob) {
1252                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1253                             }
1254                             else {
1255                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1256                             }
1257
1258                             break
1259                         default:
1260                             if (isR2RScenario(scenario)) {
1261                                 if (configuration == 'Checked' || configuration == 'Release') {
1262                                     def displayStr = getR2RDisplayName(scenario)
1263                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1264                                 }
1265                             }
1266                             break
1267                     }
1268                     break
1269                 case 'Windows_NT':
1270                     // Triggers on the non-flow jobs aren't necessary here
1271                     if (!isFlowJob) {
1272                         break
1273                     }
1274
1275                     assert isArmWindowsScenario(scenario)
1276                     switch (scenario) {
1277                         case 'default':
1278                             if (configuration == 'Checked') {
1279                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1280                             }
1281                             else {
1282                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1283                             }
1284                             break
1285                         default:
1286                             // Stress jobs will use this code path.
1287                             if (isArmWindowsScenario(scenario)) {
1288                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1289                             }
1290                             break
1291                     }
1292                     break
1293                 default:
1294                     println("NYI os: ${os}");
1295                     assert false
1296                     break
1297             }
1298             break
1299         // editor brace matching: }
1300         case 'x86': // editor brace matching: {
1301             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1302             if (os == 'Ubuntu') {
1303                 // Triggers on the non-flow jobs aren't necessary here
1304                 if (!isFlowJob) {
1305                     break
1306                 }
1307                 // on-demand only for ubuntu x86
1308                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1309                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1310                 break
1311             }
1312             switch (scenario) {
1313                 case 'default':
1314                     if (configuration == 'Checked') {
1315                         assert !job.name.contains("centos")
1316                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1317                     }
1318                     else {
1319                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1320                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1321                     }
1322                     break
1323                 case 'ilrt':
1324                     if (configuration == 'Release') {
1325                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1326                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1327                     }
1328                     break
1329                 case 'longgc':
1330                     if (configuration == 'Release') {
1331                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1332                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1333                     }
1334                     break
1335                 case 'gcsimulator':
1336                     if (configuration == 'Release') {
1337                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1338                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1339                     }
1340                     break
1341                 case 'standalone_gc':
1342                     if (configuration == 'Release' || configuration == 'Checked') {
1343                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1344                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1345                     }
1346                     break
1347                 case 'illink':
1348                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1349                     break
1350                 default:
1351                     if (isJitStressScenario(scenario)) {
1352                         def displayStr = getStressModeDisplayName(scenario)
1353                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1354                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1355                     }
1356                     else if (isR2RScenario(scenario)) {
1357                         if (configuration == 'Release' || configuration == 'Checked') {
1358                             def displayStr = getR2RDisplayName(scenario)
1359                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1360                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1361                         }
1362                     }
1363                     else {
1364                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1365                         assert false
1366                     }
1367                     break
1368             }
1369             break
1370          // editor brace matching: }
1371         case 'x64_arm64_altjit':
1372         case 'x86_arm_altjit': // editor brace matching: {
1373             assert (os == 'Windows_NT')
1374             switch (scenario) {
1375                 case 'default':
1376                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1377                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1378                     break
1379                 default:
1380                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1381                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1382                     break
1383             }
1384             break
1385         // editor brace matching: }
1386         default:
1387             println("Unknown architecture: ${architecture}");
1388             assert false
1389             break
1390     }
1391 }
1392
1393 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1394     def buildCommands = [];
1395     def osGroup = getOSGroup(os)
1396     def lowerConfiguration = configuration.toLowerCase()
1397
1398     def priority = '1'
1399     if (scenario == 'default' && isPR == true) {
1400         priority = '0'
1401     }
1402
1403     def enableCorefxTesting = isCoreFxScenario(scenario)
1404
1405     // Calculate the build steps, archival, and xunit results
1406     switch (os) {
1407         case 'Windows_NT': // editor brace matching: {
1408             switch (architecture) {
1409                 case 'x64':
1410                 case 'x86':
1411                 case 'x86_arm_altjit':
1412                 case 'x64_arm64_altjit':
1413                     def arch = architecture
1414                     def buildOpts = ''
1415                     if (architecture == 'x86_arm_altjit') {
1416                         arch = 'x86'
1417                     }
1418                     else if (architecture == 'x64_arm64_altjit') {
1419                         arch = 'x64'
1420                     }
1421
1422                     if (scenario == 'formatting') {
1423                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1424                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1425                         break
1426                     }
1427
1428                     if (scenario == 'illink') {
1429                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1430                     }
1431
1432                     // If it is a release build for windows, ensure PGO is used, else fail the build
1433                     if ((lowerConfiguration == 'release') &&
1434                         (scenario in Constants.basicScenarios) &&
1435                         (architecture != 'x86_arm_altjit') &&
1436                         (architecture != 'x64_arm64_altjit')) {
1437
1438                         buildOpts += ' -enforcepgo'
1439                     }
1440
1441                     if (enableCorefxTesting) {
1442                         buildOpts += ' skiptests';
1443                     } else {
1444                         buildOpts += " -priority=${priority}"
1445                     }
1446
1447                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1448                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1449                     // 35 characters long.
1450
1451                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1452
1453                     if (!isBuildOnly) {
1454                         def runtestArguments = ''
1455                         def testOpts = 'collectdumps'
1456
1457                         if (isR2RScenario(scenario)) {
1458
1459                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1460                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1461                             // to cause the tests to be crossgen'ed.
1462
1463                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1464                                 testOpts += ' crossgenaltjit protononjit.dll'
1465                             } else {
1466                                 testOpts += ' crossgen'
1467                             }
1468
1469                             testOpts += ' runcrossgentests'
1470
1471                             if (scenario == 'r2r_jitstress1') {
1472                                 testOpts += ' jitstress 1'
1473                             }
1474                             else if (scenario == 'r2r_jitstress2') {
1475                                 testOpts += ' jitstress 2'
1476                             }
1477                             else if (scenario == 'r2r_jitstressregs1') {
1478                                 testOpts += ' jitstressregs 1'
1479                             }
1480                             else if (scenario == 'r2r_jitstressregs2') {
1481                                 testOpts += ' jitstressregs 2'
1482                             }
1483                             else if (scenario == 'r2r_jitstressregs3') {
1484                                 testOpts += ' jitstressregs 3'
1485                             }
1486                             else if (scenario == 'r2r_jitstressregs4') {
1487                                 testOpts += ' jitstressregs 4'
1488                             }
1489                             else if (scenario == 'r2r_jitstressregs8') {
1490                                 testOpts += ' jitstressregs 8'
1491                             }
1492                             else if (scenario == 'r2r_jitstressregs0x10') {
1493                                 testOpts += ' jitstressregs 0x10'
1494                             }
1495                             else if (scenario == 'r2r_jitstressregs0x80') {
1496                                 testOpts += ' jitstressregs 0x80'
1497                             }
1498                             else if (scenario == 'r2r_jitstressregs0x1000') {
1499                                 testOpts += ' jitstressregs 0x1000'
1500                             }
1501                             else if (scenario == 'r2r_jitminopts') {
1502                                 testOpts += ' jitminopts'
1503                             }
1504                             else if (scenario == 'r2r_jitforcerelocs') {
1505                                 testOpts += ' jitforcerelocs'
1506                             }
1507                             else if (scenario == 'r2r_gcstress15') {
1508                                 testOpts += ' gcstresslevel 0xF'
1509                             }
1510                         }
1511                         else if (scenario == 'jitdiff') {
1512                             testOpts += ' jitdisasm crossgen'
1513                         }
1514                         else if (scenario == 'ilrt') {
1515                             testOpts += ' ilasmroundtrip'
1516                         }
1517                         else if (isLongGc(scenario)) {
1518                             testOpts += " ${scenario} sequential"
1519                         }
1520                         else if (scenario == 'standalone_gc') {
1521                             testOpts += ' gcname clrgc.dll'
1522                         }
1523                         else if (scenario == 'illink') {
1524                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1525                         }
1526
1527                         // If we are running a stress mode, we should write out the set of key
1528                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1529
1530                         def envScriptPath = ''
1531                         if (isJitStressScenario(scenario)) {
1532                             def buildCommandsStr = ''
1533                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1534                             buildCommandsStr += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], envScriptPath)
1535                             if (architecture == 'x86_arm_altjit') {
1536                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1537                             }
1538                             else if (architecture == 'x64_arm64_altjit') {
1539                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1540                             }
1541
1542                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1543                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1544                             // into a single script.
1545                             buildCommands += buildCommandsStr
1546                         }
1547                         else if (architecture == 'x86_arm_altjit') {
1548                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1549                         }
1550                         else if (architecture == 'x64_arm64_altjit') {
1551                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1552                         }
1553                         if (envScriptPath != '') {
1554                             testOpts += " TestEnv ${envScriptPath}"
1555                         }
1556
1557                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1558
1559                         if (enableCorefxTesting) {
1560                             def workspaceRelativeFxRoot = "_/fx"
1561                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1562
1563                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1564
1565                             setTestJobTimeOut(newJob, scenario)
1566
1567                             // Archive and process (only) the test results
1568                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1569                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1570
1571                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1572                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1573                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1574                         }
1575                         else if (isGcReliabilityFramework(scenario)) {
1576                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1577                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1578                         }
1579                         else {
1580                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1581                         }
1582                     }
1583
1584                     if (!enableCorefxTesting) {
1585                         // Run the rest of the build
1586                         // Build the mscorlib for the other OS's
1587                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1588                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1589
1590                         if (arch == "x64") {
1591                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1592                         }
1593
1594                         // Zip up the tests directory so that we don't use so much space/time copying
1595                         // 10s of thousands of files around.
1596                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1597
1598                         if (!isJitStressScenario(scenario)) {
1599                             // For windows, pull full test results and test drops for x86/x64.
1600                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1601                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1602                         }
1603
1604                         if (scenario == 'jitdiff') {
1605                             // retrive jit-dasm output for base commit, and run jit-diff
1606                             if (!isBuildOnly) {
1607                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1608                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1609                             }
1610                         }
1611
1612                         if (!isBuildOnly) {
1613                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1614                             setTestJobTimeOut(newJob, scenario)
1615                         }
1616                     }
1617                     break
1618                 case 'armlb':
1619                 case 'arm':
1620                     assert isArmWindowsScenario(scenario)
1621
1622                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1623                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1624
1625                     // Set time out
1626                     setTestJobTimeOut(newJob, scenario)
1627
1628                     if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1629                     {
1630                         // Up the timeout for arm checked testing only.
1631                         // Keep the longer timeout for gcstress.
1632                         Utilities.setJobTimeout(newJob, 240)
1633                     }
1634
1635                     def buildArchitecture = 'arm'
1636
1637                     // For 'arm' (the RyuJIT/arm32 architecture), tell build.cmd to use RyuJIT/arm32 for crossgen compilation.
1638                     // RyuJIT/arm32 is currently not the default JIT; it is an aljit. So, this is a special case.
1639                     def armCrossgenOpt = ''
1640                     if (architecture == 'arm') {
1641                         armCrossgenOpt = '-altjitcrossgen'
1642                     }
1643
1644                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1645                     priority = '1'
1646
1647                     // This is now a build only job. Do not run tests. Use the flow job.
1648                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1649
1650                     // Zip up the tests directory so that we don't use so much space/time copying
1651                     // 10s of thousands of files around.
1652                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1653
1654                     // Add archival.
1655                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1656                     break
1657                 case 'arm64':
1658                     assert isArmWindowsScenario(scenario)
1659
1660                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1661                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1662
1663                     // Set time out
1664                     setTestJobTimeOut(newJob, scenario)
1665                     if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1666                     {
1667                         // Up the timeout for arm checked testing only.
1668                         // Keep the longer timeout for gcstress.
1669                         Utilities.setJobTimeout(newJob, 240)
1670                     }
1671
1672                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1673                     priority = '1'
1674
1675                     // This is now a build only job. Do not run tests. Use the flow job.
1676                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1677
1678                     // Zip up the tests directory so that we don't use so much space/time copying
1679                     // 10s of thousands of files around.
1680                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1681
1682                     // Add archival.
1683                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1684                     break
1685                 default:
1686                     println("Unknown architecture: ${architecture}");
1687                     assert false
1688                     break
1689             }
1690             break
1691         // editor brace matching: }
1692         case 'Ubuntu':
1693         case 'Ubuntu16.04':
1694         case 'Ubuntu16.10':
1695         case 'Debian8.4':
1696         case 'OSX10.12':
1697         case 'CentOS7.1':
1698         case 'RHEL7.2':
1699         case 'Tizen':
1700         case 'Fedora24': // editor brace matching: {
1701             switch (architecture) {
1702                 case 'x64':
1703                 case 'x86':
1704                     if (architecture == 'x86' && os == 'Ubuntu') {
1705                         // build and PAL test
1706                         def dockerImage = getDockerImageName(architecture, os, true)
1707                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1708                         dockerImage = getDockerImageName(architecture, os, false)
1709                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1710                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1711                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1712                         break
1713                     }
1714
1715                     if (scenario == 'formatting') {
1716                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1717                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1718                         break
1719                     }
1720
1721                     if (scenario == 'illink') {
1722                         assert(os == 'Ubuntu')
1723                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1724                     }
1725
1726                     if (!enableCorefxTesting) {
1727                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1728                         // only on supported OS platforms.
1729                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1730                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1731                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1732                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1733
1734                         // Set time out
1735                         setTestJobTimeOut(newJob, scenario)
1736                         // Basic archiving of the build
1737                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1738                         // And pal tests
1739                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1740                     }
1741                     else {
1742                         // Corefx stress testing
1743                         assert os == 'Ubuntu'
1744                         assert architecture == 'x64'
1745                         assert lowerConfiguration == 'checked'
1746                         assert isJitStressScenario(scenario)
1747
1748                         // Build coreclr
1749                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1750
1751                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1752                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1753
1754                         // Build and text corefx
1755                         def workspaceRelativeFxRoot = "_/fx"
1756                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1757
1758                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1759
1760                         setTestJobTimeOut(newJob, scenario)
1761
1762                         // Archive and process (only) the test results
1763                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1764                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1765                     }
1766                     break
1767                 case 'arm64':
1768                     if (!enableCorefxTesting) {
1769                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1770
1771                         // HACK -- Arm64 does not have corefx jobs yet.
1772                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1773                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1774                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1775                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1776
1777                         setTestJobTimeOut(newJob, scenario)
1778                         // Basic archiving of the build
1779                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1780                     }
1781                     break
1782                 case 'armlb':
1783                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1784                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1785
1786                     // default values for Ubuntu
1787                     def arm_abi="arm"
1788                     def linuxCodeName="trusty"
1789                     if (os == 'Ubuntu16.04') {
1790                         linuxCodeName="xenial"
1791                     }
1792                     else if (os == 'Tizen') {
1793                         arm_abi="armel"
1794                         linuxCodeName="tizen"
1795                     }
1796
1797                     // Unzip the Windows test binaries first. Exit with 0
1798                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1799
1800                     // Unpack the corefx binaries
1801                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1802                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1803                     if (os != 'Tizen') {
1804                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1805                     }
1806                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1807                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1808                     if (os == 'Ubuntu') {
1809                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1810                     }
1811
1812                     // Call the ARM CI script to cross build and test using docker
1813                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1814                     --mode=docker \\
1815                     --${arm_abi} \\
1816                     --linuxCodeName=${linuxCodeName} \\
1817                     --buildConfig=${lowerConfiguration} \\
1818                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1819                     --coreFxBinDir=./bin/CoreFxBinDir \\
1820                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1821
1822                     // Basic archiving of the build, no pal tests
1823                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1824                     break
1825                 default:
1826                     println("Unknown architecture: ${architecture}");
1827                     assert false
1828                     break
1829             }
1830             break
1831         // editor brace matching: }
1832         default:
1833             println("Unknown os: ${os}");
1834             assert false
1835             break
1836     } // os
1837
1838     return buildCommands
1839 }
1840
1841 Constants.allScenarios.each { scenario ->
1842     [true, false].each { isPR ->
1843         Constants.architectureList.each { architecture ->
1844             Constants.configurationList.each { configuration ->
1845                 Constants.osList.each { os ->
1846                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1847                     // and reset the os to Windows_NT
1848                     def isBuildOnly = false
1849                     if (os == 'Windows_NT_BuildOnly') {
1850                         isBuildOnly = true
1851                         os = 'Windows_NT'
1852                     }
1853
1854                     // Tizen is only supported for arm legacy_backend architecture
1855                     if (os == 'Tizen' && architecture != 'armlb') {
1856                         return
1857                     }
1858
1859                     // Skip totally unimplemented (in CI) configurations.
1860                     switch (architecture) {
1861                         case 'arm64':
1862                             if (os == 'Ubuntu16.04') {
1863                                 os = 'Ubuntu'
1864                             }
1865
1866                             // Windows and Ubuntu only
1867                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1868                                 return
1869                             }
1870                             break
1871                         case 'armlb':
1872                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1873                                 return
1874                             }
1875                             break
1876                         case 'arm':
1877                             if (os != 'Windows_NT') {
1878                                 return
1879                             }
1880                             break
1881                         case 'x86':
1882                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1883                                 return
1884                             }
1885                             break
1886                         case 'x86_arm_altjit':
1887                         case 'x64_arm64_altjit':
1888                             if (os != 'Windows_NT') {
1889                                 return
1890                             }
1891                             break
1892                         case 'x64':
1893                             // Everything implemented
1894                             break
1895                         default:
1896                             println("Unknown architecture: ${architecture}")
1897                             assert false
1898                             break
1899                     }
1900
1901                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1902                     // with checked builds)
1903                     if (isJitStressScenario(scenario)) {
1904                         if (configuration != 'Checked') {
1905                             return
1906                         }
1907
1908                         // Since these are just execution time differences,
1909                         // skip platforms that don't execute the tests here (Windows_NT only)
1910                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
1911                         if (!isEnabledOS || isBuildOnly) {
1912                             return
1913                         }
1914
1915                         switch (architecture) {
1916                             case 'x64':
1917                             case 'x86':
1918                             case 'x86_arm_altjit':
1919                             case 'x64_arm64_altjit':
1920                                 // x86 ubuntu: default only
1921                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1922                                     return
1923                                 }
1924                                 // Windows: Everything implemented
1925                                 break
1926
1927                             default:
1928                                 // arm, arm64, armlb: stress is handled through flow jobs.
1929                                 return
1930                         }
1931                     }
1932                     else if (isR2RScenario(scenario)) {
1933                         if (os != 'Windows_NT') {
1934                             return
1935                         }
1936                         // R2R test runs are not implemented for arm/armlb/arm64.
1937                         if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
1938                             return
1939                         }
1940                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
1941                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
1942                             return
1943                         }
1944                     }
1945                     else {
1946                         // Skip scenarios
1947                         switch (scenario) {
1948                             case 'ilrt':
1949                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1950                                 // the default scenario build
1951                                 if (os != 'Windows_NT') {
1952                                     return
1953                                 }
1954                                 // Only x64 for now
1955                                 if (architecture != 'x64') {
1956                                     return
1957                                 }
1958                                 // Release only
1959                                 if (configuration != 'Release') {
1960                                     return
1961                                 }
1962                                 break
1963                             case 'jitdiff':
1964                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1965                                     return
1966                                 }
1967                                 if (architecture != 'x64') {
1968                                     return
1969                                 }
1970                                 if (configuration != 'Checked') {
1971                                     return
1972                                 }
1973                                 break
1974                             case 'longgc':
1975                             case 'gcsimulator':
1976                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1977                                     return
1978                                 }
1979                                 if (architecture != 'x64') {
1980                                     return
1981                                 }
1982                                 if (configuration != 'Release') {
1983                                     return
1984                                 }
1985                                 break
1986                             case 'gc_reliability_framework':
1987                             case 'standalone_gc':
1988                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1989                                     return
1990                                 }
1991
1992                                 if (architecture != 'x64') {
1993                                     return
1994                                 }
1995
1996                                 if (configuration != 'Release' && configuration != 'Checked') {
1997                                     return
1998                                 }
1999                                 break
2000                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2001                             case 'formatting':
2002                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2003                                     return
2004                                 }
2005                                 if (architecture != 'x64') {
2006                                     return
2007                                 }
2008                                 if (configuration != 'Checked') {
2009                                     return
2010                                 }
2011                                 if (isBuildOnly) {
2012                                     return
2013                                 }
2014                                 break
2015                             case 'illink':
2016                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2017                                     return
2018                                 }
2019                                 if (architecture != 'x64' && architecture != 'x86') {
2020                                     return
2021                                 }
2022                                 if (isBuildOnly) {
2023                                     return
2024                                 }
2025                                 break
2026                             case 'default':
2027                                 // Nothing skipped
2028                                 break
2029                             default:
2030                                 println("Unknown scenario: ${scenario}")
2031                                 assert false
2032                                 break
2033                         }
2034                     }
2035
2036                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2037                     // runtime behavior, not compile-time behavior, are not interesting.
2038                     switch (architecture) {
2039                         case 'x86_arm_altjit':
2040                         case 'x64_arm64_altjit':
2041                             if (isGCStressRelatedTesting(scenario)) {
2042                                 return
2043                             }
2044                             break
2045                         default:
2046                             break
2047                     }
2048
2049                     // Calculate names
2050                     def lowerConfiguration = configuration.toLowerCase()
2051                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2052                     def folderName = getJobFolder(scenario)
2053
2054                     // Create the new job
2055                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2056
2057                     def machineAffinityOptions = null
2058
2059                     if (os != 'Windows_NT') {
2060                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2061                     }
2062                     else {
2063                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2064                     }
2065
2066                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2067
2068                     // Add all the standard options
2069                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2070                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2071
2072                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2073                     def osGroup = getOSGroup(os)
2074
2075                     newJob.with {
2076                         steps {
2077                             if (os == 'Windows_NT') {
2078                                 buildCommands.each { buildCommand ->
2079                                     batchFile(buildCommand)
2080                                 }
2081                             }
2082                             else {
2083                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2084                                 if ( architecture == 'armlb' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2085                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2086                                     // Define the Windows Tests and Corefx build job names
2087                                     def WindowTestsName = projectFolder + '/' +
2088                                                           Utilities.getFullJobName(project,
2089                                                                                    getJobName(lowerConfiguration,
2090                                                                                               'x64' ,
2091                                                                                               'windows_nt',
2092                                                                                               'default',
2093                                                                                               true),
2094                                                                                    false)
2095                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2096                                                        Utilities.getFolderName(branch)
2097
2098                                     // Copy the Windows test binaries and the Corefx build binaries
2099                                     copyArtifacts(WindowTestsName) {
2100                                         includePatterns('bin/tests/tests.zip')
2101                                         buildSelector {
2102                                             latestSuccessful(true)
2103                                         }
2104                                     }
2105
2106                                     def arm_abi = 'arm'
2107                                     def corefx_os = 'linux'
2108                                     if (os == 'Tizen') {
2109                                         arm_abi = 'armel'
2110                                         corefx_os = 'tizen'
2111                                     }
2112
2113                                     // Let's use release CoreFX to test checked CoreCLR,
2114                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2115                                     def corefx_lowerConfiguration = lowerConfiguration
2116                                     if ( lowerConfiguration == 'checked' ) {
2117                                         corefx_lowerConfiguration='release'
2118                                     }
2119
2120                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2121                                         includePatterns('bin/build.tar.gz')
2122                                         buildSelector {
2123                                             latestSuccessful(true)
2124                                         }
2125                                     }
2126                                 }
2127
2128                                 buildCommands.each { buildCommand ->
2129                                     shell(buildCommand)
2130                                 }
2131                             }
2132                         }
2133                     } // newJob.with
2134
2135                 } // os
2136             } // configuration
2137         } // architecture
2138     } // isPR
2139 } // scenario
2140
2141
2142 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2143 Constants.allScenarios.each { scenario ->
2144     [true, false].each { isPR ->
2145         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2146             Constants.crossList.each { os ->
2147                 if (architecture == 'arm64') {
2148                     if (os != "Ubuntu" && os != "Windows_NT") {
2149                         return
2150                     }
2151                 } else if (architecture == 'arm' || architecture == 'armlb') {
2152                     if (os != 'Windows_NT') {
2153                         return
2154                     }
2155                 }
2156                 else if (architecture == 'x86') {
2157                     if (os != "Ubuntu") {
2158                         return
2159                     }
2160                 }
2161
2162                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2163
2164                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2165                     return
2166                 }
2167
2168                 Constants.configurationList.each { configuration ->
2169
2170                     // First, filter based on OS.
2171
2172                     if (os == 'Windows_NT') {
2173                         if (!isArmWindowsScenario(scenario)) {
2174                             return
2175                         }
2176                     }
2177                     else {
2178                         // Non-Windows
2179                         if (architecture == 'arm64') {
2180                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2181                                 return
2182                             }
2183                         }
2184                         else if (architecture == 'x86') {
2185                             // Linux/x86 only want default test
2186                             if (scenario != 'default') {
2187                                 return
2188                             }
2189                         }
2190                     }
2191
2192                     // For CentOS, we only want Checked/Release builds.
2193                     if (os == 'CentOS7.1') {
2194                         if (configuration != 'Checked' && configuration != 'Release') {
2195                             return
2196                         }
2197                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2198                             return
2199                         }
2200                     }
2201
2202                     // For RedHat and Debian, we only do Release builds.
2203                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2204                         if (configuration != 'Release') {
2205                             return
2206                         }
2207                         if (scenario != 'default') {
2208                             return
2209                         }
2210                     }
2211
2212                     // Next, filter based on scenario.
2213
2214                     if (isJitStressScenario(scenario)) {
2215                         if (configuration != 'Checked') {
2216                             return
2217                         }
2218                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2219                         if (isCoreFxScenario(scenario)) {
2220                             return
2221                         }
2222                     }
2223                     else if (isR2RBaselineScenario(scenario)) {
2224                         if (configuration != 'Checked' && configuration != 'Release') {
2225                             return
2226                         }
2227                     }
2228                     else if (isR2RStressScenario(scenario)) {
2229                         if (configuration != 'Checked') {
2230                             return
2231                         }
2232                     }
2233                     else {
2234                         // Skip scenarios
2235                         switch (scenario) {
2236                             case 'ilrt':
2237                             case 'longgc':
2238                             case 'gcsimulator':
2239                                 // Long GC tests take a long time on non-Release builds
2240                                 // ilrt is also Release only
2241                                 if (configuration != 'Release') {
2242                                     return
2243                                 }
2244                                 break
2245                             case 'jitdiff':
2246                                 if (configuration != 'Checked') {
2247                                     return;
2248                                 }
2249                                 break
2250                             case 'gc_reliability_framework':
2251                             case 'standalone_gc':
2252                                 if (configuration != 'Release' && configuration != 'Checked') {
2253                                     return
2254                                 }
2255                                 break
2256                             case 'formatting':
2257                                 return
2258                             case 'illink':
2259                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2260                                     return
2261                                 }
2262                                 break
2263                             case 'default':
2264                                 // Nothing skipped
2265                                 break
2266                             default:
2267                                 println("Unknown scenario: ${scenario}")
2268                                 assert false
2269                                 break
2270                         }
2271                     }
2272
2273                     // Done filtering. Now, create the jobs.
2274
2275                     def lowerConfiguration = configuration.toLowerCase()
2276                     def osGroup = getOSGroup(os)
2277                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2278
2279                     def inputCoreCLRBuildName = projectFolder + '/' +
2280                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2281
2282                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2283                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2284                     def testBuildScenario = scenario
2285                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2286                         testBuildScenario = 'default'
2287                     }
2288
2289                     def inputWindowsTestBuildArch = architecture
2290                     if (architecture == "arm64" && os != "Windows_NT") {
2291                         // Use the x64 test build for arm64 unix
2292                         inputWindowsTestBuildArch = "x64"
2293                     }
2294
2295                     def inputWindowTestsBuildName = projectFolder + '/' +
2296                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2297
2298                     // Enable Server GC for Ubuntu PR builds
2299                     def serverGCString = ''
2300                     if (os == 'Ubuntu' && isPR) {
2301                         serverGCString = '--useServerGC'
2302                     }
2303
2304                     def testOpts = ''
2305
2306                     if (isR2RScenario(scenario)) {
2307
2308                         testOpts += ' --crossgen --runcrossgentests'
2309
2310                         if (scenario == 'r2r_jitstress1') {
2311                             testOpts += ' --jitstress=1'
2312                         }
2313                         else if (scenario == 'r2r_jitstress2') {
2314                             testOpts += ' --jitstress=2'
2315                         }
2316                         else if (scenario == 'r2r_jitstressregs1') {
2317                             testOpts += ' --jitstressregs=1'
2318                         }
2319                         else if (scenario == 'r2r_jitstressregs2') {
2320                             testOpts += ' --jitstressregs=2'
2321                         }
2322                         else if (scenario == 'r2r_jitstressregs3') {
2323                             testOpts += ' --jitstressregs=3'
2324                         }
2325                         else if (scenario == 'r2r_jitstressregs4') {
2326                             testOpts += ' --jitstressregs=4'
2327                         }
2328                         else if (scenario == 'r2r_jitstressregs8') {
2329                             testOpts += ' --jitstressregs=8'
2330                         }
2331                         else if (scenario == 'r2r_jitstressregs0x10') {
2332                             testOpts += ' --jitstressregs=0x10'
2333                         }
2334                         else if (scenario == 'r2r_jitstressregs0x80') {
2335                             testOpts += ' --jitstressregs=0x80'
2336                         }
2337                         else if (scenario == 'r2r_jitstressregs0x1000') {
2338                             testOpts += ' --jitstressregs=0x1000'
2339                         }
2340                         else if (scenario == 'r2r_jitminopts') {
2341                             testOpts += ' --jitminopts'
2342                         }
2343                         else if (scenario == 'r2r_jitforcerelocs') {
2344                             testOpts += ' --jitforcerelocs'
2345                         }
2346                         else if (scenario == 'r2r_gcstress15') {
2347                             testOpts += ' --gcstresslevel=0xF'
2348                         }
2349                     }
2350                     else if (scenario == 'jitdiff') {
2351                         testOpts += ' --jitdisasm --crossgen'
2352                     }
2353                     else if (scenario == 'illink') {
2354                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2355                     }
2356                     else if (isLongGc(scenario)) {
2357                         // Long GC tests behave very poorly when they are not
2358                         // the only test running (many of them allocate until OOM).
2359                         testOpts += ' --sequential'
2360
2361                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2362                         // for running long GC and GCSimulator tests, respectively. We don't use them
2363                         // here because using a playlist file produces much more readable output on the CI machines
2364                         // and reduces running time.
2365                         //
2366                         // The Long GC playlist contains all of the tests that are
2367                         // going to be run. The GCSimulator playlist contains all of
2368                         // the GC simulator tests.
2369                         if (scenario == 'longgc') {
2370                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2371                         }
2372                         else if (scenario == 'gcsimulator') {
2373                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2374                         }
2375                     }
2376                     else if (isGcReliabilityFramework(scenario)) {
2377                         testOpts += ' --build-overlay-only'
2378                     }
2379                     else if (scenario == 'standalone_gc') {
2380                         if (osGroup == 'OSX') {
2381                             testOpts += ' --gcname=libclrgc.dylib'
2382                         }
2383                         else if (osGroup == 'Linux') {
2384                             testOpts += ' --gcname=libclrgc.so'
2385                         }
2386                         else {
2387                             println("Unexpected OS group: ${osGroup} for os ${os}")
2388                             assert false
2389                         }
2390                     }
2391
2392                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2393
2394                     def folder = getJobFolder(scenario)
2395                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2396                         // Add parameters for the inputs
2397
2398                         if (windowsArmJob == true) {
2399                             parameters {
2400                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2401                             }
2402                         }
2403                         else {
2404                             parameters {
2405                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2406                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2407                             }
2408                         }
2409
2410                         steps {
2411                             // Set up the copies
2412
2413                             // Coreclr build containing the tests and mscorlib
2414
2415                             if (windowsArmJob != true) {
2416                                 copyArtifacts(inputWindowTestsBuildName) {
2417                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2418                                     buildSelector {
2419                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2420                                     }
2421                                 }
2422                             }
2423
2424                             // Coreclr build we are trying to test
2425
2426                             copyArtifacts(inputCoreCLRBuildName) {
2427                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2428                                 buildSelector {
2429                                     buildNumber('${CORECLR_BUILD}')
2430                                 }
2431                             }
2432
2433                             // Windows CoreCLR Arm(64) will restore corefx
2434                             // packages correctly.
2435                             //
2436                             // In addition, test steps are entirely different
2437                             // because we do not have a unified runner
2438                             if (windowsArmJob != true) {
2439                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2440
2441                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2442                                 def osJobName
2443                                 if (os == 'Ubuntu') {
2444                                     osJobName = 'ubuntu14.04'
2445                                 }
2446                                 else if (architecture == 'x86') {
2447                                     if (os == 'Ubuntu') {
2448                                         // Linux/x86 corefx jobs does not build managed yet
2449                                         // Clone linux/arm corefx managed packages and overwrite linux/x86 native
2450                                         osJobName = "linux_arm_cross"
2451                                     }
2452                                 }
2453                                 else {
2454                                     osJobName = os.toLowerCase()
2455                                 }
2456                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2457                                     includePatterns('bin/build.tar.gz')
2458                                     buildSelector {
2459                                         latestSuccessful(true)
2460                                     }
2461                                 }
2462
2463                                 shell("mkdir ./bin/CoreFxBinDir")
2464                                 // Unpack the corefx binaries
2465                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2466
2467                                 // HACK -- Arm64 does not have corefx jobs yet.
2468                                 // Clone corefx and build the native packages overwriting the x64 packages.
2469                                 if (architecture == 'arm64') {
2470                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2471                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2472                                 }
2473                                 else if (architecture == 'x86') {
2474                                     shell("mkdir ./bin/CoreFxNative")
2475
2476                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2477                                         includePatterns('bin/build.tar.gz')
2478                                         targetDirectory('bin/CoreFxNative')
2479                                         buildSelector {
2480                                             latestSuccessful(true)
2481                                         }
2482                                     }
2483
2484                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2485                                 }
2486
2487                                 // Unzip the tests first.  Exit with 0
2488                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2489
2490                                 // Execute the tests
2491                                 def runDocker = isNeedDocker(architecture, os, false)
2492                                 def dockerPrefix = ""
2493                                 def dockerCmd = ""
2494                                 if (runDocker) {
2495                                     def dockerImage = getDockerImageName(architecture, os, false)
2496                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2497                                     dockerCmd = dockerPrefix + "${dockerImage} "
2498                                 }
2499
2500                                 // If we are running a stress mode, we'll set those variables first
2501                                 def testEnvOpt = ""
2502                                 if (isJitStressScenario(scenario)) {
2503                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2504                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2505                                     shell("${createScriptCmds}")
2506                                     testEnvOpt = "--test-env=" + scriptFileName
2507                                 }
2508
2509                                 if (isGCStressRelatedTesting(scenario)) {
2510                                     shell('./init-tools.sh')
2511                                 }
2512
2513                                 shell("""${dockerCmd}./tests/runtest.sh \\
2514                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2515                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2516                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2517                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2518                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2519                 --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2520
2521                                 if (isGcReliabilityFramework(scenario)) {
2522                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2523                                     if (serverGCString != '') {
2524                                         if (runDocker) {
2525                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2526                                         }
2527                                         else {
2528                                             shell("export COMPlus_gcServer=1")
2529                                         }
2530                                     }
2531
2532                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2533                                 }
2534                             }
2535
2536                             else { // windowsArmJob == true
2537                                 // Unzip tests.
2538                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2539
2540                                 // Build the build commands
2541                                 def buildCommands = ""
2542
2543                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2544                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2545                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2546
2547                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2548                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2549
2550                                 // For all jobs
2551                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2552
2553                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2554                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2555
2556                                 // Arm(32) ryujit
2557                                 if (architecture == "arm") {
2558                                     // **This is an AltJit**
2559
2560                                     addEnvVariable("COMPlus_AltJit", "*")
2561                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2562                                     addEnvVariable("COMPlus_AltJitName", "protojit.dll")
2563                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2564                                 }
2565
2566                                 // If we are running a stress mode, we'll set those variables as well
2567                                 def stressValues = null
2568                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2569                                     if (isJitStressScenario(scenario)) {
2570                                         stressValues = Constants.jitStressModeScenarios[scenario]
2571                                     }
2572                                     else {
2573                                         stressValues = Constants.r2rStressScenarios[scenario]
2574                                     }
2575
2576                                     stressValues.each { key, value ->
2577                                         addEnvVariable(key, value)
2578                                     }
2579                                 }
2580
2581                                 // TODO: do whatever is necessary to support enabling R2R testing. Environment variables,
2582                                 // crossgen the framework assemblies, etc.
2583
2584                                 // Create the smarty command
2585                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2586                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2587                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2588
2589                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "arm") { addExclude("PROTOJIT_" + exclude) } else { addExclude(exclude) } }
2590
2591                                 if (architecture == "arm") {
2592                                     addExclude("PROTOJIT_FAIL")
2593                                 }
2594
2595                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2596                                     def failTag = "JITSTRESS_FAIL"
2597                                     def excludeTag = "JITSTRESS_EXCLUDE"
2598
2599                                     if (scenario.contains('gc')) {
2600                                         failTag = "GCSTRESS_FAIL"
2601                                         excludeTag = "GCSTRESS_EXCLUDE"
2602                                     }
2603
2604                                     addArchSpecificExclude(architecture, failTag)
2605                                     addArchSpecificExclude(architecture, excludeTag)
2606                                 }
2607
2608                                 else {
2609                                     addExclude("pri1")
2610                                 }
2611
2612                                 smartyCommand += "/lstFile Tests.lst"
2613
2614                                 def testListArch = [
2615                                     'arm64': 'arm64',
2616                                     'arm': 'arm',
2617                                     'armlb': 'arm'
2618                                 ]
2619
2620                                 def archLocation = testListArch[architecture]
2621
2622                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2623                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2624                                 addCommand("${smartyCommand}")
2625
2626                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2627                                 // However, we also need to remove all the variables that were set during this batch file, so we
2628                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2629                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2630                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2631                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2632                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2633                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2634
2635                                 // ZIP up the smarty output, no matter what the smarty result.
2636                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2637
2638                                 addCommand("echo %errorlevel%")
2639                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2640
2641                                 // Use the smarty errorlevel as the script errorlevel.
2642                                 addCommand("exit /b %__save_smarty_errorlevel%")
2643
2644                                 batchFile(buildCommands)
2645                             }
2646                         }
2647                     }
2648
2649                     if (scenario == 'jitdiff') {
2650                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2651                     }
2652
2653                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2654                     if (os in ['Ubuntu']) {
2655                         SummaryBuilder summaries = new SummaryBuilder()
2656                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2657                         summaries.emit(newJob)
2658                     }
2659
2660                     def affinityOptions = null
2661
2662                     if (windowsArmJob == true) {
2663                         affinityOptions = [
2664                             "use_arm64_build_machine" : false
2665                         ]
2666                     }
2667
2668                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2669                         affinityOptions = [
2670                             "large_pages" : false
2671                         ]
2672                     }
2673
2674                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2675                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2676
2677                     // REVIEW: Should the scenario-base timeout be set in the "else" clause of the "architecture==arm64"?
2678                     //         Should the "architecture==arm64" include arm/armlb?
2679                     //         Should we be overriding possibly higher timeouts that would be set by setTestJobTimeOut()?
2680                     setTestJobTimeOut(newJob, scenario)
2681                     if (architecture == 'arm64') {
2682                         Utilities.setJobTimeout(newJob, 240)
2683                     }
2684
2685                     if (windowsArmJob != true) {
2686                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2687                     }
2688                     else {
2689                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2690
2691                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2692                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2693                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2694                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2695                     }
2696
2697                     // Create a build flow to join together the build and tests required to run this test.
2698                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2699                     // Linux CoreCLR test
2700                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2701                     def fullTestJobName = projectFolder + '/' + newJob.name
2702                     // Add a reference to the input jobs for report purposes
2703                     JobReport.Report.addReference(inputCoreCLRBuildName)
2704                     JobReport.Report.addReference(inputWindowTestsBuildName)
2705                     JobReport.Report.addReference(fullTestJobName)
2706                     def newFlowJob = null
2707
2708                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2709                         // Do not create the flow job for RHEL jobs.
2710                         return
2711                     }
2712
2713                     if (windowsArmJob == true) {
2714                         // For Windows arm jobs there is no reason to build a parallel test job.
2715                         // The product build supports building and archiving the tests.
2716
2717                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2718                         buildFlow("""
2719 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2720
2721 // And then build the test build
2722 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2723 """)
2724                         }
2725                     }
2726                     else {
2727                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2728                         buildFlow("""
2729 // Build the input jobs in parallel
2730 parallel (
2731 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2732 { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2733 )
2734
2735 // And then build the test build
2736 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2737             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2738 """)
2739                         }
2740                     }
2741
2742                     // For the flow jobs set the machine affinity as x64 if an armarch.
2743                     def flowArch = architecture
2744
2745                     if (flowArch in validWindowsNTCrossArches) {
2746                         flowArch = 'x64'
2747                         affinityOptions = null
2748                     }
2749
2750                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2751                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2752                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2753                 } // configuration
2754             } // os
2755         } // architecture
2756     } // isPR
2757 } // scenario
2758
2759 JobReport.Report.generateJobReport(out)
2760
2761 // Make the call to generate the help job
2762 Utilities.createHelperJob(this, project, branch,
2763     "Welcome to the ${project} Repository",  // This is prepended to the help message
2764     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2765
2766 Utilities.addCROSSCheck(this, project, branch)