netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'CentOS7.1',
  49                'RHEL7.2',
  50                'Ubuntu16.04',
  51                'Ubuntu16.10',
  52                'Fedora24',
  53                'Tizen']
  54
  55     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  56
  57     // This is a set of JIT stress modes combined with the set of variables that
  58     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  59     // the values are the environment variables
  60     def static jitStressModeScenarios = [
  61                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  62                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  63                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  64                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  65                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  66                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  67                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  68                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  69                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  70                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  71                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  72                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  73                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  74                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  75                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  76                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  77                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  78                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  79                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  80                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  81                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  82                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  83                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  84                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  85                'corefx_baseline'                : [ : ], // corefx baseline
  86                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  87                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  88                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  89                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  90                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  91                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  92                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  93                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  94                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  95                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  96                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  97                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  98                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  99                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 100                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 101                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 102                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 103                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 104                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 105                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 106                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 107                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 108     ]
 109
 110     // This is a set of ReadyToRun stress scenarios
 111     def static r2rStressScenarios = [
 112                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 113                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 114                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 115                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 116                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 117                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 118                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 119                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 120                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 121                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 122                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 123                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 124                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 125     ]
 126
 127     // This is the basic set of scenarios
 128     def static basicScenarios = [
 129                'default',
 130                'ilrt',
 131                'r2r',
 132                'longgc',
 133                'formatting',
 134                'gcsimulator',
 135                'jitdiff',
 136                'standalone_gc',
 137                'gc_reliability_framework',
 138                'illink'] + r2rStressScenarios.keySet()
 139
 140     def static allScenarios = basicScenarios + jitStressModeScenarios.keySet()
 141
 142     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 143     // to Tests.lst file categories to exclude.
 144     //
 145     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 146     // and with the same values, with some commented out, for easier maintenance.
 147     //
 148     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 149     //
 150     def static validArmWindowsScenarios = [
 151                'default':                                [],
 152                // 'ilrt'
 153                // 'r2r':                                    ["R2R_FAIL"],
 154                // 'longgc'
 155                // 'formatting'
 156                // 'gcsimulator'
 157                // 'jitdiff'
 158                // 'standalone_gc'
 159                // 'gc_reliability_framework'
 160                // 'illink'
 161                // 'r2r_jitstress1'
 162                // 'r2r_jitstress2'
 163                // 'r2r_jitstressregs1'
 164                // 'r2r_jitstressregs2'
 165                // 'r2r_jitstressregs3'
 166                // 'r2r_jitstressregs4'
 167                // 'r2r_jitstressregs8'
 168                // 'r2r_jitstressregs0x10'
 169                // 'r2r_jitstressregs0x80'
 170                // 'r2r_jitstressregs0x1000'
 171                // 'r2r_jitminopts'
 172                // 'r2r_jitforcerelocs'
 173                // 'r2r_gcstress15'
 174                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 175                'tieredcompilation':                      [],
 176                'forcerelocs':                            [],
 177                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 179                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 180                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 181                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 196                // 'jitsse2only'                          // Only relevant to xarch
 197                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 198                // 'corefx_baseline'
 199                // 'corefx_minopts'
 200                // 'corefx_tieredcompilation'
 201                // 'corefx_jitstress1'
 202                // 'corefx_jitstress2'
 203                // 'corefx_jitstressregs1'
 204                // 'corefx_jitstressregs2'
 205                // 'corefx_jitstressregs3'
 206                // 'corefx_jitstressregs4'
 207                // 'corefx_jitstressregs8'
 208                // 'corefx_jitstressregs0x10'
 209                // 'corefx_jitstressregs0x80'
 210                // 'corefx_jitstressregs0x1000'
 211                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 212                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 213                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 214                'heapverify1':                            [],
 215                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 216                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 217                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 219                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 220                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 221
 222                //
 223                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 224                //
 225
 226                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 227                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 228                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 245     ]
 246
 247     def static configurationList = ['Debug', 'Checked', 'Release']
 248
 249     // This is the set of architectures
 250     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 251 }
 252
 253 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 254     assert os instanceof String
 255     assert architecture instanceof String
 256
 257     def armArches = ['arm', 'armlb', 'arm64']
 258     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 259
 260     if (!(architecture in armArches)) {
 261         assert options == null
 262         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 263
 264         return
 265     }
 266
 267     // This is an arm(64) job.
 268     //
 269     // There are several options.
 270     //
 271     // Windows_NT
 272     //
 273     // Arm32 (Build) -> latest-arm64
 274     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 275     // Arm32 (Test)  -> arm64-windows_nt
 276     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 277     //
 278     // Arm64 (Build) -> latest-arm64
 279     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 280     // Arm64 (Test)  -> arm64-windows_nt
 281     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 282     //
 283     // Ubuntu
 284     //
 285     // Arm32 (Build) -> arm-cross-latest
 286     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 287     // Arm32 (Test)  -> NYI Arch not supported
 288     //       |->
 289     //
 290     // Arm64 (Build) -> arm64-cross-latest
 291     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 292     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 293     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 294     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 295     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 296
 297     // This has to be a arm arch
 298     assert architecture in armArches
 299     if (os == "Windows_NT") {
 300         // Arm(64) windows jobs share the same machines for now
 301         def isBuild = options['use_arm64_build_machine'] == true
 302
 303         if (isBuild == true) {
 304             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 305         } else {
 306             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 307         }
 308     } else {
 309         assert os != 'Windows_NT'
 310         assert os in supportedArmLinuxOs
 311
 312         if (architecture == 'arm' || architecture == 'armlb') {
 313             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 314         } else {
 315             // Arm64 Linux
 316             if (options['is_build_only'] == true) {
 317                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 318             } else {
 319                 // Arm64 Test Machines
 320                 if (options['large_pages'] == false) {
 321                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 322                 } else {
 323                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 324                 }
 325             }
 326         }
 327     }
 328 }
 329
 330 def static isGCStressRelatedTesting(def scenario) {
 331     // The 'r2r_gcstress15' scenario is a basic scenario.
 332     // Detect it and make it a GCStress related.
 333     if (scenario == 'r2r_gcstress15')
 334     {
 335         return true;
 336     }
 337
 338     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 339     def scenarioName = scenario.toLowerCase()
 340     def isGCStressTesting = false
 341     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 342         if (k in gcStressTestEnvVars) {
 343             isGCStressTesting = true;
 344         }
 345     }
 346     return isGCStressTesting
 347 }
 348
 349 def static isCoreFxScenario(def scenario) {
 350     def corefx_prefix = 'corefx_'
 351     if (scenario.length() < corefx_prefix.length()) {
 352         return false
 353     }
 354     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 355 }
 356
 357 def static isR2RBaselineScenario(def scenario) {
 358     return (scenario == 'r2r')
 359 }
 360
 361 def static isR2RStressScenario(def scenario) {
 362     return Constants.r2rStressScenarios.containsKey(scenario)
 363 }
 364
 365 def static isR2RScenario(def scenario) {
 366     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 367 }
 368
 369 def static isJitStressScenario(def scenario) {
 370     return Constants.jitStressModeScenarios.containsKey(scenario)
 371 }
 372
 373 def static isLongGc(def scenario) {
 374     return (scenario == 'longgc' || scenario == 'gcsimulator')
 375 }
 376
 377 def static isJitDiff(def scenario) {
 378     return (scenario == 'jitdiff')
 379 }
 380
 381 def static isGcReliabilityFramework(def scenario) {
 382     return (scenario == 'gc_reliability_framework')
 383 }
 384
 385 def static isArmWindowsScenario(def scenario) {
 386     return Constants.validArmWindowsScenarios.containsKey(scenario)
 387 }
 388
 389 def static setTestJobTimeOut(newJob, scenario) {
 390     if (isGCStressRelatedTesting(scenario)) {
 391         Utilities.setJobTimeout(newJob, 4320)
 392     }
 393     else if (isCoreFxScenario(scenario)) {
 394         Utilities.setJobTimeout(newJob, 360)
 395     }
 396     else if (isJitStressScenario(scenario)) {
 397         Utilities.setJobTimeout(newJob, 240)
 398     }
 399     else if (isR2RBaselineScenario(scenario)) {
 400         Utilities.setJobTimeout(newJob, 240)
 401     }
 402     else if (isLongGc(scenario)) {
 403         Utilities.setJobTimeout(newJob, 1440)
 404     }
 405     else if (isJitDiff(scenario)) {
 406         Utilities.setJobTimeout(newJob, 240)
 407     }
 408     else if (isGcReliabilityFramework(scenario)) {
 409         Utilities.setJobTimeout(newJob, 1440)
 410     }
 411     // Non-test jobs use the default timeout value.
 412 }
 413
 414 def static getJobFolder(def scenario) {
 415     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 416         return 'jitstress'
 417     }
 418     if (scenario == 'illink') {
 419         return 'illink'
 420     }
 421     return ''
 422 }
 423
 424 def static getStressModeDisplayName(def scenario) {
 425     def displayStr = ''
 426     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 427         def prefixLength = 'COMPlus_'.length()
 428         if (k.length() >= prefixLength) {
 429             def modeName = k.substring(prefixLength, k.length())
 430             displayStr += ' ' + modeName + '=' + v
 431         }
 432     }
 433
 434     if (isCoreFxScenario(scenario)) {
 435         displayStr = ('CoreFx ' + displayStr).trim()
 436     }
 437
 438     return displayStr
 439 }
 440
 441 def static getR2RDisplayName(def scenario) {
 442     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 443     def displayStr = scenario
 444     def prefixLength = 'r2r_'.length()
 445     if (displayStr.length() >= prefixLength) {
 446         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 447     } else if (scenario == 'r2r') {
 448         displayStr = "R2R"
 449     }
 450     return displayStr
 451 }
 452
 453 // Generates the string for creating a file that sets environment variables
 454 // that makes it possible to run stress modes.  Writes the script to the file
 455 // specified by the stepScriptLocation parameter.
 456 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 457     def stepScript = ''
 458     if (os == 'Windows_NT') {
 459         // Timeout in ms, default is 10 minutes. For stress modes up this to 30 minutes.
 460         // BUG?: it seems this is ignored, as this script will be run in an environment where
 461         //       environment variables will be discarded before this is used.
 462         def timeout = 1800000
 463         stepScript += "set __TestTimeout=${timeout}\r\n"
 464
 465         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 466         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 467
 468         // Create at least an empty script.
 469         stepScript += "echo. > ${stepScriptLocation}\r\n"
 470
 471         stressModeVars.each{ k, v ->
 472             // Write out what we are writing to the script file
 473             stepScript += "echo Setting ${k}=${v}\r\n"
 474             // Write out the set itself to the script file`
 475             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 476         }
 477     }
 478     else {
 479         stepScript += "echo Setting variables for ${stressModeName}\n"
 480         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 481         stressModeVars.each{ k, v ->
 482             // Write out what we are writing to the script file
 483             stepScript += "echo Setting ${k}=${v}\n"
 484             // Write out the set itself to the script file`
 485             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 486         }
 487         stepScript += "chmod +x ${stepScriptLocation}\n"
 488     }
 489
 490     return stepScript
 491 }
 492
 493 // Append an existing script to a stress mode script already created by genStressModeScriptStep().
 494 // Returns string of commands to do this.
 495 def static appendStressModeScriptStep(def os, def appendScript, def stepScriptLocation) {
 496     assert (os == 'Windows_NT')
 497     def stepScript = ''
 498     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 499     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 500
 501     // Display the resulting script. This is useful when looking at the output log file.
 502     stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 503     stepScript += "type ${stepScriptLocation}\r\n"
 504
 505     return stepScript
 506 }
 507
 508 def static isNeedDocker(def architecture, def os, def isBuild) {
 509     if (isBuild) {
 510         if (architecture == 'x86' && os == 'Ubuntu') {
 511             return true
 512         }
 513         else if (architecture == 'arm') {
 514             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 515                 return true
 516             }
 517         }
 518     }
 519     else {
 520         if (architecture == 'x86' && os == 'Ubuntu') {
 521             return true
 522         }
 523     }
 524     return false
 525 }
 526
 527 def static getDockerImageName(def architecture, def os, def isBuild) {
 528     // We must change some docker private images to official later
 529     if (isBuild) {
 530         if (architecture == 'x86' && os == 'Ubuntu') {
 531             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 532         }
 533         else if (architecture == 'arm') {
 534             if (os == 'Ubuntu') {
 535                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 536             }
 537             else if (os == 'Ubuntu16.04') {
 538                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 539             }
 540             else if (os == 'Tizen') {
 541                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 542             }
 543         }
 544     }
 545     else {
 546         if (architecture == 'x86' && os == 'Ubuntu') {
 547             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 548         }
 549     }
 550     println("Unknown architecture to use docker: ${architecture} ${os}");
 551     assert false
 552 }
 553
 554 // Calculates the name of the build job based on some typical parameters.
 555 //
 556 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 557     // If the architecture is x64, do not add that info into the build name.
 558     // Need to change around some systems and other builds to pick up the right builds
 559     // to do that.
 560
 561     def suffix = scenario != 'default' ? "_${scenario}" : '';
 562     if (isBuildOnly) {
 563         suffix += '_bld'
 564     }
 565     def baseName = ''
 566     switch (architecture) {
 567         case 'x64':
 568             if (scenario == 'default') {
 569                 // For now we leave x64 off of the name for compatibility with other jobs
 570                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 571             }
 572             else if (scenario == 'formatting') {
 573                 // we don't care about the configuration for the formatting job. It runs all configs
 574                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 575             }
 576             else {
 577                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 578             }
 579             break
 580         case 'arm64':
 581             if (os.toLowerCase() == "windows_nt") {
 582                 // These are cross builds
 583                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 584             }
 585             else {
 586                 // Defaults to a small page size set of machines.
 587                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 588             }
 589             break
 590         case 'arm':
 591             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 592             break
 593         case 'armlb':
 594             // These are cross builds
 595             if (os == 'Tizen') {
 596                 // ABI: softfp
 597                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 598             }
 599             else {
 600                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 601             }
 602             break
 603         case 'x86':
 604         case 'x86_arm_altjit':
 605         case 'x64_arm64_altjit':
 606             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 607             break
 608         default:
 609             println("Unknown architecture: ${architecture}");
 610             assert false
 611             break
 612     }
 613
 614     return baseName + suffix
 615 }
 616
 617 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 618
 619     // Check scenario.
 620     switch (scenario) {
 621         case 'default':
 622             switch (architecture) {
 623                 case 'x64':
 624                 case 'x86':
 625                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 626                         Utilities.addPeriodicTrigger(job, '@daily')
 627                     }
 628                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 629                         Utilities.addGithubPushTrigger(job)
 630                     }
 631                     break
 632                 case 'arm':
 633                 case 'armlb':
 634                 case 'x86_arm_altjit':
 635                 case 'x64_arm64_altjit':
 636                     Utilities.addGithubPushTrigger(job)
 637                     break
 638                 case 'arm64':
 639                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 640                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 641                     break
 642                 default:
 643                     println("Unknown architecture: ${architecture}");
 644                     assert false
 645                     break
 646             }
 647             break
 648         case 'r2r':
 649             assert !(os in bidailyCrossList)
 650             // r2r gets a push trigger for checked/release
 651             if (configuration == 'Checked' || configuration == 'Release') {
 652                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 653                 if (architecture == 'x64' && os != 'OSX10.12') {
 654                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 655                     if (isFlowJob || os == 'Windows_NT') {
 656                         Utilities.addGithubPushTrigger(job)
 657                     }
 658                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 659                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 660                     if (isFlowJob) {
 661                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 662                     }
 663                 }
 664                 // For x86, only add per-commit jobs for Windows
 665                 else if (architecture == 'x86') {
 666                     if (os == 'Windows_NT') {
 667                         Utilities.addGithubPushTrigger(job)
 668                     }
 669                 }
 670                 // arm64 r2r jobs should only run daily.
 671                 else if (architecture == 'arm64') {
 672                     if (os == 'Windows_NT') {
 673                         Utilities.addPeriodicTrigger(job, '@daily')
 674                     }
 675                 }
 676             }
 677             break
 678         case 'r2r_jitstress1':
 679         case 'r2r_jitstress2':
 680         case 'r2r_jitstressregs1':
 681         case 'r2r_jitstressregs2':
 682         case 'r2r_jitstressregs3':
 683         case 'r2r_jitstressregs4':
 684         case 'r2r_jitstressregs8':
 685         case 'r2r_jitstressregs0x10':
 686         case 'r2r_jitstressregs0x80':
 687         case 'r2r_jitstressregs0x1000':
 688         case 'r2r_jitminopts':
 689         case 'r2r_jitforcerelocs':
 690         case 'r2r_gcstress15':
 691             assert !(os in bidailyCrossList)
 692
 693             // GCStress=C is currently not supported on OS X
 694             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 695                 break
 696             }
 697
 698             // GC Stress 15 r2r gets a push trigger for checked/release
 699             if (configuration == 'Checked' || configuration == 'Release') {
 700                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 701                 if (architecture == 'x64') {
 702                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 703                     if (isFlowJob || os == 'Windows_NT') {
 704                         // Add a weekly periodic trigger
 705                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 706                     }
 707                 }
 708                 // For x86, only add per-commit jobs for Windows
 709                 else if (architecture == 'x86') {
 710                     if (os == 'Windows_NT') {
 711                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 712                     }
 713                 }
 714             }
 715             break
 716         case 'longgc':
 717             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 718             assert configuration == 'Release'
 719             assert architecture == 'x64'
 720             Utilities.addPeriodicTrigger(job, '@daily')
 721             // TODO: Add once external email sending is available again
 722             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 723             break
 724         case 'gcsimulator':
 725             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 726             assert configuration == 'Release'
 727             assert architecture == 'x64'
 728             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 729             // TODO: Add once external email sending is available again
 730             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 731             break
 732         case 'standalone_gc':
 733             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 734             assert (configuration == 'Release' || configuration == 'Checked')
 735             // TODO: Add once external email sending is available again
 736             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 737             Utilities.addPeriodicTrigger(job, '@daily')
 738             break
 739         case 'gc_reliability_framework':
 740             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 741             assert (configuration == 'Release' || configuration == 'Checked')
 742             // Only triggered by phrase.
 743             break
 744         case 'ilrt':
 745             assert !(os in bidailyCrossList)
 746             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 747             if (architecture == 'x64' && configuration == 'Release') {
 748                 // We don't expect to see a job generated except in these scenarios
 749                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 750                 if (isFlowJob || os == 'Windows_NT') {
 751                     Utilities.addPeriodicTrigger(job, '@daily')
 752                 }
 753             }
 754             break
 755         case 'jitdiff':
 756             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 757             assert configuration == 'Checked'
 758             assert (architecture == 'x64' || architecture == 'x86')
 759             Utilities.addGithubPushTrigger(job)
 760             break
 761         case 'formatting':
 762             assert (os == 'Windows_NT' || os == "Ubuntu")
 763             assert architecture == 'x64'
 764             Utilities.addGithubPushTrigger(job)
 765             break
 766         case 'jitstressregs1':
 767         case 'jitstressregs2':
 768         case 'jitstressregs3':
 769         case 'jitstressregs4':
 770         case 'jitstressregs8':
 771         case 'jitstressregs0x10':
 772         case 'jitstressregs0x80':
 773         case 'jitstressregs0x1000':
 774         case 'minopts':
 775         case 'forcerelocs':
 776         case 'jitstress1':
 777         case 'jitstress2':
 778         case 'jitstress2_jitstressregs1':
 779         case 'jitstress2_jitstressregs2':
 780         case 'jitstress2_jitstressregs3':
 781         case 'jitstress2_jitstressregs4':
 782         case 'jitstress2_jitstressregs8':
 783         case 'jitstress2_jitstressregs0x10':
 784         case 'jitstress2_jitstressregs0x80':
 785         case 'jitstress2_jitstressregs0x1000':
 786         case 'tailcallstress':
 787         case 'jitsse2only':
 788         case 'jitnosimd':
 789         case 'corefx_baseline':
 790         case 'corefx_minopts':
 791         case 'corefx_jitstress1':
 792         case 'corefx_jitstress2':
 793         case 'corefx_jitstressregs1':
 794         case 'corefx_jitstressregs2':
 795         case 'corefx_jitstressregs3':
 796         case 'corefx_jitstressregs4':
 797         case 'corefx_jitstressregs8':
 798         case 'corefx_jitstressregs0x10':
 799         case 'corefx_jitstressregs0x80':
 800         case 'corefx_jitstressregs0x1000':
 801         case 'zapdisable':
 802             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 803                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 804                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 805                     if (os == 'Windows_NT') {
 806                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 807                         Utilities.addPeriodicTrigger(job, '@weekly')
 808                     }
 809                 }
 810                 else {
 811                     Utilities.addPeriodicTrigger(job, '@daily')
 812                 }
 813             }
 814             break
 815         case 'heapverify1':
 816         case 'gcstress0x3':
 817             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 818                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 819                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 820                     if (os == 'Windows_NT') {
 821                         Utilities.addPeriodicTrigger(job, '@daily')
 822                     }
 823                     // TODO: Add once external email sending is available again
 824                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 825                 }
 826                 else {
 827                     Utilities.addPeriodicTrigger(job, '@weekly')
 828                 }
 829             }
 830             break
 831         case 'gcstress0xc':
 832         case 'gcstress0xc_zapdisable':
 833         case 'gcstress0xc_zapdisable_jitstress2':
 834         case 'gcstress0xc_zapdisable_heapverify1':
 835         case 'gcstress0xc_jitstress1':
 836         case 'gcstress0xc_jitstress2':
 837         case 'gcstress0xc_minopts_heapverify1':
 838             // GCStress=C is currently not supported on OS X
 839             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 840                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 841                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 842                     if (os == 'Windows_NT') {
 843                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 844                         Utilities.addPeriodicTrigger(job, '@weekly')
 845                     }
 846                     // TODO: Add once external email sending is available again
 847                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 848                 }
 849                 else {
 850                     Utilities.addPeriodicTrigger(job, '@weekly')
 851                 }
 852             }
 853             break
 854
 855         case 'illink':
 856             // Testing on other operating systems TBD
 857             assert (os == 'Windows_NT' || os == 'Ubuntu')
 858             if (architecture == 'x64' || architecture == 'x86') {
 859                 if (configuration == 'Checked') {
 860                     Utilities.addPeriodicTrigger(job, '@daily')
 861                 }
 862             }
 863             break
 864
 865         case 'tieredcompilation':
 866         case 'corefx_tieredcompilation':
 867             // No periodic jobs just yet, still testing
 868             break
 869
 870         default:
 871             println("Unknown scenario: ${scenario}");
 872             assert false
 873             break
 874     }
 875     return
 876 }
 877
 878 // **************************
 879 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 880 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 881 // builds will do a couple extra steps.
 882 // **************************
 883
 884 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 885 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 886 // it's just used for internal builds
 887 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 888 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 889     if (isWindowsBuildOnlyJob) {
 890         return
 891     }
 892
 893     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 894     // Non pull request builds.
 895     if (!isPR) {
 896         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 897         return
 898     }
 899
 900      def arm64Users = [
 901         'AndyAyersMS',
 902         'briansull',
 903         'BruceForstall',
 904         'CarolEidt',
 905         'cmckinsey',
 906         'erozenfeld',
 907         'janvorli',
 908         'jashook',
 909         'JosephTremoulet',
 910         'pgodeq',
 911         'pgavlin',
 912         'rartemev',
 913         'russellhadley',
 914         'RussKeldorph',
 915         'sandreenko',
 916         'sdmaclea',
 917         'swaroop-sridhar',
 918         'jkotas',
 919         'markwilkie',
 920         'weshaggard'
 921     ]
 922
 923     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 924     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 925     def osGroup = getOSGroup(os)
 926     switch (architecture) {
 927         case 'x64': // editor brace matching: {
 928             if (scenario == 'formatting') {
 929                 assert configuration == 'Checked'
 930                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 931                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
 932                 }
 933                 break
 934             }
 935
 936             switch (os) {
 937                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 938                 case 'Debian8.4':
 939                 case 'RHEL7.2':
 940                     if (scenario == 'default') {
 941                         assert !isFlowJob
 942                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 943                     }
 944                     break
 945                 case 'Ubuntu16.04':
 946                     assert !isFlowJob
 947                     assert scenario == 'default'
 948                     // Distinguish with the other architectures (arm and x86)
 949                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 950                     break
 951                 case 'Fedora24':
 952                 case 'Ubuntu16.10':
 953                     assert !isFlowJob
 954                     assert scenario == 'default'
 955                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 956                     break
 957                 case 'Ubuntu':
 958                     if (scenario == 'illink') {
 959                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 960                         break
 961                     }
 962                     // fall through
 963                 case 'OSX10.12':
 964                     // Triggers on the non-flow jobs aren't necessary here
 965                     // Corefx testing uses non-flow jobs.
 966                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
 967                         break
 968                     }
 969                     switch (scenario) {
 970                         case 'default':
 971                             // OSX uses checked for default PR tests
 972                             if (configuration == 'Checked') {
 973                                 // Default trigger
 974                                 assert !job.name.contains("centos")
 975                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
 976                             }
 977                             break
 978                         case 'jitdiff':
 979                             if (configuration == 'Checked') {
 980                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 981                             }
 982                             break
 983                         case 'ilrt':
 984                             if (configuration == 'Release') {
 985                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 986                             }
 987                             break
 988                         case 'longgc':
 989                             if (configuration == 'Release') {
 990                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 991                             }
 992                             break
 993                         case 'gcsimulator':
 994                             if (configuration == 'Release') {
 995                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 996                             }
 997                             break
 998                         case 'standalone_gc':
 999                             if (configuration == 'Release' || configuration == 'Checked') {
1000                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1001                             }
1002                             break
1003                         case 'gc_reliability_framework':
1004                             if (configuration == 'Release' || configuration == 'Checked') {
1005                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1006                             }
1007                             break
1008                         default:
1009                             if (isJitStressScenario(scenario)) {
1010                                 def displayStr = getStressModeDisplayName(scenario)
1011                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1012                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1013                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1014                             }
1015                             else if (isR2RScenario(scenario)) {
1016                                 if (configuration == 'Release' || configuration == 'Checked') {
1017                                     def displayStr = getR2RDisplayName(scenario)
1018                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1019                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1020                                 }
1021                             }
1022                             else {
1023                                 println("Unknown scenario: ${scenario}");
1024                                 assert false
1025                             }
1026                             break
1027                     }
1028                     break
1029
1030                 case 'CentOS7.1':
1031                     switch (scenario) {
1032                         case 'default':
1033                             // CentOS uses checked for default PR tests while debug is build only
1034                             if (configuration == 'Debug') {
1035                                 // Default trigger
1036                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1037                             }
1038
1039                             // Make sure this is a flow job to get build and test.
1040                             if (configuration == 'Checked' && isFlowJob) {
1041                                 assert job.name.contains("flow")
1042                                 // Default trigger
1043                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1044                             }
1045                             break
1046                         default:
1047                             if (isR2RScenario(scenario)) {
1048                                 if (configuration == 'Release' || configuration == 'Checked') {
1049                                     def displayStr = getR2RDisplayName(scenario)
1050                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1051                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1052                                 }
1053                             }
1054                             break
1055                     }
1056                     break
1057
1058                 case 'Windows_NT':
1059                     switch (scenario) {
1060                         case 'default':
1061                             // Default trigger
1062                             if (configuration == 'Checked') {
1063                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1064                             }
1065                             break
1066                         case 'jitdiff':
1067                             if (configuration == 'Checked') {
1068                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1069                             }
1070                             break
1071                         case 'ilrt':
1072                             if (configuration == 'Release') {
1073                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1074                             }
1075                             break
1076                         case 'longgc':
1077                             if (configuration == 'Release') {
1078                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1079                             }
1080                             break
1081                         case 'gcsimulator':
1082                             if (configuration == 'Release') {
1083                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1084                             }
1085                             break
1086                         case 'standalone_gc':
1087                             if (configuration == 'Release' || configuration == 'Checked') {
1088                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1089                             }
1090                             break
1091                         case 'gc_reliability_framework':
1092                             if (configuration == 'Release' || configuration == 'Checked') {
1093                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1094                             }
1095                             break
1096                         case 'illink':
1097                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1098                             break
1099                         default:
1100                             if (isJitStressScenario(scenario)) {
1101                                 def displayStr = getStressModeDisplayName(scenario)
1102                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1103                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1104                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1105                             }
1106                             else if (isR2RScenario(scenario)) {
1107                                 if (configuration == 'Release' || configuration == 'Checked') {
1108                                     def displayStr = getR2RDisplayName(scenario)
1109                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1110                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1111                                 }
1112                             }
1113                             else {
1114                                 println("Unknown scenario: ${scenario}");
1115                                 assert false
1116                             }
1117                             break
1118                     }
1119                     break
1120                 default:
1121                     println("Unknown os: ${os}");
1122                     assert false
1123                     break
1124             }
1125             break
1126         // editor brace matching: }
1127         case 'armlb':
1128         case 'arm': // editor brace matching: {
1129             switch (os) {
1130                 case 'Ubuntu':
1131                 case 'Ubuntu16.04':
1132                     if (architecture == 'armlb') { // Ubuntu arm is only for armlb currently
1133                         assert scenario == 'default'
1134                         job.with {
1135                             publishers {
1136                                 azureVMAgentPostBuildAction {
1137                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1138                                 }
1139                             }
1140                         }
1141                         if ((os == 'Ubuntu' && configuration == 'Debug') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1142                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1143                         }
1144                         else {
1145                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1146                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1147                         }
1148                     }
1149                     break
1150                 case 'Tizen':
1151                     if (architecture == 'armlb') { // Tizen armel is only for armlb currently
1152                         architecture='armel'
1153                         job.with {
1154                             publishers {
1155                                 azureVMAgentPostBuildAction {
1156                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1157                                 }
1158                             }
1159                         }
1160                         // Removing the regex will cause this to run on each PR.
1161                         if (configuration == 'Checked') {
1162                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1163                         }
1164                         else {
1165                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1166                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1167                         }
1168                     }
1169                     break
1170                 case 'Windows_NT':
1171                     // Triggers on the non-flow jobs aren't necessary here
1172                     if (!isFlowJob) {
1173                         break
1174                     }
1175
1176                     // Set up a private trigger
1177                     def contextString = "${os} ${architecture} Cross ${configuration}"
1178                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1179                     if (scenario == 'default') {
1180                         contextString += " Innerloop"
1181                         triggerString += "\\W+Innerloop"
1182                     }
1183                     else {
1184                         contextString += " ${scenario}"
1185                         triggerString += "\\W+${scenario}"
1186                     }
1187
1188                     if (configuration == 'Debug') {
1189                         contextString += " Build"
1190                         triggerString += "\\W+Build"
1191                     } else {
1192                         contextString += " Build and Test"
1193                         triggerString += "\\W+Build and Test"
1194                     }
1195
1196                     triggerString += ".*"
1197
1198                     switch (scenario) {
1199                         case 'default':
1200                             // Only Checked is a default trigger.
1201                             if (configuration == 'Checked')
1202                             {
1203                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1204                             }
1205                             else
1206                             {
1207                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1208                             }
1209                             break
1210                         default:
1211                             // Stress jobs will use this code path.
1212                             if (isArmWindowsScenario(scenario)) {
1213                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1214                             }
1215                             break
1216                     }
1217                     break
1218                 default:
1219                     println("NYI os: ${os}");
1220                     assert false
1221                     break
1222             }
1223             break
1224         // editor brace matching: }
1225         case 'arm64': // editor brace matching: {
1226             // Set up a private trigger
1227             def contextString = "${os} ${architecture} Cross ${configuration}"
1228             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1229             if (scenario == 'default') {
1230                 contextString += " Innerloop"
1231                 triggerString += "\\W+Innerloop"
1232             }
1233             else {
1234                 contextString += " ${scenario}"
1235                 triggerString += "\\W+${scenario}"
1236             }
1237
1238             if (configuration == 'Debug') {
1239                 contextString += " Build"
1240                 triggerString += "\\W+Build"
1241             } else {
1242                 contextString += " Build and Test"
1243                 triggerString += "\\W+Build and Test"
1244             }
1245
1246             triggerString += ".*"
1247
1248             switch (os) {
1249                 case 'Ubuntu':
1250                 case 'Ubuntu16.04':
1251                     switch (scenario) {
1252                         case 'default':
1253                             if (configuration == 'Debug' && !isFlowJob) {
1254                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1255                             }
1256                             else {
1257                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1258                             }
1259
1260                             break
1261                         default:
1262                             if (isR2RScenario(scenario)) {
1263                                 if (configuration == 'Checked' || configuration == 'Release') {
1264                                     def displayStr = getR2RDisplayName(scenario)
1265                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1266                                 }
1267                             }
1268                             break
1269                     }
1270                     break
1271                 case 'Windows_NT':
1272                     // Triggers on the non-flow jobs aren't necessary here
1273                     if (!isFlowJob) {
1274                         break
1275                     }
1276
1277                     assert isArmWindowsScenario(scenario)
1278                     switch (scenario) {
1279                         case 'default':
1280                             if (configuration == 'Checked') {
1281                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1282                             }
1283                             else {
1284                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1285                             }
1286                             break
1287                         default:
1288                             // Stress jobs will use this code path.
1289                             if (isArmWindowsScenario(scenario)) {
1290                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1291                             }
1292                             break
1293                     }
1294                     break
1295                 default:
1296                     println("NYI os: ${os}");
1297                     assert false
1298                     break
1299             }
1300             break
1301         // editor brace matching: }
1302         case 'x86': // editor brace matching: {
1303             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1304             if (os == 'Ubuntu') {
1305                 // Triggers on the non-flow jobs aren't necessary here
1306                 if (!isFlowJob) {
1307                     break
1308                 }
1309                 // on-demand only for ubuntu x86
1310                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1311                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1312                 break
1313             }
1314             switch (scenario) {
1315                 case 'default':
1316                     if (configuration == 'Checked') {
1317                         assert !job.name.contains("centos")
1318                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1319                     }
1320                     else {
1321                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1322                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1323                     }
1324                     break
1325                 case 'ilrt':
1326                     if (configuration == 'Release') {
1327                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1328                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1329                     }
1330                     break
1331                 case 'longgc':
1332                     if (configuration == 'Release') {
1333                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1334                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1335                     }
1336                     break
1337                 case 'gcsimulator':
1338                     if (configuration == 'Release') {
1339                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1340                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1341                     }
1342                     break
1343                 case 'standalone_gc':
1344                     if (configuration == 'Release' || configuration == 'Checked') {
1345                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1346                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1347                     }
1348                     break
1349                 case 'illink':
1350                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1351                     break
1352                 default:
1353                     if (isJitStressScenario(scenario)) {
1354                         def displayStr = getStressModeDisplayName(scenario)
1355                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1356                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1357                     }
1358                     else if (isR2RScenario(scenario)) {
1359                         if (configuration == 'Release' || configuration == 'Checked') {
1360                             def displayStr = getR2RDisplayName(scenario)
1361                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1362                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1363                         }
1364                     }
1365                     else {
1366                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1367                         assert false
1368                     }
1369                     break
1370             }
1371             break
1372          // editor brace matching: }
1373         case 'x64_arm64_altjit':
1374         case 'x86_arm_altjit': // editor brace matching: {
1375             assert (os == 'Windows_NT')
1376             switch (scenario) {
1377                 case 'default':
1378                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1379                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1380                     break
1381                 default:
1382                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1383                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1384                     break
1385             }
1386             break
1387         // editor brace matching: }
1388         default:
1389             println("Unknown architecture: ${architecture}");
1390             assert false
1391             break
1392     }
1393 }
1394
1395 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1396     def buildCommands = [];
1397     def osGroup = getOSGroup(os)
1398     def lowerConfiguration = configuration.toLowerCase()
1399
1400     def priority = '1'
1401     if (scenario == 'default' && isPR == true) {
1402         priority = '0'
1403     }
1404
1405     def enableCorefxTesting = isCoreFxScenario(scenario)
1406
1407     // Calculate the build steps, archival, and xunit results
1408     switch (os) {
1409         case 'Windows_NT': // editor brace matching: {
1410             switch (architecture) {
1411                 case 'x64':
1412                 case 'x86':
1413                 case 'x86_arm_altjit':
1414                 case 'x64_arm64_altjit':
1415                     def arch = architecture
1416                     def buildOpts = ''
1417                     if (architecture == 'x86_arm_altjit') {
1418                         arch = 'x86'
1419                     }
1420                     else if (architecture == 'x64_arm64_altjit') {
1421                         arch = 'x64'
1422                     }
1423
1424                     if (scenario == 'formatting') {
1425                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1426                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1427                         break
1428                     }
1429
1430                     if (scenario == 'illink') {
1431                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1432                     }
1433
1434                     // If it is a release build for windows, ensure PGO is used, else fail the build
1435                     if ((lowerConfiguration == 'release') &&
1436                         (scenario in Constants.basicScenarios) &&
1437                         (architecture != 'x86_arm_altjit') &&
1438                         (architecture != 'x64_arm64_altjit')) {
1439
1440                         buildOpts += ' -enforcepgo'
1441                     }
1442
1443                     if (enableCorefxTesting) {
1444                         buildOpts += ' skiptests';
1445                     } else {
1446                         buildOpts += " -priority=${priority}"
1447                     }
1448
1449                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1450                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1451                     // 35 characters long.
1452
1453                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1454
1455                     if (!isBuildOnly) {
1456                         def runtestArguments = ''
1457                         def testOpts = 'collectdumps'
1458
1459                         if (isR2RScenario(scenario)) {
1460
1461                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1462                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1463                             // to cause the tests to be crossgen'ed.
1464
1465                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1466                                 testOpts += ' crossgenaltjit protononjit.dll'
1467                             } else {
1468                                 testOpts += ' crossgen'
1469                             }
1470
1471                             testOpts += ' runcrossgentests'
1472
1473                             if (scenario == 'r2r_jitstress1') {
1474                                 testOpts += ' jitstress 1'
1475                             }
1476                             else if (scenario == 'r2r_jitstress2') {
1477                                 testOpts += ' jitstress 2'
1478                             }
1479                             else if (scenario == 'r2r_jitstressregs1') {
1480                                 testOpts += ' jitstressregs 1'
1481                             }
1482                             else if (scenario == 'r2r_jitstressregs2') {
1483                                 testOpts += ' jitstressregs 2'
1484                             }
1485                             else if (scenario == 'r2r_jitstressregs3') {
1486                                 testOpts += ' jitstressregs 3'
1487                             }
1488                             else if (scenario == 'r2r_jitstressregs4') {
1489                                 testOpts += ' jitstressregs 4'
1490                             }
1491                             else if (scenario == 'r2r_jitstressregs8') {
1492                                 testOpts += ' jitstressregs 8'
1493                             }
1494                             else if (scenario == 'r2r_jitstressregs0x10') {
1495                                 testOpts += ' jitstressregs 0x10'
1496                             }
1497                             else if (scenario == 'r2r_jitstressregs0x80') {
1498                                 testOpts += ' jitstressregs 0x80'
1499                             }
1500                             else if (scenario == 'r2r_jitstressregs0x1000') {
1501                                 testOpts += ' jitstressregs 0x1000'
1502                             }
1503                             else if (scenario == 'r2r_jitminopts') {
1504                                 testOpts += ' jitminopts'
1505                             }
1506                             else if (scenario == 'r2r_jitforcerelocs') {
1507                                 testOpts += ' jitforcerelocs'
1508                             }
1509                             else if (scenario == 'r2r_gcstress15') {
1510                                 testOpts += ' gcstresslevel 0xF'
1511                             }
1512                         }
1513                         else if (scenario == 'jitdiff') {
1514                             testOpts += ' jitdisasm crossgen'
1515                         }
1516                         else if (scenario == 'ilrt') {
1517                             testOpts += ' ilasmroundtrip'
1518                         }
1519                         else if (isLongGc(scenario)) {
1520                             testOpts += " ${scenario} sequential"
1521                         }
1522                         else if (scenario == 'standalone_gc') {
1523                             testOpts += ' gcname clrgc.dll'
1524                         }
1525                         else if (scenario == 'illink') {
1526                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1527                         }
1528
1529                         // If we are running a stress mode, we should write out the set of key
1530                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1531
1532                         def envScriptPath = ''
1533                         if (isJitStressScenario(scenario)) {
1534                             def buildCommandsStr = ''
1535                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1536                             buildCommandsStr += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], envScriptPath)
1537                             if (architecture == 'x86_arm_altjit') {
1538                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1539                             }
1540                             else if (architecture == 'x64_arm64_altjit') {
1541                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1542                             }
1543
1544                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1545                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1546                             // into a single script.
1547                             buildCommands += buildCommandsStr
1548                         }
1549                         else if (architecture == 'x86_arm_altjit') {
1550                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1551                         }
1552                         else if (architecture == 'x64_arm64_altjit') {
1553                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1554                         }
1555                         if (envScriptPath != '') {
1556                             testOpts += " TestEnv ${envScriptPath}"
1557                         }
1558
1559                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1560
1561                         if (enableCorefxTesting) {
1562                             def workspaceRelativeFxRoot = "_/fx"
1563                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1564
1565                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1566
1567                             setTestJobTimeOut(newJob, scenario)
1568
1569                             // Archive and process (only) the test results
1570                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1571                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1572
1573                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1574                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1575                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1576                         }
1577                         else if (isGcReliabilityFramework(scenario)) {
1578                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1579                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1580                         }
1581                         else {
1582                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1583                         }
1584                     }
1585
1586                     if (!enableCorefxTesting) {
1587                         // Run the rest of the build
1588                         // Build the mscorlib for the other OS's
1589                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1590                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1591
1592                         if (arch == "x64") {
1593                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1594                         }
1595
1596                         // Zip up the tests directory so that we don't use so much space/time copying
1597                         // 10s of thousands of files around.
1598                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1599
1600                         if (!isJitStressScenario(scenario)) {
1601                             // For windows, pull full test results and test drops for x86/x64.
1602                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1603                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1604                         }
1605
1606                         if (scenario == 'jitdiff') {
1607                             // retrive jit-dasm output for base commit, and run jit-diff
1608                             if (!isBuildOnly) {
1609                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1610                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1611                             }
1612                         }
1613
1614                         if (!isBuildOnly) {
1615                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1616                             setTestJobTimeOut(newJob, scenario)
1617                         }
1618                     }
1619                     break
1620                 case 'armlb':
1621                 case 'arm':
1622                     assert isArmWindowsScenario(scenario)
1623
1624                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1625                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1626
1627                     // Set time out
1628                     setTestJobTimeOut(newJob, scenario)
1629
1630                     if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1631                     {
1632                         // Up the timeout for arm checked testing only.
1633                         // Keep the longer timeout for gcstress.
1634                         Utilities.setJobTimeout(newJob, 240)
1635                     }
1636
1637                     def buildArchitecture = 'arm'
1638
1639                     // For 'arm' (the RyuJIT/arm32 architecture), tell build.cmd to use RyuJIT/arm32 for crossgen compilation.
1640                     // RyuJIT/arm32 is currently not the default JIT; it is an aljit. So, this is a special case.
1641                     def armCrossgenOpt = ''
1642                     if (architecture == 'arm') {
1643                         armCrossgenOpt = '-altjitcrossgen'
1644                     }
1645
1646                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1647                     priority = '1'
1648
1649                     // This is now a build only job. Do not run tests. Use the flow job.
1650                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1651
1652                     // Zip up the tests directory so that we don't use so much space/time copying
1653                     // 10s of thousands of files around.
1654                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1655
1656                     // Add archival.
1657                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1658                     break
1659                 case 'arm64':
1660                     assert isArmWindowsScenario(scenario)
1661
1662                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1663                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1664
1665                     // Set time out
1666                     setTestJobTimeOut(newJob, scenario)
1667                     if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1668                     {
1669                         // Up the timeout for arm checked testing only.
1670                         // Keep the longer timeout for gcstress.
1671                         Utilities.setJobTimeout(newJob, 240)
1672                     }
1673
1674                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1675                     priority = '1'
1676
1677                     // This is now a build only job. Do not run tests. Use the flow job.
1678                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1679
1680                     // Zip up the tests directory so that we don't use so much space/time copying
1681                     // 10s of thousands of files around.
1682                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1683
1684                     // Add archival.
1685                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1686                     break
1687                 default:
1688                     println("Unknown architecture: ${architecture}");
1689                     assert false
1690                     break
1691             }
1692             break
1693         // editor brace matching: }
1694         case 'Ubuntu':
1695         case 'Ubuntu16.04':
1696         case 'Ubuntu16.10':
1697         case 'Debian8.4':
1698         case 'OSX10.12':
1699         case 'CentOS7.1':
1700         case 'RHEL7.2':
1701         case 'Tizen':
1702         case 'Fedora24': // editor brace matching: {
1703             switch (architecture) {
1704                 case 'x64':
1705                 case 'x86':
1706                     if (architecture == 'x86' && os == 'Ubuntu') {
1707                         // build and PAL test
1708                         def dockerImage = getDockerImageName(architecture, os, true)
1709                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1710                         dockerImage = getDockerImageName(architecture, os, false)
1711                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1712                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1713                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1714                         break
1715                     }
1716
1717                     if (scenario == 'formatting') {
1718                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1719                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1720                         break
1721                     }
1722
1723                     if (scenario == 'illink') {
1724                         assert(os == 'Ubuntu')
1725                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1726                     }
1727
1728                     if (!enableCorefxTesting) {
1729                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1730                         // only on supported OS platforms.
1731                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1732                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1733                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1734                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1735
1736                         // Set time out
1737                         setTestJobTimeOut(newJob, scenario)
1738                         // Basic archiving of the build
1739                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1740                         // And pal tests
1741                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1742                     }
1743                     else {
1744                         // Corefx stress testing
1745                         assert os == 'Ubuntu'
1746                         assert architecture == 'x64'
1747                         assert lowerConfiguration == 'checked'
1748                         assert isJitStressScenario(scenario)
1749
1750                         // Build coreclr
1751                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1752
1753                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1754                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1755
1756                         // Build and text corefx
1757                         def workspaceRelativeFxRoot = "_/fx"
1758                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1759
1760                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1761
1762                         setTestJobTimeOut(newJob, scenario)
1763
1764                         // Archive and process (only) the test results
1765                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1766                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1767                     }
1768                     break
1769                 case 'arm64':
1770                     if (!enableCorefxTesting) {
1771                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1772
1773                         // HACK -- Arm64 does not have corefx jobs yet.
1774                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1775                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1776                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1777                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1778
1779                         setTestJobTimeOut(newJob, scenario)
1780                         // Basic archiving of the build
1781                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1782                     }
1783                     break
1784                 case 'armlb':
1785                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1786                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1787
1788                     // default values for Ubuntu
1789                     def arm_abi="arm"
1790                     def linuxCodeName="trusty"
1791                     if (os == 'Ubuntu16.04') {
1792                         linuxCodeName="xenial"
1793                     }
1794                     else if (os == 'Tizen') {
1795                         arm_abi="armel"
1796                         linuxCodeName="tizen"
1797                     }
1798
1799                     // Unzip the Windows test binaries first. Exit with 0
1800                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1801
1802                     // Unpack the corefx binaries
1803                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1804                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1805                     if (os != 'Tizen') {
1806                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1807                     }
1808                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1809                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1810                     if (os == 'Ubuntu') {
1811                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1812                     }
1813
1814                     // Call the ARM CI script to cross build and test using docker
1815                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1816                     --mode=docker \\
1817                     --${arm_abi} \\
1818                     --linuxCodeName=${linuxCodeName} \\
1819                     --buildConfig=${lowerConfiguration} \\
1820                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1821                     --coreFxBinDir=./bin/CoreFxBinDir \\
1822                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1823
1824                     // Basic archiving of the build, no pal tests
1825                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1826                     break
1827                 default:
1828                     println("Unknown architecture: ${architecture}");
1829                     assert false
1830                     break
1831             }
1832             break
1833         // editor brace matching: }
1834         default:
1835             println("Unknown os: ${os}");
1836             assert false
1837             break
1838     } // os
1839
1840     return buildCommands
1841 }
1842
1843 Constants.allScenarios.each { scenario ->
1844     [true, false].each { isPR ->
1845         Constants.architectureList.each { architecture ->
1846             Constants.configurationList.each { configuration ->
1847                 Constants.osList.each { os ->
1848                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1849                     // and reset the os to Windows_NT
1850                     def isBuildOnly = false
1851                     if (os == 'Windows_NT_BuildOnly') {
1852                         isBuildOnly = true
1853                         os = 'Windows_NT'
1854                     }
1855
1856                     // Tizen is only supported for arm legacy_backend architecture
1857                     if (os == 'Tizen' && architecture != 'armlb') {
1858                         return
1859                     }
1860
1861                     // Skip totally unimplemented (in CI) configurations.
1862                     switch (architecture) {
1863                         case 'arm64':
1864                             if (os == 'Ubuntu16.04') {
1865                                 os = 'Ubuntu'
1866                             }
1867
1868                             // Windows and Ubuntu only
1869                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1870                                 return
1871                             }
1872                             break
1873                         case 'armlb':
1874                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1875                                 return
1876                             }
1877                             break
1878                         case 'arm':
1879                             if (os != 'Windows_NT') {
1880                                 return
1881                             }
1882                             break
1883                         case 'x86':
1884                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1885                                 return
1886                             }
1887                             break
1888                         case 'x86_arm_altjit':
1889                         case 'x64_arm64_altjit':
1890                             if (os != 'Windows_NT') {
1891                                 return
1892                             }
1893                             break
1894                         case 'x64':
1895                             // Everything implemented
1896                             break
1897                         default:
1898                             println("Unknown architecture: ${architecture}")
1899                             assert false
1900                             break
1901                     }
1902
1903                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1904                     // with checked builds)
1905                     if (isJitStressScenario(scenario)) {
1906                         if (configuration != 'Checked') {
1907                             return
1908                         }
1909
1910                         // Since these are just execution time differences,
1911                         // skip platforms that don't execute the tests here (Windows_NT only)
1912                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
1913                         if (!isEnabledOS || isBuildOnly) {
1914                             return
1915                         }
1916
1917                         switch (architecture) {
1918                             case 'x64':
1919                             case 'x86':
1920                             case 'x86_arm_altjit':
1921                             case 'x64_arm64_altjit':
1922                                 // x86 ubuntu: default only
1923                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1924                                     return
1925                                 }
1926                                 // Windows: Everything implemented
1927                                 break
1928
1929                             default:
1930                                 // arm, arm64, armlb: stress is handled through flow jobs.
1931                                 return
1932                         }
1933                     }
1934                     else if (isR2RScenario(scenario)) {
1935                         if (os != 'Windows_NT') {
1936                             return
1937                         }
1938                         // R2R test runs are not implemented for arm/armlb/arm64.
1939                         if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
1940                             return
1941                         }
1942                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
1943                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
1944                             return
1945                         }
1946                     }
1947                     else {
1948                         // Skip scenarios
1949                         switch (scenario) {
1950                             case 'ilrt':
1951                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1952                                 // the default scenario build
1953                                 if (os != 'Windows_NT') {
1954                                     return
1955                                 }
1956                                 // Only x64 for now
1957                                 if (architecture != 'x64') {
1958                                     return
1959                                 }
1960                                 // Release only
1961                                 if (configuration != 'Release') {
1962                                     return
1963                                 }
1964                                 break
1965                             case 'jitdiff':
1966                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1967                                     return
1968                                 }
1969                                 if (architecture != 'x64') {
1970                                     return
1971                                 }
1972                                 if (configuration != 'Checked') {
1973                                     return
1974                                 }
1975                                 break
1976                             case 'longgc':
1977                             case 'gcsimulator':
1978                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1979                                     return
1980                                 }
1981                                 if (architecture != 'x64') {
1982                                     return
1983                                 }
1984                                 if (configuration != 'Release') {
1985                                     return
1986                                 }
1987                                 break
1988                             case 'gc_reliability_framework':
1989                             case 'standalone_gc':
1990                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1991                                     return
1992                                 }
1993
1994                                 if (architecture != 'x64') {
1995                                     return
1996                                 }
1997
1998                                 if (configuration != 'Release' && configuration != 'Checked') {
1999                                     return
2000                                 }
2001                                 break
2002                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2003                             case 'formatting':
2004                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2005                                     return
2006                                 }
2007                                 if (architecture != 'x64') {
2008                                     return
2009                                 }
2010                                 if (configuration != 'Checked') {
2011                                     return
2012                                 }
2013                                 if (isBuildOnly) {
2014                                     return
2015                                 }
2016                                 break
2017                             case 'illink':
2018                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2019                                     return
2020                                 }
2021                                 if (architecture != 'x64' && architecture != 'x86') {
2022                                     return
2023                                 }
2024                                 if (isBuildOnly) {
2025                                     return
2026                                 }
2027                                 break
2028                             case 'default':
2029                                 // Nothing skipped
2030                                 break
2031                             default:
2032                                 println("Unknown scenario: ${scenario}")
2033                                 assert false
2034                                 break
2035                         }
2036                     }
2037
2038                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2039                     // runtime behavior, not compile-time behavior, are not interesting.
2040                     switch (architecture) {
2041                         case 'x86_arm_altjit':
2042                         case 'x64_arm64_altjit':
2043                             if (isGCStressRelatedTesting(scenario)) {
2044                                 return
2045                             }
2046                             break
2047                         default:
2048                             break
2049                     }
2050
2051                     // Calculate names
2052                     def lowerConfiguration = configuration.toLowerCase()
2053                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2054                     def folderName = getJobFolder(scenario)
2055
2056                     // Create the new job
2057                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2058
2059                     def machineAffinityOptions = null
2060
2061                     if (os != 'Windows_NT') {
2062                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2063                     }
2064                     else {
2065                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2066                     }
2067
2068                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2069
2070                     // Add all the standard options
2071                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2072                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2073
2074                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2075                     def osGroup = getOSGroup(os)
2076
2077                     newJob.with {
2078                         steps {
2079                             if (os == 'Windows_NT') {
2080                                 buildCommands.each { buildCommand ->
2081                                     batchFile(buildCommand)
2082                                 }
2083                             }
2084                             else {
2085                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2086                                 if ( architecture == 'armlb' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2087                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2088                                     // Define the Windows Tests and Corefx build job names
2089                                     def WindowTestsName = projectFolder + '/' +
2090                                                           Utilities.getFullJobName(project,
2091                                                                                    getJobName(lowerConfiguration,
2092                                                                                               'x64' ,
2093                                                                                               'windows_nt',
2094                                                                                               'default',
2095                                                                                               true),
2096                                                                                    false)
2097                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2098                                                        Utilities.getFolderName(branch)
2099
2100                                     // Copy the Windows test binaries and the Corefx build binaries
2101                                     copyArtifacts(WindowTestsName) {
2102                                         includePatterns('bin/tests/tests.zip')
2103                                         buildSelector {
2104                                             latestSuccessful(true)
2105                                         }
2106                                     }
2107
2108                                     def arm_abi = 'arm'
2109                                     def corefx_os = 'linux'
2110                                     if (os == 'Tizen') {
2111                                         arm_abi = 'armel'
2112                                         corefx_os = 'tizen'
2113                                     }
2114
2115                                     // Let's use release CoreFX to test checked CoreCLR,
2116                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2117                                     def corefx_lowerConfiguration = lowerConfiguration
2118                                     if ( lowerConfiguration == 'checked' ) {
2119                                         corefx_lowerConfiguration='release'
2120                                     }
2121
2122                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2123                                         includePatterns('bin/build.tar.gz')
2124                                         buildSelector {
2125                                             latestSuccessful(true)
2126                                         }
2127                                     }
2128                                 }
2129
2130                                 buildCommands.each { buildCommand ->
2131                                     shell(buildCommand)
2132                                 }
2133                             }
2134                         }
2135                     } // newJob.with
2136
2137                 } // os
2138             } // configuration
2139         } // architecture
2140     } // isPR
2141 } // scenario
2142
2143
2144 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2145 Constants.allScenarios.each { scenario ->
2146     [true, false].each { isPR ->
2147         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2148             Constants.crossList.each { os ->
2149                 if (architecture == 'arm64') {
2150                     if (os != "Ubuntu" && os != "Windows_NT") {
2151                         return
2152                     }
2153                 } else if (architecture == 'arm' || architecture == 'armlb') {
2154                     if (os != 'Windows_NT') {
2155                         return
2156                     }
2157                 }
2158                 else if (architecture == 'x86') {
2159                     if (os != "Ubuntu") {
2160                         return
2161                     }
2162                 }
2163
2164                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2165
2166                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2167                     return
2168                 }
2169
2170                 Constants.configurationList.each { configuration ->
2171
2172                     // First, filter based on OS.
2173
2174                     if (os == 'Windows_NT') {
2175                         if (!isArmWindowsScenario(scenario)) {
2176                             return
2177                         }
2178                     }
2179                     else {
2180                         // Non-Windows
2181                         if (architecture == 'arm64') {
2182                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2183                                 return
2184                             }
2185                         }
2186                         else if (architecture == 'x86') {
2187                             // Linux/x86 only want default test
2188                             if (scenario != 'default') {
2189                                 return
2190                             }
2191                         }
2192                     }
2193
2194                     // For CentOS, we only want Checked/Release builds.
2195                     if (os == 'CentOS7.1') {
2196                         if (configuration != 'Checked' && configuration != 'Release') {
2197                             return
2198                         }
2199                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2200                             return
2201                         }
2202                     }
2203
2204                     // For RedHat and Debian, we only do Release builds.
2205                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2206                         if (configuration != 'Release') {
2207                             return
2208                         }
2209                         if (scenario != 'default') {
2210                             return
2211                         }
2212                     }
2213
2214                     // Next, filter based on scenario.
2215
2216                     if (isJitStressScenario(scenario)) {
2217                         if (configuration != 'Checked') {
2218                             return
2219                         }
2220                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2221                         if (isCoreFxScenario(scenario)) {
2222                             return
2223                         }
2224                     }
2225                     else if (isR2RBaselineScenario(scenario)) {
2226                         if (configuration != 'Checked' && configuration != 'Release') {
2227                             return
2228                         }
2229                     }
2230                     else if (isR2RStressScenario(scenario)) {
2231                         if (configuration != 'Checked') {
2232                             return
2233                         }
2234                     }
2235                     else {
2236                         // Skip scenarios
2237                         switch (scenario) {
2238                             case 'ilrt':
2239                             case 'longgc':
2240                             case 'gcsimulator':
2241                                 // Long GC tests take a long time on non-Release builds
2242                                 // ilrt is also Release only
2243                                 if (configuration != 'Release') {
2244                                     return
2245                                 }
2246                                 break
2247                             case 'jitdiff':
2248                                 if (configuration != 'Checked') {
2249                                     return;
2250                                 }
2251                                 break
2252                             case 'gc_reliability_framework':
2253                             case 'standalone_gc':
2254                                 if (configuration != 'Release' && configuration != 'Checked') {
2255                                     return
2256                                 }
2257                                 break
2258                             case 'formatting':
2259                                 return
2260                             case 'illink':
2261                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2262                                     return
2263                                 }
2264                                 break
2265                             case 'default':
2266                                 // Nothing skipped
2267                                 break
2268                             default:
2269                                 println("Unknown scenario: ${scenario}")
2270                                 assert false
2271                                 break
2272                         }
2273                     }
2274
2275                     // Done filtering. Now, create the jobs.
2276
2277                     def lowerConfiguration = configuration.toLowerCase()
2278                     def osGroup = getOSGroup(os)
2279                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2280
2281                     def inputCoreCLRBuildName = projectFolder + '/' +
2282                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2283
2284                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2285                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2286                     def testBuildScenario = scenario
2287                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2288                         testBuildScenario = 'default'
2289                     }
2290
2291                     def inputWindowsTestBuildArch = architecture
2292                     if (architecture == "arm64" && os != "Windows_NT") {
2293                         // Use the x64 test build for arm64 unix
2294                         inputWindowsTestBuildArch = "x64"
2295                     }
2296
2297                     def inputWindowTestsBuildName = projectFolder + '/' +
2298                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2299
2300                     // Enable Server GC for Ubuntu PR builds
2301                     def serverGCString = ''
2302                     if (os == 'Ubuntu' && isPR) {
2303                         serverGCString = '--useServerGC'
2304                     }
2305
2306                     def testOpts = ''
2307
2308                     if (isR2RScenario(scenario)) {
2309
2310                         testOpts += ' --crossgen --runcrossgentests'
2311
2312                         if (scenario == 'r2r_jitstress1') {
2313                             testOpts += ' --jitstress=1'
2314                         }
2315                         else if (scenario == 'r2r_jitstress2') {
2316                             testOpts += ' --jitstress=2'
2317                         }
2318                         else if (scenario == 'r2r_jitstressregs1') {
2319                             testOpts += ' --jitstressregs=1'
2320                         }
2321                         else if (scenario == 'r2r_jitstressregs2') {
2322                             testOpts += ' --jitstressregs=2'
2323                         }
2324                         else if (scenario == 'r2r_jitstressregs3') {
2325                             testOpts += ' --jitstressregs=3'
2326                         }
2327                         else if (scenario == 'r2r_jitstressregs4') {
2328                             testOpts += ' --jitstressregs=4'
2329                         }
2330                         else if (scenario == 'r2r_jitstressregs8') {
2331                             testOpts += ' --jitstressregs=8'
2332                         }
2333                         else if (scenario == 'r2r_jitstressregs0x10') {
2334                             testOpts += ' --jitstressregs=0x10'
2335                         }
2336                         else if (scenario == 'r2r_jitstressregs0x80') {
2337                             testOpts += ' --jitstressregs=0x80'
2338                         }
2339                         else if (scenario == 'r2r_jitstressregs0x1000') {
2340                             testOpts += ' --jitstressregs=0x1000'
2341                         }
2342                         else if (scenario == 'r2r_jitminopts') {
2343                             testOpts += ' --jitminopts'
2344                         }
2345                         else if (scenario == 'r2r_jitforcerelocs') {
2346                             testOpts += ' --jitforcerelocs'
2347                         }
2348                         else if (scenario == 'r2r_gcstress15') {
2349                             testOpts += ' --gcstresslevel=0xF'
2350                         }
2351                     }
2352                     else if (scenario == 'jitdiff') {
2353                         testOpts += ' --jitdisasm --crossgen'
2354                     }
2355                     else if (scenario == 'illink') {
2356                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2357                     }
2358                     else if (isLongGc(scenario)) {
2359                         // Long GC tests behave very poorly when they are not
2360                         // the only test running (many of them allocate until OOM).
2361                         testOpts += ' --sequential'
2362
2363                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2364                         // for running long GC and GCSimulator tests, respectively. We don't use them
2365                         // here because using a playlist file produces much more readable output on the CI machines
2366                         // and reduces running time.
2367                         //
2368                         // The Long GC playlist contains all of the tests that are
2369                         // going to be run. The GCSimulator playlist contains all of
2370                         // the GC simulator tests.
2371                         if (scenario == 'longgc') {
2372                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2373                         }
2374                         else if (scenario == 'gcsimulator') {
2375                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2376                         }
2377                     }
2378                     else if (isGcReliabilityFramework(scenario)) {
2379                         testOpts += ' --build-overlay-only'
2380                     }
2381                     else if (scenario == 'standalone_gc') {
2382                         if (osGroup == 'OSX') {
2383                             testOpts += ' --gcname=libclrgc.dylib'
2384                         }
2385                         else if (osGroup == 'Linux') {
2386                             testOpts += ' --gcname=libclrgc.so'
2387                         }
2388                         else {
2389                             println("Unexpected OS group: ${osGroup} for os ${os}")
2390                             assert false
2391                         }
2392                     }
2393
2394                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2395
2396                     def folder = getJobFolder(scenario)
2397                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2398                         // Add parameters for the inputs
2399
2400                         if (windowsArmJob == true) {
2401                             parameters {
2402                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2403                             }
2404                         }
2405                         else {
2406                             parameters {
2407                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2408                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2409                             }
2410                         }
2411
2412                         steps {
2413                             // Set up the copies
2414
2415                             // Coreclr build containing the tests and mscorlib
2416
2417                             if (windowsArmJob != true) {
2418                                 copyArtifacts(inputWindowTestsBuildName) {
2419                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2420                                     buildSelector {
2421                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2422                                     }
2423                                 }
2424                             }
2425
2426                             // Coreclr build we are trying to test
2427
2428                             copyArtifacts(inputCoreCLRBuildName) {
2429                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2430                                 buildSelector {
2431                                     buildNumber('${CORECLR_BUILD}')
2432                                 }
2433                             }
2434
2435                             // Windows CoreCLR Arm(64) will restore corefx
2436                             // packages correctly.
2437                             //
2438                             // In addition, test steps are entirely different
2439                             // because we do not have a unified runner
2440                             if (windowsArmJob != true) {
2441                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2442
2443                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2444                                 def osJobName
2445                                 if (os == 'Ubuntu') {
2446                                     osJobName = 'ubuntu14.04'
2447                                 }
2448                                 else if (architecture == 'x86') {
2449                                     if (os == 'Ubuntu') {
2450                                         // Linux/x86 corefx jobs does not build managed yet
2451                                         // Clone linux/arm corefx managed packages and overwrite linux/x86 native
2452                                         osJobName = "linux_arm_cross"
2453                                     }
2454                                 }
2455                                 else {
2456                                     osJobName = os.toLowerCase()
2457                                 }
2458                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2459                                     includePatterns('bin/build.tar.gz')
2460                                     buildSelector {
2461                                         latestSuccessful(true)
2462                                     }
2463                                 }
2464
2465                                 shell("mkdir ./bin/CoreFxBinDir")
2466                                 // Unpack the corefx binaries
2467                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2468
2469                                 // HACK -- Arm64 does not have corefx jobs yet.
2470                                 // Clone corefx and build the native packages overwriting the x64 packages.
2471                                 if (architecture == 'arm64') {
2472                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2473                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2474                                 }
2475                                 else if (architecture == 'x86') {
2476                                     shell("mkdir ./bin/CoreFxNative")
2477
2478                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2479                                         includePatterns('bin/build.tar.gz')
2480                                         targetDirectory('bin/CoreFxNative')
2481                                         buildSelector {
2482                                             latestSuccessful(true)
2483                                         }
2484                                     }
2485
2486                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2487                                 }
2488
2489                                 // Unzip the tests first.  Exit with 0
2490                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2491
2492                                 // Execute the tests
2493                                 def runDocker = isNeedDocker(architecture, os, false)
2494                                 def dockerPrefix = ""
2495                                 def dockerCmd = ""
2496                                 if (runDocker) {
2497                                     def dockerImage = getDockerImageName(architecture, os, false)
2498                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2499                                     dockerCmd = dockerPrefix + "${dockerImage} "
2500                                 }
2501
2502                                 // If we are running a stress mode, we'll set those variables first
2503                                 def testEnvOpt = ""
2504                                 if (isJitStressScenario(scenario)) {
2505                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2506                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2507                                     shell("${createScriptCmds}")
2508                                     testEnvOpt = "--test-env=" + scriptFileName
2509                                 }
2510
2511                                 if (isGCStressRelatedTesting(scenario)) {
2512                                     shell('./init-tools.sh')
2513                                 }
2514
2515                                 shell("""${dockerCmd}./tests/runtest.sh \\
2516                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2517                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2518                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2519                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2520                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2521                 --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2522
2523                                 if (isGcReliabilityFramework(scenario)) {
2524                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2525                                     if (serverGCString != '') {
2526                                         if (runDocker) {
2527                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2528                                         }
2529                                         else {
2530                                             shell("export COMPlus_gcServer=1")
2531                                         }
2532                                     }
2533
2534                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2535                                 }
2536                             }
2537
2538                             else { // windowsArmJob == true
2539                                 // Unzip tests.
2540                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2541
2542                                 // Build the build commands
2543                                 def buildCommands = ""
2544
2545                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2546                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2547                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2548
2549                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2550                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2551
2552                                 // For all jobs
2553                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2554
2555                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2556                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2557
2558                                 // Arm(32) ryujit
2559                                 if (architecture == "arm") {
2560                                     // **This is an AltJit**
2561
2562                                     addEnvVariable("COMPlus_AltJit", "*")
2563                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2564                                     addEnvVariable("COMPlus_AltJitName", "protojit.dll")
2565                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2566                                 }
2567
2568                                 // If we are running a stress mode, we'll set those variables as well
2569                                 def stressValues = null
2570                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2571                                     if (isJitStressScenario(scenario)) {
2572                                         stressValues = Constants.jitStressModeScenarios[scenario]
2573                                     }
2574                                     else {
2575                                         stressValues = Constants.r2rStressScenarios[scenario]
2576                                     }
2577
2578                                     stressValues.each { key, value ->
2579                                         addEnvVariable(key, value)
2580                                     }
2581                                 }
2582
2583                                 // TODO: do whatever is necessary to support enabling R2R testing. Environment variables,
2584                                 // crossgen the framework assemblies, etc.
2585
2586                                 // Create the smarty command
2587                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2588                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2589                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2590
2591                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "arm") { addExclude("PROTOJIT_" + exclude) } else { addExclude(exclude) } }
2592
2593                                 if (architecture == "arm") {
2594                                     addExclude("PROTOJIT_FAIL")
2595                                 }
2596
2597                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2598                                     def failTag = "JITSTRESS_FAIL"
2599                                     def excludeTag = "JITSTRESS_EXCLUDE"
2600
2601                                     if (scenario.contains('gc')) {
2602                                         failTag = "GCSTRESS_FAIL"
2603                                         excludeTag = "GCSTRESS_EXCLUDE"
2604                                     }
2605
2606                                     addArchSpecificExclude(architecture, failTag)
2607                                     addArchSpecificExclude(architecture, excludeTag)
2608                                 }
2609                                 else {
2610                                     addExclude("pri1")
2611                                 }
2612
2613                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2614                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2615                                 addExclude("LONG_RUNNING")
2616
2617                                 smartyCommand += "/lstFile Tests.lst"
2618
2619                                 def testListArch = [
2620                                     'arm64': 'arm64',
2621                                     'arm': 'arm',
2622                                     'armlb': 'arm'
2623                                 ]
2624
2625                                 def archLocation = testListArch[architecture]
2626
2627                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2628                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2629                                 addCommand("${smartyCommand}")
2630
2631                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2632                                 // However, we also need to remove all the variables that were set during this batch file, so we
2633                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2634                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2635                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2636                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2637                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2638                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2639
2640                                 // ZIP up the smarty output, no matter what the smarty result.
2641                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2642
2643                                 addCommand("echo %errorlevel%")
2644                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2645
2646                                 // Use the smarty errorlevel as the script errorlevel.
2647                                 addCommand("exit /b %__save_smarty_errorlevel%")
2648
2649                                 batchFile(buildCommands)
2650                             }
2651                         }
2652                     }
2653
2654                     if (scenario == 'jitdiff') {
2655                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2656                     }
2657
2658                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2659                     if (os in ['Ubuntu']) {
2660                         SummaryBuilder summaries = new SummaryBuilder()
2661                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2662                         summaries.emit(newJob)
2663                     }
2664
2665                     def affinityOptions = null
2666
2667                     if (windowsArmJob == true) {
2668                         affinityOptions = [
2669                             "use_arm64_build_machine" : false
2670                         ]
2671                     }
2672
2673                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2674                         affinityOptions = [
2675                             "large_pages" : false
2676                         ]
2677                     }
2678
2679                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2680                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2681
2682                     // REVIEW: Should the scenario-base timeout be set in the "else" clause of the "architecture==arm64"?
2683                     //         Should the "architecture==arm64" include arm/armlb?
2684                     //         Should we be overriding possibly higher timeouts that would be set by setTestJobTimeOut()?
2685                     setTestJobTimeOut(newJob, scenario)
2686                     if (architecture == 'arm64') {
2687                         Utilities.setJobTimeout(newJob, 240)
2688                     }
2689
2690                     if (windowsArmJob != true) {
2691                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2692                     }
2693                     else {
2694                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2695
2696                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2697                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2698                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2699                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2700                     }
2701
2702                     // Create a build flow to join together the build and tests required to run this test.
2703                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2704                     // Linux CoreCLR test
2705                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2706                     def fullTestJobName = projectFolder + '/' + newJob.name
2707                     // Add a reference to the input jobs for report purposes
2708                     JobReport.Report.addReference(inputCoreCLRBuildName)
2709                     JobReport.Report.addReference(inputWindowTestsBuildName)
2710                     JobReport.Report.addReference(fullTestJobName)
2711                     def newFlowJob = null
2712
2713                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2714                         // Do not create the flow job for RHEL jobs.
2715                         return
2716                     }
2717
2718                     if (windowsArmJob == true) {
2719                         // For Windows arm jobs there is no reason to build a parallel test job.
2720                         // The product build supports building and archiving the tests.
2721
2722                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2723                         buildFlow("""
2724 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2725
2726 // And then build the test build
2727 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2728 """)
2729                         }
2730                     }
2731                     else {
2732                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2733                         buildFlow("""
2734 // Build the input jobs in parallel
2735 parallel (
2736 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2737 { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2738 )
2739
2740 // And then build the test build
2741 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2742             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2743 """)
2744                         }
2745                     }
2746
2747                     // For the flow jobs set the machine affinity as x64 if an armarch.
2748                     def flowArch = architecture
2749
2750                     if (flowArch in validWindowsNTCrossArches) {
2751                         flowArch = 'x64'
2752                         affinityOptions = null
2753                     }
2754
2755                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2756                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2757                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2758                 } // configuration
2759             } // os
2760         } // architecture
2761     } // isPR
2762 } // scenario
2763
2764 JobReport.Report.generateJobReport(out)
2765
2766 // Make the call to generate the help job
2767 Utilities.createHelperJob(this, project, branch,
2768     "Welcome to the ${project} Repository",  // This is prepended to the help message
2769     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2770
2771 Utilities.addCROSSCheck(this, project, branch)