netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'CentOS7.1',
  49                'RHEL7.2',
  50                'Ubuntu16.04',
  51                'Ubuntu16.10',
  52                'Fedora24',
  53                'Tizen']
  54
  55     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  56
  57     // This is a set of JIT stress modes combined with the set of variables that
  58     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  59     // the values are the environment variables
  60     def static jitStressModeScenarios = [
  61                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  62                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  63                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  64                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  65                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  66                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  67                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  68                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  69                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  70                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  71                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  72                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  73                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  74                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  75                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  76                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  77                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  78                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  79                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  80                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  81                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  82                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  83                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  84                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  85                'corefx_baseline'                : [ : ], // corefx baseline
  86                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  87                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  88                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  89                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  90                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  91                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  92                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  93                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  94                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  95                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  96                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  97                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  98                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  99                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 100                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 101                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 102                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 103                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 104                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 105                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 106                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 107                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 108     ]
 109
 110     // This is a set of ReadyToRun stress scenarios
 111     def static r2rStressScenarios = [
 112                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 113                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 114                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 115                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 116                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 117                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 118                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 119                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 120                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 121                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 122                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 123                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 124                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 125     ]
 126
 127     // This is the basic set of scenarios
 128     def static basicScenarios = [
 129                'default',
 130                'ilrt',
 131                'r2r',
 132                'longgc',
 133                'formatting',
 134                'gcsimulator',
 135                'jitdiff',
 136                'standalone_gc',
 137                'gc_reliability_framework',
 138                'illink'] + r2rStressScenarios.keySet()
 139
 140     def static allScenarios = basicScenarios + jitStressModeScenarios.keySet()
 141
 142     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 143     // to Tests.lst file categories to exclude.
 144     //
 145     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 146     // and with the same values, with some commented out, for easier maintenance.
 147     //
 148     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 149     //
 150     def static validArmWindowsScenarios = [
 151                'default':                                [],
 152                // 'ilrt'
 153                // 'r2r':                                    ["R2R_FAIL"],
 154                // 'longgc'
 155                // 'formatting'
 156                // 'gcsimulator'
 157                // 'jitdiff'
 158                // 'standalone_gc'
 159                // 'gc_reliability_framework'
 160                // 'illink'
 161                // 'r2r_jitstress1'
 162                // 'r2r_jitstress2'
 163                // 'r2r_jitstressregs1'
 164                // 'r2r_jitstressregs2'
 165                // 'r2r_jitstressregs3'
 166                // 'r2r_jitstressregs4'
 167                // 'r2r_jitstressregs8'
 168                // 'r2r_jitstressregs0x10'
 169                // 'r2r_jitstressregs0x80'
 170                // 'r2r_jitstressregs0x1000'
 171                // 'r2r_jitminopts'
 172                // 'r2r_jitforcerelocs'
 173                // 'r2r_gcstress15'
 174                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 175                'tieredcompilation':                      [],
 176                'forcerelocs':                            [],
 177                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 179                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 180                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 181                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 196                // 'jitsse2only'                          // Only relevant to xarch
 197                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 198                // 'corefx_baseline'
 199                // 'corefx_minopts'
 200                // 'corefx_tieredcompilation'
 201                // 'corefx_jitstress1'
 202                // 'corefx_jitstress2'
 203                // 'corefx_jitstressregs1'
 204                // 'corefx_jitstressregs2'
 205                // 'corefx_jitstressregs3'
 206                // 'corefx_jitstressregs4'
 207                // 'corefx_jitstressregs8'
 208                // 'corefx_jitstressregs0x10'
 209                // 'corefx_jitstressregs0x80'
 210                // 'corefx_jitstressregs0x1000'
 211                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 212                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 213                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 214                'heapverify1':                            [],
 215                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 216                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 217                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 219                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 220                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 221
 222                //
 223                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 224                //
 225
 226                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 227                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 228                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 245     ]
 246
 247     def static configurationList = ['Debug', 'Checked', 'Release']
 248
 249     // This is the set of architectures
 250     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 251 }
 252
 253 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 254     assert os instanceof String
 255     assert architecture instanceof String
 256
 257     def armArches = ['arm', 'armlb', 'arm64']
 258     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 259
 260     if (!(architecture in armArches)) {
 261         assert options == null
 262         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 263
 264         return
 265     }
 266
 267     // This is an arm(64) job.
 268     //
 269     // There are several options.
 270     //
 271     // Windows_NT
 272     //
 273     // Arm32 (Build) -> latest-arm64
 274     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 275     // Arm32 (Test)  -> arm64-windows_nt
 276     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 277     //
 278     // Arm64 (Build) -> latest-arm64
 279     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 280     // Arm64 (Test)  -> arm64-windows_nt
 281     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 282     //
 283     // Ubuntu
 284     //
 285     // Arm32 (Build) -> arm-cross-latest
 286     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 287     // Arm32 (Test)  -> NYI Arch not supported
 288     //       |->
 289     //
 290     // Arm64 (Build) -> arm64-cross-latest
 291     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 292     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 293     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 294     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 295     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 296
 297     // This has to be a arm arch
 298     assert architecture in armArches
 299     if (os == "Windows_NT") {
 300         // Arm(64) windows jobs share the same machines for now
 301         def isBuild = options['use_arm64_build_machine'] == true
 302
 303         if (isBuild == true) {
 304             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 305         } else {
 306             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 307         }
 308     } else {
 309         assert os != 'Windows_NT'
 310         assert os in supportedArmLinuxOs
 311
 312         if (architecture == 'arm' || architecture == 'armlb') {
 313             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 314         } else {
 315             // Arm64 Linux
 316             if (options['is_build_only'] == true) {
 317                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 318             } else {
 319                 // Arm64 Test Machines
 320                 if (options['large_pages'] == false) {
 321                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 322                 } else {
 323                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 324                 }
 325             }
 326         }
 327     }
 328 }
 329
 330 def static isGCStressRelatedTesting(def scenario) {
 331     // The 'r2r_gcstress15' scenario is a basic scenario.
 332     // Detect it and make it a GCStress related.
 333     if (scenario == 'r2r_gcstress15')
 334     {
 335         return true;
 336     }
 337
 338     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 339     def scenarioName = scenario.toLowerCase()
 340     def isGCStressTesting = false
 341     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 342         if (k in gcStressTestEnvVars) {
 343             isGCStressTesting = true;
 344         }
 345     }
 346     return isGCStressTesting
 347 }
 348
 349 def static isCoreFxScenario(def scenario) {
 350     def corefx_prefix = 'corefx_'
 351     if (scenario.length() < corefx_prefix.length()) {
 352         return false
 353     }
 354     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 355 }
 356
 357 def static isR2RBaselineScenario(def scenario) {
 358     return (scenario == 'r2r')
 359 }
 360
 361 def static isR2RStressScenario(def scenario) {
 362     return Constants.r2rStressScenarios.containsKey(scenario)
 363 }
 364
 365 def static isR2RScenario(def scenario) {
 366     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 367 }
 368
 369 def static isJitStressScenario(def scenario) {
 370     return Constants.jitStressModeScenarios.containsKey(scenario)
 371 }
 372
 373 def static isLongGc(def scenario) {
 374     return (scenario == 'longgc' || scenario == 'gcsimulator')
 375 }
 376
 377 def static isJitDiff(def scenario) {
 378     return (scenario == 'jitdiff')
 379 }
 380
 381 def static isGcReliabilityFramework(def scenario) {
 382     return (scenario == 'gc_reliability_framework')
 383 }
 384
 385 def static isArmWindowsScenario(def scenario) {
 386     return Constants.validArmWindowsScenarios.containsKey(scenario)
 387 }
 388
 389 def static setTestJobTimeOut(newJob, scenario) {
 390     if (isGCStressRelatedTesting(scenario)) {
 391         Utilities.setJobTimeout(newJob, 4320)
 392     }
 393     else if (isCoreFxScenario(scenario)) {
 394         Utilities.setJobTimeout(newJob, 360)
 395     }
 396     else if (isJitStressScenario(scenario)) {
 397         Utilities.setJobTimeout(newJob, 240)
 398     }
 399     else if (isR2RBaselineScenario(scenario)) {
 400         Utilities.setJobTimeout(newJob, 240)
 401     }
 402     else if (isLongGc(scenario)) {
 403         Utilities.setJobTimeout(newJob, 1440)
 404     }
 405     else if (isJitDiff(scenario)) {
 406         Utilities.setJobTimeout(newJob, 240)
 407     }
 408     else if (isGcReliabilityFramework(scenario)) {
 409         Utilities.setJobTimeout(newJob, 1440)
 410     }
 411     // Non-test jobs use the default timeout value.
 412 }
 413
 414 def static getJobFolder(def scenario) {
 415     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 416         return 'jitstress'
 417     }
 418     if (scenario == 'illink') {
 419         return 'illink'
 420     }
 421     return ''
 422 }
 423
 424 def static getStressModeDisplayName(def scenario) {
 425     def displayStr = ''
 426     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 427         def prefixLength = 'COMPlus_'.length()
 428         if (k.length() >= prefixLength) {
 429             def modeName = k.substring(prefixLength, k.length())
 430             displayStr += ' ' + modeName + '=' + v
 431         }
 432     }
 433
 434     if (isCoreFxScenario(scenario)) {
 435         displayStr = ('CoreFx ' + displayStr).trim()
 436     }
 437
 438     return displayStr
 439 }
 440
 441 def static getR2RDisplayName(def scenario) {
 442     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 443     def displayStr = scenario
 444     def prefixLength = 'r2r_'.length()
 445     if (displayStr.length() >= prefixLength) {
 446         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 447     } else if (scenario == 'r2r') {
 448         displayStr = "R2R"
 449     }
 450     return displayStr
 451 }
 452
 453 // Generates the string for creating a file that sets environment variables
 454 // that makes it possible to run stress modes.  Writes the script to the file
 455 // specified by the stepScriptLocation parameter.
 456 def static genStressModeScriptStep(def os, def stressModeName, def stressModeVars, def stepScriptLocation) {
 457     def stepScript = ''
 458     if (os == 'Windows_NT') {
 459         // Timeout in ms, default is 10 minutes. For stress modes up this to 30 minutes.
 460         // BUG?: it seems this is ignored, as this script will be run in an environment where
 461         //       environment variables will be discarded before this is used.
 462         def timeout = 1800000
 463         stepScript += "set __TestTimeout=${timeout}\r\n"
 464
 465         stepScript += "echo Creating TestEnv Script for ${stressModeName}\r\n"
 466         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 467
 468         // Create at least an empty script.
 469         stepScript += "echo. > ${stepScriptLocation}\r\n"
 470
 471         stressModeVars.each{ k, v ->
 472             // Write out what we are writing to the script file
 473             stepScript += "echo Setting ${k}=${v}\r\n"
 474             // Write out the set itself to the script file`
 475             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 476         }
 477     }
 478     else {
 479         stepScript += "echo Setting variables for ${stressModeName}\n"
 480         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 481         stressModeVars.each{ k, v ->
 482             // Write out what we are writing to the script file
 483             stepScript += "echo Setting ${k}=${v}\n"
 484             // Write out the set itself to the script file`
 485             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 486         }
 487         stepScript += "chmod +x ${stepScriptLocation}\n"
 488     }
 489
 490     return stepScript
 491 }
 492
 493 // Append an existing script to a stress mode script already created by genStressModeScriptStep().
 494 // Returns string of commands to do this.
 495 def static appendStressModeScriptStep(def os, def appendScript, def stepScriptLocation) {
 496     assert (os == 'Windows_NT')
 497     def stepScript = ''
 498     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 499     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 500
 501     // Display the resulting script. This is useful when looking at the output log file.
 502     stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 503     stepScript += "type ${stepScriptLocation}\r\n"
 504
 505     return stepScript
 506 }
 507
 508 def static isNeedDocker(def architecture, def os, def isBuild) {
 509     if (isBuild) {
 510         if (architecture == 'x86' && os == 'Ubuntu') {
 511             return true
 512         }
 513         else if (architecture == 'arm') {
 514             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 515                 return true
 516             }
 517         }
 518     }
 519     else {
 520         if (architecture == 'x86' && os == 'Ubuntu') {
 521             return true
 522         }
 523     }
 524     return false
 525 }
 526
 527 def static getDockerImageName(def architecture, def os, def isBuild) {
 528     // We must change some docker private images to official later
 529     if (isBuild) {
 530         if (architecture == 'x86' && os == 'Ubuntu') {
 531             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 532         }
 533         else if (architecture == 'arm') {
 534             if (os == 'Ubuntu') {
 535                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 536             }
 537             else if (os == 'Ubuntu16.04') {
 538                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 539             }
 540             else if (os == 'Tizen') {
 541                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 542             }
 543         }
 544     }
 545     else {
 546         if (architecture == 'x86' && os == 'Ubuntu') {
 547             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 548         }
 549     }
 550     println("Unknown architecture to use docker: ${architecture} ${os}");
 551     assert false
 552 }
 553
 554 // Calculates the name of the build job based on some typical parameters.
 555 //
 556 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 557     // If the architecture is x64, do not add that info into the build name.
 558     // Need to change around some systems and other builds to pick up the right builds
 559     // to do that.
 560
 561     def suffix = scenario != 'default' ? "_${scenario}" : '';
 562     if (isBuildOnly) {
 563         suffix += '_bld'
 564     }
 565     def baseName = ''
 566     switch (architecture) {
 567         case 'x64':
 568             if (scenario == 'default') {
 569                 // For now we leave x64 off of the name for compatibility with other jobs
 570                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 571             }
 572             else if (scenario == 'formatting') {
 573                 // we don't care about the configuration for the formatting job. It runs all configs
 574                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 575             }
 576             else {
 577                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 578             }
 579             break
 580         case 'arm64':
 581             if (os.toLowerCase() == "windows_nt") {
 582                 // These are cross builds
 583                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 584             }
 585             else {
 586                 // Defaults to a small page size set of machines.
 587                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 588             }
 589             break
 590         case 'arm':
 591             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 592             break
 593         case 'armlb':
 594             // These are cross builds
 595             if (os == 'Tizen') {
 596                 // ABI: softfp
 597                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 598             }
 599             else {
 600                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 601             }
 602             break
 603         case 'x86':
 604         case 'x86_arm_altjit':
 605         case 'x64_arm64_altjit':
 606             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 607             break
 608         default:
 609             println("Unknown architecture: ${architecture}");
 610             assert false
 611             break
 612     }
 613
 614     return baseName + suffix
 615 }
 616
 617 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 618
 619     // Check scenario.
 620     switch (scenario) {
 621         case 'default':
 622             switch (architecture) {
 623                 case 'x64':
 624                 case 'x86':
 625                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 626                         Utilities.addPeriodicTrigger(job, '@daily')
 627                     }
 628                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 629                         Utilities.addGithubPushTrigger(job)
 630                     }
 631                     break
 632                 case 'arm':
 633                 case 'armlb':
 634                 case 'x86_arm_altjit':
 635                 case 'x64_arm64_altjit':
 636                     Utilities.addGithubPushTrigger(job)
 637                     break
 638                 case 'arm64':
 639                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 640                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 641                     break
 642                 default:
 643                     println("Unknown architecture: ${architecture}");
 644                     assert false
 645                     break
 646             }
 647             break
 648         case 'r2r':
 649             assert !(os in bidailyCrossList)
 650             // r2r gets a push trigger for checked/release
 651             if (configuration == 'Checked' || configuration == 'Release') {
 652                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 653                 if (architecture == 'x64' && os != 'OSX10.12') {
 654                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 655                     if (isFlowJob || os == 'Windows_NT') {
 656                         Utilities.addGithubPushTrigger(job)
 657                     }
 658                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 659                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 660                     if (isFlowJob) {
 661                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 662                     }
 663                 }
 664                 // For x86, only add per-commit jobs for Windows
 665                 else if (architecture == 'x86') {
 666                     if (os == 'Windows_NT') {
 667                         Utilities.addGithubPushTrigger(job)
 668                     }
 669                 }
 670                 // arm64 r2r jobs should only run daily.
 671                 else if (architecture == 'arm64') {
 672                     if (os == 'Windows_NT') {
 673                         Utilities.addPeriodicTrigger(job, '@daily')
 674                     }
 675                 }
 676             }
 677             break
 678         case 'r2r_jitstress1':
 679         case 'r2r_jitstress2':
 680         case 'r2r_jitstressregs1':
 681         case 'r2r_jitstressregs2':
 682         case 'r2r_jitstressregs3':
 683         case 'r2r_jitstressregs4':
 684         case 'r2r_jitstressregs8':
 685         case 'r2r_jitstressregs0x10':
 686         case 'r2r_jitstressregs0x80':
 687         case 'r2r_jitstressregs0x1000':
 688         case 'r2r_jitminopts':
 689         case 'r2r_jitforcerelocs':
 690         case 'r2r_gcstress15':
 691             assert !(os in bidailyCrossList)
 692
 693             // GCStress=C is currently not supported on OS X
 694             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 695                 break
 696             }
 697
 698             // GC Stress 15 r2r gets a push trigger for checked/release
 699             if (configuration == 'Checked' || configuration == 'Release') {
 700                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 701                 if (architecture == 'x64') {
 702                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 703                     if (isFlowJob || os == 'Windows_NT') {
 704                         // Add a weekly periodic trigger
 705                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 706                     }
 707                 }
 708                 // For x86, only add per-commit jobs for Windows
 709                 else if (architecture == 'x86') {
 710                     if (os == 'Windows_NT') {
 711                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 712                     }
 713                 }
 714             }
 715             break
 716         case 'longgc':
 717             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 718             assert configuration == 'Release'
 719             assert architecture == 'x64'
 720             Utilities.addPeriodicTrigger(job, '@daily')
 721             // TODO: Add once external email sending is available again
 722             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 723             break
 724         case 'gcsimulator':
 725             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 726             assert configuration == 'Release'
 727             assert architecture == 'x64'
 728             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 729             // TODO: Add once external email sending is available again
 730             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 731             break
 732         case 'standalone_gc':
 733             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 734             assert (configuration == 'Release' || configuration == 'Checked')
 735             // TODO: Add once external email sending is available again
 736             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 737             Utilities.addPeriodicTrigger(job, '@daily')
 738             break
 739         case 'gc_reliability_framework':
 740             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 741             assert (configuration == 'Release' || configuration == 'Checked')
 742             // Only triggered by phrase.
 743             break
 744         case 'ilrt':
 745             assert !(os in bidailyCrossList)
 746             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 747             if (architecture == 'x64' && configuration == 'Release') {
 748                 // We don't expect to see a job generated except in these scenarios
 749                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 750                 if (isFlowJob || os == 'Windows_NT') {
 751                     Utilities.addPeriodicTrigger(job, '@daily')
 752                 }
 753             }
 754             break
 755         case 'jitdiff':
 756             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 757             assert configuration == 'Checked'
 758             assert (architecture == 'x64' || architecture == 'x86')
 759             Utilities.addGithubPushTrigger(job)
 760             break
 761         case 'formatting':
 762             assert (os == 'Windows_NT' || os == "Ubuntu")
 763             assert architecture == 'x64'
 764             Utilities.addGithubPushTrigger(job)
 765             break
 766         case 'jitstressregs1':
 767         case 'jitstressregs2':
 768         case 'jitstressregs3':
 769         case 'jitstressregs4':
 770         case 'jitstressregs8':
 771         case 'jitstressregs0x10':
 772         case 'jitstressregs0x80':
 773         case 'jitstressregs0x1000':
 774         case 'minopts':
 775         case 'forcerelocs':
 776         case 'jitstress1':
 777         case 'jitstress2':
 778         case 'jitstress2_jitstressregs1':
 779         case 'jitstress2_jitstressregs2':
 780         case 'jitstress2_jitstressregs3':
 781         case 'jitstress2_jitstressregs4':
 782         case 'jitstress2_jitstressregs8':
 783         case 'jitstress2_jitstressregs0x10':
 784         case 'jitstress2_jitstressregs0x80':
 785         case 'jitstress2_jitstressregs0x1000':
 786         case 'tailcallstress':
 787         case 'jitsse2only':
 788         case 'jitnosimd':
 789         case 'corefx_baseline':
 790         case 'corefx_minopts':
 791         case 'corefx_jitstress1':
 792         case 'corefx_jitstress2':
 793         case 'corefx_jitstressregs1':
 794         case 'corefx_jitstressregs2':
 795         case 'corefx_jitstressregs3':
 796         case 'corefx_jitstressregs4':
 797         case 'corefx_jitstressregs8':
 798         case 'corefx_jitstressregs0x10':
 799         case 'corefx_jitstressregs0x80':
 800         case 'corefx_jitstressregs0x1000':
 801         case 'zapdisable':
 802             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 803                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 804                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 805                     if (os == 'Windows_NT') {
 806                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 807                         Utilities.addPeriodicTrigger(job, '@weekly')
 808                     }
 809                 }
 810                 else {
 811                     Utilities.addPeriodicTrigger(job, '@daily')
 812                 }
 813             }
 814             break
 815         case 'heapverify1':
 816         case 'gcstress0x3':
 817             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 818                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 819                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 820                     if (os == 'Windows_NT') {
 821                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 822                         Utilities.addPeriodicTrigger(job, '@weekly')
 823                     }
 824                     // TODO: Add once external email sending is available again
 825                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 826                 }
 827                 else {
 828                     Utilities.addPeriodicTrigger(job, '@weekly')
 829                 }
 830             }
 831             break
 832         case 'gcstress0xc':
 833         case 'gcstress0xc_zapdisable':
 834         case 'gcstress0xc_zapdisable_jitstress2':
 835         case 'gcstress0xc_zapdisable_heapverify1':
 836         case 'gcstress0xc_jitstress1':
 837         case 'gcstress0xc_jitstress2':
 838         case 'gcstress0xc_minopts_heapverify1':
 839             // GCStress=C is currently not supported on OS X
 840             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 841                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 842                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 843                     if (os == 'Windows_NT') {
 844                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 845                         Utilities.addPeriodicTrigger(job, '@weekly')
 846                     }
 847                     // TODO: Add once external email sending is available again
 848                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 849                 }
 850                 else {
 851                     Utilities.addPeriodicTrigger(job, '@weekly')
 852                 }
 853             }
 854             break
 855
 856         case 'illink':
 857             // Testing on other operating systems TBD
 858             assert (os == 'Windows_NT' || os == 'Ubuntu')
 859             if (architecture == 'x64' || architecture == 'x86') {
 860                 if (configuration == 'Checked') {
 861                     Utilities.addPeriodicTrigger(job, '@daily')
 862                 }
 863             }
 864             break
 865
 866         case 'tieredcompilation':
 867         case 'corefx_tieredcompilation':
 868             // No periodic jobs just yet, still testing
 869             break
 870
 871         default:
 872             println("Unknown scenario: ${scenario}");
 873             assert false
 874             break
 875     }
 876     return
 877 }
 878
 879 // **************************
 880 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 881 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 882 // builds will do a couple extra steps.
 883 // **************************
 884
 885 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 886 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 887 // it's just used for internal builds
 888 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 889 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 890     if (isWindowsBuildOnlyJob) {
 891         return
 892     }
 893
 894     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 895     // Non pull request builds.
 896     if (!isPR) {
 897         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 898         return
 899     }
 900
 901      def arm64Users = [
 902         'AndyAyersMS',
 903         'briansull',
 904         'BruceForstall',
 905         'CarolEidt',
 906         'cmckinsey',
 907         'erozenfeld',
 908         'janvorli',
 909         'jashook',
 910         'JosephTremoulet',
 911         'pgodeq',
 912         'pgavlin',
 913         'rartemev',
 914         'russellhadley',
 915         'RussKeldorph',
 916         'sandreenko',
 917         'sdmaclea',
 918         'swaroop-sridhar',
 919         'jkotas',
 920         'markwilkie',
 921         'weshaggard'
 922     ]
 923
 924     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 925     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 926     def osGroup = getOSGroup(os)
 927     switch (architecture) {
 928         case 'x64': // editor brace matching: {
 929             if (scenario == 'formatting') {
 930                 assert configuration == 'Checked'
 931                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 932                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
 933                 }
 934                 break
 935             }
 936
 937             switch (os) {
 938                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 939                 case 'Debian8.4':
 940                 case 'RHEL7.2':
 941                     if (scenario == 'default') {
 942                         assert !isFlowJob
 943                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 944                     }
 945                     break
 946                 case 'Ubuntu16.04':
 947                     assert !isFlowJob
 948                     assert scenario == 'default'
 949                     // Distinguish with the other architectures (arm and x86)
 950                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 951                     break
 952                 case 'Fedora24':
 953                 case 'Ubuntu16.10':
 954                     assert !isFlowJob
 955                     assert scenario == 'default'
 956                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 957                     break
 958                 case 'Ubuntu':
 959                     if (scenario == 'illink') {
 960                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
 961                         break
 962                     }
 963                     // fall through
 964                 case 'OSX10.12':
 965                     // Triggers on the non-flow jobs aren't necessary here
 966                     // Corefx testing uses non-flow jobs.
 967                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
 968                         break
 969                     }
 970                     switch (scenario) {
 971                         case 'default':
 972                             // OSX uses checked for default PR tests
 973                             if (configuration == 'Checked') {
 974                                 // Default trigger
 975                                 assert !job.name.contains("centos")
 976                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
 977                             }
 978                             break
 979                         case 'jitdiff':
 980                             if (configuration == 'Checked') {
 981                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 982                             }
 983                             break
 984                         case 'ilrt':
 985                             if (configuration == 'Release') {
 986                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
 987                             }
 988                             break
 989                         case 'longgc':
 990                             if (configuration == 'Release') {
 991                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 992                             }
 993                             break
 994                         case 'gcsimulator':
 995                             if (configuration == 'Release') {
 996                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
 997                             }
 998                             break
 999                         case 'standalone_gc':
1000                             if (configuration == 'Release' || configuration == 'Checked') {
1001                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1002                             }
1003                             break
1004                         case 'gc_reliability_framework':
1005                             if (configuration == 'Release' || configuration == 'Checked') {
1006                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1007                             }
1008                             break
1009                         default:
1010                             if (isJitStressScenario(scenario)) {
1011                                 def displayStr = getStressModeDisplayName(scenario)
1012                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1013                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1014                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1015                             }
1016                             else if (isR2RScenario(scenario)) {
1017                                 if (configuration == 'Release' || configuration == 'Checked') {
1018                                     def displayStr = getR2RDisplayName(scenario)
1019                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1020                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1021                                 }
1022                             }
1023                             else {
1024                                 println("Unknown scenario: ${scenario}");
1025                                 assert false
1026                             }
1027                             break
1028                     }
1029                     break
1030
1031                 case 'CentOS7.1':
1032                     switch (scenario) {
1033                         case 'default':
1034                             // CentOS uses checked for default PR tests while debug is build only
1035                             if (configuration == 'Debug') {
1036                                 // Default trigger
1037                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1038                             }
1039
1040                             // Make sure this is a flow job to get build and test.
1041                             if (configuration == 'Checked' && isFlowJob) {
1042                                 assert job.name.contains("flow")
1043                                 // Default trigger
1044                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1045                             }
1046                             break
1047                         default:
1048                             if (isR2RScenario(scenario)) {
1049                                 if (configuration == 'Release' || configuration == 'Checked') {
1050                                     def displayStr = getR2RDisplayName(scenario)
1051                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1052                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1053                                 }
1054                             }
1055                             break
1056                     }
1057                     break
1058
1059                 case 'Windows_NT':
1060                     switch (scenario) {
1061                         case 'default':
1062                             // Default trigger
1063                             if (configuration == 'Checked') {
1064                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1065                             }
1066                             break
1067                         case 'jitdiff':
1068                             if (configuration == 'Checked') {
1069                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1070                             }
1071                             break
1072                         case 'ilrt':
1073                             if (configuration == 'Release') {
1074                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1075                             }
1076                             break
1077                         case 'longgc':
1078                             if (configuration == 'Release') {
1079                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1080                             }
1081                             break
1082                         case 'gcsimulator':
1083                             if (configuration == 'Release') {
1084                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1085                             }
1086                             break
1087                         case 'standalone_gc':
1088                             if (configuration == 'Release' || configuration == 'Checked') {
1089                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1090                             }
1091                             break
1092                         case 'gc_reliability_framework':
1093                             if (configuration == 'Release' || configuration == 'Checked') {
1094                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1095                             }
1096                             break
1097                         case 'illink':
1098                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1099                             break
1100                         default:
1101                             if (isJitStressScenario(scenario)) {
1102                                 def displayStr = getStressModeDisplayName(scenario)
1103                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1104                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1105                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1106                             }
1107                             else if (isR2RScenario(scenario)) {
1108                                 if (configuration == 'Release' || configuration == 'Checked') {
1109                                     def displayStr = getR2RDisplayName(scenario)
1110                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1111                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1112                                 }
1113                             }
1114                             else {
1115                                 println("Unknown scenario: ${scenario}");
1116                                 assert false
1117                             }
1118                             break
1119                     }
1120                     break
1121                 default:
1122                     println("Unknown os: ${os}");
1123                     assert false
1124                     break
1125             }
1126             break
1127         // editor brace matching: }
1128         case 'armlb':
1129         case 'arm': // editor brace matching: {
1130             switch (os) {
1131                 case 'Ubuntu':
1132                 case 'Ubuntu16.04':
1133                     if (architecture == 'armlb') { // Ubuntu arm is only for armlb currently
1134                         assert scenario == 'default'
1135                         job.with {
1136                             publishers {
1137                                 azureVMAgentPostBuildAction {
1138                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1139                                 }
1140                             }
1141                         }
1142                         if ((os == 'Ubuntu' && configuration == 'Debug') || (os == 'Ubuntu16.04' && configuration == 'Debug')) {
1143                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1144                         }
1145                         else {
1146                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1147                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1148                         }
1149                     }
1150                     break
1151                 case 'Tizen':
1152                     if (architecture == 'armlb') { // Tizen armel is only for armlb currently
1153                         architecture='armel'
1154                         job.with {
1155                             publishers {
1156                                 azureVMAgentPostBuildAction {
1157                                     agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1158                                 }
1159                             }
1160                         }
1161                         // Removing the regex will cause this to run on each PR.
1162                         if (configuration == 'Checked') {
1163                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1164                         }
1165                         else {
1166                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1167                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1168                         }
1169                     }
1170                     break
1171                 case 'Windows_NT':
1172                     // Triggers on the non-flow jobs aren't necessary here
1173                     if (!isFlowJob) {
1174                         break
1175                     }
1176
1177                     // Set up a private trigger
1178                     def contextString = "${os} ${architecture} Cross ${configuration}"
1179                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1180                     if (scenario == 'default') {
1181                         contextString += " Innerloop"
1182                         triggerString += "\\W+Innerloop"
1183                     }
1184                     else {
1185                         contextString += " ${scenario}"
1186                         triggerString += "\\W+${scenario}"
1187                     }
1188
1189                     if (configuration == 'Debug') {
1190                         contextString += " Build"
1191                         triggerString += "\\W+Build"
1192                     } else {
1193                         contextString += " Build and Test"
1194                         triggerString += "\\W+Build and Test"
1195                     }
1196
1197                     triggerString += ".*"
1198
1199                     switch (scenario) {
1200                         case 'default':
1201                             // Only Checked is a default trigger.
1202                             if (configuration == 'Checked')
1203                             {
1204                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1205                             }
1206                             else
1207                             {
1208                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1209                             }
1210                             break
1211                         default:
1212                             // Stress jobs will use this code path.
1213                             if (isArmWindowsScenario(scenario)) {
1214                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1215                             }
1216                             break
1217                     }
1218                     break
1219                 default:
1220                     println("NYI os: ${os}");
1221                     assert false
1222                     break
1223             }
1224             break
1225         // editor brace matching: }
1226         case 'arm64': // editor brace matching: {
1227             // Set up a private trigger
1228             def contextString = "${os} ${architecture} Cross ${configuration}"
1229             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1230             if (scenario == 'default') {
1231                 contextString += " Innerloop"
1232                 triggerString += "\\W+Innerloop"
1233             }
1234             else {
1235                 contextString += " ${scenario}"
1236                 triggerString += "\\W+${scenario}"
1237             }
1238
1239             if (configuration == 'Debug') {
1240                 contextString += " Build"
1241                 triggerString += "\\W+Build"
1242             } else {
1243                 contextString += " Build and Test"
1244                 triggerString += "\\W+Build and Test"
1245             }
1246
1247             triggerString += ".*"
1248
1249             switch (os) {
1250                 case 'Ubuntu':
1251                 case 'Ubuntu16.04':
1252                     switch (scenario) {
1253                         case 'default':
1254                             if (configuration == 'Debug' && !isFlowJob) {
1255                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1256                             }
1257                             else {
1258                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1259                             }
1260
1261                             break
1262                         default:
1263                             if (isR2RScenario(scenario)) {
1264                                 if (configuration == 'Checked' || configuration == 'Release') {
1265                                     def displayStr = getR2RDisplayName(scenario)
1266                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1267                                 }
1268                             }
1269                             break
1270                     }
1271                     break
1272                 case 'Windows_NT':
1273                     // Triggers on the non-flow jobs aren't necessary here
1274                     if (!isFlowJob) {
1275                         break
1276                     }
1277
1278                     assert isArmWindowsScenario(scenario)
1279                     switch (scenario) {
1280                         case 'default':
1281                             if (configuration == 'Checked') {
1282                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1283                             }
1284                             else {
1285                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1286                             }
1287                             break
1288                         default:
1289                             // Stress jobs will use this code path.
1290                             if (isArmWindowsScenario(scenario)) {
1291                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1292                             }
1293                             break
1294                     }
1295                     break
1296                 default:
1297                     println("NYI os: ${os}");
1298                     assert false
1299                     break
1300             }
1301             break
1302         // editor brace matching: }
1303         case 'x86': // editor brace matching: {
1304             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1305             if (os == 'Ubuntu') {
1306                 // Triggers on the non-flow jobs aren't necessary here
1307                 if (!isFlowJob) {
1308                     break
1309                 }
1310                 // on-demand only for ubuntu x86
1311                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1312                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1313                 break
1314             }
1315             switch (scenario) {
1316                 case 'default':
1317                     if (configuration == 'Checked') {
1318                         assert !job.name.contains("centos")
1319                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1320                     }
1321                     else {
1322                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1323                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1324                     }
1325                     break
1326                 case 'ilrt':
1327                     if (configuration == 'Release') {
1328                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1329                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1330                     }
1331                     break
1332                 case 'longgc':
1333                     if (configuration == 'Release') {
1334                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1335                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1336                     }
1337                     break
1338                 case 'gcsimulator':
1339                     if (configuration == 'Release') {
1340                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1341                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1342                     }
1343                     break
1344                 case 'standalone_gc':
1345                     if (configuration == 'Release' || configuration == 'Checked') {
1346                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1347                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1348                     }
1349                     break
1350                 case 'illink':
1351                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1352                     break
1353                 default:
1354                     if (isJitStressScenario(scenario)) {
1355                         def displayStr = getStressModeDisplayName(scenario)
1356                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1357                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1358                     }
1359                     else if (isR2RScenario(scenario)) {
1360                         if (configuration == 'Release' || configuration == 'Checked') {
1361                             def displayStr = getR2RDisplayName(scenario)
1362                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1363                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1364                         }
1365                     }
1366                     else {
1367                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1368                         assert false
1369                     }
1370                     break
1371             }
1372             break
1373          // editor brace matching: }
1374         case 'x64_arm64_altjit':
1375         case 'x86_arm_altjit': // editor brace matching: {
1376             assert (os == 'Windows_NT')
1377             switch (scenario) {
1378                 case 'default':
1379                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1380                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1381                     break
1382                 default:
1383                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1384                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1385                     break
1386             }
1387             break
1388         // editor brace matching: }
1389         default:
1390             println("Unknown architecture: ${architecture}");
1391             assert false
1392             break
1393     }
1394 }
1395
1396 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1397     def buildCommands = [];
1398     def osGroup = getOSGroup(os)
1399     def lowerConfiguration = configuration.toLowerCase()
1400
1401     def priority = '1'
1402     if (scenario == 'default' && isPR == true) {
1403         priority = '0'
1404     }
1405
1406     def enableCorefxTesting = isCoreFxScenario(scenario)
1407
1408     // Calculate the build steps, archival, and xunit results
1409     switch (os) {
1410         case 'Windows_NT': // editor brace matching: {
1411             switch (architecture) {
1412                 case 'x64':
1413                 case 'x86':
1414                 case 'x86_arm_altjit':
1415                 case 'x64_arm64_altjit':
1416                     def arch = architecture
1417                     def buildOpts = ''
1418                     if (architecture == 'x86_arm_altjit') {
1419                         arch = 'x86'
1420                     }
1421                     else if (architecture == 'x64_arm64_altjit') {
1422                         arch = 'x64'
1423                     }
1424
1425                     if (scenario == 'formatting') {
1426                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1427                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1428                         break
1429                     }
1430
1431                     if (scenario == 'illink') {
1432                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1433                     }
1434
1435                     // If it is a release build for windows, ensure PGO is used, else fail the build
1436                     if ((lowerConfiguration == 'release') &&
1437                         (scenario in Constants.basicScenarios) &&
1438                         (architecture != 'x86_arm_altjit') &&
1439                         (architecture != 'x64_arm64_altjit')) {
1440
1441                         buildOpts += ' -enforcepgo'
1442                     }
1443
1444                     if (enableCorefxTesting) {
1445                         buildOpts += ' skiptests';
1446                     } else {
1447                         buildOpts += " -priority=${priority}"
1448                     }
1449
1450                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1451                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1452                     // 35 characters long.
1453
1454                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1455
1456                     if (!isBuildOnly) {
1457                         def runtestArguments = ''
1458                         def testOpts = 'collectdumps'
1459
1460                         if (isR2RScenario(scenario)) {
1461
1462                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1463                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1464                             // to cause the tests to be crossgen'ed.
1465
1466                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1467                                 testOpts += ' crossgenaltjit protononjit.dll'
1468                             } else {
1469                                 testOpts += ' crossgen'
1470                             }
1471
1472                             testOpts += ' runcrossgentests'
1473
1474                             if (scenario == 'r2r_jitstress1') {
1475                                 testOpts += ' jitstress 1'
1476                             }
1477                             else if (scenario == 'r2r_jitstress2') {
1478                                 testOpts += ' jitstress 2'
1479                             }
1480                             else if (scenario == 'r2r_jitstressregs1') {
1481                                 testOpts += ' jitstressregs 1'
1482                             }
1483                             else if (scenario == 'r2r_jitstressregs2') {
1484                                 testOpts += ' jitstressregs 2'
1485                             }
1486                             else if (scenario == 'r2r_jitstressregs3') {
1487                                 testOpts += ' jitstressregs 3'
1488                             }
1489                             else if (scenario == 'r2r_jitstressregs4') {
1490                                 testOpts += ' jitstressregs 4'
1491                             }
1492                             else if (scenario == 'r2r_jitstressregs8') {
1493                                 testOpts += ' jitstressregs 8'
1494                             }
1495                             else if (scenario == 'r2r_jitstressregs0x10') {
1496                                 testOpts += ' jitstressregs 0x10'
1497                             }
1498                             else if (scenario == 'r2r_jitstressregs0x80') {
1499                                 testOpts += ' jitstressregs 0x80'
1500                             }
1501                             else if (scenario == 'r2r_jitstressregs0x1000') {
1502                                 testOpts += ' jitstressregs 0x1000'
1503                             }
1504                             else if (scenario == 'r2r_jitminopts') {
1505                                 testOpts += ' jitminopts'
1506                             }
1507                             else if (scenario == 'r2r_jitforcerelocs') {
1508                                 testOpts += ' jitforcerelocs'
1509                             }
1510                             else if (scenario == 'r2r_gcstress15') {
1511                                 testOpts += ' gcstresslevel 0xF'
1512                             }
1513                         }
1514                         else if (scenario == 'jitdiff') {
1515                             testOpts += ' jitdisasm crossgen'
1516                         }
1517                         else if (scenario == 'ilrt') {
1518                             testOpts += ' ilasmroundtrip'
1519                         }
1520                         else if (isLongGc(scenario)) {
1521                             testOpts += " ${scenario} sequential"
1522                         }
1523                         else if (scenario == 'standalone_gc') {
1524                             testOpts += ' gcname clrgc.dll'
1525                         }
1526                         else if (scenario == 'illink') {
1527                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1528                         }
1529
1530                         // If we are running a stress mode, we should write out the set of key
1531                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1532
1533                         def envScriptPath = ''
1534                         if (isJitStressScenario(scenario)) {
1535                             def buildCommandsStr = ''
1536                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1537                             buildCommandsStr += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], envScriptPath)
1538                             if (architecture == 'x86_arm_altjit') {
1539                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1540                             }
1541                             else if (architecture == 'x64_arm64_altjit') {
1542                                 buildCommandsStr += appendStressModeScriptStep(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1543                             }
1544
1545                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1546                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1547                             // into a single script.
1548                             buildCommands += buildCommandsStr
1549                         }
1550                         else if (architecture == 'x86_arm_altjit') {
1551                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1552                         }
1553                         else if (architecture == 'x64_arm64_altjit') {
1554                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1555                         }
1556                         if (envScriptPath != '') {
1557                             testOpts += " TestEnv ${envScriptPath}"
1558                         }
1559
1560                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1561
1562                         if (enableCorefxTesting) {
1563                             def workspaceRelativeFxRoot = "_/fx"
1564                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1565
1566                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1567
1568                             setTestJobTimeOut(newJob, scenario)
1569
1570                             // Archive and process (only) the test results
1571                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1572                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1573
1574                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1575                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1576                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1577                         }
1578                         else if (isGcReliabilityFramework(scenario)) {
1579                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1580                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1581                         }
1582                         else {
1583                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1584                         }
1585                     }
1586
1587                     if (!enableCorefxTesting) {
1588                         // Run the rest of the build
1589                         // Build the mscorlib for the other OS's
1590                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1591                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1592
1593                         if (arch == "x64") {
1594                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1595                         }
1596
1597                         // Zip up the tests directory so that we don't use so much space/time copying
1598                         // 10s of thousands of files around.
1599                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1600
1601                         if (!isJitStressScenario(scenario)) {
1602                             // For windows, pull full test results and test drops for x86/x64.
1603                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1604                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1605                         }
1606
1607                         if (scenario == 'jitdiff') {
1608                             // retrive jit-dasm output for base commit, and run jit-diff
1609                             if (!isBuildOnly) {
1610                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1611                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1612                             }
1613                         }
1614
1615                         if (!isBuildOnly) {
1616                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1617                             setTestJobTimeOut(newJob, scenario)
1618                         }
1619                     }
1620                     break
1621                 case 'armlb':
1622                 case 'arm':
1623                     assert isArmWindowsScenario(scenario)
1624
1625                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1626                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1627
1628                     // Set time out
1629                     setTestJobTimeOut(newJob, scenario)
1630
1631                     if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1632                     {
1633                         // Up the timeout for arm checked testing only.
1634                         // Keep the longer timeout for gcstress.
1635                         Utilities.setJobTimeout(newJob, 240)
1636                     }
1637
1638                     def buildArchitecture = 'arm'
1639
1640                     // For 'arm' (the RyuJIT/arm32 architecture), tell build.cmd to use RyuJIT/arm32 for crossgen compilation.
1641                     // RyuJIT/arm32 is currently not the default JIT; it is an aljit. So, this is a special case.
1642                     def armCrossgenOpt = ''
1643                     if (architecture == 'arm') {
1644                         armCrossgenOpt = '-altjitcrossgen'
1645                     }
1646
1647                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1648                     priority = '1'
1649
1650                     // This is now a build only job. Do not run tests. Use the flow job.
1651                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1652
1653                     // Zip up the tests directory so that we don't use so much space/time copying
1654                     // 10s of thousands of files around.
1655                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1656
1657                     // Add archival.
1658                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1659                     break
1660                 case 'arm64':
1661                     assert isArmWindowsScenario(scenario)
1662
1663                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1664                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1665
1666                     // Set time out
1667                     setTestJobTimeOut(newJob, scenario)
1668                     if ((scenario != 'gcstress0x3') && (scenario != 'gcstress0xc'))
1669                     {
1670                         // Up the timeout for arm checked testing only.
1671                         // Keep the longer timeout for gcstress.
1672                         Utilities.setJobTimeout(newJob, 240)
1673                     }
1674
1675                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1676                     priority = '1'
1677
1678                     // This is now a build only job. Do not run tests. Use the flow job.
1679                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1680
1681                     // Zip up the tests directory so that we don't use so much space/time copying
1682                     // 10s of thousands of files around.
1683                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1684
1685                     // Add archival.
1686                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1687                     break
1688                 default:
1689                     println("Unknown architecture: ${architecture}");
1690                     assert false
1691                     break
1692             }
1693             break
1694         // editor brace matching: }
1695         case 'Ubuntu':
1696         case 'Ubuntu16.04':
1697         case 'Ubuntu16.10':
1698         case 'Debian8.4':
1699         case 'OSX10.12':
1700         case 'CentOS7.1':
1701         case 'RHEL7.2':
1702         case 'Tizen':
1703         case 'Fedora24': // editor brace matching: {
1704             switch (architecture) {
1705                 case 'x64':
1706                 case 'x86':
1707                     if (architecture == 'x86' && os == 'Ubuntu') {
1708                         // build and PAL test
1709                         def dockerImage = getDockerImageName(architecture, os, true)
1710                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1711                         dockerImage = getDockerImageName(architecture, os, false)
1712                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1713                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1714                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1715                         break
1716                     }
1717
1718                     if (scenario == 'formatting') {
1719                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1720                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1721                         break
1722                     }
1723
1724                     if (scenario == 'illink') {
1725                         assert(os == 'Ubuntu')
1726                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1727                     }
1728
1729                     if (!enableCorefxTesting) {
1730                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1731                         // only on supported OS platforms.
1732                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1733                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1734                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1735                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1736
1737                         // Set time out
1738                         setTestJobTimeOut(newJob, scenario)
1739                         // Basic archiving of the build
1740                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1741                         // And pal tests
1742                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1743                     }
1744                     else {
1745                         // Corefx stress testing
1746                         assert os == 'Ubuntu'
1747                         assert architecture == 'x64'
1748                         assert lowerConfiguration == 'checked'
1749                         assert isJitStressScenario(scenario)
1750
1751                         // Build coreclr
1752                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1753
1754                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1755                         buildCommands += genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
1756
1757                         // Build and text corefx
1758                         def workspaceRelativeFxRoot = "_/fx"
1759                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1760
1761                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1762
1763                         setTestJobTimeOut(newJob, scenario)
1764
1765                         // Archive and process (only) the test results
1766                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1767                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1768                     }
1769                     break
1770                 case 'arm64':
1771                     if (!enableCorefxTesting) {
1772                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1773
1774                         // HACK -- Arm64 does not have corefx jobs yet.
1775                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1776                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1777                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1778                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1779
1780                         setTestJobTimeOut(newJob, scenario)
1781                         // Basic archiving of the build
1782                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1783                     }
1784                     break
1785                 case 'armlb':
1786                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1787                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1788
1789                     // default values for Ubuntu
1790                     def arm_abi="arm"
1791                     def linuxCodeName="trusty"
1792                     if (os == 'Ubuntu16.04') {
1793                         linuxCodeName="xenial"
1794                     }
1795                     else if (os == 'Tizen') {
1796                         arm_abi="armel"
1797                         linuxCodeName="tizen"
1798                     }
1799
1800                     // Unzip the Windows test binaries first. Exit with 0
1801                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1802
1803                     // Unpack the corefx binaries
1804                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1805                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1806                     if (os != 'Tizen') {
1807                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1808                     }
1809                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1810                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1811                     if (os == 'Ubuntu') {
1812                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1813                     }
1814
1815                     // Call the ARM CI script to cross build and test using docker
1816                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1817                     --mode=docker \\
1818                     --${arm_abi} \\
1819                     --linuxCodeName=${linuxCodeName} \\
1820                     --buildConfig=${lowerConfiguration} \\
1821                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1822                     --coreFxBinDir=./bin/CoreFxBinDir \\
1823                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1824
1825                     // Basic archiving of the build, no pal tests
1826                     Utilities.addArchival(newJob, "bin/Product/**", "bin/Product/**/.nuget/**")
1827                     break
1828                 default:
1829                     println("Unknown architecture: ${architecture}");
1830                     assert false
1831                     break
1832             }
1833             break
1834         // editor brace matching: }
1835         default:
1836             println("Unknown os: ${os}");
1837             assert false
1838             break
1839     } // os
1840
1841     return buildCommands
1842 }
1843
1844 Constants.allScenarios.each { scenario ->
1845     [true, false].each { isPR ->
1846         Constants.architectureList.each { architecture ->
1847             Constants.configurationList.each { configuration ->
1848                 Constants.osList.each { os ->
1849                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1850                     // and reset the os to Windows_NT
1851                     def isBuildOnly = false
1852                     if (os == 'Windows_NT_BuildOnly') {
1853                         isBuildOnly = true
1854                         os = 'Windows_NT'
1855                     }
1856
1857                     // Tizen is only supported for arm legacy_backend architecture
1858                     if (os == 'Tizen' && architecture != 'armlb') {
1859                         return
1860                     }
1861
1862                     // Skip totally unimplemented (in CI) configurations.
1863                     switch (architecture) {
1864                         case 'arm64':
1865                             if (os == 'Ubuntu16.04') {
1866                                 os = 'Ubuntu'
1867                             }
1868
1869                             // Windows and Ubuntu only
1870                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1871                                 return
1872                             }
1873                             break
1874                         case 'armlb':
1875                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1876                                 return
1877                             }
1878                             break
1879                         case 'arm':
1880                             if (os != 'Windows_NT') {
1881                                 return
1882                             }
1883                             break
1884                         case 'x86':
1885                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1886                                 return
1887                             }
1888                             break
1889                         case 'x86_arm_altjit':
1890                         case 'x64_arm64_altjit':
1891                             if (os != 'Windows_NT') {
1892                                 return
1893                             }
1894                             break
1895                         case 'x64':
1896                             // Everything implemented
1897                             break
1898                         default:
1899                             println("Unknown architecture: ${architecture}")
1900                             assert false
1901                             break
1902                     }
1903
1904                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1905                     // with checked builds)
1906                     if (isJitStressScenario(scenario)) {
1907                         if (configuration != 'Checked') {
1908                             return
1909                         }
1910
1911                         // Since these are just execution time differences,
1912                         // skip platforms that don't execute the tests here (Windows_NT only)
1913                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
1914                         if (!isEnabledOS || isBuildOnly) {
1915                             return
1916                         }
1917
1918                         switch (architecture) {
1919                             case 'x64':
1920                             case 'x86':
1921                             case 'x86_arm_altjit':
1922                             case 'x64_arm64_altjit':
1923                                 // x86 ubuntu: default only
1924                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1925                                     return
1926                                 }
1927                                 // Windows: Everything implemented
1928                                 break
1929
1930                             default:
1931                                 // arm, arm64, armlb: stress is handled through flow jobs.
1932                                 return
1933                         }
1934                     }
1935                     else if (isR2RScenario(scenario)) {
1936                         if (os != 'Windows_NT') {
1937                             return
1938                         }
1939                         // R2R test runs are not implemented for arm/armlb/arm64.
1940                         if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
1941                             return
1942                         }
1943                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
1944                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
1945                             return
1946                         }
1947                     }
1948                     else {
1949                         // Skip scenarios
1950                         switch (scenario) {
1951                             case 'ilrt':
1952                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1953                                 // the default scenario build
1954                                 if (os != 'Windows_NT') {
1955                                     return
1956                                 }
1957                                 // Only x64 for now
1958                                 if (architecture != 'x64') {
1959                                     return
1960                                 }
1961                                 // Release only
1962                                 if (configuration != 'Release') {
1963                                     return
1964                                 }
1965                                 break
1966                             case 'jitdiff':
1967                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1968                                     return
1969                                 }
1970                                 if (architecture != 'x64') {
1971                                     return
1972                                 }
1973                                 if (configuration != 'Checked') {
1974                                     return
1975                                 }
1976                                 break
1977                             case 'longgc':
1978                             case 'gcsimulator':
1979                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1980                                     return
1981                                 }
1982                                 if (architecture != 'x64') {
1983                                     return
1984                                 }
1985                                 if (configuration != 'Release') {
1986                                     return
1987                                 }
1988                                 break
1989                             case 'gc_reliability_framework':
1990                             case 'standalone_gc':
1991                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1992                                     return
1993                                 }
1994
1995                                 if (architecture != 'x64') {
1996                                     return
1997                                 }
1998
1999                                 if (configuration != 'Release' && configuration != 'Checked') {
2000                                     return
2001                                 }
2002                                 break
2003                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2004                             case 'formatting':
2005                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2006                                     return
2007                                 }
2008                                 if (architecture != 'x64') {
2009                                     return
2010                                 }
2011                                 if (configuration != 'Checked') {
2012                                     return
2013                                 }
2014                                 if (isBuildOnly) {
2015                                     return
2016                                 }
2017                                 break
2018                             case 'illink':
2019                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2020                                     return
2021                                 }
2022                                 if (architecture != 'x64' && architecture != 'x86') {
2023                                     return
2024                                 }
2025                                 if (isBuildOnly) {
2026                                     return
2027                                 }
2028                                 break
2029                             case 'default':
2030                                 // Nothing skipped
2031                                 break
2032                             default:
2033                                 println("Unknown scenario: ${scenario}")
2034                                 assert false
2035                                 break
2036                         }
2037                     }
2038
2039                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2040                     // runtime behavior, not compile-time behavior, are not interesting.
2041                     switch (architecture) {
2042                         case 'x86_arm_altjit':
2043                         case 'x64_arm64_altjit':
2044                             if (isGCStressRelatedTesting(scenario)) {
2045                                 return
2046                             }
2047                             break
2048                         default:
2049                             break
2050                     }
2051
2052                     // Calculate names
2053                     def lowerConfiguration = configuration.toLowerCase()
2054                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2055                     def folderName = getJobFolder(scenario)
2056
2057                     // Create the new job
2058                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2059
2060                     def machineAffinityOptions = null
2061
2062                     if (os != 'Windows_NT') {
2063                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2064                     }
2065                     else {
2066                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2067                     }
2068
2069                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2070
2071                     // Add all the standard options
2072                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2073                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2074
2075                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2076                     def osGroup = getOSGroup(os)
2077
2078                     newJob.with {
2079                         steps {
2080                             if (os == 'Windows_NT') {
2081                                 buildCommands.each { buildCommand ->
2082                                     batchFile(buildCommand)
2083                                 }
2084                             }
2085                             else {
2086                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2087                                 if ( architecture == 'armlb' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2088                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2089                                     // Define the Windows Tests and Corefx build job names
2090                                     def WindowTestsName = projectFolder + '/' +
2091                                                           Utilities.getFullJobName(project,
2092                                                                                    getJobName(lowerConfiguration,
2093                                                                                               'x64' ,
2094                                                                                               'windows_nt',
2095                                                                                               'default',
2096                                                                                               true),
2097                                                                                    false)
2098                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2099                                                        Utilities.getFolderName(branch)
2100
2101                                     // Copy the Windows test binaries and the Corefx build binaries
2102                                     copyArtifacts(WindowTestsName) {
2103                                         includePatterns('bin/tests/tests.zip')
2104                                         buildSelector {
2105                                             latestSuccessful(true)
2106                                         }
2107                                     }
2108
2109                                     def arm_abi = 'arm'
2110                                     def corefx_os = 'linux'
2111                                     if (os == 'Tizen') {
2112                                         arm_abi = 'armel'
2113                                         corefx_os = 'tizen'
2114                                     }
2115
2116                                     // Let's use release CoreFX to test checked CoreCLR,
2117                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2118                                     def corefx_lowerConfiguration = lowerConfiguration
2119                                     if ( lowerConfiguration == 'checked' ) {
2120                                         corefx_lowerConfiguration='release'
2121                                     }
2122
2123                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2124                                         includePatterns('bin/build.tar.gz')
2125                                         buildSelector {
2126                                             latestSuccessful(true)
2127                                         }
2128                                     }
2129                                 }
2130
2131                                 buildCommands.each { buildCommand ->
2132                                     shell(buildCommand)
2133                                 }
2134                             }
2135                         }
2136                     } // newJob.with
2137
2138                 } // os
2139             } // configuration
2140         } // architecture
2141     } // isPR
2142 } // scenario
2143
2144
2145 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2146 Constants.allScenarios.each { scenario ->
2147     [true, false].each { isPR ->
2148         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2149             Constants.crossList.each { os ->
2150                 if (architecture == 'arm64') {
2151                     if (os != "Ubuntu" && os != "Windows_NT") {
2152                         return
2153                     }
2154                 } else if (architecture == 'arm' || architecture == 'armlb') {
2155                     if (os != 'Windows_NT') {
2156                         return
2157                     }
2158                 }
2159                 else if (architecture == 'x86') {
2160                     if (os != "Ubuntu") {
2161                         return
2162                     }
2163                 }
2164
2165                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2166
2167                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2168                     return
2169                 }
2170
2171                 Constants.configurationList.each { configuration ->
2172
2173                     // First, filter based on OS.
2174
2175                     if (os == 'Windows_NT') {
2176                         if (!isArmWindowsScenario(scenario)) {
2177                             return
2178                         }
2179                     }
2180                     else {
2181                         // Non-Windows
2182                         if (architecture == 'arm64') {
2183                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2184                                 return
2185                             }
2186                         }
2187                         else if (architecture == 'x86') {
2188                             // Linux/x86 only want default test
2189                             if (scenario != 'default') {
2190                                 return
2191                             }
2192                         }
2193                     }
2194
2195                     // For CentOS, we only want Checked/Release builds.
2196                     if (os == 'CentOS7.1') {
2197                         if (configuration != 'Checked' && configuration != 'Release') {
2198                             return
2199                         }
2200                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2201                             return
2202                         }
2203                     }
2204
2205                     // For RedHat and Debian, we only do Release builds.
2206                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2207                         if (configuration != 'Release') {
2208                             return
2209                         }
2210                         if (scenario != 'default') {
2211                             return
2212                         }
2213                     }
2214
2215                     // Next, filter based on scenario.
2216
2217                     if (isJitStressScenario(scenario)) {
2218                         if (configuration != 'Checked') {
2219                             return
2220                         }
2221                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2222                         if (isCoreFxScenario(scenario)) {
2223                             return
2224                         }
2225                     }
2226                     else if (isR2RBaselineScenario(scenario)) {
2227                         if (configuration != 'Checked' && configuration != 'Release') {
2228                             return
2229                         }
2230                     }
2231                     else if (isR2RStressScenario(scenario)) {
2232                         if (configuration != 'Checked') {
2233                             return
2234                         }
2235                     }
2236                     else {
2237                         // Skip scenarios
2238                         switch (scenario) {
2239                             case 'ilrt':
2240                             case 'longgc':
2241                             case 'gcsimulator':
2242                                 // Long GC tests take a long time on non-Release builds
2243                                 // ilrt is also Release only
2244                                 if (configuration != 'Release') {
2245                                     return
2246                                 }
2247                                 break
2248                             case 'jitdiff':
2249                                 if (configuration != 'Checked') {
2250                                     return;
2251                                 }
2252                                 break
2253                             case 'gc_reliability_framework':
2254                             case 'standalone_gc':
2255                                 if (configuration != 'Release' && configuration != 'Checked') {
2256                                     return
2257                                 }
2258                                 break
2259                             case 'formatting':
2260                                 return
2261                             case 'illink':
2262                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2263                                     return
2264                                 }
2265                                 break
2266                             case 'default':
2267                                 // Nothing skipped
2268                                 break
2269                             default:
2270                                 println("Unknown scenario: ${scenario}")
2271                                 assert false
2272                                 break
2273                         }
2274                     }
2275
2276                     // Done filtering. Now, create the jobs.
2277
2278                     def lowerConfiguration = configuration.toLowerCase()
2279                     def osGroup = getOSGroup(os)
2280                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2281
2282                     def inputCoreCLRBuildName = projectFolder + '/' +
2283                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2284
2285                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2286                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2287                     def testBuildScenario = scenario
2288                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2289                         testBuildScenario = 'default'
2290                     }
2291
2292                     def inputWindowsTestBuildArch = architecture
2293                     if (architecture == "arm64" && os != "Windows_NT") {
2294                         // Use the x64 test build for arm64 unix
2295                         inputWindowsTestBuildArch = "x64"
2296                     }
2297
2298                     def inputWindowTestsBuildName = projectFolder + '/' +
2299                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2300
2301                     // Enable Server GC for Ubuntu PR builds
2302                     def serverGCString = ''
2303                     if (os == 'Ubuntu' && isPR) {
2304                         serverGCString = '--useServerGC'
2305                     }
2306
2307                     def testOpts = ''
2308
2309                     if (isR2RScenario(scenario)) {
2310
2311                         testOpts += ' --crossgen --runcrossgentests'
2312
2313                         if (scenario == 'r2r_jitstress1') {
2314                             testOpts += ' --jitstress=1'
2315                         }
2316                         else if (scenario == 'r2r_jitstress2') {
2317                             testOpts += ' --jitstress=2'
2318                         }
2319                         else if (scenario == 'r2r_jitstressregs1') {
2320                             testOpts += ' --jitstressregs=1'
2321                         }
2322                         else if (scenario == 'r2r_jitstressregs2') {
2323                             testOpts += ' --jitstressregs=2'
2324                         }
2325                         else if (scenario == 'r2r_jitstressregs3') {
2326                             testOpts += ' --jitstressregs=3'
2327                         }
2328                         else if (scenario == 'r2r_jitstressregs4') {
2329                             testOpts += ' --jitstressregs=4'
2330                         }
2331                         else if (scenario == 'r2r_jitstressregs8') {
2332                             testOpts += ' --jitstressregs=8'
2333                         }
2334                         else if (scenario == 'r2r_jitstressregs0x10') {
2335                             testOpts += ' --jitstressregs=0x10'
2336                         }
2337                         else if (scenario == 'r2r_jitstressregs0x80') {
2338                             testOpts += ' --jitstressregs=0x80'
2339                         }
2340                         else if (scenario == 'r2r_jitstressregs0x1000') {
2341                             testOpts += ' --jitstressregs=0x1000'
2342                         }
2343                         else if (scenario == 'r2r_jitminopts') {
2344                             testOpts += ' --jitminopts'
2345                         }
2346                         else if (scenario == 'r2r_jitforcerelocs') {
2347                             testOpts += ' --jitforcerelocs'
2348                         }
2349                         else if (scenario == 'r2r_gcstress15') {
2350                             testOpts += ' --gcstresslevel=0xF'
2351                         }
2352                     }
2353                     else if (scenario == 'jitdiff') {
2354                         testOpts += ' --jitdisasm --crossgen'
2355                     }
2356                     else if (scenario == 'illink') {
2357                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2358                     }
2359                     else if (isLongGc(scenario)) {
2360                         // Long GC tests behave very poorly when they are not
2361                         // the only test running (many of them allocate until OOM).
2362                         testOpts += ' --sequential'
2363
2364                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2365                         // for running long GC and GCSimulator tests, respectively. We don't use them
2366                         // here because using a playlist file produces much more readable output on the CI machines
2367                         // and reduces running time.
2368                         //
2369                         // The Long GC playlist contains all of the tests that are
2370                         // going to be run. The GCSimulator playlist contains all of
2371                         // the GC simulator tests.
2372                         if (scenario == 'longgc') {
2373                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2374                         }
2375                         else if (scenario == 'gcsimulator') {
2376                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2377                         }
2378                     }
2379                     else if (isGcReliabilityFramework(scenario)) {
2380                         testOpts += ' --build-overlay-only'
2381                     }
2382                     else if (scenario == 'standalone_gc') {
2383                         if (osGroup == 'OSX') {
2384                             testOpts += ' --gcname=libclrgc.dylib'
2385                         }
2386                         else if (osGroup == 'Linux') {
2387                             testOpts += ' --gcname=libclrgc.so'
2388                         }
2389                         else {
2390                             println("Unexpected OS group: ${osGroup} for os ${os}")
2391                             assert false
2392                         }
2393                     }
2394
2395                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2396
2397                     def folder = getJobFolder(scenario)
2398                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2399                         // Add parameters for the inputs
2400
2401                         if (windowsArmJob == true) {
2402                             parameters {
2403                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2404                             }
2405                         }
2406                         else {
2407                             parameters {
2408                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2409                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2410                             }
2411                         }
2412
2413                         steps {
2414                             // Set up the copies
2415
2416                             // Coreclr build containing the tests and mscorlib
2417
2418                             if (windowsArmJob != true) {
2419                                 copyArtifacts(inputWindowTestsBuildName) {
2420                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2421                                     buildSelector {
2422                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2423                                     }
2424                                 }
2425                             }
2426
2427                             // Coreclr build we are trying to test
2428
2429                             copyArtifacts(inputCoreCLRBuildName) {
2430                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2431                                 buildSelector {
2432                                     buildNumber('${CORECLR_BUILD}')
2433                                 }
2434                             }
2435
2436                             // Windows CoreCLR Arm(64) will restore corefx
2437                             // packages correctly.
2438                             //
2439                             // In addition, test steps are entirely different
2440                             // because we do not have a unified runner
2441                             if (windowsArmJob != true) {
2442                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2443
2444                                 // Corefx components.  We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
2445                                 def osJobName
2446                                 if (os == 'Ubuntu') {
2447                                     osJobName = 'ubuntu14.04'
2448                                 }
2449                                 else if (architecture == 'x86') {
2450                                     if (os == 'Ubuntu') {
2451                                         // Linux/x86 corefx jobs does not build managed yet
2452                                         // Clone linux/arm corefx managed packages and overwrite linux/x86 native
2453                                         osJobName = "linux_arm_cross"
2454                                     }
2455                                 }
2456                                 else {
2457                                     osJobName = os.toLowerCase()
2458                                 }
2459                                 copyArtifacts("${corefxFolder}/${osJobName}_release") {
2460                                     includePatterns('bin/build.tar.gz')
2461                                     buildSelector {
2462                                         latestSuccessful(true)
2463                                     }
2464                                 }
2465
2466                                 shell("mkdir ./bin/CoreFxBinDir")
2467                                 // Unpack the corefx binaries
2468                                 shell("tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir")
2469
2470                                 // HACK -- Arm64 does not have corefx jobs yet.
2471                                 // Clone corefx and build the native packages overwriting the x64 packages.
2472                                 if (architecture == 'arm64') {
2473                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2474                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2475                                 }
2476                                 else if (architecture == 'x86') {
2477                                     shell("mkdir ./bin/CoreFxNative")
2478
2479                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2480                                         includePatterns('bin/build.tar.gz')
2481                                         targetDirectory('bin/CoreFxNative')
2482                                         buildSelector {
2483                                             latestSuccessful(true)
2484                                         }
2485                                     }
2486
2487                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2488                                 }
2489
2490                                 // Unzip the tests first.  Exit with 0
2491                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.${architecture}.${configuration} || exit 0")
2492
2493                                 // Execute the tests
2494                                 def runDocker = isNeedDocker(architecture, os, false)
2495                                 def dockerPrefix = ""
2496                                 def dockerCmd = ""
2497                                 if (runDocker) {
2498                                     def dockerImage = getDockerImageName(architecture, os, false)
2499                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2500                                     dockerCmd = dockerPrefix + "${dockerImage} "
2501                                 }
2502
2503                                 // If we are running a stress mode, we'll set those variables first
2504                                 def testEnvOpt = ""
2505                                 if (isJitStressScenario(scenario)) {
2506                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2507                                     def createScriptCmds = genStressModeScriptStep(os, scenario, Constants.jitStressModeScenarios[scenario], scriptFileName)
2508                                     shell("${createScriptCmds}")
2509                                     testEnvOpt = "--test-env=" + scriptFileName
2510                                 }
2511
2512                                 if (isGCStressRelatedTesting(scenario)) {
2513                                     shell('./init-tools.sh')
2514                                 }
2515
2516                                 shell("""${dockerCmd}./tests/runtest.sh \\
2517                 --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
2518                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2519                 --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2520                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
2521                 --coreFxBinDir=\"\${WORKSPACE}/bin/CoreFxBinDir\" \\
2522                 --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2523
2524                                 if (isGcReliabilityFramework(scenario)) {
2525                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2526                                     if (serverGCString != '') {
2527                                         if (runDocker) {
2528                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2529                                         }
2530                                         else {
2531                                             shell("export COMPlus_gcServer=1")
2532                                         }
2533                                     }
2534
2535                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2536                                 }
2537                             }
2538
2539                             else { // windowsArmJob == true
2540                                 // Unzip tests.
2541                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2542
2543                                 // Build the build commands
2544                                 def buildCommands = ""
2545
2546                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2547                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2548                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2549
2550                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2551                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2552
2553                                 // For all jobs
2554                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2555
2556                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2557                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2558
2559                                 // Arm(32) ryujit
2560                                 if (architecture == "arm") {
2561                                     // **This is an AltJit**
2562
2563                                     addEnvVariable("COMPlus_AltJit", "*")
2564                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2565                                     addEnvVariable("COMPlus_AltJitName", "protojit.dll")
2566                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2567                                 }
2568
2569                                 // If we are running a stress mode, we'll set those variables as well
2570                                 def stressValues = null
2571                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2572                                     if (isJitStressScenario(scenario)) {
2573                                         stressValues = Constants.jitStressModeScenarios[scenario]
2574                                     }
2575                                     else {
2576                                         stressValues = Constants.r2rStressScenarios[scenario]
2577                                     }
2578
2579                                     stressValues.each { key, value ->
2580                                         addEnvVariable(key, value)
2581                                     }
2582                                 }
2583
2584                                 // TODO: do whatever is necessary to support enabling R2R testing. Environment variables,
2585                                 // crossgen the framework assemblies, etc.
2586
2587                                 // Create the smarty command
2588                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2589                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2590                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2591
2592                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "arm") { addExclude("PROTOJIT_" + exclude) } else { addExclude(exclude) } }
2593
2594                                 if (architecture == "arm") {
2595                                     addExclude("PROTOJIT_FAIL")
2596                                 }
2597
2598                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2599                                     def failTag = "JITSTRESS_FAIL"
2600                                     def excludeTag = "JITSTRESS_EXCLUDE"
2601
2602                                     if (scenario.contains('gc')) {
2603                                         failTag = "GCSTRESS_FAIL"
2604                                         excludeTag = "GCSTRESS_EXCLUDE"
2605                                     }
2606
2607                                     addArchSpecificExclude(architecture, failTag)
2608                                     addArchSpecificExclude(architecture, excludeTag)
2609                                 }
2610                                 else {
2611                                     addExclude("pri1")
2612                                 }
2613
2614                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2615                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2616                                 addExclude("LONG_RUNNING")
2617
2618                                 smartyCommand += "/lstFile Tests.lst"
2619
2620                                 def testListArch = [
2621                                     'arm64': 'arm64',
2622                                     'arm': 'arm',
2623                                     'armlb': 'arm'
2624                                 ]
2625
2626                                 def archLocation = testListArch[architecture]
2627
2628                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2629                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2630                                 addCommand("${smartyCommand}")
2631
2632                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2633                                 // However, we also need to remove all the variables that were set during this batch file, so we
2634                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2635                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2636                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2637                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2638                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2639                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2640
2641                                 // ZIP up the smarty output, no matter what the smarty result.
2642                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2643
2644                                 addCommand("echo %errorlevel%")
2645                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2646
2647                                 // Use the smarty errorlevel as the script errorlevel.
2648                                 addCommand("exit /b %__save_smarty_errorlevel%")
2649
2650                                 batchFile(buildCommands)
2651                             }
2652                         }
2653                     }
2654
2655                     if (scenario == 'jitdiff') {
2656                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2657                     }
2658
2659                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2660                     if (os in ['Ubuntu']) {
2661                         SummaryBuilder summaries = new SummaryBuilder()
2662                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2663                         summaries.emit(newJob)
2664                     }
2665
2666                     def affinityOptions = null
2667
2668                     if (windowsArmJob == true) {
2669                         affinityOptions = [
2670                             "use_arm64_build_machine" : false
2671                         ]
2672                     }
2673
2674                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2675                         affinityOptions = [
2676                             "large_pages" : false
2677                         ]
2678                     }
2679
2680                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2681                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2682
2683                     // REVIEW: Should the scenario-base timeout be set in the "else" clause of the "architecture==arm64"?
2684                     //         Should the "architecture==arm64" include arm/armlb?
2685                     //         Should we be overriding possibly higher timeouts that would be set by setTestJobTimeOut()?
2686                     setTestJobTimeOut(newJob, scenario)
2687                     if (architecture == 'arm64') {
2688                         Utilities.setJobTimeout(newJob, 240)
2689                     }
2690
2691                     if (windowsArmJob != true) {
2692                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2693                     }
2694                     else {
2695                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2696
2697                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2698                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2699                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2700                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2701                     }
2702
2703                     // Create a build flow to join together the build and tests required to run this test.
2704                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2705                     // Linux CoreCLR test
2706                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2707                     def fullTestJobName = projectFolder + '/' + newJob.name
2708                     // Add a reference to the input jobs for report purposes
2709                     JobReport.Report.addReference(inputCoreCLRBuildName)
2710                     JobReport.Report.addReference(inputWindowTestsBuildName)
2711                     JobReport.Report.addReference(fullTestJobName)
2712                     def newFlowJob = null
2713
2714                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2715                         // Do not create the flow job for RHEL jobs.
2716                         return
2717                     }
2718
2719                     if (windowsArmJob == true) {
2720                         // For Windows arm jobs there is no reason to build a parallel test job.
2721                         // The product build supports building and archiving the tests.
2722
2723                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2724                         buildFlow("""
2725 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2726
2727 // And then build the test build
2728 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2729 """)
2730                         }
2731                     }
2732                     else {
2733                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2734                         buildFlow("""
2735 // Build the input jobs in parallel
2736 parallel (
2737 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2738 { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2739 )
2740
2741 // And then build the test build
2742 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2743             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2744 """)
2745                         }
2746                     }
2747
2748                     // For the flow jobs set the machine affinity as x64 if an armarch.
2749                     def flowArch = architecture
2750
2751                     if (flowArch in validWindowsNTCrossArches) {
2752                         flowArch = 'x64'
2753                         affinityOptions = null
2754                     }
2755
2756                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2757                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2758                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2759                 } // configuration
2760             } // os
2761         } // architecture
2762     } // isPR
2763 } // scenario
2764
2765 JobReport.Report.generateJobReport(out)
2766
2767 // Make the call to generate the help job
2768 Utilities.createHelperJob(this, project, branch,
2769     "Welcome to the ${project} Repository",  // This is prepended to the help message
2770     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2771
2772 Utilities.addCROSSCheck(this, project, branch)