netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without running the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'corefx_baseline'                : [ : ], // corefx baseline
  90                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  91                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  92                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  93                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  94                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  95                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  96                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  97                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  98                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  99                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 100                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 101                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 102                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 103                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 104                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 105                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 107                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 109                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 110                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 111                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 112     ]
 113
 114     // This is a set of ReadyToRun stress scenarios
 115     def static r2rStressScenarios = [
 116                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 117                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 118                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 119                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 120                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 121                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 122                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 123                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 124                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 125                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 126                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 127                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 128                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 129     ]
 130
 131     // This is the basic set of scenarios
 132     def static basicScenarios = [
 133                'default',
 134                'ilrt',
 135                'r2r',
 136                'longgc',
 137                'formatting',
 138                'gcsimulator',
 139                'jitdiff',
 140                'standalone_gc',
 141                'gc_reliability_framework',
 142                'illink']
 143
 144     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 145
 146     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 147     // to Tests.lst file categories to exclude.
 148     //
 149     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 150     // and with the same values, with some commented out, for easier maintenance.
 151     //
 152     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 153     //
 154     def static validArmWindowsScenarios = [
 155                'default':                                [],
 156                // 'ilrt'
 157                'r2r':                                    ["R2R_FAIL"],
 158                // 'longgc'
 159                // 'formatting'
 160                // 'gcsimulator'
 161                // 'jitdiff'
 162                // 'standalone_gc'
 163                // 'gc_reliability_framework'
 164                // 'illink'
 165                'r2r_jitstress1':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 166                'r2r_jitstress2':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 167                'r2r_jitstressregs1':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 168                'r2r_jitstressregs2':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 169                'r2r_jitstressregs3':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 170                'r2r_jitstressregs4':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 171                'r2r_jitstressregs8':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 172                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 173                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 174                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 175                'r2r_jitminopts':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 176                'r2r_jitforcerelocs':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 177                'r2r_gcstress15':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 179                'tieredcompilation':                      [],
 180                'forcerelocs':                            [],
 181                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 196                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 197                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 198                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 199                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 200                // 'jitsse2only'                          // Only relevant to xarch
 201                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 202                // 'corefx_baseline'
 203                // 'corefx_minopts'
 204                // 'corefx_tieredcompilation'
 205                // 'corefx_jitstress1'
 206                // 'corefx_jitstress2'
 207                // 'corefx_jitstressregs1'
 208                // 'corefx_jitstressregs2'
 209                // 'corefx_jitstressregs3'
 210                // 'corefx_jitstressregs4'
 211                // 'corefx_jitstressregs8'
 212                // 'corefx_jitstressregs0x10'
 213                // 'corefx_jitstressregs0x80'
 214                // 'corefx_jitstressregs0x1000'
 215                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 216                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 217                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'heapverify1':                            [],
 219                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 220                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 221                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 222                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 223                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 224                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 225
 226                //
 227                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 228                //
 229
 230                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 231                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 249     ]
 250
 251     def static configurationList = ['Debug', 'Checked', 'Release']
 252
 253     // This is the set of architectures
 254     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 255 }
 256
 257 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 258     assert os instanceof String
 259     assert architecture instanceof String
 260
 261     def armArches = ['arm', 'armlb', 'arm64']
 262     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 263
 264     if (!(architecture in armArches)) {
 265         assert options == null
 266         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 267
 268         return
 269     }
 270
 271     // This is an arm(64) job.
 272     //
 273     // There are several options.
 274     //
 275     // Windows_NT
 276     //
 277     // Arm32 (Build) -> latest-arm64
 278     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 279     // Arm32 (Test)  -> arm64-windows_nt
 280     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 281     //
 282     // Arm64 (Build) -> latest-arm64
 283     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true
 284     // Arm64 (Test)  -> arm64-windows_nt
 285     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 286     //
 287     // Ubuntu
 288     //
 289     // Arm32 (Build) -> arm-cross-latest
 290     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 291     // Arm32 (Test)  -> NYI Arch not supported
 292     //       |->
 293     //
 294     // Arm64 (Build) -> arm64-cross-latest
 295     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 296     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 297     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 298     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 299     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 300
 301     // This has to be a arm arch
 302     assert architecture in armArches
 303     if (os == "Windows_NT") {
 304         // Arm(64) Windows jobs share the same machines for now
 305         def isBuild = options['use_arm64_build_machine'] == true
 306
 307         if (isBuild == true) {
 308             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 309         } else {
 310             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 311         }
 312     } else {
 313         assert os != 'Windows_NT'
 314         assert os in supportedArmLinuxOs
 315
 316         if (architecture == 'arm' || architecture == 'armlb') {
 317             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 318         } else {
 319             // Arm64 Linux
 320             if (options['is_build_only'] == true) {
 321                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 322             } else {
 323                 // Arm64 Test Machines
 324                 if (options['large_pages'] == false) {
 325                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 326                 } else {
 327                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 328                 }
 329             }
 330         }
 331     }
 332 }
 333
 334 def static isGCStressRelatedTesting(def scenario) {
 335     // The 'r2r_gcstress15' scenario is a basic scenario.
 336     // Detect it and make it a GCStress related.
 337     if (scenario == 'r2r_gcstress15')
 338     {
 339         return true;
 340     }
 341
 342     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 343     def scenarioName = scenario.toLowerCase()
 344     def isGCStressTesting = false
 345     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 346         if (k in gcStressTestEnvVars) {
 347             isGCStressTesting = true;
 348         }
 349     }
 350     return isGCStressTesting
 351 }
 352
 353 def static isCoreFxScenario(def scenario) {
 354     def corefx_prefix = 'corefx_'
 355     if (scenario.length() < corefx_prefix.length()) {
 356         return false
 357     }
 358     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 359 }
 360
 361 def static isR2RBaselineScenario(def scenario) {
 362     return (scenario == 'r2r')
 363 }
 364
 365 def static isR2RStressScenario(def scenario) {
 366     return Constants.r2rStressScenarios.containsKey(scenario)
 367 }
 368
 369 def static isR2RScenario(def scenario) {
 370     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 371 }
 372
 373 def static isJitStressScenario(def scenario) {
 374     return Constants.jitStressModeScenarios.containsKey(scenario)
 375 }
 376
 377 def static isLongGc(def scenario) {
 378     return (scenario == 'longgc' || scenario == 'gcsimulator')
 379 }
 380
 381 def static isJitDiff(def scenario) {
 382     return (scenario == 'jitdiff')
 383 }
 384
 385 def static isGcReliabilityFramework(def scenario) {
 386     return (scenario == 'gc_reliability_framework')
 387 }
 388
 389 def static isArmWindowsScenario(def scenario) {
 390     return Constants.validArmWindowsScenarios.containsKey(scenario)
 391 }
 392
 393 def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) {
 394     // 2 hours (120 minutes) is the default timeout
 395     def timeout = 120
 396
 397     if (!(scenario == 'default' && isPR == true)) {
 398         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 399         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 400         timeout = 240
 401     }
 402
 403     if (!isBuildOnly) {
 404         // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above.
 405         if (isGCStressRelatedTesting(scenario)) {
 406             timeout = 4320
 407         }
 408         else if (isCoreFxScenario(scenario)) {
 409             timeout = 360
 410         }
 411         else if (isJitStressScenario(scenario)) {
 412             timeout = 240
 413         }
 414         else if (isR2RBaselineScenario(scenario)) {
 415             timeout = 240
 416         }
 417         else if (isLongGc(scenario)) {
 418             timeout = 1440
 419         }
 420         else if (isJitDiff(scenario)) {
 421             timeout = 240
 422         }
 423         else if (isGcReliabilityFramework(scenario)) {
 424             timeout = 1440
 425         }
 426         else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 427             timeout = 240
 428         }
 429     }
 430
 431     if (configuration == 'Debug') {
 432         // Debug runs can be very slow. Add an hour.
 433         timeout += 60
 434     }
 435
 436     // If we've changed the timeout from the default, set it in the job.
 437
 438     if (timeout != 120) {
 439         Utilities.setJobTimeout(newJob, timeout)
 440     }
 441 }
 442
 443 def static getJobFolder(def scenario) {
 444     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 445         return 'jitstress'
 446     }
 447     if (scenario == 'illink') {
 448         return 'illink'
 449     }
 450     return ''
 451 }
 452
 453 def static getStressModeDisplayName(def scenario) {
 454     def displayStr = ''
 455     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 456         def prefixLength = 'COMPlus_'.length()
 457         if (k.length() >= prefixLength) {
 458             def modeName = k.substring(prefixLength, k.length())
 459             displayStr += ' ' + modeName + '=' + v
 460         }
 461     }
 462
 463     if (isCoreFxScenario(scenario)) {
 464         displayStr = ('CoreFx ' + displayStr).trim()
 465     }
 466
 467     return displayStr
 468 }
 469
 470 def static getR2RDisplayName(def scenario) {
 471     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 472     def displayStr = scenario
 473     def prefixLength = 'r2r_'.length()
 474     if (displayStr.length() >= prefixLength) {
 475         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 476     } else if (scenario == 'r2r') {
 477         displayStr = "R2R"
 478     }
 479     return displayStr
 480 }
 481
 482 //
 483 // Functions to create an environment script.
 484 //      envScriptCreate -- initialize the script (call first)
 485 //      envScriptFinalize -- finalize the script (call last)
 486 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 487 //      envScriptAppendExistingScript -- append an existing script to the generated script
 488 //
 489 // Each script returns a string of commands. Concatenate all the strings together before
 490 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 491 //
 492
 493 // Initialize the environment setting script.
 494 def static envScriptCreate(def os, def stepScriptLocation) {
 495     def stepScript = ''
 496     if (os == 'Windows_NT') {
 497         stepScript += "echo Creating TestEnv script\r\n"
 498         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 499
 500         // Create at least an empty script.
 501         stepScript += "echo. > ${stepScriptLocation}\r\n"
 502     }
 503     else {
 504         stepScript += "echo Creating environment setting script\n"
 505         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 506     }
 507
 508     return stepScript
 509 }
 510
 511 // Generates the string for setting stress mode variables.
 512 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 513     def stepScript = ''
 514     if (os == 'Windows_NT') {
 515         stressModeVars.each{ k, v ->
 516             // Write out what we are writing to the script file
 517             stepScript += "echo Setting ${k}=${v}\r\n"
 518             // Write out the set itself to the script file`
 519             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 520         }
 521     }
 522     else {
 523         stressModeVars.each{ k, v ->
 524             // Write out what we are writing to the script file
 525             stepScript += "echo Setting ${k}=${v}\n"
 526             // Write out the set itself to the script file`
 527             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 528         }
 529     }
 530
 531     return stepScript
 532 }
 533
 534 // Append an existing script to an environment script.
 535 // Returns string of commands to do this.
 536 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 537     assert (os == 'Windows_NT')
 538     def stepScript = ''
 539
 540     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 541     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 542
 543     return stepScript
 544 }
 545
 546 // Finalize an environment setting script.
 547 // Returns string of commands to do this.
 548 def static envScriptFinalize(def os, def stepScriptLocation) {
 549     def stepScript = ''
 550
 551     if (os == 'Windows_NT') {
 552         // Display the resulting script. This is useful when looking at the output log file.
 553         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 554         stepScript += "type ${stepScriptLocation}\r\n"
 555     }
 556     else {
 557         stepScript += "chmod +x ${stepScriptLocation}\n"
 558     }
 559
 560     return stepScript
 561 }
 562
 563 def static isNeedDocker(def architecture, def os, def isBuild) {
 564     if (isBuild) {
 565         if (architecture == 'x86' && os == 'Ubuntu') {
 566             return true
 567         }
 568         else if (architecture == 'arm') {
 569             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 570                 return true
 571             }
 572         }
 573     }
 574     else {
 575         if (architecture == 'x86' && os == 'Ubuntu') {
 576             return true
 577         }
 578     }
 579     return false
 580 }
 581
 582 def static getDockerImageName(def architecture, def os, def isBuild) {
 583     // We must change some docker private images to official later
 584     if (isBuild) {
 585         if (architecture == 'x86' && os == 'Ubuntu') {
 586             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 587         }
 588         else if (architecture == 'arm') {
 589             if (os == 'Ubuntu') {
 590                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 591             }
 592             else if (os == 'Ubuntu16.04') {
 593                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 594             }
 595             else if (os == 'Tizen') {
 596                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 597             }
 598         }
 599     }
 600     else {
 601         if (architecture == 'x86' && os == 'Ubuntu') {
 602             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 603         }
 604     }
 605     println("Unknown architecture to use docker: ${architecture} ${os}");
 606     assert false
 607 }
 608
 609 // Calculates the name of the build job based on some typical parameters.
 610 //
 611 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 612     // If the architecture is x64, do not add that info into the build name.
 613     // Need to change around some systems and other builds to pick up the right builds
 614     // to do that.
 615
 616     def suffix = scenario != 'default' ? "_${scenario}" : '';
 617     if (isBuildOnly) {
 618         suffix += '_bld'
 619     }
 620     def baseName = ''
 621     switch (architecture) {
 622         case 'x64':
 623             if (scenario == 'default') {
 624                 // For now we leave x64 off of the name for compatibility with other jobs
 625                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 626             }
 627             else if (scenario == 'formatting') {
 628                 // we don't care about the configuration for the formatting job. It runs all configs
 629                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 630             }
 631             else {
 632                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 633             }
 634             break
 635         case 'arm64':
 636             if (os.toLowerCase() == "windows_nt") {
 637                 // These are cross builds
 638                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 639             }
 640             else {
 641                 // Defaults to a small page size set of machines.
 642                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 643             }
 644             break
 645         case 'arm':
 646             // These are cross builds
 647             if (os == 'Tizen') {
 648                 // ABI: softfp
 649                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 650             }
 651             else {
 652                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 653             }
 654             break
 655         case 'armlb':
 656             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 657             break
 658         case 'x86':
 659         case 'x86_arm_altjit':
 660         case 'x64_arm64_altjit':
 661             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 662             break
 663         default:
 664             println("Unknown architecture: ${architecture}");
 665             assert false
 666             break
 667     }
 668
 669     return baseName + suffix
 670 }
 671
 672 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 673
 674     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 675     if (os == 'Windows_NT') {
 676         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 677             if (!(branch in Constants.WindowsArm64Branches)) {
 678                 return
 679             }
 680         }
 681     }
 682
 683     // Check scenario.
 684     switch (scenario) {
 685         case 'default':
 686             switch (architecture) {
 687                 case 'x64':
 688                 case 'x86':
 689                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 690                         Utilities.addPeriodicTrigger(job, '@daily')
 691                     }
 692                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 693                         Utilities.addGithubPushTrigger(job)
 694                     }
 695                     break
 696                 case 'arm':
 697                 case 'armlb':
 698                 case 'x86_arm_altjit':
 699                 case 'x64_arm64_altjit':
 700                     Utilities.addGithubPushTrigger(job)
 701                     break
 702                 case 'arm64':
 703                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 704                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 705                     break
 706                 default:
 707                     println("Unknown architecture: ${architecture}");
 708                     assert false
 709                     break
 710             }
 711             break
 712         case 'r2r':
 713             assert !(os in bidailyCrossList)
 714             // r2r gets a push trigger for checked/release
 715             if (configuration == 'Checked' || configuration == 'Release') {
 716                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 717                 if (architecture == 'x64' && os != 'OSX10.12') {
 718                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 719                     if (isFlowJob || os == 'Windows_NT') {
 720                         Utilities.addGithubPushTrigger(job)
 721                     }
 722                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 723                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 724                     if (isFlowJob) {
 725                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 726                     }
 727                 }
 728                 // For x86, only add per-commit jobs for Windows
 729                 else if (architecture == 'x86') {
 730                     if (os == 'Windows_NT') {
 731                         Utilities.addGithubPushTrigger(job)
 732                     }
 733                 }
 734                 // arm64 r2r jobs should only run daily.
 735                 else if (architecture == 'arm64') {
 736                     if (os == 'Windows_NT') {
 737                         Utilities.addPeriodicTrigger(job, '@daily')
 738                     }
 739                 }
 740             }
 741             break
 742         case 'r2r_jitstress1':
 743         case 'r2r_jitstress2':
 744         case 'r2r_jitstressregs1':
 745         case 'r2r_jitstressregs2':
 746         case 'r2r_jitstressregs3':
 747         case 'r2r_jitstressregs4':
 748         case 'r2r_jitstressregs8':
 749         case 'r2r_jitstressregs0x10':
 750         case 'r2r_jitstressregs0x80':
 751         case 'r2r_jitstressregs0x1000':
 752         case 'r2r_jitminopts':
 753         case 'r2r_jitforcerelocs':
 754         case 'r2r_gcstress15':
 755             assert !(os in bidailyCrossList)
 756
 757             // GCStress=C is currently not supported on OS X
 758             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 759                 break
 760             }
 761
 762             // GC Stress 15 r2r gets a push trigger for checked/release
 763             if (configuration == 'Checked' || configuration == 'Release') {
 764                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 765                 if (architecture == 'x64') {
 766                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 767                     if (isFlowJob || os == 'Windows_NT') {
 768                         // Add a weekly periodic trigger
 769                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 770                     }
 771                 }
 772                 // For x86, only add per-commit jobs for Windows
 773                 else if (architecture == 'x86') {
 774                     if (os == 'Windows_NT') {
 775                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 776                     }
 777                 }
 778             }
 779             break
 780         case 'longgc':
 781             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 782             assert configuration == 'Release'
 783             assert architecture == 'x64'
 784             Utilities.addPeriodicTrigger(job, '@daily')
 785             // TODO: Add once external email sending is available again
 786             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 787             break
 788         case 'gcsimulator':
 789             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 790             assert configuration == 'Release'
 791             assert architecture == 'x64'
 792             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 793             // TODO: Add once external email sending is available again
 794             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 795             break
 796         case 'standalone_gc':
 797             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 798             assert (configuration == 'Release' || configuration == 'Checked')
 799             // TODO: Add once external email sending is available again
 800             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 801             Utilities.addPeriodicTrigger(job, '@daily')
 802             break
 803         case 'gc_reliability_framework':
 804             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 805             assert (configuration == 'Release' || configuration == 'Checked')
 806             // Only triggered by phrase.
 807             break
 808         case 'ilrt':
 809             assert !(os in bidailyCrossList)
 810             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 811             if (architecture == 'x64' && configuration == 'Release') {
 812                 // We don't expect to see a job generated except in these scenarios
 813                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 814                 if (isFlowJob || os == 'Windows_NT') {
 815                     Utilities.addPeriodicTrigger(job, '@daily')
 816                 }
 817             }
 818             break
 819         case 'jitdiff':
 820             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 821             assert configuration == 'Checked'
 822             assert (architecture == 'x64' || architecture == 'x86')
 823             Utilities.addGithubPushTrigger(job)
 824             break
 825         case 'formatting':
 826             assert (os == 'Windows_NT' || os == "Ubuntu")
 827             assert architecture == 'x64'
 828             Utilities.addGithubPushTrigger(job)
 829             break
 830         case 'jitstressregs1':
 831         case 'jitstressregs2':
 832         case 'jitstressregs3':
 833         case 'jitstressregs4':
 834         case 'jitstressregs8':
 835         case 'jitstressregs0x10':
 836         case 'jitstressregs0x80':
 837         case 'jitstressregs0x1000':
 838         case 'minopts':
 839         case 'forcerelocs':
 840         case 'jitstress1':
 841         case 'jitstress2':
 842         case 'jitstress2_jitstressregs1':
 843         case 'jitstress2_jitstressregs2':
 844         case 'jitstress2_jitstressregs3':
 845         case 'jitstress2_jitstressregs4':
 846         case 'jitstress2_jitstressregs8':
 847         case 'jitstress2_jitstressregs0x10':
 848         case 'jitstress2_jitstressregs0x80':
 849         case 'jitstress2_jitstressregs0x1000':
 850         case 'tailcallstress':
 851         case 'jitsse2only':
 852         case 'jitnosimd':
 853         case 'corefx_baseline':
 854         case 'corefx_minopts':
 855         case 'corefx_jitstress1':
 856         case 'corefx_jitstress2':
 857         case 'corefx_jitstressregs1':
 858         case 'corefx_jitstressregs2':
 859         case 'corefx_jitstressregs3':
 860         case 'corefx_jitstressregs4':
 861         case 'corefx_jitstressregs8':
 862         case 'corefx_jitstressregs0x10':
 863         case 'corefx_jitstressregs0x80':
 864         case 'corefx_jitstressregs0x1000':
 865         case 'zapdisable':
 866             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 867                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 868                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 869                     if (os == 'Windows_NT') {
 870                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 871                         Utilities.addPeriodicTrigger(job, '@weekly')
 872                     }
 873                 }
 874                 else {
 875                     Utilities.addPeriodicTrigger(job, '@daily')
 876                 }
 877             }
 878             break
 879         case 'heapverify1':
 880         case 'gcstress0x3':
 881             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 882                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 883                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 884                     if (os == 'Windows_NT') {
 885                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 886                         Utilities.addPeriodicTrigger(job, '@weekly')
 887                     }
 888                     // TODO: Add once external email sending is available again
 889                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 890                 }
 891                 else {
 892                     Utilities.addPeriodicTrigger(job, '@weekly')
 893                 }
 894             }
 895             break
 896         case 'gcstress0xc':
 897         case 'gcstress0xc_zapdisable':
 898         case 'gcstress0xc_zapdisable_jitstress2':
 899         case 'gcstress0xc_zapdisable_heapverify1':
 900         case 'gcstress0xc_jitstress1':
 901         case 'gcstress0xc_jitstress2':
 902         case 'gcstress0xc_minopts_heapverify1':
 903             // GCStress=C is currently not supported on OS X
 904             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 905                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 906                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 907                     if (os == 'Windows_NT') {
 908                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 909                         Utilities.addPeriodicTrigger(job, '@weekly')
 910                     }
 911                     // TODO: Add once external email sending is available again
 912                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 913                 }
 914                 else {
 915                     Utilities.addPeriodicTrigger(job, '@weekly')
 916                 }
 917             }
 918             break
 919
 920         case 'illink':
 921             // Testing on other operating systems TBD
 922             assert (os == 'Windows_NT' || os == 'Ubuntu')
 923             if (architecture == 'x64' || architecture == 'x86') {
 924                 if (configuration == 'Checked') {
 925                     Utilities.addPeriodicTrigger(job, '@daily')
 926                 }
 927             }
 928             break
 929
 930         case 'tieredcompilation':
 931         case 'corefx_tieredcompilation':
 932             // No periodic jobs just yet, still testing
 933             break
 934
 935         default:
 936             println("Unknown scenario: ${scenario}");
 937             assert false
 938             break
 939     }
 940     return
 941 }
 942
 943 // **************************
 944 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 945 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 946 // builds will do a couple extra steps.
 947 // **************************
 948
 949 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 950 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 951 // it's just used for internal builds
 952 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 953 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 954     if (isWindowsBuildOnlyJob) {
 955         return
 956     }
 957
 958     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 959     // Non pull request builds.
 960     if (!isPR) {
 961         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 962         return
 963     }
 964
 965      def arm64Users = [
 966         'AndyAyersMS',
 967         'briansull',
 968         'BruceForstall',
 969         'CarolEidt',
 970         'cmckinsey',
 971         'erozenfeld',
 972         'janvorli',
 973         'jashook',
 974         'JosephTremoulet',
 975         'pgodeq',
 976         'pgavlin',
 977         'rartemev',
 978         'russellhadley',
 979         'RussKeldorph',
 980         'sandreenko',
 981         'sdmaclea',
 982         'swaroop-sridhar',
 983         'jkotas',
 984         'markwilkie',
 985         'weshaggard'
 986     ]
 987
 988     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 989     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 990     def osGroup = getOSGroup(os)
 991     switch (architecture) {
 992         case 'x64': // editor brace matching: {
 993             if (scenario == 'formatting') {
 994                 assert configuration == 'Checked'
 995                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 996                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
 997                 }
 998                 break
 999             }
1000
1001             switch (os) {
1002                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
1003                 case 'Debian8.4':
1004                 case 'RHEL7.2':
1005                     if (scenario == 'default') {
1006                         assert !isFlowJob
1007                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
1008                     }
1009                     break
1010                 case 'Ubuntu16.04':
1011                     assert !isFlowJob
1012                     assert scenario == 'default'
1013                     // Distinguish with the other architectures (arm and x86)
1014                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1015                     break
1016                 case 'Fedora24':
1017                 case 'Ubuntu16.10':
1018                     assert !isFlowJob
1019                     assert scenario == 'default'
1020                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1021                     break
1022                 case 'Ubuntu':
1023                     if (scenario == 'illink') {
1024                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1025                         break
1026                     }
1027                     // fall through
1028                 case 'OSX10.12':
1029                     // Triggers on the non-flow jobs aren't necessary here
1030                     // Corefx testing uses non-flow jobs.
1031                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1032                         break
1033                     }
1034                     switch (scenario) {
1035                         case 'default':
1036                             // OSX uses checked for default PR tests
1037                             if (configuration == 'Checked') {
1038                                 // Default trigger
1039                                 assert !job.name.contains("centos")
1040                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1041                             }
1042                             break
1043                         case 'jitdiff':
1044                             if (configuration == 'Checked') {
1045                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1046                             }
1047                             break
1048                         case 'ilrt':
1049                             if (configuration == 'Release') {
1050                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1051                             }
1052                             break
1053                         case 'longgc':
1054                             if (configuration == 'Release') {
1055                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1056                             }
1057                             break
1058                         case 'gcsimulator':
1059                             if (configuration == 'Release') {
1060                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1061                             }
1062                             break
1063                         case 'standalone_gc':
1064                             if (configuration == 'Release' || configuration == 'Checked') {
1065                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1066                             }
1067                             break
1068                         case 'gc_reliability_framework':
1069                             if (configuration == 'Release' || configuration == 'Checked') {
1070                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1071                             }
1072                             break
1073                         default:
1074                             if (isJitStressScenario(scenario)) {
1075                                 def displayStr = getStressModeDisplayName(scenario)
1076                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1077                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1078                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1079                             }
1080                             else if (isR2RScenario(scenario)) {
1081                                 if (configuration == 'Release' || configuration == 'Checked') {
1082                                     def displayStr = getR2RDisplayName(scenario)
1083                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1084                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1085                                 }
1086                             }
1087                             else {
1088                                 println("Unknown scenario: ${scenario}");
1089                                 assert false
1090                             }
1091                             break
1092                     }
1093                     break
1094
1095                 case 'CentOS7.1':
1096                     switch (scenario) {
1097                         case 'default':
1098                             // CentOS uses checked for default PR tests while debug is build only
1099                             if (configuration == 'Debug') {
1100                                 // Default trigger
1101                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1102                             }
1103
1104                             // Make sure this is a flow job to get build and test.
1105                             if (configuration == 'Checked' && isFlowJob) {
1106                                 assert job.name.contains("flow")
1107                                 // Default trigger
1108                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1109                             }
1110                             break
1111                         default:
1112                             if (isR2RScenario(scenario)) {
1113                                 if (configuration == 'Release' || configuration == 'Checked') {
1114                                     def displayStr = getR2RDisplayName(scenario)
1115                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1116                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1117                                 }
1118                             }
1119                             break
1120                     }
1121                     break
1122
1123                 case 'Windows_NT':
1124                     switch (scenario) {
1125                         case 'default':
1126                             // Default trigger
1127                             if (configuration == 'Checked') {
1128                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1129                             }
1130                             break
1131                         case 'jitdiff':
1132                             if (configuration == 'Checked') {
1133                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1134                             }
1135                             break
1136                         case 'ilrt':
1137                             if (configuration == 'Release') {
1138                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1139                             }
1140                             break
1141                         case 'longgc':
1142                             if (configuration == 'Release') {
1143                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1144                             }
1145                             break
1146                         case 'gcsimulator':
1147                             if (configuration == 'Release') {
1148                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1149                             }
1150                             break
1151                         case 'standalone_gc':
1152                             if (configuration == 'Release' || configuration == 'Checked') {
1153                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1154                             }
1155                             break
1156                         case 'gc_reliability_framework':
1157                             if (configuration == 'Release' || configuration == 'Checked') {
1158                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1159                             }
1160                             break
1161                         case 'illink':
1162                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1163                             break
1164                         default:
1165                             if (isJitStressScenario(scenario)) {
1166                                 def displayStr = getStressModeDisplayName(scenario)
1167                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1168                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1169                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1170                             }
1171                             else if (isR2RScenario(scenario)) {
1172                                 if (configuration == 'Release' || configuration == 'Checked') {
1173                                     def displayStr = getR2RDisplayName(scenario)
1174                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1175                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1176                                 }
1177                             }
1178                             else {
1179                                 println("Unknown scenario: ${scenario}");
1180                                 assert false
1181                             }
1182                             break
1183                     }
1184                     break
1185                 default:
1186                     println("Unknown os: ${os}");
1187                     assert false
1188                     break
1189             }
1190             break
1191         // editor brace matching: }
1192         case 'armlb':
1193         case 'arm': // editor brace matching: {
1194             switch (os) {
1195                 case 'Ubuntu':
1196                 case 'Ubuntu16.04':
1197                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1198                         break
1199                     }
1200                     assert scenario == 'default'
1201                     job.with {
1202                         publishers {
1203                             azureVMAgentPostBuildAction {
1204                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1205                             }
1206                         }
1207                     }
1208                     if (configuration == 'Debug') {
1209                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1210                     }
1211                     else {
1212                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1213                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1214                     }
1215                     break
1216                 case 'Tizen':
1217                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1218                         break
1219                     }
1220                     architecture='armel'
1221                     job.with {
1222                         publishers {
1223                             azureVMAgentPostBuildAction {
1224                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1225                             }
1226                         }
1227                     }
1228                     if (configuration == 'Checked') {
1229                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1230                     }
1231                     else {
1232                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1233                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1234                     }
1235                     break
1236                 case 'Windows_NT':
1237                     // Triggers on the non-flow jobs aren't necessary here
1238                     if (!isFlowJob) {
1239                         break
1240                     }
1241
1242                     // Set up a private trigger
1243                     def contextString = "${os} ${architecture} Cross ${configuration}"
1244                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1245                     if (scenario == 'default') {
1246                         contextString += " Innerloop"
1247                         triggerString += "\\W+Innerloop"
1248                     }
1249                     else {
1250                         contextString += " ${scenario}"
1251                         triggerString += "\\W+${scenario}"
1252                     }
1253
1254                     if (configuration == 'Debug') {
1255                         contextString += " Build"
1256                         triggerString += "\\W+Build"
1257                     } else {
1258                         contextString += " Build and Test"
1259                         triggerString += "\\W+Build and Test"
1260                     }
1261
1262                     triggerString += ".*"
1263
1264                     switch (scenario) {
1265                         case 'default':
1266                             // Only Checked is a default trigger.
1267                             if (configuration == 'Checked')
1268                             {
1269                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1270                             }
1271                             else
1272                             {
1273                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1274                             }
1275                             break
1276                         default:
1277                             // Stress jobs will use this code path.
1278                             if (isArmWindowsScenario(scenario)) {
1279                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1280                             }
1281                             break
1282                     }
1283                     break
1284                 default:
1285                     println("NYI os: ${os}");
1286                     assert false
1287                     break
1288             }
1289             break
1290         // editor brace matching: }
1291         case 'arm64': // editor brace matching: {
1292             // Set up a private trigger
1293             def contextString = "${os} ${architecture} Cross ${configuration}"
1294             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1295             if (scenario == 'default') {
1296                 contextString += " Innerloop"
1297                 triggerString += "\\W+Innerloop"
1298             }
1299             else {
1300                 contextString += " ${scenario}"
1301                 triggerString += "\\W+${scenario}"
1302             }
1303
1304             if (configuration == 'Debug') {
1305                 contextString += " Build"
1306                 triggerString += "\\W+Build"
1307             } else {
1308                 contextString += " Build and Test"
1309                 triggerString += "\\W+Build and Test"
1310             }
1311
1312             triggerString += ".*"
1313
1314             switch (os) {
1315                 case 'Ubuntu':
1316                 case 'Ubuntu16.04':
1317                     switch (scenario) {
1318                         case 'default':
1319                             if (configuration == 'Debug' && !isFlowJob) {
1320                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1321                             }
1322                             else {
1323                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1324                             }
1325
1326                             break
1327                         default:
1328                             if (isR2RScenario(scenario)) {
1329                                 if (configuration == 'Checked' || configuration == 'Release') {
1330                                     def displayStr = getR2RDisplayName(scenario)
1331                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1332                                 }
1333                             }
1334                             break
1335                     }
1336                     break
1337                 case 'Windows_NT':
1338                     // Triggers on the non-flow jobs aren't necessary here
1339                     if (!isFlowJob) {
1340                         break
1341                     }
1342
1343                     assert isArmWindowsScenario(scenario)
1344                     switch (scenario) {
1345                         case 'default':
1346                             if (configuration == 'Checked') {
1347                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1348                             }
1349                             else {
1350                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1351                             }
1352                             break
1353                         default:
1354                             // Stress jobs will use this code path.
1355                             if (isArmWindowsScenario(scenario)) {
1356                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1357                             }
1358                             break
1359                     }
1360                     break
1361                 default:
1362                     println("NYI os: ${os}");
1363                     assert false
1364                     break
1365             }
1366             break
1367         // editor brace matching: }
1368         case 'x86': // editor brace matching: {
1369             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1370             if (os == 'Ubuntu') {
1371                 // Triggers on the non-flow jobs aren't necessary here
1372                 if (!isFlowJob) {
1373                     break
1374                 }
1375                 // on-demand only for ubuntu x86
1376                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1377                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1378                 break
1379             }
1380             switch (scenario) {
1381                 case 'default':
1382                     if (configuration == 'Checked') {
1383                         assert !job.name.contains("centos")
1384                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1385                     }
1386                     else {
1387                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1388                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1389                     }
1390                     break
1391                 case 'ilrt':
1392                     if (configuration == 'Release') {
1393                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1394                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1395                     }
1396                     break
1397                 case 'longgc':
1398                     if (configuration == 'Release') {
1399                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1400                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1401                     }
1402                     break
1403                 case 'gcsimulator':
1404                     if (configuration == 'Release') {
1405                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1406                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1407                     }
1408                     break
1409                 case 'standalone_gc':
1410                     if (configuration == 'Release' || configuration == 'Checked') {
1411                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1412                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1413                     }
1414                     break
1415                 case 'illink':
1416                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1417                     break
1418                 default:
1419                     if (isJitStressScenario(scenario)) {
1420                         def displayStr = getStressModeDisplayName(scenario)
1421                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1422                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1423                     }
1424                     else if (isR2RScenario(scenario)) {
1425                         if (configuration == 'Release' || configuration == 'Checked') {
1426                             def displayStr = getR2RDisplayName(scenario)
1427                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1428                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1429                         }
1430                     }
1431                     else {
1432                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1433                         assert false
1434                     }
1435                     break
1436             }
1437             break
1438          // editor brace matching: }
1439         case 'x64_arm64_altjit':
1440         case 'x86_arm_altjit': // editor brace matching: {
1441             assert (os == 'Windows_NT')
1442             switch (scenario) {
1443                 case 'default':
1444                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1445                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1446                     break
1447                 default:
1448                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1449                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1450                     break
1451             }
1452             break
1453         // editor brace matching: }
1454         default:
1455             println("Unknown architecture: ${architecture}");
1456             assert false
1457             break
1458     }
1459 }
1460
1461 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1462     def buildCommands = [];
1463     def osGroup = getOSGroup(os)
1464     def lowerConfiguration = configuration.toLowerCase()
1465
1466     def priority = '1'
1467     if (scenario == 'default' && isPR == true) {
1468         priority = '0'
1469     }
1470
1471     setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly)
1472
1473     def enableCorefxTesting = isCoreFxScenario(scenario)
1474
1475     // Calculate the build steps, archival, and xunit results
1476     switch (os) {
1477         case 'Windows_NT': // editor brace matching: {
1478             switch (architecture) {
1479                 case 'x64':
1480                 case 'x86':
1481                 case 'x86_arm_altjit':
1482                 case 'x64_arm64_altjit':
1483                     def arch = architecture
1484                     def buildOpts = ''
1485                     if (architecture == 'x86_arm_altjit') {
1486                         arch = 'x86'
1487                     }
1488                     else if (architecture == 'x64_arm64_altjit') {
1489                         arch = 'x64'
1490                     }
1491
1492                     if (scenario == 'formatting') {
1493                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1494                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1495                         break
1496                     }
1497
1498                     if (scenario == 'illink') {
1499                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1500                     }
1501
1502                     // If it is a release build for windows, ensure PGO is used, else fail the build
1503                     if ((lowerConfiguration == 'release') &&
1504                         (scenario in Constants.basicScenarios) &&
1505                         (architecture != 'x86_arm_altjit') &&
1506                         (architecture != 'x64_arm64_altjit')) {
1507
1508                         buildOpts += ' -enforcepgo'
1509                     }
1510
1511                     if (enableCorefxTesting) {
1512                         buildOpts += ' skiptests';
1513                     } else {
1514                         buildOpts += " -priority=${priority}"
1515                     }
1516
1517                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1518                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1519                     // 35 characters long.
1520
1521                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1522
1523                     if (!isBuildOnly) {
1524                         def runtestArguments = ''
1525                         def testOpts = 'collectdumps'
1526
1527                         if (isR2RScenario(scenario)) {
1528
1529                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1530                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1531                             // to cause the tests to be crossgen'ed.
1532
1533                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1534                                 testOpts += ' crossgenaltjit protononjit.dll'
1535                             } else {
1536                                 testOpts += ' crossgen'
1537                             }
1538
1539                             testOpts += ' runcrossgentests'
1540                         }
1541                         else if (scenario == 'jitdiff') {
1542                             testOpts += ' jitdisasm crossgen'
1543                         }
1544                         else if (scenario == 'ilrt') {
1545                             testOpts += ' ilasmroundtrip'
1546                         }
1547                         else if (isLongGc(scenario)) {
1548                             testOpts += " ${scenario} sequential"
1549                         }
1550                         else if (scenario == 'standalone_gc') {
1551                             testOpts += ' gcname clrgc.dll'
1552                         }
1553                         else if (scenario == 'illink') {
1554                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1555                         }
1556
1557                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1558                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1559                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1560                         // which is read by the xunit harness.)
1561                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1562                         {
1563                             def timeout = 1800000
1564                             testOpts += " timeout ${timeout}"
1565                         }
1566
1567                         // If we are running a stress mode, we should write out the set of key
1568                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1569
1570                         def envScriptPath = ''
1571                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1572                             def buildCommandsStr = ''
1573                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1574                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1575
1576                             if (isJitStressScenario(scenario)) {
1577                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1578                             }
1579                             else if (isR2RStressScenario(scenario)) {
1580                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1581                             }
1582
1583                             if (architecture == 'x86_arm_altjit') {
1584                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1585                             }
1586                             else if (architecture == 'x64_arm64_altjit') {
1587                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1588                             }
1589
1590                             envScriptFinalize(os, envScriptPath)
1591
1592                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1593                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1594                             // into a single script.
1595                             buildCommands += buildCommandsStr
1596                         }
1597                         else if (architecture == 'x86_arm_altjit') {
1598                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1599                         }
1600                         else if (architecture == 'x64_arm64_altjit') {
1601                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1602                         }
1603                         if (envScriptPath != '') {
1604                             testOpts += " TestEnv ${envScriptPath}"
1605                         }
1606
1607                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1608
1609                         if (enableCorefxTesting) {
1610                             def workspaceRelativeFxRoot = "_/fx"
1611                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1612
1613                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1614
1615                             // Archive and process (only) the test results
1616                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1617                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1618
1619                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1620                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1621                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1622                         }
1623                         else if (isGcReliabilityFramework(scenario)) {
1624                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1625                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1626                         }
1627                         else {
1628                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1629                         }
1630                     }
1631
1632                     if (!enableCorefxTesting) {
1633                         // Run the rest of the build
1634                         // Build the mscorlib for the other OS's
1635                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1636                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1637
1638                         if (arch == "x64") {
1639                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1640                         }
1641
1642                         // Zip up the tests directory so that we don't use so much space/time copying
1643                         // 10s of thousands of files around.
1644                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1645
1646                         if (!isJitStressScenario(scenario)) {
1647                             // For windows, pull full test results and test drops for x86/x64.
1648                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1649                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1650                         }
1651
1652                         if (scenario == 'jitdiff') {
1653                             // retrive jit-dasm output for base commit, and run jit-diff
1654                             if (!isBuildOnly) {
1655                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1656                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1657                             }
1658                         }
1659
1660                         if (!isBuildOnly) {
1661                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1662                         }
1663                     }
1664                     break
1665                 case 'armlb':
1666                 case 'arm':
1667                     assert isArmWindowsScenario(scenario)
1668
1669                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1670                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1671
1672                     def buildArchitecture = 'arm'
1673
1674                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1675                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1676                     def armCrossgenOpt = ''
1677                     if (architecture == 'armlb') {
1678                         armCrossgenOpt = '-crossgenaltjit legacyjit.dll'
1679                     }
1680
1681                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1682                     priority = '1'
1683
1684                     // This is now a build only job. Do not run tests. Use the flow job.
1685                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1686
1687                     // Zip up the tests directory so that we don't use so much space/time copying
1688                     // 10s of thousands of files around.
1689                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1690
1691                     // Add archival.
1692                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1693                     break
1694                 case 'arm64':
1695                     assert isArmWindowsScenario(scenario)
1696
1697                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1698                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1699
1700                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1701                     priority = '1'
1702
1703                     // This is now a build only job. Do not run tests. Use the flow job.
1704                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1705
1706                     // Zip up the tests directory so that we don't use so much space/time copying
1707                     // 10s of thousands of files around.
1708                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1709
1710                     // Add archival.
1711                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1712                     break
1713                 default:
1714                     println("Unknown architecture: ${architecture}");
1715                     assert false
1716                     break
1717             }
1718             break
1719         // editor brace matching: }
1720         case 'Ubuntu':
1721         case 'Ubuntu16.04':
1722         case 'Ubuntu16.10':
1723         case 'Debian8.4':
1724         case 'OSX10.12':
1725         case 'CentOS7.1':
1726         case 'RHEL7.2':
1727         case 'Tizen':
1728         case 'Fedora24': // editor brace matching: {
1729             switch (architecture) {
1730                 case 'x64':
1731                 case 'x86':
1732                     if (architecture == 'x86' && os == 'Ubuntu') {
1733                         // build and PAL test
1734                         def dockerImage = getDockerImageName(architecture, os, true)
1735                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1736                         dockerImage = getDockerImageName(architecture, os, false)
1737                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1738                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1739                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1740                         break
1741                     }
1742
1743                     if (scenario == 'formatting') {
1744                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1745                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1746                         break
1747                     }
1748
1749                     if (scenario == 'illink') {
1750                         assert(os == 'Ubuntu')
1751                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1752                     }
1753
1754                     if (!enableCorefxTesting) {
1755                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1756                         // only on supported OS platforms.
1757                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1758                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1759
1760                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1761                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1762
1763                         // Basic archiving of the build
1764                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1765                         // And pal tests
1766                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1767                     }
1768                     else {
1769                         // Corefx stress testing
1770                         assert os == 'Ubuntu'
1771                         assert architecture == 'x64'
1772                         assert lowerConfiguration == 'checked'
1773                         assert isJitStressScenario(scenario)
1774
1775                         // Build coreclr
1776                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1777
1778                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1779
1780                         def envScriptCmds = envScriptCreate(os, scriptFileName)
1781                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
1782                         envScriptCmds += envScriptFinalize(os, scriptFileName)
1783                         buildCommands += envScriptCmds
1784
1785                         // Build and text corefx
1786                         def workspaceRelativeFxRoot = "_/fx"
1787                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1788
1789                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1790
1791                         // Archive and process (only) the test results
1792                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1793                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1794                     }
1795                     break
1796                 case 'arm64':
1797                     if (!enableCorefxTesting) {
1798                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1799
1800                         // HACK -- Arm64 does not have corefx jobs yet.
1801                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1802                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1803                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1804                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1805
1806                         // Basic archiving of the build
1807                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1808                     }
1809                     break
1810                 case 'arm':
1811                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1812                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1813
1814                     // default values for Ubuntu
1815                     def arm_abi="arm"
1816                     def linuxCodeName="trusty"
1817                     if (os == 'Ubuntu16.04') {
1818                         linuxCodeName="xenial"
1819                     }
1820                     else if (os == 'Tizen') {
1821                         arm_abi="armel"
1822                         linuxCodeName="tizen"
1823                     }
1824
1825                     // Unzip the Windows test binaries first. Exit with 0
1826                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1827
1828                     // Unpack the corefx binaries
1829                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1830                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1831                     if (os != 'Tizen') {
1832                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1833                     }
1834                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1835                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1836                     if (os == 'Ubuntu') {
1837                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1838                     }
1839
1840                     // Call the ARM CI script to cross build and test using docker
1841                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1842                     --mode=docker \\
1843                     --${arm_abi} \\
1844                     --linuxCodeName=${linuxCodeName} \\
1845                     --buildConfig=${lowerConfiguration} \\
1846                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1847                     --coreFxBinDir=./bin/CoreFxBinDir \\
1848                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1849
1850                     // Basic archiving of the build, no pal tests
1851                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1852                     break
1853                 default:
1854                     println("Unknown architecture: ${architecture}");
1855                     assert false
1856                     break
1857             }
1858             break
1859         // editor brace matching: }
1860         default:
1861             println("Unknown os: ${os}");
1862             assert false
1863             break
1864     } // os
1865
1866     return buildCommands
1867 }
1868
1869 Constants.allScenarios.each { scenario ->
1870     [true, false].each { isPR ->
1871         Constants.architectureList.each { architecture ->
1872             Constants.configurationList.each { configuration ->
1873                 Constants.osList.each { os ->
1874                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1875                     // and reset the os to Windows_NT
1876                     def isBuildOnly = false
1877                     if (os == 'Windows_NT_BuildOnly') {
1878                         isBuildOnly = true
1879                         os = 'Windows_NT'
1880                     }
1881
1882                     // Tizen is only supported for arm architecture
1883                     if (os == 'Tizen' && architecture != 'arm') {
1884                         return
1885                     }
1886
1887                     // Skip totally unimplemented (in CI) configurations.
1888                     switch (architecture) {
1889                         case 'arm64':
1890                             if (os == 'Ubuntu16.04') {
1891                                 os = 'Ubuntu'
1892                             }
1893
1894                             // Windows and Ubuntu only
1895                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1896                                 return
1897                             }
1898                             break
1899                         case 'arm':
1900                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1901                                 return
1902                             }
1903                             break
1904                         case 'armlb':
1905                             if (os != 'Windows_NT') {
1906                                 return
1907                             }
1908                             break
1909                         case 'x86':
1910                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1911                                 return
1912                             }
1913                             break
1914                         case 'x86_arm_altjit':
1915                         case 'x64_arm64_altjit':
1916                             if (os != 'Windows_NT') {
1917                                 return
1918                             }
1919                             break
1920                         case 'x64':
1921                             // Everything implemented
1922                             break
1923                         default:
1924                             println("Unknown architecture: ${architecture}")
1925                             assert false
1926                             break
1927                     }
1928
1929                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1930                     // with checked builds)
1931                     if (isJitStressScenario(scenario)) {
1932                         if (configuration != 'Checked') {
1933                             return
1934                         }
1935
1936                         // Since these are just execution time differences,
1937                         // skip platforms that don't execute the tests here (Windows_NT only)
1938                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
1939                         if (!isEnabledOS || isBuildOnly) {
1940                             return
1941                         }
1942
1943                         switch (architecture) {
1944                             case 'x64':
1945                             case 'x86':
1946                             case 'x86_arm_altjit':
1947                             case 'x64_arm64_altjit':
1948                                 // x86 ubuntu: default only
1949                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1950                                     return
1951                                 }
1952                                 // Windows: Everything implemented
1953                                 break
1954
1955                             default:
1956                                 // arm, arm64, armlb: stress is handled through flow jobs.
1957                                 return
1958                         }
1959                     }
1960                     else if (isR2RScenario(scenario)) {
1961                         if (os != 'Windows_NT') {
1962                             return
1963                         }
1964                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
1965                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
1966                             return
1967                         }
1968                     }
1969                     else {
1970                         // Skip scenarios
1971                         switch (scenario) {
1972                             case 'ilrt':
1973                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1974                                 // the default scenario build
1975                                 if (os != 'Windows_NT') {
1976                                     return
1977                                 }
1978                                 // Only x64 for now
1979                                 if (architecture != 'x64') {
1980                                     return
1981                                 }
1982                                 // Release only
1983                                 if (configuration != 'Release') {
1984                                     return
1985                                 }
1986                                 break
1987                             case 'jitdiff':
1988                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1989                                     return
1990                                 }
1991                                 if (architecture != 'x64') {
1992                                     return
1993                                 }
1994                                 if (configuration != 'Checked') {
1995                                     return
1996                                 }
1997                                 break
1998                             case 'longgc':
1999                             case 'gcsimulator':
2000                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2001                                     return
2002                                 }
2003                                 if (architecture != 'x64') {
2004                                     return
2005                                 }
2006                                 if (configuration != 'Release') {
2007                                     return
2008                                 }
2009                                 break
2010                             case 'gc_reliability_framework':
2011                             case 'standalone_gc':
2012                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2013                                     return
2014                                 }
2015
2016                                 if (architecture != 'x64') {
2017                                     return
2018                                 }
2019
2020                                 if (configuration != 'Release' && configuration != 'Checked') {
2021                                     return
2022                                 }
2023                                 break
2024                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2025                             case 'formatting':
2026                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2027                                     return
2028                                 }
2029                                 if (architecture != 'x64') {
2030                                     return
2031                                 }
2032                                 if (configuration != 'Checked') {
2033                                     return
2034                                 }
2035                                 if (isBuildOnly) {
2036                                     return
2037                                 }
2038                                 break
2039                             case 'illink':
2040                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2041                                     return
2042                                 }
2043                                 if (architecture != 'x64' && architecture != 'x86') {
2044                                     return
2045                                 }
2046                                 if (isBuildOnly) {
2047                                     return
2048                                 }
2049                                 break
2050                             case 'default':
2051                                 // Nothing skipped
2052                                 break
2053                             default:
2054                                 println("Unknown scenario: ${scenario}")
2055                                 assert false
2056                                 break
2057                         }
2058                     }
2059
2060                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2061                     // runtime behavior, not compile-time behavior, are not interesting.
2062                     switch (architecture) {
2063                         case 'x86_arm_altjit':
2064                         case 'x64_arm64_altjit':
2065                             if (isGCStressRelatedTesting(scenario)) {
2066                                 return
2067                             }
2068                             break
2069                         default:
2070                             break
2071                     }
2072
2073                     // Calculate names
2074                     def lowerConfiguration = configuration.toLowerCase()
2075                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2076                     def folderName = getJobFolder(scenario)
2077
2078                     // Create the new job
2079                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2080
2081                     def machineAffinityOptions = null
2082
2083                     if (os != 'Windows_NT') {
2084                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2085                     }
2086                     else {
2087                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2088                     }
2089
2090                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2091
2092                     // Add all the standard options
2093                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2094                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2095
2096                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2097                     def osGroup = getOSGroup(os)
2098
2099                     newJob.with {
2100                         steps {
2101                             if (os == 'Windows_NT') {
2102                                 buildCommands.each { buildCommand ->
2103                                     batchFile(buildCommand)
2104                                 }
2105                             }
2106                             else {
2107                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2108                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2109                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2110                                     // Define the Windows Tests and Corefx build job names
2111                                     def WindowsTestsName = projectFolder + '/' +
2112                                                            Utilities.getFullJobName(project,
2113                                                                                     getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'default', true),
2114                                                                                     false)
2115                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2116                                                        Utilities.getFolderName(branch)
2117
2118                                     // Copy the Windows test binaries and the Corefx build binaries
2119                                     copyArtifacts(WindowsTestsName) {
2120                                         includePatterns('bin/tests/tests.zip')
2121                                         buildSelector {
2122                                             latestSuccessful(true)
2123                                         }
2124                                     }
2125
2126                                     def arm_abi = 'arm'
2127                                     def corefx_os = 'linux'
2128                                     if (os == 'Tizen') {
2129                                         arm_abi = 'armel'
2130                                         corefx_os = 'tizen'
2131                                     }
2132
2133                                     // Let's use release CoreFX to test checked CoreCLR,
2134                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2135                                     def corefx_lowerConfiguration = lowerConfiguration
2136                                     if ( lowerConfiguration == 'checked' ) {
2137                                         corefx_lowerConfiguration='release'
2138                                     }
2139
2140                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2141                                         includePatterns('bin/build.tar.gz')
2142                                         buildSelector {
2143                                             latestSuccessful(true)
2144                                         }
2145                                     }
2146                                 }
2147
2148                                 buildCommands.each { buildCommand ->
2149                                     shell(buildCommand)
2150                                 }
2151                             }
2152                         }
2153                     } // newJob.with
2154
2155                 } // os
2156             } // configuration
2157         } // architecture
2158     } // isPR
2159 } // scenario
2160
2161
2162 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2163 Constants.allScenarios.each { scenario ->
2164     [true, false].each { isPR ->
2165         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2166             Constants.crossList.each { os ->
2167                 if (architecture == 'arm64') {
2168                     if (os != "Ubuntu" && os != "Windows_NT") {
2169                         return
2170                     }
2171                 } else if (architecture == 'arm' || architecture == 'armlb') {
2172                     if (os != 'Windows_NT') {
2173                         return
2174                     }
2175                 }
2176                 else if (architecture == 'x86') {
2177                     if (os != "Ubuntu") {
2178                         return
2179                     }
2180                 }
2181
2182                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2183
2184                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2185                     return
2186                 }
2187
2188                 Constants.configurationList.each { configuration ->
2189
2190                     // First, filter based on OS.
2191
2192                     if (os == 'Windows_NT') {
2193                         if (!isArmWindowsScenario(scenario)) {
2194                             return
2195                         }
2196                     }
2197                     else {
2198                         // Non-Windows
2199                         if (architecture == 'arm64') {
2200                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2201                                 return
2202                             }
2203                         }
2204                         else if (architecture == 'x86') {
2205                             // Linux/x86 only want default test
2206                             if (scenario != 'default') {
2207                                 return
2208                             }
2209                         }
2210                     }
2211
2212                     // For CentOS, we only want Checked/Release builds.
2213                     if (os == 'CentOS7.1') {
2214                         if (configuration != 'Checked' && configuration != 'Release') {
2215                             return
2216                         }
2217                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2218                             return
2219                         }
2220                     }
2221
2222                     // For RedHat and Debian, we only do Release builds.
2223                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2224                         if (configuration != 'Release') {
2225                             return
2226                         }
2227                         if (scenario != 'default') {
2228                             return
2229                         }
2230                     }
2231
2232                     // Next, filter based on scenario.
2233
2234                     if (isJitStressScenario(scenario)) {
2235                         if (configuration != 'Checked') {
2236                             return
2237                         }
2238                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2239                         if (isCoreFxScenario(scenario)) {
2240                             return
2241                         }
2242                     }
2243                     else if (isR2RBaselineScenario(scenario)) {
2244                         if (configuration != 'Checked' && configuration != 'Release') {
2245                             return
2246                         }
2247                     }
2248                     else if (isR2RStressScenario(scenario)) {
2249                         if (configuration != 'Checked') {
2250                             return
2251                         }
2252                     }
2253                     else {
2254                         // Skip scenarios
2255                         switch (scenario) {
2256                             case 'ilrt':
2257                             case 'longgc':
2258                             case 'gcsimulator':
2259                                 // Long GC tests take a long time on non-Release builds
2260                                 // ilrt is also Release only
2261                                 if (configuration != 'Release') {
2262                                     return
2263                                 }
2264                                 break
2265                             case 'jitdiff':
2266                                 if (configuration != 'Checked') {
2267                                     return;
2268                                 }
2269                                 break
2270                             case 'gc_reliability_framework':
2271                             case 'standalone_gc':
2272                                 if (configuration != 'Release' && configuration != 'Checked') {
2273                                     return
2274                                 }
2275                                 break
2276                             case 'formatting':
2277                                 return
2278                             case 'illink':
2279                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2280                                     return
2281                                 }
2282                                 break
2283                             case 'default':
2284                                 // Nothing skipped
2285                                 break
2286                             default:
2287                                 println("Unknown scenario: ${scenario}")
2288                                 assert false
2289                                 break
2290                         }
2291                     }
2292
2293                     // Done filtering. Now, create the jobs.
2294
2295                     def lowerConfiguration = configuration.toLowerCase()
2296                     def osGroup = getOSGroup(os)
2297                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2298
2299                     def inputCoreCLRBuildName = projectFolder + '/' +
2300                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2301
2302                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2303                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2304                     def testBuildScenario = scenario
2305                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2306                         testBuildScenario = 'default'
2307                     }
2308
2309                     def inputWindowsTestBuildArch = architecture
2310                     if (architecture == "arm64" && os != "Windows_NT") {
2311                         // Use the x64 test build for arm64 unix
2312                         inputWindowsTestBuildArch = "x64"
2313                     }
2314
2315                     def inputWindowsTestsBuildName = ""
2316
2317                     if (isJitStressScenario(scenario)) {
2318                         inputWindowsTestsBuildName = projectFolder + '/' +
2319                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2320                     } else {
2321                         inputWindowsTestsBuildName = projectFolder + '/' +
2322                             Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2323
2324                     }
2325
2326
2327                     // Enable Server GC for Ubuntu PR builds
2328                     def serverGCString = ''
2329                     if (os == 'Ubuntu' && isPR) {
2330                         serverGCString = '--useServerGC'
2331                     }
2332
2333                     def testOpts = ''
2334
2335                     if (isR2RScenario(scenario)) {
2336
2337                         testOpts += ' --crossgen --runcrossgentests'
2338
2339                         if (scenario == 'r2r_jitstress1') {
2340                             testOpts += ' --jitstress=1'
2341                         }
2342                         else if (scenario == 'r2r_jitstress2') {
2343                             testOpts += ' --jitstress=2'
2344                         }
2345                         else if (scenario == 'r2r_jitstressregs1') {
2346                             testOpts += ' --jitstressregs=1'
2347                         }
2348                         else if (scenario == 'r2r_jitstressregs2') {
2349                             testOpts += ' --jitstressregs=2'
2350                         }
2351                         else if (scenario == 'r2r_jitstressregs3') {
2352                             testOpts += ' --jitstressregs=3'
2353                         }
2354                         else if (scenario == 'r2r_jitstressregs4') {
2355                             testOpts += ' --jitstressregs=4'
2356                         }
2357                         else if (scenario == 'r2r_jitstressregs8') {
2358                             testOpts += ' --jitstressregs=8'
2359                         }
2360                         else if (scenario == 'r2r_jitstressregs0x10') {
2361                             testOpts += ' --jitstressregs=0x10'
2362                         }
2363                         else if (scenario == 'r2r_jitstressregs0x80') {
2364                             testOpts += ' --jitstressregs=0x80'
2365                         }
2366                         else if (scenario == 'r2r_jitstressregs0x1000') {
2367                             testOpts += ' --jitstressregs=0x1000'
2368                         }
2369                         else if (scenario == 'r2r_jitminopts') {
2370                             testOpts += ' --jitminopts'
2371                         }
2372                         else if (scenario == 'r2r_jitforcerelocs') {
2373                             testOpts += ' --jitforcerelocs'
2374                         }
2375                         else if (scenario == 'r2r_gcstress15') {
2376                             testOpts += ' --gcstresslevel=0xF'
2377                         }
2378                     }
2379                     else if (scenario == 'jitdiff') {
2380                         testOpts += ' --jitdisasm --crossgen'
2381                     }
2382                     else if (scenario == 'illink') {
2383                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2384                     }
2385                     else if (isLongGc(scenario)) {
2386                         // Long GC tests behave very poorly when they are not
2387                         // the only test running (many of them allocate until OOM).
2388                         testOpts += ' --sequential'
2389
2390                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2391                         // for running long GC and GCSimulator tests, respectively. We don't use them
2392                         // here because using a playlist file produces much more readable output on the CI machines
2393                         // and reduces running time.
2394                         //
2395                         // The Long GC playlist contains all of the tests that are
2396                         // going to be run. The GCSimulator playlist contains all of
2397                         // the GC simulator tests.
2398                         if (scenario == 'longgc') {
2399                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2400                         }
2401                         else if (scenario == 'gcsimulator') {
2402                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2403                         }
2404                     }
2405                     else if (isGcReliabilityFramework(scenario)) {
2406                         testOpts += ' --build-overlay-only'
2407                     }
2408                     else if (scenario == 'standalone_gc') {
2409                         if (osGroup == 'OSX') {
2410                             testOpts += ' --gcname=libclrgc.dylib'
2411                         }
2412                         else if (osGroup == 'Linux') {
2413                             testOpts += ' --gcname=libclrgc.so'
2414                         }
2415                         else {
2416                             println("Unexpected OS group: ${osGroup} for os ${os}")
2417                             assert false
2418                         }
2419                     }
2420
2421                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2422
2423                     def folder = getJobFolder(scenario)
2424                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2425                         // Add parameters for the inputs
2426
2427                         if (windowsArmJob == true) {
2428                             parameters {
2429                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2430                             }
2431                         }
2432                         else {
2433                             parameters {
2434                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2435                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2436                             }
2437                         }
2438
2439                         steps {
2440                             // Set up the copies
2441
2442                             // Coreclr build containing the tests and mscorlib
2443                             // pri1 jobs still need to copy windows_nt built tests
2444                             if (windowsArmJob != true) {
2445                                 copyArtifacts(inputWindowsTestsBuildName) {
2446                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2447                                     buildSelector {
2448                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2449                                     }
2450                                 }
2451                             }
2452
2453                             // Coreclr build we are trying to test
2454                             //
2455                             //  ** NOTE ** This will, correctly, overwrite over the CORE_ROOT from the windows test archive
2456
2457                             copyArtifacts(inputCoreCLRBuildName) {
2458                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2459                                 buildSelector {
2460                                     buildNumber('${CORECLR_BUILD}')
2461                                 }
2462                             }
2463
2464                             // Windows CoreCLR Arm(64) will restore corefx
2465                             // packages correctly.
2466                             //
2467                             // In addition, test steps are entirely different
2468                             // because we do not have a unified runner
2469                             if (windowsArmJob != true) {
2470                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2471
2472                                 // HACK -- Arm64 does not have corefx jobs yet.
2473                                 // Clone corefx and build the native packages overwriting the x64 packages.
2474                                 if (architecture == 'arm64') {
2475                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2476                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2477                                 }
2478                                 else if (architecture == 'x86') {
2479                                     shell("mkdir ./bin/CoreFxNative")
2480
2481                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2482                                         includePatterns('bin/build.tar.gz')
2483                                         targetDirectory('bin/CoreFxNative')
2484                                         buildSelector {
2485                                             latestSuccessful(true)
2486                                         }
2487                                     }
2488
2489                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2490                                 }
2491
2492                                 // Unzip the tests first.  Exit with 0
2493                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2494                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2495
2496                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2497
2498                                 // Execute the tests
2499                                 def runDocker = isNeedDocker(architecture, os, false)
2500                                 def dockerPrefix = ""
2501                                 def dockerCmd = ""
2502                                 if (runDocker) {
2503                                     def dockerImage = getDockerImageName(architecture, os, false)
2504                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2505                                     dockerCmd = dockerPrefix + "${dockerImage} "
2506                                 }
2507
2508                                 // If we are running a stress mode, we'll set those variables first
2509                                 def testEnvOpt = ""
2510                                 if (isJitStressScenario(scenario)) {
2511                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2512                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2513                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2514                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2515                                     shell("${envScriptCmds}")
2516                                     testEnvOpt = "--test-env=" + scriptFileName
2517                                 }
2518
2519                                 if (isGCStressRelatedTesting(scenario)) {
2520                                     shell('./init-tools.sh')
2521                                 }
2522
2523                                 shell("""${dockerCmd}./tests/runtest.sh \\
2524                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2525                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2526                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2527                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2528
2529                                 if (isGcReliabilityFramework(scenario)) {
2530                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2531                                     if (serverGCString != '') {
2532                                         if (runDocker) {
2533                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2534                                         }
2535                                         else {
2536                                             shell("export COMPlus_gcServer=1")
2537                                         }
2538                                     }
2539
2540                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2541                                 }
2542                             }
2543
2544                             else { // windowsArmJob == true
2545                                 // Unzip tests.
2546                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2547
2548                                 // Build the build commands
2549                                 def buildCommands = ""
2550
2551                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2552                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2553                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2554
2555                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2556                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2557
2558                                 // For all jobs
2559                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2560
2561                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2562                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2563
2564                                 // ARM legacy backend; this is an altjit.
2565                                 if (architecture == "armlb") {
2566                                     addEnvVariable("COMPlus_AltJit", "*")
2567                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2568                                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2569                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2570                                 }
2571
2572                                 // If we are running a stress mode, we'll set those variables as well
2573                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2574                                     def stressValues = null
2575                                     if (isJitStressScenario(scenario)) {
2576                                         stressValues = Constants.jitStressModeScenarios[scenario]
2577                                     }
2578                                     else {
2579                                         stressValues = Constants.r2rStressScenarios[scenario]
2580                                     }
2581
2582                                     stressValues.each { key, value ->
2583                                         addEnvVariable(key, value)
2584                                     }
2585                                 }
2586
2587                                 if (isR2RScenario(scenario)) {
2588                                     // Crossgen the framework assemblies.
2589                                     buildCommands += """
2590 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2591 @goto skip_PrecompileAssembly
2592
2593 :PrecompileAssembly
2594 @REM Skip mscorlib since it is already precompiled.
2595 @if /I "%3" == "mscorlib.dll" exit /b 0
2596 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2597
2598 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2599 @if "%errorlevel%" == "-2146230517" (
2600     echo %2 is not a managed assembly.
2601 ) else if "%errorlevel%" == "-2146234344" (
2602     echo %2 is not a managed assembly.
2603 ) else if %errorlevel% neq 0 (
2604     echo Unable to precompile %2
2605 ) else (
2606     echo Precompiled %2
2607 )
2608 @exit /b 0
2609
2610 :skip_PrecompileAssembly
2611 """
2612
2613                                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2614                                     // crossgen on tests before running them.
2615                                     addEnvVariable("RunCrossGen", "true")
2616                                 }
2617
2618                                 // Create the smarty command
2619                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2620                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2621                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2622
2623                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2624
2625                                 if (architecture == "armlb") {
2626                                     addExclude("LEGACYJIT_FAIL")
2627                                 }
2628
2629                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2630                                     def failTag = "JITSTRESS_FAIL"
2631                                     def excludeTag = "JITSTRESS_EXCLUDE"
2632
2633                                     if (scenario.contains('gc')) {
2634                                         failTag = "GCSTRESS_FAIL"
2635                                         excludeTag = "GCSTRESS_EXCLUDE"
2636                                     }
2637
2638                                     addArchSpecificExclude(architecture, failTag)
2639                                     addArchSpecificExclude(architecture, excludeTag)
2640                                 }
2641                                 else {
2642                                     addExclude("pri1")
2643                                 }
2644
2645                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2646                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2647                                 addExclude("LONG_RUNNING")
2648
2649                                 smartyCommand += "/lstFile Tests.lst"
2650
2651                                 def testListArch = [
2652                                     'arm64': 'arm64',
2653                                     'arm': 'arm',
2654                                     'armlb': 'arm'
2655                                 ]
2656
2657                                 def archLocation = testListArch[architecture]
2658
2659                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2660                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2661                                 addCommand("${smartyCommand}")
2662
2663                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2664                                 // However, we also need to remove all the variables that were set during this batch file, so we
2665                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2666                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2667                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2668                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2669                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2670                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2671
2672                                 // ZIP up the smarty output, no matter what the smarty result.
2673                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2674
2675                                 addCommand("echo %errorlevel%")
2676                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2677
2678                                 // Use the smarty errorlevel as the script errorlevel.
2679                                 addCommand("exit /b %__save_smarty_errorlevel%")
2680
2681                                 batchFile(buildCommands)
2682                             }
2683                         }
2684                     }
2685
2686                     if (scenario == 'jitdiff') {
2687                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2688                     }
2689
2690                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2691                     if (os in ['Ubuntu']) {
2692                         SummaryBuilder summaries = new SummaryBuilder()
2693                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2694                         summaries.emit(newJob)
2695                     }
2696
2697                     def affinityOptions = null
2698
2699                     if (windowsArmJob == true) {
2700                         affinityOptions = [
2701                             "use_arm64_build_machine" : false
2702                         ]
2703                     }
2704
2705                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2706                         affinityOptions = [
2707                             "large_pages" : false
2708                         ]
2709                     }
2710
2711                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2712                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2713
2714                     setJobTimeout(newJob, isPR, architecture, configuration, scenario, false)
2715
2716                     if (windowsArmJob != true) {
2717                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2718                     }
2719                     else {
2720                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2721
2722                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2723                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2724                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2725                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2726                     }
2727
2728                     // Create a build flow to join together the build and tests required to run this test.
2729                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2730                     // Linux CoreCLR test
2731                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2732                     def fullTestJobName = projectFolder + '/' + newJob.name
2733                     // Add a reference to the input jobs for report purposes
2734                     JobReport.Report.addReference(inputCoreCLRBuildName)
2735                     JobReport.Report.addReference(inputWindowsTestsBuildName)
2736                     JobReport.Report.addReference(fullTestJobName)
2737                     def newFlowJob = null
2738
2739                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2740                         // Do not create the flow job for RHEL jobs.
2741                         return
2742                     }
2743
2744                     // For pri0 jobs we can build tests on unix
2745                     if (windowsArmJob) {
2746                         // For Windows arm jobs there is no reason to build a parallel test job.
2747                         // The product build supports building and archiving the tests.
2748
2749                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2750                         buildFlow("""
2751 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2752
2753 // And then build the test build
2754 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2755 """)
2756                         }
2757                     }
2758                     else {
2759                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2760                         buildFlow("""
2761 // Build the input jobs in parallel
2762 parallel (
2763 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2764 { windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') }
2765 )
2766
2767 // And then build the test build
2768 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2769             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2770 """)
2771                         }
2772                     }
2773
2774                     // For the flow jobs set the machine affinity as x64 if an armarch.
2775                     def flowArch = architecture
2776
2777                     if (flowArch in validWindowsNTCrossArches) {
2778                         flowArch = 'x64'
2779                         affinityOptions = null
2780                     }
2781
2782                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2783                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2784                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2785                 } // configuration
2786             } // os
2787         } // architecture
2788     } // isPR
2789 } // scenario
2790
2791 JobReport.Report.generateJobReport(out)
2792
2793 // Make the call to generate the help job
2794 Utilities.createHelperJob(this, project, branch,
2795     "Welcome to the ${project} Repository",  // This is prepended to the help message
2796     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2797
2798 Utilities.addCROSSCheck(this, project, branch)