netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'corefx_baseline'                : [ : ], // corefx baseline
  90                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  91                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  92                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  93                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  94                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  95                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  96                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  97                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  98                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  99                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 100                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 101                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 102                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 103                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 104                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 105                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 107                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 109                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 110                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 111                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 112     ]
 113
 114     // This is a set of ReadyToRun stress scenarios
 115     def static r2rStressScenarios = [
 116                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 117                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 118                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 119                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 120                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 121                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 122                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 123                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 124                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 125                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 126                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 127                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 128                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 129     ]
 130
 131     // This is the basic set of scenarios
 132     def static basicScenarios = [
 133                'default',
 134                'ilrt',
 135                'r2r',
 136                'longgc',
 137                'formatting',
 138                'gcsimulator',
 139                'jitdiff',
 140                'standalone_gc',
 141                'gc_reliability_framework',
 142                'illink']
 143
 144     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 145
 146     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 147     // to Tests.lst file categories to exclude.
 148     //
 149     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 150     // and with the same values, with some commented out, for easier maintenance.
 151     //
 152     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 153     //
 154     def static validArmWindowsScenarios = [
 155                'default':                                [],
 156                // 'ilrt'
 157                'r2r':                                    ["R2R_FAIL"],
 158                // 'longgc'
 159                // 'formatting'
 160                // 'gcsimulator'
 161                // 'jitdiff'
 162                // 'standalone_gc'
 163                // 'gc_reliability_framework'
 164                // 'illink'
 165                'r2r_jitstress1':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 166                'r2r_jitstress2':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 167                'r2r_jitstressregs1':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 168                'r2r_jitstressregs2':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 169                'r2r_jitstressregs3':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 170                'r2r_jitstressregs4':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 171                'r2r_jitstressregs8':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 172                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 173                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 174                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 175                'r2r_jitminopts':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 176                'r2r_jitforcerelocs':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 177                'r2r_gcstress15':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 179                'tieredcompilation':                      [],
 180                'forcerelocs':                            [],
 181                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 196                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 197                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 198                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 199                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 200                // 'jitsse2only'                          // Only relevant to xarch
 201                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 202                // 'corefx_baseline'
 203                // 'corefx_minopts'
 204                // 'corefx_tieredcompilation'
 205                // 'corefx_jitstress1'
 206                // 'corefx_jitstress2'
 207                // 'corefx_jitstressregs1'
 208                // 'corefx_jitstressregs2'
 209                // 'corefx_jitstressregs3'
 210                // 'corefx_jitstressregs4'
 211                // 'corefx_jitstressregs8'
 212                // 'corefx_jitstressregs0x10'
 213                // 'corefx_jitstressregs0x80'
 214                // 'corefx_jitstressregs0x1000'
 215                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 216                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 217                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'heapverify1':                            [],
 219                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 220                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 221                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 222                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 223                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 224                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 225
 226                //
 227                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 228                //
 229
 230                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 231                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 249     ]
 250
 251     def static configurationList = ['Debug', 'Checked', 'Release']
 252
 253     // This is the set of architectures
 254     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 255 }
 256
 257 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 258     assert os instanceof String
 259     assert architecture instanceof String
 260
 261     def armArches = ['arm', 'armlb', 'arm64']
 262     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 263
 264     if (!(architecture in armArches)) {
 265         assert options == null
 266         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 267
 268         return
 269     }
 270
 271     // This is an arm(64) job.
 272     //
 273     // There are several options.
 274     //
 275     // Windows_NT
 276     //
 277     // Arm32 (Build) -> latest-arm64
 278     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 279     // Arm32 (Test)  -> arm64-windows_nt
 280     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 281     //
 282     // Arm64 (Build) -> latest-arm64
 283     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 284     // Arm64 (Test)  -> arm64-windows_nt
 285     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 286     //
 287     // Ubuntu
 288     //
 289     // Arm32 (Build) -> arm-cross-latest
 290     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 291     // Arm32 (Test)  -> NYI Arch not supported
 292     //       |->
 293     //
 294     // Arm64 (Build) -> arm64-cross-latest
 295     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 296     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 297     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 298     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 299     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 300
 301     // This has to be a arm arch
 302     assert architecture in armArches
 303     if (os == "Windows_NT") {
 304         // Arm(64) windows jobs share the same machines for now
 305         def isBuild = options['use_arm64_build_machine'] == true
 306
 307         if (isBuild == true) {
 308             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 309         } else {
 310             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 311         }
 312     } else {
 313         assert os != 'Windows_NT'
 314         assert os in supportedArmLinuxOs
 315
 316         if (architecture == 'arm' || architecture == 'armlb') {
 317             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 318         } else {
 319             // Arm64 Linux
 320             if (options['is_build_only'] == true) {
 321                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 322             } else {
 323                 // Arm64 Test Machines
 324                 if (options['large_pages'] == false) {
 325                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 326                 } else {
 327                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 328                 }
 329             }
 330         }
 331     }
 332 }
 333
 334 def static isGCStressRelatedTesting(def scenario) {
 335     // The 'r2r_gcstress15' scenario is a basic scenario.
 336     // Detect it and make it a GCStress related.
 337     if (scenario == 'r2r_gcstress15')
 338     {
 339         return true;
 340     }
 341
 342     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 343     def scenarioName = scenario.toLowerCase()
 344     def isGCStressTesting = false
 345     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 346         if (k in gcStressTestEnvVars) {
 347             isGCStressTesting = true;
 348         }
 349     }
 350     return isGCStressTesting
 351 }
 352
 353 def static isCoreFxScenario(def scenario) {
 354     def corefx_prefix = 'corefx_'
 355     if (scenario.length() < corefx_prefix.length()) {
 356         return false
 357     }
 358     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 359 }
 360
 361 def static isR2RBaselineScenario(def scenario) {
 362     return (scenario == 'r2r')
 363 }
 364
 365 def static isR2RStressScenario(def scenario) {
 366     return Constants.r2rStressScenarios.containsKey(scenario)
 367 }
 368
 369 def static isR2RScenario(def scenario) {
 370     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 371 }
 372
 373 def static isJitStressScenario(def scenario) {
 374     return Constants.jitStressModeScenarios.containsKey(scenario)
 375 }
 376
 377 def static isLongGc(def scenario) {
 378     return (scenario == 'longgc' || scenario == 'gcsimulator')
 379 }
 380
 381 def static isJitDiff(def scenario) {
 382     return (scenario == 'jitdiff')
 383 }
 384
 385 def static isGcReliabilityFramework(def scenario) {
 386     return (scenario == 'gc_reliability_framework')
 387 }
 388
 389 def static isArmWindowsScenario(def scenario) {
 390     return Constants.validArmWindowsScenarios.containsKey(scenario)
 391 }
 392
 393 def static setTestJobTimeOut(newJob, isPR, architecture, configuration, scenario) {
 394     // 2 hours (120 minutes) is the default timeout
 395     def timeout = 120
 396
 397     if (isGCStressRelatedTesting(scenario)) {
 398         timeout = 4320
 399     }
 400     else if (isCoreFxScenario(scenario)) {
 401         timeout = 360
 402     }
 403     else if (isJitStressScenario(scenario)) {
 404         timeout = 240
 405     }
 406     else if (isR2RBaselineScenario(scenario)) {
 407         timeout = 240
 408     }
 409     else if (isLongGc(scenario)) {
 410         timeout = 1440
 411     }
 412     else if (isJitDiff(scenario)) {
 413         timeout = 240
 414     }
 415     else if (isGcReliabilityFramework(scenario)) {
 416         timeout = 1440
 417     }
 418     else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 419         timeout = 240
 420     }
 421     else if (!(scenario == 'default' && isPR == true)) {
 422         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 423         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 424         timeout = 240
 425     }
 426
 427     if (configuration == 'Debug') {
 428         // Debug runs can be very slow. Add an hour.
 429         timeout += 60
 430     }
 431
 432     // If we've changed the timeout from the default, set it in the job.
 433
 434     if (timeout != 120) {
 435         Utilities.setJobTimeout(newJob, timeout)
 436     }
 437 }
 438
 439 def static getJobFolder(def scenario) {
 440     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 441         return 'jitstress'
 442     }
 443     if (scenario == 'illink') {
 444         return 'illink'
 445     }
 446     return ''
 447 }
 448
 449 def static getStressModeDisplayName(def scenario) {
 450     def displayStr = ''
 451     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 452         def prefixLength = 'COMPlus_'.length()
 453         if (k.length() >= prefixLength) {
 454             def modeName = k.substring(prefixLength, k.length())
 455             displayStr += ' ' + modeName + '=' + v
 456         }
 457     }
 458
 459     if (isCoreFxScenario(scenario)) {
 460         displayStr = ('CoreFx ' + displayStr).trim()
 461     }
 462
 463     return displayStr
 464 }
 465
 466 def static getR2RDisplayName(def scenario) {
 467     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 468     def displayStr = scenario
 469     def prefixLength = 'r2r_'.length()
 470     if (displayStr.length() >= prefixLength) {
 471         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 472     } else if (scenario == 'r2r') {
 473         displayStr = "R2R"
 474     }
 475     return displayStr
 476 }
 477
 478 //
 479 // Functions to create an environment script.
 480 //      envScriptCreate -- initialize the script (call first)
 481 //      envScriptFinalize -- finalize the script (call last)
 482 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 483 //      envScriptAppendExistingScript -- append an existing script to the generated script
 484 //
 485 // Each script returns a string of commands. Concatenate all the strings together before
 486 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 487 //
 488
 489 // Initialize the environment setting script.
 490 def static envScriptCreate(def os, def stepScriptLocation) {
 491     def stepScript = ''
 492     if (os == 'Windows_NT') {
 493         stepScript += "echo Creating TestEnv script\r\n"
 494         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 495
 496         // Create at least an empty script.
 497         stepScript += "echo. > ${stepScriptLocation}\r\n"
 498     }
 499     else {
 500         stepScript += "echo Creating environment setting script\n"
 501         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 502     }
 503
 504     return stepScript
 505 }
 506
 507 // Generates the string for setting stress mode variables.
 508 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 509     def stepScript = ''
 510     if (os == 'Windows_NT') {
 511         stressModeVars.each{ k, v ->
 512             // Write out what we are writing to the script file
 513             stepScript += "echo Setting ${k}=${v}\r\n"
 514             // Write out the set itself to the script file`
 515             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 516         }
 517     }
 518     else {
 519         stressModeVars.each{ k, v ->
 520             // Write out what we are writing to the script file
 521             stepScript += "echo Setting ${k}=${v}\n"
 522             // Write out the set itself to the script file`
 523             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 524         }
 525     }
 526
 527     return stepScript
 528 }
 529
 530 // Append an existing script to an environment script.
 531 // Returns string of commands to do this.
 532 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 533     assert (os == 'Windows_NT')
 534     def stepScript = ''
 535
 536     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 537     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 538
 539     return stepScript
 540 }
 541
 542 // Finalize an environment setting script.
 543 // Returns string of commands to do this.
 544 def static envScriptFinalize(def os, def stepScriptLocation) {
 545     def stepScript = ''
 546
 547     if (os == 'Windows_NT') {
 548         // Display the resulting script. This is useful when looking at the output log file.
 549         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 550         stepScript += "type ${stepScriptLocation}\r\n"
 551     }
 552     else {
 553         stepScript += "chmod +x ${stepScriptLocation}\n"
 554     }
 555
 556     return stepScript
 557 }
 558
 559 def static isNeedDocker(def architecture, def os, def isBuild) {
 560     if (isBuild) {
 561         if (architecture == 'x86' && os == 'Ubuntu') {
 562             return true
 563         }
 564         else if (architecture == 'arm') {
 565             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 566                 return true
 567             }
 568         }
 569     }
 570     else {
 571         if (architecture == 'x86' && os == 'Ubuntu') {
 572             return true
 573         }
 574     }
 575     return false
 576 }
 577
 578 def static getDockerImageName(def architecture, def os, def isBuild) {
 579     // We must change some docker private images to official later
 580     if (isBuild) {
 581         if (architecture == 'x86' && os == 'Ubuntu') {
 582             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 583         }
 584         else if (architecture == 'arm') {
 585             if (os == 'Ubuntu') {
 586                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 587             }
 588             else if (os == 'Ubuntu16.04') {
 589                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 590             }
 591             else if (os == 'Tizen') {
 592                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 593             }
 594         }
 595     }
 596     else {
 597         if (architecture == 'x86' && os == 'Ubuntu') {
 598             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 599         }
 600     }
 601     println("Unknown architecture to use docker: ${architecture} ${os}");
 602     assert false
 603 }
 604
 605 // Calculates the name of the build job based on some typical parameters.
 606 //
 607 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 608     // If the architecture is x64, do not add that info into the build name.
 609     // Need to change around some systems and other builds to pick up the right builds
 610     // to do that.
 611
 612     def suffix = scenario != 'default' ? "_${scenario}" : '';
 613     if (isBuildOnly) {
 614         suffix += '_bld'
 615     }
 616     def baseName = ''
 617     switch (architecture) {
 618         case 'x64':
 619             if (scenario == 'default') {
 620                 // For now we leave x64 off of the name for compatibility with other jobs
 621                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 622             }
 623             else if (scenario == 'formatting') {
 624                 // we don't care about the configuration for the formatting job. It runs all configs
 625                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 626             }
 627             else {
 628                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 629             }
 630             break
 631         case 'arm64':
 632             if (os.toLowerCase() == "windows_nt") {
 633                 // These are cross builds
 634                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 635             }
 636             else {
 637                 // Defaults to a small page size set of machines.
 638                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 639             }
 640             break
 641         case 'arm':
 642             // These are cross builds
 643             if (os == 'Tizen') {
 644                 // ABI: softfp
 645                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 646             }
 647             else {
 648                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 649             }
 650             break
 651         case 'armlb':
 652             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 653             break
 654         case 'x86':
 655         case 'x86_arm_altjit':
 656         case 'x64_arm64_altjit':
 657             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 658             break
 659         default:
 660             println("Unknown architecture: ${architecture}");
 661             assert false
 662             break
 663     }
 664
 665     return baseName + suffix
 666 }
 667
 668 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 669
 670     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 671     if (os == 'Windows_NT') {
 672         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 673             if (!(branch in Constants.WindowsArm64Branches)) {
 674                 return
 675             }
 676         }
 677     }
 678
 679     // Check scenario.
 680     switch (scenario) {
 681         case 'default':
 682             switch (architecture) {
 683                 case 'x64':
 684                 case 'x86':
 685                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 686                         Utilities.addPeriodicTrigger(job, '@daily')
 687                     }
 688                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 689                         Utilities.addGithubPushTrigger(job)
 690                     }
 691                     break
 692                 case 'arm':
 693                 case 'armlb':
 694                 case 'x86_arm_altjit':
 695                 case 'x64_arm64_altjit':
 696                     Utilities.addGithubPushTrigger(job)
 697                     break
 698                 case 'arm64':
 699                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 700                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 701                     break
 702                 default:
 703                     println("Unknown architecture: ${architecture}");
 704                     assert false
 705                     break
 706             }
 707             break
 708         case 'r2r':
 709             assert !(os in bidailyCrossList)
 710             // r2r gets a push trigger for checked/release
 711             if (configuration == 'Checked' || configuration == 'Release') {
 712                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 713                 if (architecture == 'x64' && os != 'OSX10.12') {
 714                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 715                     if (isFlowJob || os == 'Windows_NT') {
 716                         Utilities.addGithubPushTrigger(job)
 717                     }
 718                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 719                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 720                     if (isFlowJob) {
 721                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 722                     }
 723                 }
 724                 // For x86, only add per-commit jobs for Windows
 725                 else if (architecture == 'x86') {
 726                     if (os == 'Windows_NT') {
 727                         Utilities.addGithubPushTrigger(job)
 728                     }
 729                 }
 730                 // arm64 r2r jobs should only run daily.
 731                 else if (architecture == 'arm64') {
 732                     if (os == 'Windows_NT') {
 733                         Utilities.addPeriodicTrigger(job, '@daily')
 734                     }
 735                 }
 736             }
 737             break
 738         case 'r2r_jitstress1':
 739         case 'r2r_jitstress2':
 740         case 'r2r_jitstressregs1':
 741         case 'r2r_jitstressregs2':
 742         case 'r2r_jitstressregs3':
 743         case 'r2r_jitstressregs4':
 744         case 'r2r_jitstressregs8':
 745         case 'r2r_jitstressregs0x10':
 746         case 'r2r_jitstressregs0x80':
 747         case 'r2r_jitstressregs0x1000':
 748         case 'r2r_jitminopts':
 749         case 'r2r_jitforcerelocs':
 750         case 'r2r_gcstress15':
 751             assert !(os in bidailyCrossList)
 752
 753             // GCStress=C is currently not supported on OS X
 754             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 755                 break
 756             }
 757
 758             // GC Stress 15 r2r gets a push trigger for checked/release
 759             if (configuration == 'Checked' || configuration == 'Release') {
 760                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 761                 if (architecture == 'x64') {
 762                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 763                     if (isFlowJob || os == 'Windows_NT') {
 764                         // Add a weekly periodic trigger
 765                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 766                     }
 767                 }
 768                 // For x86, only add per-commit jobs for Windows
 769                 else if (architecture == 'x86') {
 770                     if (os == 'Windows_NT') {
 771                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 772                     }
 773                 }
 774             }
 775             break
 776         case 'longgc':
 777             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 778             assert configuration == 'Release'
 779             assert architecture == 'x64'
 780             Utilities.addPeriodicTrigger(job, '@daily')
 781             // TODO: Add once external email sending is available again
 782             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 783             break
 784         case 'gcsimulator':
 785             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 786             assert configuration == 'Release'
 787             assert architecture == 'x64'
 788             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 789             // TODO: Add once external email sending is available again
 790             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 791             break
 792         case 'standalone_gc':
 793             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 794             assert (configuration == 'Release' || configuration == 'Checked')
 795             // TODO: Add once external email sending is available again
 796             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 797             Utilities.addPeriodicTrigger(job, '@daily')
 798             break
 799         case 'gc_reliability_framework':
 800             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 801             assert (configuration == 'Release' || configuration == 'Checked')
 802             // Only triggered by phrase.
 803             break
 804         case 'ilrt':
 805             assert !(os in bidailyCrossList)
 806             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 807             if (architecture == 'x64' && configuration == 'Release') {
 808                 // We don't expect to see a job generated except in these scenarios
 809                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 810                 if (isFlowJob || os == 'Windows_NT') {
 811                     Utilities.addPeriodicTrigger(job, '@daily')
 812                 }
 813             }
 814             break
 815         case 'jitdiff':
 816             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 817             assert configuration == 'Checked'
 818             assert (architecture == 'x64' || architecture == 'x86')
 819             Utilities.addGithubPushTrigger(job)
 820             break
 821         case 'formatting':
 822             assert (os == 'Windows_NT' || os == "Ubuntu")
 823             assert architecture == 'x64'
 824             Utilities.addGithubPushTrigger(job)
 825             break
 826         case 'jitstressregs1':
 827         case 'jitstressregs2':
 828         case 'jitstressregs3':
 829         case 'jitstressregs4':
 830         case 'jitstressregs8':
 831         case 'jitstressregs0x10':
 832         case 'jitstressregs0x80':
 833         case 'jitstressregs0x1000':
 834         case 'minopts':
 835         case 'forcerelocs':
 836         case 'jitstress1':
 837         case 'jitstress2':
 838         case 'jitstress2_jitstressregs1':
 839         case 'jitstress2_jitstressregs2':
 840         case 'jitstress2_jitstressregs3':
 841         case 'jitstress2_jitstressregs4':
 842         case 'jitstress2_jitstressregs8':
 843         case 'jitstress2_jitstressregs0x10':
 844         case 'jitstress2_jitstressregs0x80':
 845         case 'jitstress2_jitstressregs0x1000':
 846         case 'tailcallstress':
 847         case 'jitsse2only':
 848         case 'jitnosimd':
 849         case 'corefx_baseline':
 850         case 'corefx_minopts':
 851         case 'corefx_jitstress1':
 852         case 'corefx_jitstress2':
 853         case 'corefx_jitstressregs1':
 854         case 'corefx_jitstressregs2':
 855         case 'corefx_jitstressregs3':
 856         case 'corefx_jitstressregs4':
 857         case 'corefx_jitstressregs8':
 858         case 'corefx_jitstressregs0x10':
 859         case 'corefx_jitstressregs0x80':
 860         case 'corefx_jitstressregs0x1000':
 861         case 'zapdisable':
 862             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 863                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 864                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 865                     if (os == 'Windows_NT') {
 866                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 867                         Utilities.addPeriodicTrigger(job, '@weekly')
 868                     }
 869                 }
 870                 else {
 871                     Utilities.addPeriodicTrigger(job, '@daily')
 872                 }
 873             }
 874             break
 875         case 'heapverify1':
 876         case 'gcstress0x3':
 877             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 878                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 879                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 880                     if (os == 'Windows_NT') {
 881                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 882                         Utilities.addPeriodicTrigger(job, '@weekly')
 883                     }
 884                     // TODO: Add once external email sending is available again
 885                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 886                 }
 887                 else {
 888                     Utilities.addPeriodicTrigger(job, '@weekly')
 889                 }
 890             }
 891             break
 892         case 'gcstress0xc':
 893         case 'gcstress0xc_zapdisable':
 894         case 'gcstress0xc_zapdisable_jitstress2':
 895         case 'gcstress0xc_zapdisable_heapverify1':
 896         case 'gcstress0xc_jitstress1':
 897         case 'gcstress0xc_jitstress2':
 898         case 'gcstress0xc_minopts_heapverify1':
 899             // GCStress=C is currently not supported on OS X
 900             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 901                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 902                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 903                     if (os == 'Windows_NT') {
 904                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 905                         Utilities.addPeriodicTrigger(job, '@weekly')
 906                     }
 907                     // TODO: Add once external email sending is available again
 908                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 909                 }
 910                 else {
 911                     Utilities.addPeriodicTrigger(job, '@weekly')
 912                 }
 913             }
 914             break
 915
 916         case 'illink':
 917             // Testing on other operating systems TBD
 918             assert (os == 'Windows_NT' || os == 'Ubuntu')
 919             if (architecture == 'x64' || architecture == 'x86') {
 920                 if (configuration == 'Checked') {
 921                     Utilities.addPeriodicTrigger(job, '@daily')
 922                 }
 923             }
 924             break
 925
 926         case 'tieredcompilation':
 927         case 'corefx_tieredcompilation':
 928             // No periodic jobs just yet, still testing
 929             break
 930
 931         default:
 932             println("Unknown scenario: ${scenario}");
 933             assert false
 934             break
 935     }
 936     return
 937 }
 938
 939 // **************************
 940 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 941 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 942 // builds will do a couple extra steps.
 943 // **************************
 944
 945 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 946 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 947 // it's just used for internal builds
 948 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 949 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 950     if (isWindowsBuildOnlyJob) {
 951         return
 952     }
 953
 954     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 955     // Non pull request builds.
 956     if (!isPR) {
 957         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 958         return
 959     }
 960
 961      def arm64Users = [
 962         'AndyAyersMS',
 963         'briansull',
 964         'BruceForstall',
 965         'CarolEidt',
 966         'cmckinsey',
 967         'erozenfeld',
 968         'janvorli',
 969         'jashook',
 970         'JosephTremoulet',
 971         'pgodeq',
 972         'pgavlin',
 973         'rartemev',
 974         'russellhadley',
 975         'RussKeldorph',
 976         'sandreenko',
 977         'sdmaclea',
 978         'swaroop-sridhar',
 979         'jkotas',
 980         'markwilkie',
 981         'weshaggard'
 982     ]
 983
 984     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 985     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 986     def osGroup = getOSGroup(os)
 987     switch (architecture) {
 988         case 'x64': // editor brace matching: {
 989             if (scenario == 'formatting') {
 990                 assert configuration == 'Checked'
 991                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 992                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
 993                 }
 994                 break
 995             }
 996
 997             switch (os) {
 998                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 999                 case 'Debian8.4':
1000                 case 'RHEL7.2':
1001                     if (scenario == 'default') {
1002                         assert !isFlowJob
1003                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
1004                     }
1005                     break
1006                 case 'Ubuntu16.04':
1007                     assert !isFlowJob
1008                     assert scenario == 'default'
1009                     // Distinguish with the other architectures (arm and x86)
1010                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
1011                     break
1012                 case 'Fedora24':
1013                 case 'Ubuntu16.10':
1014                     assert !isFlowJob
1015                     assert scenario == 'default'
1016                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1017                     break
1018                 case 'Ubuntu':
1019                     if (scenario == 'illink') {
1020                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1021                         break
1022                     }
1023                     // fall through
1024                 case 'OSX10.12':
1025                     // Triggers on the non-flow jobs aren't necessary here
1026                     // Corefx testing uses non-flow jobs.
1027                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1028                         break
1029                     }
1030                     switch (scenario) {
1031                         case 'default':
1032                             // OSX uses checked for default PR tests
1033                             if (configuration == 'Checked') {
1034                                 // Default trigger
1035                                 assert !job.name.contains("centos")
1036                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1037                             }
1038                             break
1039                         case 'jitdiff':
1040                             if (configuration == 'Checked') {
1041                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1042                             }
1043                             break
1044                         case 'ilrt':
1045                             if (configuration == 'Release') {
1046                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1047                             }
1048                             break
1049                         case 'longgc':
1050                             if (configuration == 'Release') {
1051                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1052                             }
1053                             break
1054                         case 'gcsimulator':
1055                             if (configuration == 'Release') {
1056                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1057                             }
1058                             break
1059                         case 'standalone_gc':
1060                             if (configuration == 'Release' || configuration == 'Checked') {
1061                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1062                             }
1063                             break
1064                         case 'gc_reliability_framework':
1065                             if (configuration == 'Release' || configuration == 'Checked') {
1066                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1067                             }
1068                             break
1069                         default:
1070                             if (isJitStressScenario(scenario)) {
1071                                 def displayStr = getStressModeDisplayName(scenario)
1072                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1073                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1074                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1075                             }
1076                             else if (isR2RScenario(scenario)) {
1077                                 if (configuration == 'Release' || configuration == 'Checked') {
1078                                     def displayStr = getR2RDisplayName(scenario)
1079                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1080                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1081                                 }
1082                             }
1083                             else {
1084                                 println("Unknown scenario: ${scenario}");
1085                                 assert false
1086                             }
1087                             break
1088                     }
1089                     break
1090
1091                 case 'CentOS7.1':
1092                     switch (scenario) {
1093                         case 'default':
1094                             // CentOS uses checked for default PR tests while debug is build only
1095                             if (configuration == 'Debug') {
1096                                 // Default trigger
1097                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1098                             }
1099
1100                             // Make sure this is a flow job to get build and test.
1101                             if (configuration == 'Checked' && isFlowJob) {
1102                                 assert job.name.contains("flow")
1103                                 // Default trigger
1104                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1105                             }
1106                             break
1107                         default:
1108                             if (isR2RScenario(scenario)) {
1109                                 if (configuration == 'Release' || configuration == 'Checked') {
1110                                     def displayStr = getR2RDisplayName(scenario)
1111                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1112                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1113                                 }
1114                             }
1115                             break
1116                     }
1117                     break
1118
1119                 case 'Windows_NT':
1120                     switch (scenario) {
1121                         case 'default':
1122                             // Default trigger
1123                             if (configuration == 'Checked') {
1124                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1125                             }
1126                             break
1127                         case 'jitdiff':
1128                             if (configuration == 'Checked') {
1129                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1130                             }
1131                             break
1132                         case 'ilrt':
1133                             if (configuration == 'Release') {
1134                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1135                             }
1136                             break
1137                         case 'longgc':
1138                             if (configuration == 'Release') {
1139                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1140                             }
1141                             break
1142                         case 'gcsimulator':
1143                             if (configuration == 'Release') {
1144                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1145                             }
1146                             break
1147                         case 'standalone_gc':
1148                             if (configuration == 'Release' || configuration == 'Checked') {
1149                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1150                             }
1151                             break
1152                         case 'gc_reliability_framework':
1153                             if (configuration == 'Release' || configuration == 'Checked') {
1154                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1155                             }
1156                             break
1157                         case 'illink':
1158                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1159                             break
1160                         default:
1161                             if (isJitStressScenario(scenario)) {
1162                                 def displayStr = getStressModeDisplayName(scenario)
1163                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1164                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1165                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1166                             }
1167                             else if (isR2RScenario(scenario)) {
1168                                 if (configuration == 'Release' || configuration == 'Checked') {
1169                                     def displayStr = getR2RDisplayName(scenario)
1170                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1171                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1172                                 }
1173                             }
1174                             else {
1175                                 println("Unknown scenario: ${scenario}");
1176                                 assert false
1177                             }
1178                             break
1179                     }
1180                     break
1181                 default:
1182                     println("Unknown os: ${os}");
1183                     assert false
1184                     break
1185             }
1186             break
1187         // editor brace matching: }
1188         case 'armlb':
1189         case 'arm': // editor brace matching: {
1190             switch (os) {
1191                 case 'Ubuntu':
1192                 case 'Ubuntu16.04':
1193                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1194                         break
1195                     }
1196                     assert scenario == 'default'
1197                     job.with {
1198                         publishers {
1199                             azureVMAgentPostBuildAction {
1200                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1201                             }
1202                         }
1203                     }
1204                     if (configuration == 'Debug') {
1205                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1206                     }
1207                     else {
1208                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1209                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1210                     }
1211                     break
1212                 case 'Tizen':
1213                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1214                         break
1215                     }
1216                     architecture='armel'
1217                     job.with {
1218                         publishers {
1219                             azureVMAgentPostBuildAction {
1220                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1221                             }
1222                         }
1223                     }
1224                     if (configuration == 'Checked') {
1225                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1226                     }
1227                     else {
1228                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1229                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1230                     }
1231                     break
1232                 case 'Windows_NT':
1233                     // Triggers on the non-flow jobs aren't necessary here
1234                     if (!isFlowJob) {
1235                         break
1236                     }
1237
1238                     // Set up a private trigger
1239                     def contextString = "${os} ${architecture} Cross ${configuration}"
1240                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1241                     if (scenario == 'default') {
1242                         contextString += " Innerloop"
1243                         triggerString += "\\W+Innerloop"
1244                     }
1245                     else {
1246                         contextString += " ${scenario}"
1247                         triggerString += "\\W+${scenario}"
1248                     }
1249
1250                     if (configuration == 'Debug') {
1251                         contextString += " Build"
1252                         triggerString += "\\W+Build"
1253                     } else {
1254                         contextString += " Build and Test"
1255                         triggerString += "\\W+Build and Test"
1256                     }
1257
1258                     triggerString += ".*"
1259
1260                     switch (scenario) {
1261                         case 'default':
1262                             // Only Checked is a default trigger.
1263                             if (configuration == 'Checked')
1264                             {
1265                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1266                             }
1267                             else
1268                             {
1269                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1270                             }
1271                             break
1272                         default:
1273                             // Stress jobs will use this code path.
1274                             if (isArmWindowsScenario(scenario)) {
1275                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1276                             }
1277                             break
1278                     }
1279                     break
1280                 default:
1281                     println("NYI os: ${os}");
1282                     assert false
1283                     break
1284             }
1285             break
1286         // editor brace matching: }
1287         case 'arm64': // editor brace matching: {
1288             // Set up a private trigger
1289             def contextString = "${os} ${architecture} Cross ${configuration}"
1290             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1291             if (scenario == 'default') {
1292                 contextString += " Innerloop"
1293                 triggerString += "\\W+Innerloop"
1294             }
1295             else {
1296                 contextString += " ${scenario}"
1297                 triggerString += "\\W+${scenario}"
1298             }
1299
1300             if (configuration == 'Debug') {
1301                 contextString += " Build"
1302                 triggerString += "\\W+Build"
1303             } else {
1304                 contextString += " Build and Test"
1305                 triggerString += "\\W+Build and Test"
1306             }
1307
1308             triggerString += ".*"
1309
1310             switch (os) {
1311                 case 'Ubuntu':
1312                 case 'Ubuntu16.04':
1313                     switch (scenario) {
1314                         case 'default':
1315                             if (configuration == 'Debug' && !isFlowJob) {
1316                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1317                             }
1318                             else {
1319                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1320                             }
1321
1322                             break
1323                         default:
1324                             if (isR2RScenario(scenario)) {
1325                                 if (configuration == 'Checked' || configuration == 'Release') {
1326                                     def displayStr = getR2RDisplayName(scenario)
1327                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1328                                 }
1329                             }
1330                             break
1331                     }
1332                     break
1333                 case 'Windows_NT':
1334                     // Triggers on the non-flow jobs aren't necessary here
1335                     if (!isFlowJob) {
1336                         break
1337                     }
1338
1339                     assert isArmWindowsScenario(scenario)
1340                     switch (scenario) {
1341                         case 'default':
1342                             if (configuration == 'Checked') {
1343                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1344                             }
1345                             else {
1346                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1347                             }
1348                             break
1349                         default:
1350                             // Stress jobs will use this code path.
1351                             if (isArmWindowsScenario(scenario)) {
1352                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1353                             }
1354                             break
1355                     }
1356                     break
1357                 default:
1358                     println("NYI os: ${os}");
1359                     assert false
1360                     break
1361             }
1362             break
1363         // editor brace matching: }
1364         case 'x86': // editor brace matching: {
1365             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1366             if (os == 'Ubuntu') {
1367                 // Triggers on the non-flow jobs aren't necessary here
1368                 if (!isFlowJob) {
1369                     break
1370                 }
1371                 // on-demand only for ubuntu x86
1372                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1373                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1374                 break
1375             }
1376             switch (scenario) {
1377                 case 'default':
1378                     if (configuration == 'Checked') {
1379                         assert !job.name.contains("centos")
1380                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1381                     }
1382                     else {
1383                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1384                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1385                     }
1386                     break
1387                 case 'ilrt':
1388                     if (configuration == 'Release') {
1389                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1390                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1391                     }
1392                     break
1393                 case 'longgc':
1394                     if (configuration == 'Release') {
1395                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1396                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1397                     }
1398                     break
1399                 case 'gcsimulator':
1400                     if (configuration == 'Release') {
1401                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1402                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1403                     }
1404                     break
1405                 case 'standalone_gc':
1406                     if (configuration == 'Release' || configuration == 'Checked') {
1407                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1408                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1409                     }
1410                     break
1411                 case 'illink':
1412                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1413                     break
1414                 default:
1415                     if (isJitStressScenario(scenario)) {
1416                         def displayStr = getStressModeDisplayName(scenario)
1417                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1418                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1419                     }
1420                     else if (isR2RScenario(scenario)) {
1421                         if (configuration == 'Release' || configuration == 'Checked') {
1422                             def displayStr = getR2RDisplayName(scenario)
1423                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1424                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1425                         }
1426                     }
1427                     else {
1428                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1429                         assert false
1430                     }
1431                     break
1432             }
1433             break
1434          // editor brace matching: }
1435         case 'x64_arm64_altjit':
1436         case 'x86_arm_altjit': // editor brace matching: {
1437             assert (os == 'Windows_NT')
1438             switch (scenario) {
1439                 case 'default':
1440                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1441                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1442                     break
1443                 default:
1444                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1445                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1446                     break
1447             }
1448             break
1449         // editor brace matching: }
1450         default:
1451             println("Unknown architecture: ${architecture}");
1452             assert false
1453             break
1454     }
1455 }
1456
1457 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1458     def buildCommands = [];
1459     def osGroup = getOSGroup(os)
1460     def lowerConfiguration = configuration.toLowerCase()
1461
1462     def priority = '1'
1463     if (scenario == 'default' && isPR == true) {
1464         priority = '0'
1465     }
1466
1467     def enableCorefxTesting = isCoreFxScenario(scenario)
1468
1469     // Calculate the build steps, archival, and xunit results
1470     switch (os) {
1471         case 'Windows_NT': // editor brace matching: {
1472             switch (architecture) {
1473                 case 'x64':
1474                 case 'x86':
1475                 case 'x86_arm_altjit':
1476                 case 'x64_arm64_altjit':
1477                     def arch = architecture
1478                     def buildOpts = ''
1479                     if (architecture == 'x86_arm_altjit') {
1480                         arch = 'x86'
1481                     }
1482                     else if (architecture == 'x64_arm64_altjit') {
1483                         arch = 'x64'
1484                     }
1485
1486                     if (scenario == 'formatting') {
1487                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1488                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1489                         break
1490                     }
1491
1492                     if (scenario == 'illink') {
1493                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1494                     }
1495
1496                     // If it is a release build for windows, ensure PGO is used, else fail the build
1497                     if ((lowerConfiguration == 'release') &&
1498                         (scenario in Constants.basicScenarios) &&
1499                         (architecture != 'x86_arm_altjit') &&
1500                         (architecture != 'x64_arm64_altjit')) {
1501
1502                         buildOpts += ' -enforcepgo'
1503                     }
1504
1505                     if (enableCorefxTesting) {
1506                         buildOpts += ' skiptests';
1507                     } else {
1508                         buildOpts += " -priority=${priority}"
1509                     }
1510
1511                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1512                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1513                     // 35 characters long.
1514
1515                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1516
1517                     if (!isBuildOnly) {
1518                         def runtestArguments = ''
1519                         def testOpts = 'collectdumps'
1520
1521                         if (isR2RScenario(scenario)) {
1522
1523                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1524                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1525                             // to cause the tests to be crossgen'ed.
1526
1527                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1528                                 testOpts += ' crossgenaltjit protononjit.dll'
1529                             } else {
1530                                 testOpts += ' crossgen'
1531                             }
1532
1533                             testOpts += ' runcrossgentests'
1534                         }
1535                         else if (scenario == 'jitdiff') {
1536                             testOpts += ' jitdisasm crossgen'
1537                         }
1538                         else if (scenario == 'ilrt') {
1539                             testOpts += ' ilasmroundtrip'
1540                         }
1541                         else if (isLongGc(scenario)) {
1542                             testOpts += " ${scenario} sequential"
1543                         }
1544                         else if (scenario == 'standalone_gc') {
1545                             testOpts += ' gcname clrgc.dll'
1546                         }
1547                         else if (scenario == 'illink') {
1548                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1549                         }
1550
1551                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1552                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1553                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1554                         // which is read by the xunit harness.)
1555                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1556                         {
1557                             def timeout = 1800000
1558                             testOpts += " timeout ${timeout}"
1559                         }
1560
1561                         // If we are running a stress mode, we should write out the set of key
1562                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1563
1564                         def envScriptPath = ''
1565                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1566                             def buildCommandsStr = ''
1567                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1568                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1569
1570                             if (isJitStressScenario(scenario)) {
1571                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1572                             }
1573                             else if (isR2RStressScenario(scenario)) {
1574                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1575                             }
1576
1577                             if (architecture == 'x86_arm_altjit') {
1578                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1579                             }
1580                             else if (architecture == 'x64_arm64_altjit') {
1581                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1582                             }
1583
1584                             envScriptFinalize(os, envScriptPath)
1585
1586                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1587                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1588                             // into a single script.
1589                             buildCommands += buildCommandsStr
1590                         }
1591                         else if (architecture == 'x86_arm_altjit') {
1592                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1593                         }
1594                         else if (architecture == 'x64_arm64_altjit') {
1595                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1596                         }
1597                         if (envScriptPath != '') {
1598                             testOpts += " TestEnv ${envScriptPath}"
1599                         }
1600
1601                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1602
1603                         if (enableCorefxTesting) {
1604                             def workspaceRelativeFxRoot = "_/fx"
1605                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1606
1607                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1608
1609                             setTestJobTimeOut(newJob, isPR, architecture, configuration, scenario)
1610
1611                             // Archive and process (only) the test results
1612                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1613                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1614
1615                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1616                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1617                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1618                         }
1619                         else if (isGcReliabilityFramework(scenario)) {
1620                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1621                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1622                         }
1623                         else {
1624                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1625                         }
1626                     }
1627
1628                     if (!enableCorefxTesting) {
1629                         // Run the rest of the build
1630                         // Build the mscorlib for the other OS's
1631                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1632                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1633
1634                         if (arch == "x64") {
1635                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1636                         }
1637
1638                         // Zip up the tests directory so that we don't use so much space/time copying
1639                         // 10s of thousands of files around.
1640                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1641
1642                         if (!isJitStressScenario(scenario)) {
1643                             // For windows, pull full test results and test drops for x86/x64.
1644                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1645                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1646                         }
1647
1648                         if (scenario == 'jitdiff') {
1649                             // retrive jit-dasm output for base commit, and run jit-diff
1650                             if (!isBuildOnly) {
1651                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1652                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1653                             }
1654                         }
1655
1656                         if (!isBuildOnly) {
1657                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1658                             setTestJobTimeOut(newJob, isPR, architecture, configuration, scenario)
1659                         }
1660                     }
1661                     break
1662                 case 'armlb':
1663                 case 'arm':
1664                     assert isArmWindowsScenario(scenario)
1665
1666                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1667                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1668
1669                     setTestJobTimeOut(newJob, isPR, architecture, configuration, scenario)
1670
1671                     def buildArchitecture = 'arm'
1672
1673                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1674                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1675                     def armCrossgenOpt = ''
1676                     if (architecture == 'armlb') {
1677                         armCrossgenOpt = '-crossgenaltjit legacyjit.dll'
1678                     }
1679
1680                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1681                     priority = '1'
1682
1683                     // This is now a build only job. Do not run tests. Use the flow job.
1684                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1685
1686                     // Zip up the tests directory so that we don't use so much space/time copying
1687                     // 10s of thousands of files around.
1688                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1689
1690                     // Add archival.
1691                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1692                     break
1693                 case 'arm64':
1694                     assert isArmWindowsScenario(scenario)
1695
1696                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1697                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1698
1699                     setTestJobTimeOut(newJob, isPR, architecture, configuration, scenario)
1700
1701                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1702                     priority = '1'
1703
1704                     // This is now a build only job. Do not run tests. Use the flow job.
1705                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1706
1707                     // Zip up the tests directory so that we don't use so much space/time copying
1708                     // 10s of thousands of files around.
1709                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1710
1711                     // Add archival.
1712                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1713                     break
1714                 default:
1715                     println("Unknown architecture: ${architecture}");
1716                     assert false
1717                     break
1718             }
1719             break
1720         // editor brace matching: }
1721         case 'Ubuntu':
1722         case 'Ubuntu16.04':
1723         case 'Ubuntu16.10':
1724         case 'Debian8.4':
1725         case 'OSX10.12':
1726         case 'CentOS7.1':
1727         case 'RHEL7.2':
1728         case 'Tizen':
1729         case 'Fedora24': // editor brace matching: {
1730             switch (architecture) {
1731                 case 'x64':
1732                 case 'x86':
1733                     if (architecture == 'x86' && os == 'Ubuntu') {
1734                         // build and PAL test
1735                         def dockerImage = getDockerImageName(architecture, os, true)
1736                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1737                         dockerImage = getDockerImageName(architecture, os, false)
1738                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1739                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1740                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1741                         break
1742                     }
1743
1744                     if (scenario == 'formatting') {
1745                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1746                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1747                         break
1748                     }
1749
1750                     if (scenario == 'illink') {
1751                         assert(os == 'Ubuntu')
1752                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1753                     }
1754
1755                     if (!enableCorefxTesting) {
1756                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1757                         // only on supported OS platforms.
1758                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1759                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1760
1761                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1762                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1763
1764                         setTestJobTimeOut(newJob, isPR, architecture, configuration, scenario)
1765
1766                         // Basic archiving of the build
1767                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1768                         // And pal tests
1769                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1770                     }
1771                     else {
1772                         // Corefx stress testing
1773                         assert os == 'Ubuntu'
1774                         assert architecture == 'x64'
1775                         assert lowerConfiguration == 'checked'
1776                         assert isJitStressScenario(scenario)
1777
1778                         // Build coreclr
1779                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1780
1781                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1782
1783                         def envScriptCmds = envScriptCreate(os, scriptFileName)
1784                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
1785                         envScriptCmds += envScriptFinalize(os, scriptFileName)
1786                         buildCommands += envScriptCmds
1787
1788                         // Build and text corefx
1789                         def workspaceRelativeFxRoot = "_/fx"
1790                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1791
1792                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1793
1794                         setTestJobTimeOut(newJob, isPR, architecture, configuration, scenario)
1795
1796                         // Archive and process (only) the test results
1797                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1798                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1799                     }
1800                     break
1801                 case 'arm64':
1802                     if (!enableCorefxTesting) {
1803                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1804
1805                         // HACK -- Arm64 does not have corefx jobs yet.
1806                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1807                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1808                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1809                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1810
1811                         setTestJobTimeOut(newJob, isPR, architecture, configuration, scenario)
1812                         // Basic archiving of the build
1813                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1814                     }
1815                     break
1816                 case 'arm':
1817                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1818                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1819
1820                     // default values for Ubuntu
1821                     def arm_abi="arm"
1822                     def linuxCodeName="trusty"
1823                     if (os == 'Ubuntu16.04') {
1824                         linuxCodeName="xenial"
1825                     }
1826                     else if (os == 'Tizen') {
1827                         arm_abi="armel"
1828                         linuxCodeName="tizen"
1829                     }
1830
1831                     // Unzip the Windows test binaries first. Exit with 0
1832                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1833
1834                     // Unpack the corefx binaries
1835                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1836                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1837                     if (os != 'Tizen') {
1838                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1839                     }
1840                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1841                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1842                     if (os == 'Ubuntu') {
1843                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1844                     }
1845
1846                     // Call the ARM CI script to cross build and test using docker
1847                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1848                     --mode=docker \\
1849                     --${arm_abi} \\
1850                     --linuxCodeName=${linuxCodeName} \\
1851                     --buildConfig=${lowerConfiguration} \\
1852                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1853                     --coreFxBinDir=./bin/CoreFxBinDir \\
1854                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1855
1856                     // Basic archiving of the build, no pal tests
1857                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1858                     break
1859                 default:
1860                     println("Unknown architecture: ${architecture}");
1861                     assert false
1862                     break
1863             }
1864             break
1865         // editor brace matching: }
1866         default:
1867             println("Unknown os: ${os}");
1868             assert false
1869             break
1870     } // os
1871
1872     return buildCommands
1873 }
1874
1875 Constants.allScenarios.each { scenario ->
1876     [true, false].each { isPR ->
1877         Constants.architectureList.each { architecture ->
1878             Constants.configurationList.each { configuration ->
1879                 Constants.osList.each { os ->
1880                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1881                     // and reset the os to Windows_NT
1882                     def isBuildOnly = false
1883                     if (os == 'Windows_NT_BuildOnly') {
1884                         isBuildOnly = true
1885                         os = 'Windows_NT'
1886                     }
1887
1888                     // Tizen is only supported for arm architecture
1889                     if (os == 'Tizen' && architecture != 'arm') {
1890                         return
1891                     }
1892
1893                     // Skip totally unimplemented (in CI) configurations.
1894                     switch (architecture) {
1895                         case 'arm64':
1896                             if (os == 'Ubuntu16.04') {
1897                                 os = 'Ubuntu'
1898                             }
1899
1900                             // Windows and Ubuntu only
1901                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1902                                 return
1903                             }
1904                             break
1905                         case 'arm':
1906                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1907                                 return
1908                             }
1909                             break
1910                         case 'armlb':
1911                             if (os != 'Windows_NT') {
1912                                 return
1913                             }
1914                             break
1915                         case 'x86':
1916                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1917                                 return
1918                             }
1919                             break
1920                         case 'x86_arm_altjit':
1921                         case 'x64_arm64_altjit':
1922                             if (os != 'Windows_NT') {
1923                                 return
1924                             }
1925                             break
1926                         case 'x64':
1927                             // Everything implemented
1928                             break
1929                         default:
1930                             println("Unknown architecture: ${architecture}")
1931                             assert false
1932                             break
1933                     }
1934
1935                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1936                     // with checked builds)
1937                     if (isJitStressScenario(scenario)) {
1938                         if (configuration != 'Checked') {
1939                             return
1940                         }
1941
1942                         // Since these are just execution time differences,
1943                         // skip platforms that don't execute the tests here (Windows_NT only)
1944                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
1945                         if (!isEnabledOS || isBuildOnly) {
1946                             return
1947                         }
1948
1949                         switch (architecture) {
1950                             case 'x64':
1951                             case 'x86':
1952                             case 'x86_arm_altjit':
1953                             case 'x64_arm64_altjit':
1954                                 // x86 ubuntu: default only
1955                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1956                                     return
1957                                 }
1958                                 // Windows: Everything implemented
1959                                 break
1960
1961                             default:
1962                                 // arm, arm64, armlb: stress is handled through flow jobs.
1963                                 return
1964                         }
1965                     }
1966                     else if (isR2RScenario(scenario)) {
1967                         if (os != 'Windows_NT') {
1968                             return
1969                         }
1970                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
1971                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
1972                             return
1973                         }
1974                     }
1975                     else {
1976                         // Skip scenarios
1977                         switch (scenario) {
1978                             case 'ilrt':
1979                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1980                                 // the default scenario build
1981                                 if (os != 'Windows_NT') {
1982                                     return
1983                                 }
1984                                 // Only x64 for now
1985                                 if (architecture != 'x64') {
1986                                     return
1987                                 }
1988                                 // Release only
1989                                 if (configuration != 'Release') {
1990                                     return
1991                                 }
1992                                 break
1993                             case 'jitdiff':
1994                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1995                                     return
1996                                 }
1997                                 if (architecture != 'x64') {
1998                                     return
1999                                 }
2000                                 if (configuration != 'Checked') {
2001                                     return
2002                                 }
2003                                 break
2004                             case 'longgc':
2005                             case 'gcsimulator':
2006                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2007                                     return
2008                                 }
2009                                 if (architecture != 'x64') {
2010                                     return
2011                                 }
2012                                 if (configuration != 'Release') {
2013                                     return
2014                                 }
2015                                 break
2016                             case 'gc_reliability_framework':
2017                             case 'standalone_gc':
2018                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2019                                     return
2020                                 }
2021
2022                                 if (architecture != 'x64') {
2023                                     return
2024                                 }
2025
2026                                 if (configuration != 'Release' && configuration != 'Checked') {
2027                                     return
2028                                 }
2029                                 break
2030                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2031                             case 'formatting':
2032                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2033                                     return
2034                                 }
2035                                 if (architecture != 'x64') {
2036                                     return
2037                                 }
2038                                 if (configuration != 'Checked') {
2039                                     return
2040                                 }
2041                                 if (isBuildOnly) {
2042                                     return
2043                                 }
2044                                 break
2045                             case 'illink':
2046                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2047                                     return
2048                                 }
2049                                 if (architecture != 'x64' && architecture != 'x86') {
2050                                     return
2051                                 }
2052                                 if (isBuildOnly) {
2053                                     return
2054                                 }
2055                                 break
2056                             case 'default':
2057                                 // Nothing skipped
2058                                 break
2059                             default:
2060                                 println("Unknown scenario: ${scenario}")
2061                                 assert false
2062                                 break
2063                         }
2064                     }
2065
2066                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2067                     // runtime behavior, not compile-time behavior, are not interesting.
2068                     switch (architecture) {
2069                         case 'x86_arm_altjit':
2070                         case 'x64_arm64_altjit':
2071                             if (isGCStressRelatedTesting(scenario)) {
2072                                 return
2073                             }
2074                             break
2075                         default:
2076                             break
2077                     }
2078
2079                     // Calculate names
2080                     def lowerConfiguration = configuration.toLowerCase()
2081                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2082                     def folderName = getJobFolder(scenario)
2083
2084                     // Create the new job
2085                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2086
2087                     def machineAffinityOptions = null
2088
2089                     if (os != 'Windows_NT') {
2090                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2091                     }
2092                     else {
2093                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2094                     }
2095
2096                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2097
2098                     // Add all the standard options
2099                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2100                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2101
2102                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2103                     def osGroup = getOSGroup(os)
2104
2105                     newJob.with {
2106                         steps {
2107                             if (os == 'Windows_NT') {
2108                                 buildCommands.each { buildCommand ->
2109                                     batchFile(buildCommand)
2110                                 }
2111                             }
2112                             else {
2113                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2114                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2115                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2116                                     // Define the Windows Tests and Corefx build job names
2117                                     def WindowTestsName = projectFolder + '/' +
2118                                                           Utilities.getFullJobName(project,
2119                                                                                    getJobName(lowerConfiguration,
2120                                                                                               'x64' ,
2121                                                                                               'windows_nt',
2122                                                                                               'default',
2123                                                                                               true),
2124                                                                                    false)
2125                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2126                                                        Utilities.getFolderName(branch)
2127
2128                                     // Copy the Windows test binaries and the Corefx build binaries
2129                                     copyArtifacts(WindowTestsName) {
2130                                         includePatterns('bin/tests/tests.zip')
2131                                         buildSelector {
2132                                             latestSuccessful(true)
2133                                         }
2134                                     }
2135
2136                                     def arm_abi = 'arm'
2137                                     def corefx_os = 'linux'
2138                                     if (os == 'Tizen') {
2139                                         arm_abi = 'armel'
2140                                         corefx_os = 'tizen'
2141                                     }
2142
2143                                     // Let's use release CoreFX to test checked CoreCLR,
2144                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2145                                     def corefx_lowerConfiguration = lowerConfiguration
2146                                     if ( lowerConfiguration == 'checked' ) {
2147                                         corefx_lowerConfiguration='release'
2148                                     }
2149
2150                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2151                                         includePatterns('bin/build.tar.gz')
2152                                         buildSelector {
2153                                             latestSuccessful(true)
2154                                         }
2155                                     }
2156                                 }
2157
2158                                 buildCommands.each { buildCommand ->
2159                                     shell(buildCommand)
2160                                 }
2161                             }
2162                         }
2163                     } // newJob.with
2164
2165                 } // os
2166             } // configuration
2167         } // architecture
2168     } // isPR
2169 } // scenario
2170
2171
2172 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2173 Constants.allScenarios.each { scenario ->
2174     [true, false].each { isPR ->
2175         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2176             Constants.crossList.each { os ->
2177                 if (architecture == 'arm64') {
2178                     if (os != "Ubuntu" && os != "Windows_NT") {
2179                         return
2180                     }
2181                 } else if (architecture == 'arm' || architecture == 'armlb') {
2182                     if (os != 'Windows_NT') {
2183                         return
2184                     }
2185                 }
2186                 else if (architecture == 'x86') {
2187                     if (os != "Ubuntu") {
2188                         return
2189                     }
2190                 }
2191
2192                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2193
2194                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2195                     return
2196                 }
2197
2198                 Constants.configurationList.each { configuration ->
2199
2200                     // First, filter based on OS.
2201
2202                     if (os == 'Windows_NT') {
2203                         if (!isArmWindowsScenario(scenario)) {
2204                             return
2205                         }
2206                     }
2207                     else {
2208                         // Non-Windows
2209                         if (architecture == 'arm64') {
2210                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2211                                 return
2212                             }
2213                         }
2214                         else if (architecture == 'x86') {
2215                             // Linux/x86 only want default test
2216                             if (scenario != 'default') {
2217                                 return
2218                             }
2219                         }
2220                     }
2221
2222                     // For CentOS, we only want Checked/Release builds.
2223                     if (os == 'CentOS7.1') {
2224                         if (configuration != 'Checked' && configuration != 'Release') {
2225                             return
2226                         }
2227                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2228                             return
2229                         }
2230                     }
2231
2232                     // For RedHat and Debian, we only do Release builds.
2233                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2234                         if (configuration != 'Release') {
2235                             return
2236                         }
2237                         if (scenario != 'default') {
2238                             return
2239                         }
2240                     }
2241
2242                     // Next, filter based on scenario.
2243
2244                     if (isJitStressScenario(scenario)) {
2245                         if (configuration != 'Checked') {
2246                             return
2247                         }
2248                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2249                         if (isCoreFxScenario(scenario)) {
2250                             return
2251                         }
2252                     }
2253                     else if (isR2RBaselineScenario(scenario)) {
2254                         if (configuration != 'Checked' && configuration != 'Release') {
2255                             return
2256                         }
2257                     }
2258                     else if (isR2RStressScenario(scenario)) {
2259                         if (configuration != 'Checked') {
2260                             return
2261                         }
2262                     }
2263                     else {
2264                         // Skip scenarios
2265                         switch (scenario) {
2266                             case 'ilrt':
2267                             case 'longgc':
2268                             case 'gcsimulator':
2269                                 // Long GC tests take a long time on non-Release builds
2270                                 // ilrt is also Release only
2271                                 if (configuration != 'Release') {
2272                                     return
2273                                 }
2274                                 break
2275                             case 'jitdiff':
2276                                 if (configuration != 'Checked') {
2277                                     return;
2278                                 }
2279                                 break
2280                             case 'gc_reliability_framework':
2281                             case 'standalone_gc':
2282                                 if (configuration != 'Release' && configuration != 'Checked') {
2283                                     return
2284                                 }
2285                                 break
2286                             case 'formatting':
2287                                 return
2288                             case 'illink':
2289                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2290                                     return
2291                                 }
2292                                 break
2293                             case 'default':
2294                                 // Nothing skipped
2295                                 break
2296                             default:
2297                                 println("Unknown scenario: ${scenario}")
2298                                 assert false
2299                                 break
2300                         }
2301                     }
2302
2303                     // Done filtering. Now, create the jobs.
2304
2305                     def lowerConfiguration = configuration.toLowerCase()
2306                     def osGroup = getOSGroup(os)
2307                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2308
2309                     def inputCoreCLRBuildName = projectFolder + '/' +
2310                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2311
2312                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2313                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2314                     def testBuildScenario = scenario
2315                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2316                         testBuildScenario = 'default'
2317                     }
2318
2319                     def inputWindowsTestBuildArch = architecture
2320                     if (architecture == "arm64" && os != "Windows_NT") {
2321                         // Use the x64 test build for arm64 unix
2322                         inputWindowsTestBuildArch = "x64"
2323                     }
2324
2325                     def inputWindowTestsBuildName = ""
2326
2327                     if (isJitStressScenario(scenario)) {
2328                         inputWindowTestsBuildName = projectFolder + '/' +
2329                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2330                     } else {
2331                         inputWindowTestsBuildName = projectFolder + '/' +
2332                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2333
2334                     }
2335
2336
2337                     // Enable Server GC for Ubuntu PR builds
2338                     def serverGCString = ''
2339                     if (os == 'Ubuntu' && isPR) {
2340                         serverGCString = '--useServerGC'
2341                     }
2342
2343                     def testOpts = ''
2344
2345                     if (isR2RScenario(scenario)) {
2346
2347                         testOpts += ' --crossgen --runcrossgentests'
2348
2349                         if (scenario == 'r2r_jitstress1') {
2350                             testOpts += ' --jitstress=1'
2351                         }
2352                         else if (scenario == 'r2r_jitstress2') {
2353                             testOpts += ' --jitstress=2'
2354                         }
2355                         else if (scenario == 'r2r_jitstressregs1') {
2356                             testOpts += ' --jitstressregs=1'
2357                         }
2358                         else if (scenario == 'r2r_jitstressregs2') {
2359                             testOpts += ' --jitstressregs=2'
2360                         }
2361                         else if (scenario == 'r2r_jitstressregs3') {
2362                             testOpts += ' --jitstressregs=3'
2363                         }
2364                         else if (scenario == 'r2r_jitstressregs4') {
2365                             testOpts += ' --jitstressregs=4'
2366                         }
2367                         else if (scenario == 'r2r_jitstressregs8') {
2368                             testOpts += ' --jitstressregs=8'
2369                         }
2370                         else if (scenario == 'r2r_jitstressregs0x10') {
2371                             testOpts += ' --jitstressregs=0x10'
2372                         }
2373                         else if (scenario == 'r2r_jitstressregs0x80') {
2374                             testOpts += ' --jitstressregs=0x80'
2375                         }
2376                         else if (scenario == 'r2r_jitstressregs0x1000') {
2377                             testOpts += ' --jitstressregs=0x1000'
2378                         }
2379                         else if (scenario == 'r2r_jitminopts') {
2380                             testOpts += ' --jitminopts'
2381                         }
2382                         else if (scenario == 'r2r_jitforcerelocs') {
2383                             testOpts += ' --jitforcerelocs'
2384                         }
2385                         else if (scenario == 'r2r_gcstress15') {
2386                             testOpts += ' --gcstresslevel=0xF'
2387                         }
2388                     }
2389                     else if (scenario == 'jitdiff') {
2390                         testOpts += ' --jitdisasm --crossgen'
2391                     }
2392                     else if (scenario == 'illink') {
2393                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2394                     }
2395                     else if (isLongGc(scenario)) {
2396                         // Long GC tests behave very poorly when they are not
2397                         // the only test running (many of them allocate until OOM).
2398                         testOpts += ' --sequential'
2399
2400                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2401                         // for running long GC and GCSimulator tests, respectively. We don't use them
2402                         // here because using a playlist file produces much more readable output on the CI machines
2403                         // and reduces running time.
2404                         //
2405                         // The Long GC playlist contains all of the tests that are
2406                         // going to be run. The GCSimulator playlist contains all of
2407                         // the GC simulator tests.
2408                         if (scenario == 'longgc') {
2409                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2410                         }
2411                         else if (scenario == 'gcsimulator') {
2412                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2413                         }
2414                     }
2415                     else if (isGcReliabilityFramework(scenario)) {
2416                         testOpts += ' --build-overlay-only'
2417                     }
2418                     else if (scenario == 'standalone_gc') {
2419                         if (osGroup == 'OSX') {
2420                             testOpts += ' --gcname=libclrgc.dylib'
2421                         }
2422                         else if (osGroup == 'Linux') {
2423                             testOpts += ' --gcname=libclrgc.so'
2424                         }
2425                         else {
2426                             println("Unexpected OS group: ${osGroup} for os ${os}")
2427                             assert false
2428                         }
2429                     }
2430
2431                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2432
2433                     def folder = getJobFolder(scenario)
2434                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2435                         // Add parameters for the inputs
2436
2437                         if (windowsArmJob == true) {
2438                             parameters {
2439                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2440                             }
2441                         }
2442                         else {
2443                             parameters {
2444                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2445                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2446                             }
2447                         }
2448
2449                         steps {
2450                             // Set up the copies
2451
2452                             // Coreclr build containing the tests and mscorlib
2453                             // pri1 jobs still need to copy windows_nt built tests
2454                             if (windowsArmJob != true) {
2455                                 copyArtifacts(inputWindowTestsBuildName) {
2456                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2457                                     buildSelector {
2458                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2459                                     }
2460                                 }
2461                             }
2462
2463                             // Coreclr build we are trying to test
2464                             //
2465                             //  ** NOTE ** This will, correctly, overwrite over the CORE_ROOT from the windows test archive
2466
2467                             copyArtifacts(inputCoreCLRBuildName) {
2468                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2469                                 buildSelector {
2470                                     buildNumber('${CORECLR_BUILD}')
2471                                 }
2472                             }
2473
2474                             // Windows CoreCLR Arm(64) will restore corefx
2475                             // packages correctly.
2476                             //
2477                             // In addition, test steps are entirely different
2478                             // because we do not have a unified runner
2479                             if (windowsArmJob != true) {
2480                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2481
2482                                 // HACK -- Arm64 does not have corefx jobs yet.
2483                                 // Clone corefx and build the native packages overwriting the x64 packages.
2484                                 if (architecture == 'arm64') {
2485                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2486                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2487                                 }
2488                                 else if (architecture == 'x86') {
2489                                     shell("mkdir ./bin/CoreFxNative")
2490
2491                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2492                                         includePatterns('bin/build.tar.gz')
2493                                         targetDirectory('bin/CoreFxNative')
2494                                         buildSelector {
2495                                             latestSuccessful(true)
2496                                         }
2497                                     }
2498
2499                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2500                                 }
2501
2502                                 // Unzip the tests first.  Exit with 0
2503                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2504                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2505
2506                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2507
2508                                 // Execute the tests
2509                                 def runDocker = isNeedDocker(architecture, os, false)
2510                                 def dockerPrefix = ""
2511                                 def dockerCmd = ""
2512                                 if (runDocker) {
2513                                     def dockerImage = getDockerImageName(architecture, os, false)
2514                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2515                                     dockerCmd = dockerPrefix + "${dockerImage} "
2516                                 }
2517
2518                                 // If we are running a stress mode, we'll set those variables first
2519                                 def testEnvOpt = ""
2520                                 if (isJitStressScenario(scenario)) {
2521                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2522                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2523                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2524                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2525                                     shell("${envScriptCmds}")
2526                                     testEnvOpt = "--test-env=" + scriptFileName
2527                                 }
2528
2529                                 if (isGCStressRelatedTesting(scenario)) {
2530                                     shell('./init-tools.sh')
2531                                 }
2532
2533                                 shell("""${dockerCmd}./tests/runtest.sh \\
2534                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2535                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2536                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2537                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2538
2539                                 if (isGcReliabilityFramework(scenario)) {
2540                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2541                                     if (serverGCString != '') {
2542                                         if (runDocker) {
2543                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2544                                         }
2545                                         else {
2546                                             shell("export COMPlus_gcServer=1")
2547                                         }
2548                                     }
2549
2550                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2551                                 }
2552                             }
2553
2554                             else { // windowsArmJob == true
2555                                 // Unzip tests.
2556                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2557
2558                                 // Build the build commands
2559                                 def buildCommands = ""
2560
2561                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2562                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2563                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2564
2565                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2566                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2567
2568                                 // For all jobs
2569                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2570
2571                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2572                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2573
2574                                 // ARM legacy backend; this is an altjit.
2575                                 if (architecture == "armlb") {
2576                                     addEnvVariable("COMPlus_AltJit", "*")
2577                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2578                                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2579                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2580                                 }
2581
2582                                 // If we are running a stress mode, we'll set those variables as well
2583                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2584                                     def stressValues = null
2585                                     if (isJitStressScenario(scenario)) {
2586                                         stressValues = Constants.jitStressModeScenarios[scenario]
2587                                     }
2588                                     else {
2589                                         stressValues = Constants.r2rStressScenarios[scenario]
2590                                     }
2591
2592                                     stressValues.each { key, value ->
2593                                         addEnvVariable(key, value)
2594                                     }
2595                                 }
2596
2597                                 if (isR2RScenario(scenario)) {
2598                                     // Crossgen the framework assemblies.
2599                                     buildCommands += """
2600 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2601 @goto skip_PrecompileAssembly
2602
2603 :PrecompileAssembly
2604 @REM Skip mscorlib since it is already precompiled.
2605 @if /I "%3" == "mscorlib.dll" exit /b 0
2606 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2607
2608 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2609 @if "%errorlevel%" == "-2146230517" (
2610     echo %2 is not a managed assembly.
2611 ) else if "%errorlevel%" == "-2146234344" (
2612     echo %2 is not a managed assembly.
2613 ) else if %errorlevel% neq 0 (
2614     echo Unable to precompile %2
2615 ) else (
2616     echo Precompiled %2
2617 )
2618 @exit /b 0
2619
2620 :skip_PrecompileAssembly
2621 """
2622
2623                                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2624                                     // crossgen on tests before running them.
2625                                     addEnvVariable("RunCrossGen", "true")
2626                                 }
2627
2628                                 // Create the smarty command
2629                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2630                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2631                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2632
2633                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2634
2635                                 if (architecture == "armlb") {
2636                                     addExclude("LEGACYJIT_FAIL")
2637                                 }
2638
2639                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2640                                     def failTag = "JITSTRESS_FAIL"
2641                                     def excludeTag = "JITSTRESS_EXCLUDE"
2642
2643                                     if (scenario.contains('gc')) {
2644                                         failTag = "GCSTRESS_FAIL"
2645                                         excludeTag = "GCSTRESS_EXCLUDE"
2646                                     }
2647
2648                                     addArchSpecificExclude(architecture, failTag)
2649                                     addArchSpecificExclude(architecture, excludeTag)
2650                                 }
2651                                 else {
2652                                     addExclude("pri1")
2653                                 }
2654
2655                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2656                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2657                                 addExclude("LONG_RUNNING")
2658
2659                                 smartyCommand += "/lstFile Tests.lst"
2660
2661                                 def testListArch = [
2662                                     'arm64': 'arm64',
2663                                     'arm': 'arm',
2664                                     'armlb': 'arm'
2665                                 ]
2666
2667                                 def archLocation = testListArch[architecture]
2668
2669                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2670                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2671                                 addCommand("${smartyCommand}")
2672
2673                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2674                                 // However, we also need to remove all the variables that were set during this batch file, so we
2675                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2676                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2677                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2678                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2679                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2680                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2681
2682                                 // ZIP up the smarty output, no matter what the smarty result.
2683                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2684
2685                                 addCommand("echo %errorlevel%")
2686                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2687
2688                                 // Use the smarty errorlevel as the script errorlevel.
2689                                 addCommand("exit /b %__save_smarty_errorlevel%")
2690
2691                                 batchFile(buildCommands)
2692                             }
2693                         }
2694                     }
2695
2696                     if (scenario == 'jitdiff') {
2697                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2698                     }
2699
2700                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2701                     if (os in ['Ubuntu']) {
2702                         SummaryBuilder summaries = new SummaryBuilder()
2703                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2704                         summaries.emit(newJob)
2705                     }
2706
2707                     def affinityOptions = null
2708
2709                     if (windowsArmJob == true) {
2710                         affinityOptions = [
2711                             "use_arm64_build_machine" : false
2712                         ]
2713                     }
2714
2715                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2716                         affinityOptions = [
2717                             "large_pages" : false
2718                         ]
2719                     }
2720
2721                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2722                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2723
2724                     setTestJobTimeOut(newJob, isPR, architecture, configuration, scenario)
2725
2726                     if (windowsArmJob != true) {
2727                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2728                     }
2729                     else {
2730                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2731
2732                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2733                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2734                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2735                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2736                     }
2737
2738                     // Create a build flow to join together the build and tests required to run this test.
2739                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2740                     // Linux CoreCLR test
2741                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2742                     def fullTestJobName = projectFolder + '/' + newJob.name
2743                     // Add a reference to the input jobs for report purposes
2744                     JobReport.Report.addReference(inputCoreCLRBuildName)
2745                     JobReport.Report.addReference(inputWindowTestsBuildName)
2746                     JobReport.Report.addReference(fullTestJobName)
2747                     def newFlowJob = null
2748
2749                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2750                         // Do not create the flow job for RHEL jobs.
2751                         return
2752                     }
2753
2754                     // For pri0 jobs we can build tests on unix
2755                     if (windowsArmJob) {
2756                         // For Windows arm jobs there is no reason to build a parallel test job.
2757                         // The product build supports building and archiving the tests.
2758
2759                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2760                         buildFlow("""
2761 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2762
2763 // And then build the test build
2764 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2765 """)
2766                         }
2767                     }
2768                     else {
2769                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2770                         buildFlow("""
2771 // Build the input jobs in parallel
2772 parallel (
2773 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2774 { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2775 )
2776
2777 // And then build the test build
2778 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2779             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2780 """)
2781                         }
2782                     }
2783
2784                     // For the flow jobs set the machine affinity as x64 if an armarch.
2785                     def flowArch = architecture
2786
2787                     if (flowArch in validWindowsNTCrossArches) {
2788                         flowArch = 'x64'
2789                         affinityOptions = null
2790                     }
2791
2792                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2793                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2794                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2795                 } // configuration
2796             } // os
2797         } // architecture
2798     } // isPR
2799 } // scenario
2800
2801 JobReport.Report.generateJobReport(out)
2802
2803 // Make the call to generate the help job
2804 Utilities.createHelperJob(this, project, branch,
2805     "Welcome to the ${project} Repository",  // This is prepended to the help message
2806     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2807
2808 Utilities.addCROSSCheck(this, project, branch)