netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // We have very limited ARM64 hardware (used for ARM/ARMLB/ARM64 testing). So only allow certain branches to use it.
  39     def static WindowsArm64Branches = [
  40                'master']
  41
  42     // Innerloop build OS's
  43     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  44     // test execution in the build flow runs.  It generates the exact same build
  45     // as Windows_NT but without the tests.
  46     def static osList = [
  47                'Ubuntu',
  48                'Debian8.4',
  49                'OSX10.12',
  50                'Windows_NT',
  51                'Windows_NT_BuildOnly',
  52                'CentOS7.1',
  53                'RHEL7.2',
  54                'Ubuntu16.04',
  55                'Ubuntu16.10',
  56                'Fedora24',
  57                'Tizen']
  58
  59     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  60
  61     // This is a set of JIT stress modes combined with the set of variables that
  62     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  63     // the values are the environment variables
  64     def static jitStressModeScenarios = [
  65                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  66                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  67                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  68                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  69                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  70                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  71                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  72                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  73                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  74                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  75                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  76                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  77                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  78                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  79                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  80                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  81                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  82                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  83                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  84                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  85                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  86                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  87                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  88                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  89                'corefx_baseline'                : [ : ], // corefx baseline
  90                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  91                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  92                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  93                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  94                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  95                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  96                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  97                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  98                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  99                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
 100                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
 101                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
 102                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
 103                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 104                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 105                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 106                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 107                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 108                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 109                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 110                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 111                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 112     ]
 113
 114     // This is a set of ReadyToRun stress scenarios
 115     def static r2rStressScenarios = [
 116                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 117                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 118                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 119                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 120                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 121                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 122                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 123                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 124                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 125                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 126                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 127                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 128                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 129     ]
 130
 131     // This is the basic set of scenarios
 132     def static basicScenarios = [
 133                'default',
 134                'ilrt',
 135                'r2r',
 136                'longgc',
 137                'formatting',
 138                'gcsimulator',
 139                'jitdiff',
 140                'standalone_gc',
 141                'gc_reliability_framework',
 142                'illink']
 143
 144     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 145
 146     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 147     // to Tests.lst file categories to exclude.
 148     //
 149     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 150     // and with the same values, with some commented out, for easier maintenance.
 151     //
 152     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 153     //
 154     def static validArmWindowsScenarios = [
 155                'default':                                [],
 156                // 'ilrt'
 157                'r2r':                                    ["R2R_FAIL"],
 158                // 'longgc'
 159                // 'formatting'
 160                // 'gcsimulator'
 161                // 'jitdiff'
 162                // 'standalone_gc'
 163                // 'gc_reliability_framework'
 164                // 'illink'
 165                'r2r_jitstress1':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 166                'r2r_jitstress2':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 167                'r2r_jitstressregs1':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 168                'r2r_jitstressregs2':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 169                'r2r_jitstressregs3':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 170                'r2r_jitstressregs4':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 171                'r2r_jitstressregs8':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 172                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 173                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 174                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 175                'r2r_jitminopts':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 176                'r2r_jitforcerelocs':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 177                'r2r_gcstress15':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 179                'tieredcompilation':                      [],
 180                'forcerelocs':                            [],
 181                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 196                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 197                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 198                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 199                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 200                // 'jitsse2only'                          // Only relevant to xarch
 201                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 202                // 'corefx_baseline'
 203                // 'corefx_minopts'
 204                // 'corefx_tieredcompilation'
 205                // 'corefx_jitstress1'
 206                // 'corefx_jitstress2'
 207                // 'corefx_jitstressregs1'
 208                // 'corefx_jitstressregs2'
 209                // 'corefx_jitstressregs3'
 210                // 'corefx_jitstressregs4'
 211                // 'corefx_jitstressregs8'
 212                // 'corefx_jitstressregs0x10'
 213                // 'corefx_jitstressregs0x80'
 214                // 'corefx_jitstressregs0x1000'
 215                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 216                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 217                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'heapverify1':                            [],
 219                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 220                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 221                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 222                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 223                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 224                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 225
 226                //
 227                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 228                //
 229
 230                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 231                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 245                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 246                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 247                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 248                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 249     ]
 250
 251     def static configurationList = ['Debug', 'Checked', 'Release']
 252
 253     // This is the set of architectures
 254     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 255 }
 256
 257 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 258     assert os instanceof String
 259     assert architecture instanceof String
 260
 261     def armArches = ['arm', 'armlb', 'arm64']
 262     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 263
 264     if (!(architecture in armArches)) {
 265         assert options == null
 266         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 267
 268         return
 269     }
 270
 271     // This is an arm(64) job.
 272     //
 273     // There are several options.
 274     //
 275     // Windows_NT
 276     //
 277     // Arm32 (Build) -> latest-arm64
 278     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 279     // Arm32 (Test)  -> arm64-windows_nt
 280     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 281     //
 282     // Arm64 (Build) -> latest-arm64
 283     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 284     // Arm64 (Test)  -> arm64-windows_nt
 285     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 286     //
 287     // Ubuntu
 288     //
 289     // Arm32 (Build) -> arm-cross-latest
 290     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 291     // Arm32 (Test)  -> NYI Arch not supported
 292     //       |->
 293     //
 294     // Arm64 (Build) -> arm64-cross-latest
 295     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 296     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 297     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 298     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 299     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 300
 301     // This has to be a arm arch
 302     assert architecture in armArches
 303     if (os == "Windows_NT") {
 304         // Arm(64) windows jobs share the same machines for now
 305         def isBuild = options['use_arm64_build_machine'] == true
 306
 307         if (isBuild == true) {
 308             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 309         } else {
 310             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 311         }
 312     } else {
 313         assert os != 'Windows_NT'
 314         assert os in supportedArmLinuxOs
 315
 316         if (architecture == 'arm' || architecture == 'armlb') {
 317             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 318         } else {
 319             // Arm64 Linux
 320             if (options['is_build_only'] == true) {
 321                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 322             } else {
 323                 // Arm64 Test Machines
 324                 if (options['large_pages'] == false) {
 325                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 326                 } else {
 327                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 328                 }
 329             }
 330         }
 331     }
 332 }
 333
 334 def static isGCStressRelatedTesting(def scenario) {
 335     // The 'r2r_gcstress15' scenario is a basic scenario.
 336     // Detect it and make it a GCStress related.
 337     if (scenario == 'r2r_gcstress15')
 338     {
 339         return true;
 340     }
 341
 342     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 343     def scenarioName = scenario.toLowerCase()
 344     def isGCStressTesting = false
 345     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 346         if (k in gcStressTestEnvVars) {
 347             isGCStressTesting = true;
 348         }
 349     }
 350     return isGCStressTesting
 351 }
 352
 353 def static isCoreFxScenario(def scenario) {
 354     def corefx_prefix = 'corefx_'
 355     if (scenario.length() < corefx_prefix.length()) {
 356         return false
 357     }
 358     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 359 }
 360
 361 def static isR2RBaselineScenario(def scenario) {
 362     return (scenario == 'r2r')
 363 }
 364
 365 def static isR2RStressScenario(def scenario) {
 366     return Constants.r2rStressScenarios.containsKey(scenario)
 367 }
 368
 369 def static isR2RScenario(def scenario) {
 370     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 371 }
 372
 373 def static isJitStressScenario(def scenario) {
 374     return Constants.jitStressModeScenarios.containsKey(scenario)
 375 }
 376
 377 def static isLongGc(def scenario) {
 378     return (scenario == 'longgc' || scenario == 'gcsimulator')
 379 }
 380
 381 def static isJitDiff(def scenario) {
 382     return (scenario == 'jitdiff')
 383 }
 384
 385 def static isGcReliabilityFramework(def scenario) {
 386     return (scenario == 'gc_reliability_framework')
 387 }
 388
 389 def static isArmWindowsScenario(def scenario) {
 390     return Constants.validArmWindowsScenarios.containsKey(scenario)
 391 }
 392
 393 def static setTestJobTimeOut(newJob, isPR, architecture, scenario) {
 394     if (isGCStressRelatedTesting(scenario)) {
 395         Utilities.setJobTimeout(newJob, 4320)
 396     }
 397     else if (isCoreFxScenario(scenario)) {
 398         Utilities.setJobTimeout(newJob, 360)
 399     }
 400     else if (isJitStressScenario(scenario)) {
 401         Utilities.setJobTimeout(newJob, 240)
 402     }
 403     else if (isR2RBaselineScenario(scenario)) {
 404         Utilities.setJobTimeout(newJob, 240)
 405     }
 406     else if (isLongGc(scenario)) {
 407         Utilities.setJobTimeout(newJob, 1440)
 408     }
 409     else if (isJitDiff(scenario)) {
 410         Utilities.setJobTimeout(newJob, 240)
 411     }
 412     else if (isGcReliabilityFramework(scenario)) {
 413         Utilities.setJobTimeout(newJob, 1440)
 414     }
 415     else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 416         Utilities.setJobTimeout(newJob, 240)
 417     }
 418     else if (!(scenario == 'default' && isPR == true)) {
 419         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 420         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 421         Utilities.setJobTimeout(newJob, 240)
 422     }
 423     // Non-test jobs use the default timeout value.
 424 }
 425
 426 def static getJobFolder(def scenario) {
 427     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 428         return 'jitstress'
 429     }
 430     if (scenario == 'illink') {
 431         return 'illink'
 432     }
 433     return ''
 434 }
 435
 436 def static getStressModeDisplayName(def scenario) {
 437     def displayStr = ''
 438     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 439         def prefixLength = 'COMPlus_'.length()
 440         if (k.length() >= prefixLength) {
 441             def modeName = k.substring(prefixLength, k.length())
 442             displayStr += ' ' + modeName + '=' + v
 443         }
 444     }
 445
 446     if (isCoreFxScenario(scenario)) {
 447         displayStr = ('CoreFx ' + displayStr).trim()
 448     }
 449
 450     return displayStr
 451 }
 452
 453 def static getR2RDisplayName(def scenario) {
 454     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 455     def displayStr = scenario
 456     def prefixLength = 'r2r_'.length()
 457     if (displayStr.length() >= prefixLength) {
 458         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 459     } else if (scenario == 'r2r') {
 460         displayStr = "R2R"
 461     }
 462     return displayStr
 463 }
 464
 465 //
 466 // Functions to create an environment script.
 467 //      envScriptCreate -- initialize the script (call first)
 468 //      envScriptFinalize -- finalize the script (call last)
 469 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 470 //      envScriptAppendExistingScript -- append an existing script to the generated script
 471 //
 472 // Each script returns a string of commands. Concatenate all the strings together before
 473 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 474 //
 475
 476 // Initialize the environment setting script.
 477 def static envScriptCreate(def os, def stepScriptLocation) {
 478     def stepScript = ''
 479     if (os == 'Windows_NT') {
 480         stepScript += "echo Creating TestEnv script\r\n"
 481         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 482
 483         // Create at least an empty script.
 484         stepScript += "echo. > ${stepScriptLocation}\r\n"
 485     }
 486     else {
 487         stepScript += "echo Creating environment setting script\n"
 488         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 489     }
 490
 491     return stepScript
 492 }
 493
 494 // Generates the string for setting stress mode variables.
 495 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 496     def stepScript = ''
 497     if (os == 'Windows_NT') {
 498         stressModeVars.each{ k, v ->
 499             // Write out what we are writing to the script file
 500             stepScript += "echo Setting ${k}=${v}\r\n"
 501             // Write out the set itself to the script file`
 502             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 503         }
 504     }
 505     else {
 506         stressModeVars.each{ k, v ->
 507             // Write out what we are writing to the script file
 508             stepScript += "echo Setting ${k}=${v}\n"
 509             // Write out the set itself to the script file`
 510             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 511         }
 512     }
 513
 514     return stepScript
 515 }
 516
 517 // Append an existing script to an environment script.
 518 // Returns string of commands to do this.
 519 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 520     assert (os == 'Windows_NT')
 521     def stepScript = ''
 522
 523     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 524     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 525
 526     return stepScript
 527 }
 528
 529 // Finalize an environment setting script.
 530 // Returns string of commands to do this.
 531 def static envScriptFinalize(def os, def stepScriptLocation) {
 532     def stepScript = ''
 533
 534     if (os == 'Windows_NT') {
 535         // Display the resulting script. This is useful when looking at the output log file.
 536         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 537         stepScript += "type ${stepScriptLocation}\r\n"
 538     }
 539     else {
 540         stepScript += "chmod +x ${stepScriptLocation}\n"
 541     }
 542
 543     return stepScript
 544 }
 545
 546 def static isNeedDocker(def architecture, def os, def isBuild) {
 547     if (isBuild) {
 548         if (architecture == 'x86' && os == 'Ubuntu') {
 549             return true
 550         }
 551         else if (architecture == 'arm') {
 552             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 553                 return true
 554             }
 555         }
 556     }
 557     else {
 558         if (architecture == 'x86' && os == 'Ubuntu') {
 559             return true
 560         }
 561     }
 562     return false
 563 }
 564
 565 def static getDockerImageName(def architecture, def os, def isBuild) {
 566     // We must change some docker private images to official later
 567     if (isBuild) {
 568         if (architecture == 'x86' && os == 'Ubuntu') {
 569             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 570         }
 571         else if (architecture == 'arm') {
 572             if (os == 'Ubuntu') {
 573                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 574             }
 575             else if (os == 'Ubuntu16.04') {
 576                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 577             }
 578             else if (os == 'Tizen') {
 579                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 580             }
 581         }
 582     }
 583     else {
 584         if (architecture == 'x86' && os == 'Ubuntu') {
 585             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 586         }
 587     }
 588     println("Unknown architecture to use docker: ${architecture} ${os}");
 589     assert false
 590 }
 591
 592 // Calculates the name of the build job based on some typical parameters.
 593 //
 594 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 595     // If the architecture is x64, do not add that info into the build name.
 596     // Need to change around some systems and other builds to pick up the right builds
 597     // to do that.
 598
 599     def suffix = scenario != 'default' ? "_${scenario}" : '';
 600     if (isBuildOnly) {
 601         suffix += '_bld'
 602     }
 603     def baseName = ''
 604     switch (architecture) {
 605         case 'x64':
 606             if (scenario == 'default') {
 607                 // For now we leave x64 off of the name for compatibility with other jobs
 608                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 609             }
 610             else if (scenario == 'formatting') {
 611                 // we don't care about the configuration for the formatting job. It runs all configs
 612                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 613             }
 614             else {
 615                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 616             }
 617             break
 618         case 'arm64':
 619             if (os.toLowerCase() == "windows_nt") {
 620                 // These are cross builds
 621                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 622             }
 623             else {
 624                 // Defaults to a small page size set of machines.
 625                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 626             }
 627             break
 628         case 'arm':
 629             // These are cross builds
 630             if (os == 'Tizen') {
 631                 // ABI: softfp
 632                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 633             }
 634             else {
 635                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 636             }
 637             break
 638         case 'armlb':
 639             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 640             break
 641         case 'x86':
 642         case 'x86_arm_altjit':
 643         case 'x64_arm64_altjit':
 644             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 645             break
 646         default:
 647             println("Unknown architecture: ${architecture}");
 648             assert false
 649             break
 650     }
 651
 652     return baseName + suffix
 653 }
 654
 655 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 656
 657     // Limited Windows ARM64 hardware is restricted for non-PR triggers to certain branches.
 658     if (os == 'Windows_NT') {
 659         if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 660             if (!(branch in Constants.WindowsArm64Branches)) {
 661                 return
 662             }
 663         }
 664     }
 665
 666     // Check scenario.
 667     switch (scenario) {
 668         case 'default':
 669             switch (architecture) {
 670                 case 'x64':
 671                 case 'x86':
 672                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 673                         Utilities.addPeriodicTrigger(job, '@daily')
 674                     }
 675                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 676                         Utilities.addGithubPushTrigger(job)
 677                     }
 678                     break
 679                 case 'arm':
 680                 case 'armlb':
 681                 case 'x86_arm_altjit':
 682                 case 'x64_arm64_altjit':
 683                     Utilities.addGithubPushTrigger(job)
 684                     break
 685                 case 'arm64':
 686                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 687                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 688                     break
 689                 default:
 690                     println("Unknown architecture: ${architecture}");
 691                     assert false
 692                     break
 693             }
 694             break
 695         case 'r2r':
 696             assert !(os in bidailyCrossList)
 697             // r2r gets a push trigger for checked/release
 698             if (configuration == 'Checked' || configuration == 'Release') {
 699                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 700                 if (architecture == 'x64' && os != 'OSX10.12') {
 701                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 702                     if (isFlowJob || os == 'Windows_NT') {
 703                         Utilities.addGithubPushTrigger(job)
 704                     }
 705                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 706                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 707                     if (isFlowJob) {
 708                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 709                     }
 710                 }
 711                 // For x86, only add per-commit jobs for Windows
 712                 else if (architecture == 'x86') {
 713                     if (os == 'Windows_NT') {
 714                         Utilities.addGithubPushTrigger(job)
 715                     }
 716                 }
 717                 // arm64 r2r jobs should only run daily.
 718                 else if (architecture == 'arm64') {
 719                     if (os == 'Windows_NT') {
 720                         Utilities.addPeriodicTrigger(job, '@daily')
 721                     }
 722                 }
 723             }
 724             break
 725         case 'r2r_jitstress1':
 726         case 'r2r_jitstress2':
 727         case 'r2r_jitstressregs1':
 728         case 'r2r_jitstressregs2':
 729         case 'r2r_jitstressregs3':
 730         case 'r2r_jitstressregs4':
 731         case 'r2r_jitstressregs8':
 732         case 'r2r_jitstressregs0x10':
 733         case 'r2r_jitstressregs0x80':
 734         case 'r2r_jitstressregs0x1000':
 735         case 'r2r_jitminopts':
 736         case 'r2r_jitforcerelocs':
 737         case 'r2r_gcstress15':
 738             assert !(os in bidailyCrossList)
 739
 740             // GCStress=C is currently not supported on OS X
 741             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 742                 break
 743             }
 744
 745             // GC Stress 15 r2r gets a push trigger for checked/release
 746             if (configuration == 'Checked' || configuration == 'Release') {
 747                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 748                 if (architecture == 'x64') {
 749                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 750                     if (isFlowJob || os == 'Windows_NT') {
 751                         // Add a weekly periodic trigger
 752                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 753                     }
 754                 }
 755                 // For x86, only add per-commit jobs for Windows
 756                 else if (architecture == 'x86') {
 757                     if (os == 'Windows_NT') {
 758                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 759                     }
 760                 }
 761             }
 762             break
 763         case 'longgc':
 764             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 765             assert configuration == 'Release'
 766             assert architecture == 'x64'
 767             Utilities.addPeriodicTrigger(job, '@daily')
 768             // TODO: Add once external email sending is available again
 769             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 770             break
 771         case 'gcsimulator':
 772             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 773             assert configuration == 'Release'
 774             assert architecture == 'x64'
 775             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 776             // TODO: Add once external email sending is available again
 777             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 778             break
 779         case 'standalone_gc':
 780             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 781             assert (configuration == 'Release' || configuration == 'Checked')
 782             // TODO: Add once external email sending is available again
 783             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 784             Utilities.addPeriodicTrigger(job, '@daily')
 785             break
 786         case 'gc_reliability_framework':
 787             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 788             assert (configuration == 'Release' || configuration == 'Checked')
 789             // Only triggered by phrase.
 790             break
 791         case 'ilrt':
 792             assert !(os in bidailyCrossList)
 793             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 794             if (architecture == 'x64' && configuration == 'Release') {
 795                 // We don't expect to see a job generated except in these scenarios
 796                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 797                 if (isFlowJob || os == 'Windows_NT') {
 798                     Utilities.addPeriodicTrigger(job, '@daily')
 799                 }
 800             }
 801             break
 802         case 'jitdiff':
 803             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 804             assert configuration == 'Checked'
 805             assert (architecture == 'x64' || architecture == 'x86')
 806             Utilities.addGithubPushTrigger(job)
 807             break
 808         case 'formatting':
 809             assert (os == 'Windows_NT' || os == "Ubuntu")
 810             assert architecture == 'x64'
 811             Utilities.addGithubPushTrigger(job)
 812             break
 813         case 'jitstressregs1':
 814         case 'jitstressregs2':
 815         case 'jitstressregs3':
 816         case 'jitstressregs4':
 817         case 'jitstressregs8':
 818         case 'jitstressregs0x10':
 819         case 'jitstressregs0x80':
 820         case 'jitstressregs0x1000':
 821         case 'minopts':
 822         case 'forcerelocs':
 823         case 'jitstress1':
 824         case 'jitstress2':
 825         case 'jitstress2_jitstressregs1':
 826         case 'jitstress2_jitstressregs2':
 827         case 'jitstress2_jitstressregs3':
 828         case 'jitstress2_jitstressregs4':
 829         case 'jitstress2_jitstressregs8':
 830         case 'jitstress2_jitstressregs0x10':
 831         case 'jitstress2_jitstressregs0x80':
 832         case 'jitstress2_jitstressregs0x1000':
 833         case 'tailcallstress':
 834         case 'jitsse2only':
 835         case 'jitnosimd':
 836         case 'corefx_baseline':
 837         case 'corefx_minopts':
 838         case 'corefx_jitstress1':
 839         case 'corefx_jitstress2':
 840         case 'corefx_jitstressregs1':
 841         case 'corefx_jitstressregs2':
 842         case 'corefx_jitstressregs3':
 843         case 'corefx_jitstressregs4':
 844         case 'corefx_jitstressregs8':
 845         case 'corefx_jitstressregs0x10':
 846         case 'corefx_jitstressregs0x80':
 847         case 'corefx_jitstressregs0x1000':
 848         case 'zapdisable':
 849             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 850                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 851                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 852                     if (os == 'Windows_NT') {
 853                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 854                         Utilities.addPeriodicTrigger(job, '@weekly')
 855                     }
 856                 }
 857                 else {
 858                     Utilities.addPeriodicTrigger(job, '@daily')
 859                 }
 860             }
 861             break
 862         case 'heapverify1':
 863         case 'gcstress0x3':
 864             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 865                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 866                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 867                     if (os == 'Windows_NT') {
 868                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 869                         Utilities.addPeriodicTrigger(job, '@weekly')
 870                     }
 871                     // TODO: Add once external email sending is available again
 872                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 873                 }
 874                 else {
 875                     Utilities.addPeriodicTrigger(job, '@weekly')
 876                 }
 877             }
 878             break
 879         case 'gcstress0xc':
 880         case 'gcstress0xc_zapdisable':
 881         case 'gcstress0xc_zapdisable_jitstress2':
 882         case 'gcstress0xc_zapdisable_heapverify1':
 883         case 'gcstress0xc_jitstress1':
 884         case 'gcstress0xc_jitstress2':
 885         case 'gcstress0xc_minopts_heapverify1':
 886             // GCStress=C is currently not supported on OS X
 887             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 888                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 889                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 890                     if (os == 'Windows_NT') {
 891                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 892                         Utilities.addPeriodicTrigger(job, '@weekly')
 893                     }
 894                     // TODO: Add once external email sending is available again
 895                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 896                 }
 897                 else {
 898                     Utilities.addPeriodicTrigger(job, '@weekly')
 899                 }
 900             }
 901             break
 902
 903         case 'illink':
 904             // Testing on other operating systems TBD
 905             assert (os == 'Windows_NT' || os == 'Ubuntu')
 906             if (architecture == 'x64' || architecture == 'x86') {
 907                 if (configuration == 'Checked') {
 908                     Utilities.addPeriodicTrigger(job, '@daily')
 909                 }
 910             }
 911             break
 912
 913         case 'tieredcompilation':
 914         case 'corefx_tieredcompilation':
 915             // No periodic jobs just yet, still testing
 916             break
 917
 918         default:
 919             println("Unknown scenario: ${scenario}");
 920             assert false
 921             break
 922     }
 923     return
 924 }
 925
 926 // **************************
 927 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 928 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 929 // builds will do a couple extra steps.
 930 // **************************
 931
 932 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 933 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 934 // it's just used for internal builds
 935 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 936 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 937     if (isWindowsBuildOnlyJob) {
 938         return
 939     }
 940
 941     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 942     // Non pull request builds.
 943     if (!isPR) {
 944         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 945         return
 946     }
 947
 948      def arm64Users = [
 949         'AndyAyersMS',
 950         'briansull',
 951         'BruceForstall',
 952         'CarolEidt',
 953         'cmckinsey',
 954         'erozenfeld',
 955         'janvorli',
 956         'jashook',
 957         'JosephTremoulet',
 958         'pgodeq',
 959         'pgavlin',
 960         'rartemev',
 961         'russellhadley',
 962         'RussKeldorph',
 963         'sandreenko',
 964         'sdmaclea',
 965         'swaroop-sridhar',
 966         'jkotas',
 967         'markwilkie',
 968         'weshaggard'
 969     ]
 970
 971     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 972     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 973     def osGroup = getOSGroup(os)
 974     switch (architecture) {
 975         case 'x64': // editor brace matching: {
 976             if (scenario == 'formatting') {
 977                 assert configuration == 'Checked'
 978                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 979                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
 980                 }
 981                 break
 982             }
 983
 984             switch (os) {
 985                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 986                 case 'Debian8.4':
 987                 case 'RHEL7.2':
 988                     if (scenario == 'default') {
 989                         assert !isFlowJob
 990                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 991                     }
 992                     break
 993                 case 'Ubuntu16.04':
 994                     assert !isFlowJob
 995                     assert scenario == 'default'
 996                     // Distinguish with the other architectures (arm and x86)
 997                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 998                     break
 999                 case 'Fedora24':
1000                 case 'Ubuntu16.10':
1001                     assert !isFlowJob
1002                     assert scenario == 'default'
1003                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
1004                     break
1005                 case 'Ubuntu':
1006                     if (scenario == 'illink') {
1007                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1008                         break
1009                     }
1010                     // fall through
1011                 case 'OSX10.12':
1012                     // Triggers on the non-flow jobs aren't necessary here
1013                     // Corefx testing uses non-flow jobs.
1014                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1015                         break
1016                     }
1017                     switch (scenario) {
1018                         case 'default':
1019                             // OSX uses checked for default PR tests
1020                             if (configuration == 'Checked') {
1021                                 // Default trigger
1022                                 assert !job.name.contains("centos")
1023                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1024                             }
1025                             break
1026                         case 'jitdiff':
1027                             if (configuration == 'Checked') {
1028                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1029                             }
1030                             break
1031                         case 'ilrt':
1032                             if (configuration == 'Release') {
1033                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1034                             }
1035                             break
1036                         case 'longgc':
1037                             if (configuration == 'Release') {
1038                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1039                             }
1040                             break
1041                         case 'gcsimulator':
1042                             if (configuration == 'Release') {
1043                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1044                             }
1045                             break
1046                         case 'standalone_gc':
1047                             if (configuration == 'Release' || configuration == 'Checked') {
1048                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1049                             }
1050                             break
1051                         case 'gc_reliability_framework':
1052                             if (configuration == 'Release' || configuration == 'Checked') {
1053                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1054                             }
1055                             break
1056                         default:
1057                             if (isJitStressScenario(scenario)) {
1058                                 def displayStr = getStressModeDisplayName(scenario)
1059                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1060                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1061                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1062                             }
1063                             else if (isR2RScenario(scenario)) {
1064                                 if (configuration == 'Release' || configuration == 'Checked') {
1065                                     def displayStr = getR2RDisplayName(scenario)
1066                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1067                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1068                                 }
1069                             }
1070                             else {
1071                                 println("Unknown scenario: ${scenario}");
1072                                 assert false
1073                             }
1074                             break
1075                     }
1076                     break
1077
1078                 case 'CentOS7.1':
1079                     switch (scenario) {
1080                         case 'default':
1081                             // CentOS uses checked for default PR tests while debug is build only
1082                             if (configuration == 'Debug') {
1083                                 // Default trigger
1084                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1085                             }
1086
1087                             // Make sure this is a flow job to get build and test.
1088                             if (configuration == 'Checked' && isFlowJob) {
1089                                 assert job.name.contains("flow")
1090                                 // Default trigger
1091                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1092                             }
1093                             break
1094                         default:
1095                             if (isR2RScenario(scenario)) {
1096                                 if (configuration == 'Release' || configuration == 'Checked') {
1097                                     def displayStr = getR2RDisplayName(scenario)
1098                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1099                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1100                                 }
1101                             }
1102                             break
1103                     }
1104                     break
1105
1106                 case 'Windows_NT':
1107                     switch (scenario) {
1108                         case 'default':
1109                             // Default trigger
1110                             if (configuration == 'Checked') {
1111                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1112                             }
1113                             break
1114                         case 'jitdiff':
1115                             if (configuration == 'Checked') {
1116                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1117                             }
1118                             break
1119                         case 'ilrt':
1120                             if (configuration == 'Release') {
1121                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1122                             }
1123                             break
1124                         case 'longgc':
1125                             if (configuration == 'Release') {
1126                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1127                             }
1128                             break
1129                         case 'gcsimulator':
1130                             if (configuration == 'Release') {
1131                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1132                             }
1133                             break
1134                         case 'standalone_gc':
1135                             if (configuration == 'Release' || configuration == 'Checked') {
1136                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1137                             }
1138                             break
1139                         case 'gc_reliability_framework':
1140                             if (configuration == 'Release' || configuration == 'Checked') {
1141                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1142                             }
1143                             break
1144                         case 'illink':
1145                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1146                             break
1147                         default:
1148                             if (isJitStressScenario(scenario)) {
1149                                 def displayStr = getStressModeDisplayName(scenario)
1150                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1151                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1152                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1153                             }
1154                             else if (isR2RScenario(scenario)) {
1155                                 if (configuration == 'Release' || configuration == 'Checked') {
1156                                     def displayStr = getR2RDisplayName(scenario)
1157                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1158                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1159                                 }
1160                             }
1161                             else {
1162                                 println("Unknown scenario: ${scenario}");
1163                                 assert false
1164                             }
1165                             break
1166                     }
1167                     break
1168                 default:
1169                     println("Unknown os: ${os}");
1170                     assert false
1171                     break
1172             }
1173             break
1174         // editor brace matching: }
1175         case 'armlb':
1176         case 'arm': // editor brace matching: {
1177             switch (os) {
1178                 case 'Ubuntu':
1179                 case 'Ubuntu16.04':
1180                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1181                         break
1182                     }
1183                     assert scenario == 'default'
1184                     job.with {
1185                         publishers {
1186                             azureVMAgentPostBuildAction {
1187                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1188                             }
1189                         }
1190                     }
1191                     if (configuration == 'Debug') {
1192                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1193                     }
1194                     else {
1195                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1196                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1197                     }
1198                     break
1199                 case 'Tizen':
1200                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1201                         break
1202                     }
1203                     architecture='armel'
1204                     job.with {
1205                         publishers {
1206                             azureVMAgentPostBuildAction {
1207                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1208                             }
1209                         }
1210                     }
1211                     if (configuration == 'Checked') {
1212                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1213                     }
1214                     else {
1215                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1216                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1217                     }
1218                     break
1219                 case 'Windows_NT':
1220                     // Triggers on the non-flow jobs aren't necessary here
1221                     if (!isFlowJob) {
1222                         break
1223                     }
1224
1225                     // Set up a private trigger
1226                     def contextString = "${os} ${architecture} Cross ${configuration}"
1227                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1228                     if (scenario == 'default') {
1229                         contextString += " Innerloop"
1230                         triggerString += "\\W+Innerloop"
1231                     }
1232                     else {
1233                         contextString += " ${scenario}"
1234                         triggerString += "\\W+${scenario}"
1235                     }
1236
1237                     if (configuration == 'Debug') {
1238                         contextString += " Build"
1239                         triggerString += "\\W+Build"
1240                     } else {
1241                         contextString += " Build and Test"
1242                         triggerString += "\\W+Build and Test"
1243                     }
1244
1245                     triggerString += ".*"
1246
1247                     switch (scenario) {
1248                         case 'default':
1249                             // Only Checked is a default trigger.
1250                             if (configuration == 'Checked')
1251                             {
1252                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1253                             }
1254                             else
1255                             {
1256                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1257                             }
1258                             break
1259                         default:
1260                             // Stress jobs will use this code path.
1261                             if (isArmWindowsScenario(scenario)) {
1262                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1263                             }
1264                             break
1265                     }
1266                     break
1267                 default:
1268                     println("NYI os: ${os}");
1269                     assert false
1270                     break
1271             }
1272             break
1273         // editor brace matching: }
1274         case 'arm64': // editor brace matching: {
1275             // Set up a private trigger
1276             def contextString = "${os} ${architecture} Cross ${configuration}"
1277             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1278             if (scenario == 'default') {
1279                 contextString += " Innerloop"
1280                 triggerString += "\\W+Innerloop"
1281             }
1282             else {
1283                 contextString += " ${scenario}"
1284                 triggerString += "\\W+${scenario}"
1285             }
1286
1287             if (configuration == 'Debug') {
1288                 contextString += " Build"
1289                 triggerString += "\\W+Build"
1290             } else {
1291                 contextString += " Build and Test"
1292                 triggerString += "\\W+Build and Test"
1293             }
1294
1295             triggerString += ".*"
1296
1297             switch (os) {
1298                 case 'Ubuntu':
1299                 case 'Ubuntu16.04':
1300                     switch (scenario) {
1301                         case 'default':
1302                             if (configuration == 'Debug' && !isFlowJob) {
1303                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1304                             }
1305                             else {
1306                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1307                             }
1308
1309                             break
1310                         default:
1311                             if (isR2RScenario(scenario)) {
1312                                 if (configuration == 'Checked' || configuration == 'Release') {
1313                                     def displayStr = getR2RDisplayName(scenario)
1314                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1315                                 }
1316                             }
1317                             break
1318                     }
1319                     break
1320                 case 'Windows_NT':
1321                     // Triggers on the non-flow jobs aren't necessary here
1322                     if (!isFlowJob) {
1323                         break
1324                     }
1325
1326                     assert isArmWindowsScenario(scenario)
1327                     switch (scenario) {
1328                         case 'default':
1329                             if (configuration == 'Checked') {
1330                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1331                             }
1332                             else {
1333                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1334                             }
1335                             break
1336                         default:
1337                             // Stress jobs will use this code path.
1338                             if (isArmWindowsScenario(scenario)) {
1339                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1340                             }
1341                             break
1342                     }
1343                     break
1344                 default:
1345                     println("NYI os: ${os}");
1346                     assert false
1347                     break
1348             }
1349             break
1350         // editor brace matching: }
1351         case 'x86': // editor brace matching: {
1352             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1353             if (os == 'Ubuntu') {
1354                 // Triggers on the non-flow jobs aren't necessary here
1355                 if (!isFlowJob) {
1356                     break
1357                 }
1358                 // on-demand only for ubuntu x86
1359                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1360                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1361                 break
1362             }
1363             switch (scenario) {
1364                 case 'default':
1365                     if (configuration == 'Checked') {
1366                         assert !job.name.contains("centos")
1367                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1368                     }
1369                     else {
1370                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1371                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1372                     }
1373                     break
1374                 case 'ilrt':
1375                     if (configuration == 'Release') {
1376                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1377                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1378                     }
1379                     break
1380                 case 'longgc':
1381                     if (configuration == 'Release') {
1382                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1383                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1384                     }
1385                     break
1386                 case 'gcsimulator':
1387                     if (configuration == 'Release') {
1388                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1389                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1390                     }
1391                     break
1392                 case 'standalone_gc':
1393                     if (configuration == 'Release' || configuration == 'Checked') {
1394                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1395                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1396                     }
1397                     break
1398                 case 'illink':
1399                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1400                     break
1401                 default:
1402                     if (isJitStressScenario(scenario)) {
1403                         def displayStr = getStressModeDisplayName(scenario)
1404                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1405                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1406                     }
1407                     else if (isR2RScenario(scenario)) {
1408                         if (configuration == 'Release' || configuration == 'Checked') {
1409                             def displayStr = getR2RDisplayName(scenario)
1410                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1411                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1412                         }
1413                     }
1414                     else {
1415                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1416                         assert false
1417                     }
1418                     break
1419             }
1420             break
1421          // editor brace matching: }
1422         case 'x64_arm64_altjit':
1423         case 'x86_arm_altjit': // editor brace matching: {
1424             assert (os == 'Windows_NT')
1425             switch (scenario) {
1426                 case 'default':
1427                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1428                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1429                     break
1430                 default:
1431                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1432                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1433                     break
1434             }
1435             break
1436         // editor brace matching: }
1437         default:
1438             println("Unknown architecture: ${architecture}");
1439             assert false
1440             break
1441     }
1442 }
1443
1444 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1445     def buildCommands = [];
1446     def osGroup = getOSGroup(os)
1447     def lowerConfiguration = configuration.toLowerCase()
1448
1449     def priority = '1'
1450     if ((scenario == 'default' && isPR == true) || (scenario == 'default' && isBuildOnly == true)) {
1451         priority = '0'
1452     }
1453
1454     def enableCorefxTesting = isCoreFxScenario(scenario)
1455
1456     // Calculate the build steps, archival, and xunit results
1457     switch (os) {
1458         case 'Windows_NT': // editor brace matching: {
1459             switch (architecture) {
1460                 case 'x64':
1461                 case 'x86':
1462                 case 'x86_arm_altjit':
1463                 case 'x64_arm64_altjit':
1464                     def arch = architecture
1465                     def buildOpts = ''
1466                     if (architecture == 'x86_arm_altjit') {
1467                         arch = 'x86'
1468                     }
1469                     else if (architecture == 'x64_arm64_altjit') {
1470                         arch = 'x64'
1471                     }
1472
1473                     if (scenario == 'formatting') {
1474                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1475                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1476                         break
1477                     }
1478
1479                     if (scenario == 'illink') {
1480                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1481                     }
1482
1483                     // If it is a release build for windows, ensure PGO is used, else fail the build
1484                     if ((lowerConfiguration == 'release') &&
1485                         (scenario in Constants.basicScenarios) &&
1486                         (architecture != 'x86_arm_altjit') &&
1487                         (architecture != 'x64_arm64_altjit')) {
1488
1489                         buildOpts += ' -enforcepgo'
1490                     }
1491
1492                     if (enableCorefxTesting) {
1493                         buildOpts += ' skiptests';
1494                     } else {
1495                         buildOpts += " -priority=${priority}"
1496                     }
1497
1498                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1499                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1500                     // 35 characters long.
1501
1502                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1503
1504                     if (!isBuildOnly) {
1505                         def runtestArguments = ''
1506                         def testOpts = 'collectdumps'
1507
1508                         if (isR2RScenario(scenario)) {
1509
1510                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1511                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1512                             // to cause the tests to be crossgen'ed.
1513
1514                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1515                                 testOpts += ' crossgenaltjit protononjit.dll'
1516                             } else {
1517                                 testOpts += ' crossgen'
1518                             }
1519
1520                             testOpts += ' runcrossgentests'
1521                         }
1522                         else if (scenario == 'jitdiff') {
1523                             testOpts += ' jitdisasm crossgen'
1524                         }
1525                         else if (scenario == 'ilrt') {
1526                             testOpts += ' ilasmroundtrip'
1527                         }
1528                         else if (isLongGc(scenario)) {
1529                             testOpts += " ${scenario} sequential"
1530                         }
1531                         else if (scenario == 'standalone_gc') {
1532                             testOpts += ' gcname clrgc.dll'
1533                         }
1534                         else if (scenario == 'illink') {
1535                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1536                         }
1537
1538                         // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this
1539                         // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by
1540                         // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable,
1541                         // which is read by the xunit harness.)
1542                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug'))
1543                         {
1544                             def timeout = 1800000
1545                             testOpts += " timeout ${timeout}"
1546                         }
1547
1548                         // If we are running a stress mode, we should write out the set of key
1549                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1550
1551                         def envScriptPath = ''
1552                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1553                             def buildCommandsStr = ''
1554                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1555                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1556
1557                             if (isJitStressScenario(scenario)) {
1558                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1559                             }
1560                             else if (isR2RStressScenario(scenario)) {
1561                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1562                             }
1563
1564                             if (architecture == 'x86_arm_altjit') {
1565                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1566                             }
1567                             else if (architecture == 'x64_arm64_altjit') {
1568                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1569                             }
1570
1571                             envScriptFinalize(os, envScriptPath)
1572
1573                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1574                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1575                             // into a single script.
1576                             buildCommands += buildCommandsStr
1577                         }
1578                         else if (architecture == 'x86_arm_altjit') {
1579                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1580                         }
1581                         else if (architecture == 'x64_arm64_altjit') {
1582                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1583                         }
1584                         if (envScriptPath != '') {
1585                             testOpts += " TestEnv ${envScriptPath}"
1586                         }
1587
1588                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1589
1590                         if (enableCorefxTesting) {
1591                             def workspaceRelativeFxRoot = "_/fx"
1592                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1593
1594                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1595
1596                             setTestJobTimeOut(newJob, isPR, architecture, scenario)
1597
1598                             // Archive and process (only) the test results
1599                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1600                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1601
1602                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1603                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1604                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1605                         }
1606                         else if (isGcReliabilityFramework(scenario)) {
1607                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1608                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1609                         }
1610                         else {
1611                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1612                         }
1613                     }
1614
1615                     if (!enableCorefxTesting) {
1616                         // Run the rest of the build
1617                         // Build the mscorlib for the other OS's
1618                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1619                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1620
1621                         if (arch == "x64") {
1622                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1623                         }
1624
1625                         // Zip up the tests directory so that we don't use so much space/time copying
1626                         // 10s of thousands of files around.
1627                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1628
1629                         if (!isJitStressScenario(scenario)) {
1630                             // For windows, pull full test results and test drops for x86/x64.
1631                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1632                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1633                         }
1634
1635                         if (scenario == 'jitdiff') {
1636                             // retrive jit-dasm output for base commit, and run jit-diff
1637                             if (!isBuildOnly) {
1638                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1639                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1640                             }
1641                         }
1642
1643                         if (!isBuildOnly) {
1644                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1645                             setTestJobTimeOut(newJob, isPR, architecture, scenario)
1646                         }
1647                     }
1648                     break
1649                 case 'armlb':
1650                 case 'arm':
1651                     assert isArmWindowsScenario(scenario)
1652
1653                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1654                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1655
1656                     setTestJobTimeOut(newJob, isPR, architecture, scenario)
1657
1658                     def buildArchitecture = 'arm'
1659
1660                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1661                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1662                     def armCrossgenOpt = ''
1663                     if (architecture == 'armlb') {
1664                         armCrossgenOpt = '-crossgenaltjit legacyjit.dll'
1665                     }
1666
1667                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1668                     priority = '1'
1669
1670                     // This is now a build only job. Do not run tests. Use the flow job.
1671                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1672
1673                     // Zip up the tests directory so that we don't use so much space/time copying
1674                     // 10s of thousands of files around.
1675                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1676
1677                     // Add archival.
1678                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1679                     break
1680                 case 'arm64':
1681                     assert isArmWindowsScenario(scenario)
1682
1683                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1684                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1685
1686                     setTestJobTimeOut(newJob, isPR, architecture, scenario)
1687
1688                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1689                     priority = '1'
1690
1691                     // This is now a build only job. Do not run tests. Use the flow job.
1692                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1693
1694                     // Zip up the tests directory so that we don't use so much space/time copying
1695                     // 10s of thousands of files around.
1696                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1697
1698                     // Add archival.
1699                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1700                     break
1701                 default:
1702                     println("Unknown architecture: ${architecture}");
1703                     assert false
1704                     break
1705             }
1706             break
1707         // editor brace matching: }
1708         case 'Ubuntu':
1709         case 'Ubuntu16.04':
1710         case 'Ubuntu16.10':
1711         case 'Debian8.4':
1712         case 'OSX10.12':
1713         case 'CentOS7.1':
1714         case 'RHEL7.2':
1715         case 'Tizen':
1716         case 'Fedora24': // editor brace matching: {
1717             switch (architecture) {
1718                 case 'x64':
1719                 case 'x86':
1720                     if (architecture == 'x86' && os == 'Ubuntu') {
1721                         // build and PAL test
1722                         def dockerImage = getDockerImageName(architecture, os, true)
1723                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1724                         dockerImage = getDockerImageName(architecture, os, false)
1725                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1726                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1727                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1728                         break
1729                     }
1730
1731                     if (scenario == 'formatting') {
1732                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1733                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1734                         break
1735                     }
1736
1737                     if (scenario == 'illink') {
1738                         assert(os == 'Ubuntu')
1739                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1740                     }
1741
1742                     if (!enableCorefxTesting) {
1743                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1744                         // only on supported OS platforms.
1745                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1746                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1747
1748                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1749                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1750
1751                         setTestJobTimeOut(newJob, isPR, architecture, scenario)
1752
1753                         // Basic archiving of the build
1754                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1755                         // And pal tests
1756                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1757                     }
1758                     else {
1759                         // Corefx stress testing
1760                         assert os == 'Ubuntu'
1761                         assert architecture == 'x64'
1762                         assert lowerConfiguration == 'checked'
1763                         assert isJitStressScenario(scenario)
1764
1765                         // Build coreclr
1766                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1767
1768                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1769
1770                         def envScriptCmds = envScriptCreate(os, scriptFileName)
1771                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
1772                         envScriptCmds += envScriptFinalize(os, scriptFileName)
1773                         buildCommands += envScriptCmds
1774
1775                         // Build and text corefx
1776                         def workspaceRelativeFxRoot = "_/fx"
1777                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1778
1779                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1780
1781                         setTestJobTimeOut(newJob, isPR, architecture, scenario)
1782
1783                         // Archive and process (only) the test results
1784                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1785                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1786                     }
1787                     break
1788                 case 'arm64':
1789                     if (!enableCorefxTesting) {
1790                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1791
1792                         // HACK -- Arm64 does not have corefx jobs yet.
1793                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1794                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1795                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1796                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1797
1798                         setTestJobTimeOut(newJob, isPR, architecture, scenario)
1799                         // Basic archiving of the build
1800                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1801                     }
1802                     break
1803                 case 'arm':
1804                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1805                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1806
1807                     // default values for Ubuntu
1808                     def arm_abi="arm"
1809                     def linuxCodeName="trusty"
1810                     if (os == 'Ubuntu16.04') {
1811                         linuxCodeName="xenial"
1812                     }
1813                     else if (os == 'Tizen') {
1814                         arm_abi="armel"
1815                         linuxCodeName="tizen"
1816                     }
1817
1818                     // Unzip the Windows test binaries first. Exit with 0
1819                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1820
1821                     // Unpack the corefx binaries
1822                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1823                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1824                     if (os != 'Tizen') {
1825                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1826                     }
1827                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1828                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1829                     if (os == 'Ubuntu') {
1830                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1831                     }
1832
1833                     // Call the ARM CI script to cross build and test using docker
1834                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1835                     --mode=docker \\
1836                     --${arm_abi} \\
1837                     --linuxCodeName=${linuxCodeName} \\
1838                     --buildConfig=${lowerConfiguration} \\
1839                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1840                     --coreFxBinDir=./bin/CoreFxBinDir \\
1841                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1842
1843                     // Basic archiving of the build, no pal tests
1844                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1845                     break
1846                 default:
1847                     println("Unknown architecture: ${architecture}");
1848                     assert false
1849                     break
1850             }
1851             break
1852         // editor brace matching: }
1853         default:
1854             println("Unknown os: ${os}");
1855             assert false
1856             break
1857     } // os
1858
1859     return buildCommands
1860 }
1861
1862 Constants.allScenarios.each { scenario ->
1863     [true, false].each { isPR ->
1864         Constants.architectureList.each { architecture ->
1865             Constants.configurationList.each { configuration ->
1866                 Constants.osList.each { os ->
1867                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1868                     // and reset the os to Windows_NT
1869                     def isBuildOnly = false
1870                     if (os == 'Windows_NT_BuildOnly') {
1871                         isBuildOnly = true
1872                         os = 'Windows_NT'
1873                     }
1874
1875                     // Tizen is only supported for arm architecture
1876                     if (os == 'Tizen' && architecture != 'arm') {
1877                         return
1878                     }
1879
1880                     // Skip totally unimplemented (in CI) configurations.
1881                     switch (architecture) {
1882                         case 'arm64':
1883                             if (os == 'Ubuntu16.04') {
1884                                 os = 'Ubuntu'
1885                             }
1886
1887                             // Windows and Ubuntu only
1888                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1889                                 return
1890                             }
1891                             break
1892                         case 'arm':
1893                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1894                                 return
1895                             }
1896                             break
1897                         case 'armlb':
1898                             if (os != 'Windows_NT') {
1899                                 return
1900                             }
1901                             break
1902                         case 'x86':
1903                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1904                                 return
1905                             }
1906                             break
1907                         case 'x86_arm_altjit':
1908                         case 'x64_arm64_altjit':
1909                             if (os != 'Windows_NT') {
1910                                 return
1911                             }
1912                             break
1913                         case 'x64':
1914                             // Everything implemented
1915                             break
1916                         default:
1917                             println("Unknown architecture: ${architecture}")
1918                             assert false
1919                             break
1920                     }
1921
1922                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1923                     // with checked builds)
1924                     if (isJitStressScenario(scenario)) {
1925                         if (configuration != 'Checked') {
1926                             return
1927                         }
1928
1929                         // Since these are just execution time differences,
1930                         // skip platforms that don't execute the tests here (Windows_NT only)
1931                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
1932                         if (!isEnabledOS || isBuildOnly) {
1933                             return
1934                         }
1935
1936                         switch (architecture) {
1937                             case 'x64':
1938                             case 'x86':
1939                             case 'x86_arm_altjit':
1940                             case 'x64_arm64_altjit':
1941                                 // x86 ubuntu: default only
1942                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1943                                     return
1944                                 }
1945                                 // Windows: Everything implemented
1946                                 break
1947
1948                             default:
1949                                 // arm, arm64, armlb: stress is handled through flow jobs.
1950                                 return
1951                         }
1952                     }
1953                     else if (isR2RScenario(scenario)) {
1954                         if (os != 'Windows_NT') {
1955                             return
1956                         }
1957                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
1958                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
1959                             return
1960                         }
1961                     }
1962                     else {
1963                         // Skip scenarios
1964                         switch (scenario) {
1965                             case 'ilrt':
1966                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1967                                 // the default scenario build
1968                                 if (os != 'Windows_NT') {
1969                                     return
1970                                 }
1971                                 // Only x64 for now
1972                                 if (architecture != 'x64') {
1973                                     return
1974                                 }
1975                                 // Release only
1976                                 if (configuration != 'Release') {
1977                                     return
1978                                 }
1979                                 break
1980                             case 'jitdiff':
1981                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1982                                     return
1983                                 }
1984                                 if (architecture != 'x64') {
1985                                     return
1986                                 }
1987                                 if (configuration != 'Checked') {
1988                                     return
1989                                 }
1990                                 break
1991                             case 'longgc':
1992                             case 'gcsimulator':
1993                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1994                                     return
1995                                 }
1996                                 if (architecture != 'x64') {
1997                                     return
1998                                 }
1999                                 if (configuration != 'Release') {
2000                                     return
2001                                 }
2002                                 break
2003                             case 'gc_reliability_framework':
2004                             case 'standalone_gc':
2005                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
2006                                     return
2007                                 }
2008
2009                                 if (architecture != 'x64') {
2010                                     return
2011                                 }
2012
2013                                 if (configuration != 'Release' && configuration != 'Checked') {
2014                                     return
2015                                 }
2016                                 break
2017                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2018                             case 'formatting':
2019                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2020                                     return
2021                                 }
2022                                 if (architecture != 'x64') {
2023                                     return
2024                                 }
2025                                 if (configuration != 'Checked') {
2026                                     return
2027                                 }
2028                                 if (isBuildOnly) {
2029                                     return
2030                                 }
2031                                 break
2032                             case 'illink':
2033                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2034                                     return
2035                                 }
2036                                 if (architecture != 'x64' && architecture != 'x86') {
2037                                     return
2038                                 }
2039                                 if (isBuildOnly) {
2040                                     return
2041                                 }
2042                                 break
2043                             case 'default':
2044                                 // Nothing skipped
2045                                 break
2046                             default:
2047                                 println("Unknown scenario: ${scenario}")
2048                                 assert false
2049                                 break
2050                         }
2051                     }
2052
2053                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2054                     // runtime behavior, not compile-time behavior, are not interesting.
2055                     switch (architecture) {
2056                         case 'x86_arm_altjit':
2057                         case 'x64_arm64_altjit':
2058                             if (isGCStressRelatedTesting(scenario)) {
2059                                 return
2060                             }
2061                             break
2062                         default:
2063                             break
2064                     }
2065
2066                     // Calculate names
2067                     def lowerConfiguration = configuration.toLowerCase()
2068                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2069                     def folderName = getJobFolder(scenario)
2070
2071                     // Create the new job
2072                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2073
2074                     def machineAffinityOptions = null
2075
2076                     if (os != 'Windows_NT') {
2077                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2078                     }
2079                     else {
2080                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2081                     }
2082
2083                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2084
2085                     // Add all the standard options
2086                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2087                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2088
2089                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2090                     def osGroup = getOSGroup(os)
2091
2092                     newJob.with {
2093                         steps {
2094                             if (os == 'Windows_NT') {
2095                                 buildCommands.each { buildCommand ->
2096                                     batchFile(buildCommand)
2097                                 }
2098                             }
2099                             else {
2100                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2101                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2102                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2103                                     // Define the Windows Tests and Corefx build job names
2104                                     def WindowTestsName = projectFolder + '/' +
2105                                                           Utilities.getFullJobName(project,
2106                                                                                    getJobName(lowerConfiguration,
2107                                                                                               'x64' ,
2108                                                                                               'windows_nt',
2109                                                                                               'default',
2110                                                                                               true),
2111                                                                                    false)
2112                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2113                                                        Utilities.getFolderName(branch)
2114
2115                                     // Copy the Windows test binaries and the Corefx build binaries
2116                                     copyArtifacts(WindowTestsName) {
2117                                         includePatterns('bin/tests/tests.zip')
2118                                         buildSelector {
2119                                             latestSuccessful(true)
2120                                         }
2121                                     }
2122
2123                                     def arm_abi = 'arm'
2124                                     def corefx_os = 'linux'
2125                                     if (os == 'Tizen') {
2126                                         arm_abi = 'armel'
2127                                         corefx_os = 'tizen'
2128                                     }
2129
2130                                     // Let's use release CoreFX to test checked CoreCLR,
2131                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2132                                     def corefx_lowerConfiguration = lowerConfiguration
2133                                     if ( lowerConfiguration == 'checked' ) {
2134                                         corefx_lowerConfiguration='release'
2135                                     }
2136
2137                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2138                                         includePatterns('bin/build.tar.gz')
2139                                         buildSelector {
2140                                             latestSuccessful(true)
2141                                         }
2142                                     }
2143                                 }
2144
2145                                 buildCommands.each { buildCommand ->
2146                                     shell(buildCommand)
2147                                 }
2148                             }
2149                         }
2150                     } // newJob.with
2151
2152                 } // os
2153             } // configuration
2154         } // architecture
2155     } // isPR
2156 } // scenario
2157
2158
2159 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2160 Constants.allScenarios.each { scenario ->
2161     [true, false].each { isPR ->
2162         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2163             Constants.crossList.each { os ->
2164                 if (architecture == 'arm64') {
2165                     if (os != "Ubuntu" && os != "Windows_NT") {
2166                         return
2167                     }
2168                 } else if (architecture == 'arm' || architecture == 'armlb') {
2169                     if (os != 'Windows_NT') {
2170                         return
2171                     }
2172                 }
2173                 else if (architecture == 'x86') {
2174                     if (os != "Ubuntu") {
2175                         return
2176                     }
2177                 }
2178
2179                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2180
2181                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2182                     return
2183                 }
2184
2185                 Constants.configurationList.each { configuration ->
2186
2187                     // First, filter based on OS.
2188
2189                     if (os == 'Windows_NT') {
2190                         if (!isArmWindowsScenario(scenario)) {
2191                             return
2192                         }
2193                     }
2194                     else {
2195                         // Non-Windows
2196                         if (architecture == 'arm64') {
2197                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2198                                 return
2199                             }
2200                         }
2201                         else if (architecture == 'x86') {
2202                             // Linux/x86 only want default test
2203                             if (scenario != 'default') {
2204                                 return
2205                             }
2206                         }
2207                     }
2208
2209                     // For CentOS, we only want Checked/Release builds.
2210                     if (os == 'CentOS7.1') {
2211                         if (configuration != 'Checked' && configuration != 'Release') {
2212                             return
2213                         }
2214                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2215                             return
2216                         }
2217                     }
2218
2219                     // For RedHat and Debian, we only do Release builds.
2220                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2221                         if (configuration != 'Release') {
2222                             return
2223                         }
2224                         if (scenario != 'default') {
2225                             return
2226                         }
2227                     }
2228
2229                     // Next, filter based on scenario.
2230
2231                     if (isJitStressScenario(scenario)) {
2232                         if (configuration != 'Checked') {
2233                             return
2234                         }
2235                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2236                         if (isCoreFxScenario(scenario)) {
2237                             return
2238                         }
2239                     }
2240                     else if (isR2RBaselineScenario(scenario)) {
2241                         if (configuration != 'Checked' && configuration != 'Release') {
2242                             return
2243                         }
2244                     }
2245                     else if (isR2RStressScenario(scenario)) {
2246                         if (configuration != 'Checked') {
2247                             return
2248                         }
2249                     }
2250                     else {
2251                         // Skip scenarios
2252                         switch (scenario) {
2253                             case 'ilrt':
2254                             case 'longgc':
2255                             case 'gcsimulator':
2256                                 // Long GC tests take a long time on non-Release builds
2257                                 // ilrt is also Release only
2258                                 if (configuration != 'Release') {
2259                                     return
2260                                 }
2261                                 break
2262                             case 'jitdiff':
2263                                 if (configuration != 'Checked') {
2264                                     return;
2265                                 }
2266                                 break
2267                             case 'gc_reliability_framework':
2268                             case 'standalone_gc':
2269                                 if (configuration != 'Release' && configuration != 'Checked') {
2270                                     return
2271                                 }
2272                                 break
2273                             case 'formatting':
2274                                 return
2275                             case 'illink':
2276                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2277                                     return
2278                                 }
2279                                 break
2280                             case 'default':
2281                                 // Nothing skipped
2282                                 break
2283                             default:
2284                                 println("Unknown scenario: ${scenario}")
2285                                 assert false
2286                                 break
2287                         }
2288                     }
2289
2290                     // Done filtering. Now, create the jobs.
2291
2292                     def lowerConfiguration = configuration.toLowerCase()
2293                     def osGroup = getOSGroup(os)
2294                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2295
2296                     def inputCoreCLRBuildName = projectFolder + '/' +
2297                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2298
2299                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2300                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2301                     def testBuildScenario = scenario
2302                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2303                         testBuildScenario = 'default'
2304                     }
2305
2306                     def inputWindowsTestBuildArch = architecture
2307                     if (architecture == "arm64" && os != "Windows_NT") {
2308                         // Use the x64 test build for arm64 unix
2309                         inputWindowsTestBuildArch = "x64"
2310                     }
2311
2312                     def inputWindowTestsBuildName = ""
2313
2314                     if (isJitStressScenario(scenario)) {
2315                         inputWindowTestsBuildName = projectFolder + '/' +
2316                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2317                     } else {
2318                         inputWindowTestsBuildName = projectFolder + '/' +
2319                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2320
2321                     }
2322
2323
2324                     // Enable Server GC for Ubuntu PR builds
2325                     def serverGCString = ''
2326                     if (os == 'Ubuntu' && isPR) {
2327                         serverGCString = '--useServerGC'
2328                     }
2329
2330                     def testOpts = ''
2331
2332                     if (isR2RScenario(scenario)) {
2333
2334                         testOpts += ' --crossgen --runcrossgentests'
2335
2336                         if (scenario == 'r2r_jitstress1') {
2337                             testOpts += ' --jitstress=1'
2338                         }
2339                         else if (scenario == 'r2r_jitstress2') {
2340                             testOpts += ' --jitstress=2'
2341                         }
2342                         else if (scenario == 'r2r_jitstressregs1') {
2343                             testOpts += ' --jitstressregs=1'
2344                         }
2345                         else if (scenario == 'r2r_jitstressregs2') {
2346                             testOpts += ' --jitstressregs=2'
2347                         }
2348                         else if (scenario == 'r2r_jitstressregs3') {
2349                             testOpts += ' --jitstressregs=3'
2350                         }
2351                         else if (scenario == 'r2r_jitstressregs4') {
2352                             testOpts += ' --jitstressregs=4'
2353                         }
2354                         else if (scenario == 'r2r_jitstressregs8') {
2355                             testOpts += ' --jitstressregs=8'
2356                         }
2357                         else if (scenario == 'r2r_jitstressregs0x10') {
2358                             testOpts += ' --jitstressregs=0x10'
2359                         }
2360                         else if (scenario == 'r2r_jitstressregs0x80') {
2361                             testOpts += ' --jitstressregs=0x80'
2362                         }
2363                         else if (scenario == 'r2r_jitstressregs0x1000') {
2364                             testOpts += ' --jitstressregs=0x1000'
2365                         }
2366                         else if (scenario == 'r2r_jitminopts') {
2367                             testOpts += ' --jitminopts'
2368                         }
2369                         else if (scenario == 'r2r_jitforcerelocs') {
2370                             testOpts += ' --jitforcerelocs'
2371                         }
2372                         else if (scenario == 'r2r_gcstress15') {
2373                             testOpts += ' --gcstresslevel=0xF'
2374                         }
2375                     }
2376                     else if (scenario == 'jitdiff') {
2377                         testOpts += ' --jitdisasm --crossgen'
2378                     }
2379                     else if (scenario == 'illink') {
2380                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2381                     }
2382                     else if (isLongGc(scenario)) {
2383                         // Long GC tests behave very poorly when they are not
2384                         // the only test running (many of them allocate until OOM).
2385                         testOpts += ' --sequential'
2386
2387                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2388                         // for running long GC and GCSimulator tests, respectively. We don't use them
2389                         // here because using a playlist file produces much more readable output on the CI machines
2390                         // and reduces running time.
2391                         //
2392                         // The Long GC playlist contains all of the tests that are
2393                         // going to be run. The GCSimulator playlist contains all of
2394                         // the GC simulator tests.
2395                         if (scenario == 'longgc') {
2396                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2397                         }
2398                         else if (scenario == 'gcsimulator') {
2399                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2400                         }
2401                     }
2402                     else if (isGcReliabilityFramework(scenario)) {
2403                         testOpts += ' --build-overlay-only'
2404                     }
2405                     else if (scenario == 'standalone_gc') {
2406                         if (osGroup == 'OSX') {
2407                             testOpts += ' --gcname=libclrgc.dylib'
2408                         }
2409                         else if (osGroup == 'Linux') {
2410                             testOpts += ' --gcname=libclrgc.so'
2411                         }
2412                         else {
2413                             println("Unexpected OS group: ${osGroup} for os ${os}")
2414                             assert false
2415                         }
2416                     }
2417
2418                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2419
2420                     def folder = getJobFolder(scenario)
2421                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2422                         // Add parameters for the inputs
2423
2424                         if (windowsArmJob == true) {
2425                             parameters {
2426                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2427                             }
2428                         }
2429                         else {
2430                             parameters {
2431                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2432                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2433                             }
2434                         }
2435
2436                         steps {
2437                             // Set up the copies
2438
2439                             // Coreclr build containing the tests and mscorlib
2440                             // pri1 jobs still need to copy windows_nt built tests
2441                             if (windowsArmJob != true) {
2442                                 copyArtifacts(inputWindowTestsBuildName) {
2443                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2444                                     buildSelector {
2445                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2446                                     }
2447                                 }
2448                             }
2449
2450                             // Coreclr build we are trying to test
2451                             //
2452                             //  ** NOTE ** This will, correctly, overwrite over the CORE_ROOT from the windows test archive
2453
2454                             copyArtifacts(inputCoreCLRBuildName) {
2455                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2456                                 buildSelector {
2457                                     buildNumber('${CORECLR_BUILD}')
2458                                 }
2459                             }
2460
2461                             // Windows CoreCLR Arm(64) will restore corefx
2462                             // packages correctly.
2463                             //
2464                             // In addition, test steps are entirely different
2465                             // because we do not have a unified runner
2466                             if (windowsArmJob != true) {
2467                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2468
2469                                 // HACK -- Arm64 does not have corefx jobs yet.
2470                                 // Clone corefx and build the native packages overwriting the x64 packages.
2471                                 if (architecture == 'arm64') {
2472                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2473                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2474                                 }
2475                                 else if (architecture == 'x86') {
2476                                     shell("mkdir ./bin/CoreFxNative")
2477
2478                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2479                                         includePatterns('bin/build.tar.gz')
2480                                         targetDirectory('bin/CoreFxNative')
2481                                         buildSelector {
2482                                             latestSuccessful(true)
2483                                         }
2484                                     }
2485
2486                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2487                                 }
2488
2489                                 // Unzip the tests first.  Exit with 0
2490                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2491                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2492
2493                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2494
2495                                 // Execute the tests
2496                                 def runDocker = isNeedDocker(architecture, os, false)
2497                                 def dockerPrefix = ""
2498                                 def dockerCmd = ""
2499                                 if (runDocker) {
2500                                     def dockerImage = getDockerImageName(architecture, os, false)
2501                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2502                                     dockerCmd = dockerPrefix + "${dockerImage} "
2503                                 }
2504
2505                                 // If we are running a stress mode, we'll set those variables first
2506                                 def testEnvOpt = ""
2507                                 if (isJitStressScenario(scenario)) {
2508                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2509                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2510                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2511                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2512                                     shell("${envScriptCmds}")
2513                                     testEnvOpt = "--test-env=" + scriptFileName
2514                                 }
2515
2516                                 if (isGCStressRelatedTesting(scenario)) {
2517                                     shell('./init-tools.sh')
2518                                 }
2519
2520                                 shell("""${dockerCmd}./tests/runtest.sh \\
2521                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2522                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2523                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2524                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2525
2526                                 if (isGcReliabilityFramework(scenario)) {
2527                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2528                                     if (serverGCString != '') {
2529                                         if (runDocker) {
2530                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2531                                         }
2532                                         else {
2533                                             shell("export COMPlus_gcServer=1")
2534                                         }
2535                                     }
2536
2537                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2538                                 }
2539                             }
2540
2541                             else { // windowsArmJob == true
2542                                 // Unzip tests.
2543                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2544
2545                                 // Build the build commands
2546                                 def buildCommands = ""
2547
2548                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2549                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2550                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2551
2552                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2553                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2554
2555                                 // For all jobs
2556                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2557
2558                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2559                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2560
2561                                 // ARM legacy backend; this is an altjit.
2562                                 if (architecture == "armlb") {
2563                                     addEnvVariable("COMPlus_AltJit", "*")
2564                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2565                                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2566                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2567                                 }
2568
2569                                 // If we are running a stress mode, we'll set those variables as well
2570                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2571                                     def stressValues = null
2572                                     if (isJitStressScenario(scenario)) {
2573                                         stressValues = Constants.jitStressModeScenarios[scenario]
2574                                     }
2575                                     else {
2576                                         stressValues = Constants.r2rStressScenarios[scenario]
2577                                     }
2578
2579                                     stressValues.each { key, value ->
2580                                         addEnvVariable(key, value)
2581                                     }
2582                                 }
2583
2584                                 if (isR2RScenario(scenario)) {
2585                                     // Crossgen the framework assemblies.
2586                                     buildCommands += """
2587 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2588 @goto skip_PrecompileAssembly
2589
2590 :PrecompileAssembly
2591 @REM Skip mscorlib since it is already precompiled.
2592 @if /I "%3" == "mscorlib.dll" exit /b 0
2593 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2594
2595 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2596 @if "%errorlevel%" == "-2146230517" (
2597     echo %2 is not a managed assembly.
2598 ) else if "%errorlevel%" == "-2146234344" (
2599     echo %2 is not a managed assembly.
2600 ) else if %errorlevel% neq 0 (
2601     echo Unable to precompile %2
2602 ) else (
2603     echo Precompiled %2
2604 )
2605 @exit /b 0
2606
2607 :skip_PrecompileAssembly
2608 """
2609
2610                                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2611                                     // crossgen on tests before running them.
2612                                     addEnvVariable("RunCrossGen", "true")
2613                                 }
2614
2615                                 // Create the smarty command
2616                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2617                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2618                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2619
2620                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2621
2622                                 if (architecture == "armlb") {
2623                                     addExclude("LEGACYJIT_FAIL")
2624                                 }
2625
2626                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2627                                     def failTag = "JITSTRESS_FAIL"
2628                                     def excludeTag = "JITSTRESS_EXCLUDE"
2629
2630                                     if (scenario.contains('gc')) {
2631                                         failTag = "GCSTRESS_FAIL"
2632                                         excludeTag = "GCSTRESS_EXCLUDE"
2633                                     }
2634
2635                                     addArchSpecificExclude(architecture, failTag)
2636                                     addArchSpecificExclude(architecture, excludeTag)
2637                                 }
2638                                 else {
2639                                     addExclude("pri1")
2640                                 }
2641
2642                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2643                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2644                                 addExclude("LONG_RUNNING")
2645
2646                                 smartyCommand += "/lstFile Tests.lst"
2647
2648                                 def testListArch = [
2649                                     'arm64': 'arm64',
2650                                     'arm': 'arm',
2651                                     'armlb': 'arm'
2652                                 ]
2653
2654                                 def archLocation = testListArch[architecture]
2655
2656                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2657                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2658                                 addCommand("${smartyCommand}")
2659
2660                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2661                                 // However, we also need to remove all the variables that were set during this batch file, so we
2662                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2663                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2664                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2665                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2666                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2667                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2668
2669                                 // ZIP up the smarty output, no matter what the smarty result.
2670                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2671
2672                                 addCommand("echo %errorlevel%")
2673                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2674
2675                                 // Use the smarty errorlevel as the script errorlevel.
2676                                 addCommand("exit /b %__save_smarty_errorlevel%")
2677
2678                                 batchFile(buildCommands)
2679                             }
2680                         }
2681                     }
2682
2683                     if (scenario == 'jitdiff') {
2684                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2685                     }
2686
2687                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2688                     if (os in ['Ubuntu']) {
2689                         SummaryBuilder summaries = new SummaryBuilder()
2690                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2691                         summaries.emit(newJob)
2692                     }
2693
2694                     def affinityOptions = null
2695
2696                     if (windowsArmJob == true) {
2697                         affinityOptions = [
2698                             "use_arm64_build_machine" : false
2699                         ]
2700                     }
2701
2702                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2703                         affinityOptions = [
2704                             "large_pages" : false
2705                         ]
2706                     }
2707
2708                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2709                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2710
2711                     setTestJobTimeOut(newJob, isPR, architecture, scenario)
2712
2713                     if (windowsArmJob != true) {
2714                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2715                     }
2716                     else {
2717                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2718
2719                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2720                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2721                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2722                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2723                     }
2724
2725                     // Create a build flow to join together the build and tests required to run this test.
2726                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2727                     // Linux CoreCLR test
2728                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2729                     def fullTestJobName = projectFolder + '/' + newJob.name
2730                     // Add a reference to the input jobs for report purposes
2731                     JobReport.Report.addReference(inputCoreCLRBuildName)
2732                     JobReport.Report.addReference(inputWindowTestsBuildName)
2733                     JobReport.Report.addReference(fullTestJobName)
2734                     def newFlowJob = null
2735
2736                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2737                         // Do not create the flow job for RHEL jobs.
2738                         return
2739                     }
2740
2741                     // For pri0 jobs we can build tests on unix
2742                     if (windowsArmJob) {
2743                         // For Windows arm jobs there is no reason to build a parallel test job.
2744                         // The product build supports building and archiving the tests.
2745
2746                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2747                         buildFlow("""
2748 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2749
2750 // And then build the test build
2751 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2752 """)
2753                         }
2754                     }
2755                     else {
2756                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2757                         buildFlow("""
2758 // Build the input jobs in parallel
2759 parallel (
2760 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2761 { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2762 )
2763
2764 // And then build the test build
2765 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2766             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2767 """)
2768                         }
2769                     }
2770
2771                     // For the flow jobs set the machine affinity as x64 if an armarch.
2772                     def flowArch = architecture
2773
2774                     if (flowArch in validWindowsNTCrossArches) {
2775                         flowArch = 'x64'
2776                         affinityOptions = null
2777                     }
2778
2779                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2780                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2781                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2782                 } // configuration
2783             } // os
2784         } // architecture
2785     } // isPR
2786 } // scenario
2787
2788 JobReport.Report.generateJobReport(out)
2789
2790 // Make the call to generate the help job
2791 Utilities.createHelperJob(this, project, branch,
2792     "Welcome to the ${project} Repository",  // This is prepended to the help message
2793     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2794
2795 Utilities.addCROSSCheck(this, project, branch)