netci.groovy

   1 // Import the utility functionality.
   2
   3 import jobs.generation.*
   4
   5 // The input project name (e.g. dotnet/coreclr)
   6 def project = GithubProject
   7 // The input branch name (e.g. master)
   8 def branch = GithubBranchName
   9 def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch)
  10
  11 // Create a folder for JIT stress jobs and associated folder views
  12 folder('jitstress')
  13 Utilities.addStandardFolderView(this, 'jitstress', project)
  14
  15 // Create a folder for testing via illink
  16 folder('illink')
  17 Utilities.addStandardFolderView(this, 'illink', project)
  18
  19 def static getOSGroup(def os) {
  20     def osGroupMap = ['Ubuntu':'Linux',
  21         'RHEL7.2': 'Linux',
  22         'Ubuntu16.04': 'Linux',
  23         'Ubuntu16.10': 'Linux',
  24         'Debian8.4':'Linux',
  25         'Fedora24':'Linux',
  26         'OSX10.12':'OSX',
  27         'Windows_NT':'Windows_NT',
  28         'CentOS7.1': 'Linux',
  29         'Tizen': 'Linux']
  30     def osGroup = osGroupMap.get(os, null)
  31     assert osGroup != null : "Could not find os group for ${os}"
  32     return osGroupMap[os]
  33 }
  34
  35 // We use this class (vs variables) so that the static functions can access data here.
  36 class Constants {
  37
  38     // Innerloop build OS's
  39     // The Windows_NT_BuildOnly OS is a way to speed up the Non-NT builds temporarily by avoiding
  40     // test execution in the build flow runs.  It generates the exact same build
  41     // as Windows_NT but without the tests.
  42     def static osList = [
  43                'Ubuntu',
  44                'Debian8.4',
  45                'OSX10.12',
  46                'Windows_NT',
  47                'Windows_NT_BuildOnly',
  48                'CentOS7.1',
  49                'RHEL7.2',
  50                'Ubuntu16.04',
  51                'Ubuntu16.10',
  52                'Fedora24',
  53                'Tizen']
  54
  55     def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT']
  56
  57     // This is a set of JIT stress modes combined with the set of variables that
  58     // need to be set to actually enable that stress mode.  The key of the map is the stress mode and
  59     // the values are the environment variables
  60     def static jitStressModeScenarios = [
  61                'minopts'                        : ['COMPlus_JITMinOpts' : '1'],
  62                'tieredcompilation'              : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  63                'forcerelocs'                    : ['COMPlus_ForceRelocs' : '1'],
  64                'jitstress1'                     : ['COMPlus_JitStress' : '1'],
  65                'jitstress2'                     : ['COMPlus_JitStress' : '2'],
  66                'jitstressregs1'                 : ['COMPlus_JitStressRegs' : '1'],
  67                'jitstressregs2'                 : ['COMPlus_JitStressRegs' : '2'],
  68                'jitstressregs3'                 : ['COMPlus_JitStressRegs' : '3'],
  69                'jitstressregs4'                 : ['COMPlus_JitStressRegs' : '4'],
  70                'jitstressregs8'                 : ['COMPlus_JitStressRegs' : '8'],
  71                'jitstressregs0x10'              : ['COMPlus_JitStressRegs' : '0x10'],
  72                'jitstressregs0x80'              : ['COMPlus_JitStressRegs' : '0x80'],
  73                'jitstressregs0x1000'            : ['COMPlus_JitStressRegs' : '0x1000'],
  74                'jitstress2_jitstressregs1'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'],
  75                'jitstress2_jitstressregs2'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'],
  76                'jitstress2_jitstressregs3'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'],
  77                'jitstress2_jitstressregs4'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'],
  78                'jitstress2_jitstressregs8'      : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'],
  79                'jitstress2_jitstressregs0x10'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'],
  80                'jitstress2_jitstressregs0x80'   : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'],
  81                'jitstress2_jitstressregs0x1000' : ['COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'],
  82                'tailcallstress'                 : ['COMPlus_TailcallStress' : '1'],
  83                'jitsse2only'                    : ['COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'],
  84                'jitnosimd'                      : ['COMPlus_FeatureSIMD' : '0'],
  85                'corefx_baseline'                : [ : ], // corefx baseline
  86                'corefx_minopts'                 : ['COMPlus_JITMinOpts' : '1'],
  87                'corefx_tieredcompilation'       : ['COMPlus_EXPERIMENTAL_TieredCompilation' : '1'],
  88                'corefx_jitstress1'              : ['COMPlus_JitStress' : '1'],
  89                'corefx_jitstress2'              : ['COMPlus_JitStress' : '2'],
  90                'corefx_jitstressregs1'          : ['COMPlus_JitStressRegs' : '1'],
  91                'corefx_jitstressregs2'          : ['COMPlus_JitStressRegs' : '2'],
  92                'corefx_jitstressregs3'          : ['COMPlus_JitStressRegs' : '3'],
  93                'corefx_jitstressregs4'          : ['COMPlus_JitStressRegs' : '4'],
  94                'corefx_jitstressregs8'          : ['COMPlus_JitStressRegs' : '8'],
  95                'corefx_jitstressregs0x10'       : ['COMPlus_JitStressRegs' : '0x10'],
  96                'corefx_jitstressregs0x80'       : ['COMPlus_JitStressRegs' : '0x80'],
  97                'corefx_jitstressregs0x1000'     : ['COMPlus_JitStressRegs' : '0x1000'],
  98                'gcstress0x3'                    : ['COMPlus_GCStress' : '0x3'],
  99                'gcstress0xc'                    : ['COMPlus_GCStress' : '0xC'],
 100                'zapdisable'                     : ['COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 101                'heapverify1'                    : ['COMPlus_HeapVerify' : '1'],
 102                'gcstress0xc_zapdisable'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'],
 103                'gcstress0xc_zapdisable_jitstress2'  : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress'  : '2'],
 104                'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'],
 105                'gcstress0xc_jitstress1'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '1'],
 106                'gcstress0xc_jitstress2'             : ['COMPlus_GCStress' : '0xC', 'COMPlus_JitStress'  : '2'],
 107                'gcstress0xc_minopts_heapverify1'    : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1']
 108     ]
 109
 110     // This is a set of ReadyToRun stress scenarios
 111     def static r2rStressScenarios = [
 112                'r2r_jitstress1'             : ["COMPlus_JitStress": "1"],
 113                'r2r_jitstress2'             : ["COMPlus_JitStress": "2"],
 114                'r2r_jitstressregs1'         : ["COMPlus_JitStressRegs": "1"],
 115                'r2r_jitstressregs2'         : ["COMPlus_JitStressRegs": "2"],
 116                'r2r_jitstressregs3'         : ["COMPlus_JitStressRegs": "3"],
 117                'r2r_jitstressregs4'         : ["COMPlus_JitStressRegs": "4"],
 118                'r2r_jitstressregs8'         : ["COMPlus_JitStressRegs": "8"],
 119                'r2r_jitstressregs0x10'      : ["COMPlus_JitStressRegs": "0x10"],
 120                'r2r_jitstressregs0x80'      : ["COMPlus_JitStressRegs": "0x80"],
 121                'r2r_jitstressregs0x1000'    : ["COMPlus_JitStressRegs": "0x1000"],
 122                'r2r_jitminopts'             : ["COMPlus_JITMinOpts": "1"],
 123                'r2r_jitforcerelocs'         : ["COMPlus_ForceRelocs": "1"],
 124                'r2r_gcstress15'             : ["COMPlus_GCStress": "0xF"]
 125     ]
 126
 127     // This is the basic set of scenarios
 128     def static basicScenarios = [
 129                'default',
 130                'ilrt',
 131                'r2r',
 132                'longgc',
 133                'formatting',
 134                'gcsimulator',
 135                'jitdiff',
 136                'standalone_gc',
 137                'gc_reliability_framework',
 138                'illink']
 139
 140     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 141
 142     // A set of scenarios that are valid for arm/arm64/armlb tests run on hardware. This is a map from valid scenario name
 143     // to Tests.lst file categories to exclude.
 144     //
 145     // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that,
 146     // and with the same values, with some commented out, for easier maintenance.
 147     //
 148     // Note that some scenarios that are commented out should be enabled, but haven't yet been.
 149     //
 150     def static validArmWindowsScenarios = [
 151                'default':                                [],
 152                // 'ilrt'
 153                'r2r':                                    ["R2R_FAIL"],
 154                // 'longgc'
 155                // 'formatting'
 156                // 'gcsimulator'
 157                // 'jitdiff'
 158                // 'standalone_gc'
 159                // 'gc_reliability_framework'
 160                // 'illink'
 161                'r2r_jitstress1':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 162                'r2r_jitstress2':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 163                'r2r_jitstressregs1':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 164                'r2r_jitstressregs2':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 165                'r2r_jitstressregs3':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 166                'r2r_jitstressregs4':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 167                'r2r_jitstressregs8':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 168                'r2r_jitstressregs0x10':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 169                'r2r_jitstressregs0x80':                  ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 170                'r2r_jitstressregs0x1000':                ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 171                'r2r_jitminopts':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 172                'r2r_jitforcerelocs':                     ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 173                'r2r_gcstress15':                         ["R2R_FAIL", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 174                'minopts':                                ["MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 175                'tieredcompilation':                      [],
 176                'forcerelocs':                            [],
 177                'jitstress1':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 178                'jitstress2':                             ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 179                'jitstressregs1':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 180                'jitstressregs2':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 181                'jitstressregs3':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 182                'jitstressregs4':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 183                'jitstressregs8':                         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 184                'jitstressregs0x10':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 185                'jitstressregs0x80':                      ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 186                'jitstressregs0x1000':                    ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 187                'jitstress2_jitstressregs1':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 188                'jitstress2_jitstressregs2':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 189                'jitstress2_jitstressregs3':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 190                'jitstress2_jitstressregs4':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 191                'jitstress2_jitstressregs8':              ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 192                'jitstress2_jitstressregs0x10':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 193                'jitstress2_jitstressregs0x80':           ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 194                'jitstress2_jitstressregs0x1000':         ["JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 195                'tailcallstress':                         ["TAILCALLSTRESS_FAIL", "TAILCALLSTRESS_EXCLUDE"],
 196                // 'jitsse2only'                          // Only relevant to xarch
 197                'jitnosimd':                              [],    // Only interesting on platforms where SIMD support exists.
 198                // 'corefx_baseline'
 199                // 'corefx_minopts'
 200                // 'corefx_tieredcompilation'
 201                // 'corefx_jitstress1'
 202                // 'corefx_jitstress2'
 203                // 'corefx_jitstressregs1'
 204                // 'corefx_jitstressregs2'
 205                // 'corefx_jitstressregs3'
 206                // 'corefx_jitstressregs4'
 207                // 'corefx_jitstressregs8'
 208                // 'corefx_jitstressregs0x10'
 209                // 'corefx_jitstressregs0x80'
 210                // 'corefx_jitstressregs0x1000'
 211                'gcstress0x3':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 212                'gcstress0xc':                            ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE"],
 213                'zapdisable':                             ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 214                'heapverify1':                            [],
 215                'gcstress0xc_zapdisable':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 216                'gcstress0xc_zapdisable_jitstress2':      ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 217                'gcstress0xc_zapdisable_heapverify1':     ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE"],
 218                'gcstress0xc_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 219                'gcstress0xc_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 220                'gcstress0xc_minopts_heapverify1':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 221
 222                //
 223                // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug?
 224                //
 225
 226                'minopts_zapdisable':                     ["ZAPDISABLE_FAIL", "ZAPDISABLE_EXCLUDE", "MINOPTS_FAIL", "MINOPTS_EXCLUDE"],
 227                'gcstress0x3_jitstress1':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 228                'gcstress0x3_jitstress2':                 ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 229                'gcstress0x3_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 230                'gcstress0x3_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 231                'gcstress0x3_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 232                'gcstress0x3_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 233                'gcstress0x3_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 234                'gcstress0x3_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 235                'gcstress0x3_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 236                'gcstress0x3_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 237                'gcstress0xc_jitstressregs1':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 238                'gcstress0xc_jitstressregs2':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 239                'gcstress0xc_jitstressregs3':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 240                'gcstress0xc_jitstressregs4':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 241                'gcstress0xc_jitstressregs8':             ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 242                'gcstress0xc_jitstressregs0x10':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 243                'gcstress0xc_jitstressregs0x80':          ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
 244                'gcstress0xc_jitstressregs0x1000':        ["GCSTRESS_FAIL", "GCSTRESS_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"]
 245     ]
 246
 247     def static configurationList = ['Debug', 'Checked', 'Release']
 248
 249     // This is the set of architectures
 250     def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86']
 251 }
 252
 253 def static setMachineAffinity(def job, def os, def architecture, def options = null) {
 254     assert os instanceof String
 255     assert architecture instanceof String
 256
 257     def armArches = ['arm', 'armlb', 'arm64']
 258     def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen']
 259
 260     if (!(architecture in armArches)) {
 261         assert options == null
 262         Utilities.setMachineAffinity(job, os, 'latest-or-auto')
 263
 264         return
 265     }
 266
 267     // This is an arm(64) job.
 268     //
 269     // There are several options.
 270     //
 271     // Windows_NT
 272     //
 273     // Arm32 (Build) -> latest-arm64
 274     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true
 275     // Arm32 (Test)  -> arm64-windows_nt
 276     //       |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false
 277     //
 278     // Arm64 (Build) -> latest-arm64
 279     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 280     // Arm64 (Test)  -> arm64-windows_nt
 281     //       |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false
 282     //
 283     // Ubuntu
 284     //
 285     // Arm32 (Build) -> arm-cross-latest
 286     //       |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb"
 287     // Arm32 (Test)  -> NYI Arch not supported
 288     //       |->
 289     //
 290     // Arm64 (Build) -> arm64-cross-latest
 291     //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
 292     // Arm64 Small Page Size (Test) -> arm64-small-page-size
 293     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
 294     // Arm64 Large Page Size (Test) -> arm64-huge-page-size
 295     //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
 296
 297     // This has to be a arm arch
 298     assert architecture in armArches
 299     if (os == "Windows_NT") {
 300         // Arm(64) windows jobs share the same machines for now
 301         def isBuild = options['use_arm64_build_machine'] == true
 302
 303         if (isBuild == true) {
 304             Utilities.setMachineAffinity(job, os, 'latest-arm64')
 305         } else {
 306             Utilities.setMachineAffinity(job, os, 'arm64-windows_nt')
 307         }
 308     } else {
 309         assert os != 'Windows_NT'
 310         assert os in supportedArmLinuxOs
 311
 312         if (architecture == 'arm' || architecture == 'armlb') {
 313             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
 314         } else {
 315             // Arm64 Linux
 316             if (options['is_build_only'] == true) {
 317                 Utilities.setMachineAffinity(job, os, 'arm64-cross-latest')
 318             } else {
 319                 // Arm64 Test Machines
 320                 if (options['large_pages'] == false) {
 321                     Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
 322                 } else {
 323                     Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
 324                 }
 325             }
 326         }
 327     }
 328 }
 329
 330 def static isGCStressRelatedTesting(def scenario) {
 331     // The 'r2r_gcstress15' scenario is a basic scenario.
 332     // Detect it and make it a GCStress related.
 333     if (scenario == 'r2r_gcstress15')
 334     {
 335         return true;
 336     }
 337
 338     def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify']
 339     def scenarioName = scenario.toLowerCase()
 340     def isGCStressTesting = false
 341     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 342         if (k in gcStressTestEnvVars) {
 343             isGCStressTesting = true;
 344         }
 345     }
 346     return isGCStressTesting
 347 }
 348
 349 def static isCoreFxScenario(def scenario) {
 350     def corefx_prefix = 'corefx_'
 351     if (scenario.length() < corefx_prefix.length()) {
 352         return false
 353     }
 354     return scenario.substring(0,corefx_prefix.length()) == corefx_prefix
 355 }
 356
 357 def static isR2RBaselineScenario(def scenario) {
 358     return (scenario == 'r2r')
 359 }
 360
 361 def static isR2RStressScenario(def scenario) {
 362     return Constants.r2rStressScenarios.containsKey(scenario)
 363 }
 364
 365 def static isR2RScenario(def scenario) {
 366     return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario)
 367 }
 368
 369 def static isJitStressScenario(def scenario) {
 370     return Constants.jitStressModeScenarios.containsKey(scenario)
 371 }
 372
 373 def static isLongGc(def scenario) {
 374     return (scenario == 'longgc' || scenario == 'gcsimulator')
 375 }
 376
 377 def static isJitDiff(def scenario) {
 378     return (scenario == 'jitdiff')
 379 }
 380
 381 def static isGcReliabilityFramework(def scenario) {
 382     return (scenario == 'gc_reliability_framework')
 383 }
 384
 385 def static isArmWindowsScenario(def scenario) {
 386     return Constants.validArmWindowsScenarios.containsKey(scenario)
 387 }
 388
 389 def static setTestJobTimeOut(newJob, isPR, architecture, scenario) {
 390     if (isGCStressRelatedTesting(scenario)) {
 391         Utilities.setJobTimeout(newJob, 4320)
 392     }
 393     else if (isCoreFxScenario(scenario)) {
 394         Utilities.setJobTimeout(newJob, 360)
 395     }
 396     else if (isJitStressScenario(scenario)) {
 397         Utilities.setJobTimeout(newJob, 240)
 398     }
 399     else if (isR2RBaselineScenario(scenario)) {
 400         Utilities.setJobTimeout(newJob, 240)
 401     }
 402     else if (isLongGc(scenario)) {
 403         Utilities.setJobTimeout(newJob, 1440)
 404     }
 405     else if (isJitDiff(scenario)) {
 406         Utilities.setJobTimeout(newJob, 240)
 407     }
 408     else if (isGcReliabilityFramework(scenario)) {
 409         Utilities.setJobTimeout(newJob, 1440)
 410     }
 411     else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') {
 412         Utilities.setJobTimeout(newJob, 240)
 413     }
 414     else if (!(scenario == 'default' && isPR == true)) {
 415         // Pri-1 test builds take a long time. Default PR jobs are Pri-0; everything else is Pri-1
 416         // (see calculateBuildCommands()). So up the Pri-1 build jobs timeout.
 417         Utilities.setJobTimeout(newJob, 240)
 418     }
 419     // Non-test jobs use the default timeout value.
 420 }
 421
 422 def static getJobFolder(def scenario) {
 423     if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
 424         return 'jitstress'
 425     }
 426     if (scenario == 'illink') {
 427         return 'illink'
 428     }
 429     return ''
 430 }
 431
 432 def static getStressModeDisplayName(def scenario) {
 433     def displayStr = ''
 434     Constants.jitStressModeScenarios[scenario].each{ k, v ->
 435         def prefixLength = 'COMPlus_'.length()
 436         if (k.length() >= prefixLength) {
 437             def modeName = k.substring(prefixLength, k.length())
 438             displayStr += ' ' + modeName + '=' + v
 439         }
 440     }
 441
 442     if (isCoreFxScenario(scenario)) {
 443         displayStr = ('CoreFx ' + displayStr).trim()
 444     }
 445
 446     return displayStr
 447 }
 448
 449 def static getR2RDisplayName(def scenario) {
 450     // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix.
 451     def displayStr = scenario
 452     def prefixLength = 'r2r_'.length()
 453     if (displayStr.length() >= prefixLength) {
 454         displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length())
 455     } else if (scenario == 'r2r') {
 456         displayStr = "R2R"
 457     }
 458     return displayStr
 459 }
 460
 461 //
 462 // Functions to create an environment script.
 463 //      envScriptCreate -- initialize the script (call first)
 464 //      envScriptFinalize -- finalize the script (call last)
 465 //      envScriptSetStressModeVariables -- set stress mode variables in the env script
 466 //      envScriptAppendExistingScript -- append an existing script to the generated script
 467 //
 468 // Each script returns a string of commands. Concatenate all the strings together before
 469 // adding them to the builds commands, to make sure they get executed as one Jenkins script.
 470 //
 471
 472 // Initialize the environment setting script.
 473 def static envScriptCreate(def os, def stepScriptLocation) {
 474     def stepScript = ''
 475     if (os == 'Windows_NT') {
 476         // Timeout in ms, default is 10 minutes. For stress modes up this to 30 minutes.
 477         // BUG?: it seems this is ignored, as this script will be run in an environment where
 478         //       environment variables will be discarded before this is used.
 479         def timeout = 1800000
 480         stepScript += "set __TestTimeout=${timeout}\r\n"
 481
 482         stepScript += "echo Creating TestEnv script\r\n"
 483         stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n"
 484
 485         // Create at least an empty script.
 486         stepScript += "echo. > ${stepScriptLocation}\r\n"
 487     }
 488     else {
 489         stepScript += "echo Creating environment setting script\n"
 490         stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n"
 491     }
 492
 493     return stepScript
 494 }
 495
 496 // Generates the string for setting stress mode variables.
 497 def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) {
 498     def stepScript = ''
 499     if (os == 'Windows_NT') {
 500         stressModeVars.each{ k, v ->
 501             // Write out what we are writing to the script file
 502             stepScript += "echo Setting ${k}=${v}\r\n"
 503             // Write out the set itself to the script file`
 504             stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n"
 505         }
 506     }
 507     else {
 508         stressModeVars.each{ k, v ->
 509             // Write out what we are writing to the script file
 510             stepScript += "echo Setting ${k}=${v}\n"
 511             // Write out the set itself to the script file`
 512             stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n"
 513         }
 514     }
 515
 516     return stepScript
 517 }
 518
 519 // Append an existing script to an environment script.
 520 // Returns string of commands to do this.
 521 def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) {
 522     assert (os == 'Windows_NT')
 523     def stepScript = ''
 524
 525     stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n"
 526     stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n"
 527
 528     return stepScript
 529 }
 530
 531 // Finalize an environment setting script.
 532 // Returns string of commands to do this.
 533 def static envScriptFinalize(def os, def stepScriptLocation) {
 534     def stepScript = ''
 535
 536     if (os == 'Windows_NT') {
 537         // Display the resulting script. This is useful when looking at the output log file.
 538         stepScript += "echo Display the total script ${stepScriptLocation}\r\n"
 539         stepScript += "type ${stepScriptLocation}\r\n"
 540     }
 541     else {
 542         stepScript += "chmod +x ${stepScriptLocation}\n"
 543     }
 544
 545     return stepScript
 546 }
 547
 548 def static isNeedDocker(def architecture, def os, def isBuild) {
 549     if (isBuild) {
 550         if (architecture == 'x86' && os == 'Ubuntu') {
 551             return true
 552         }
 553         else if (architecture == 'arm') {
 554             if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') {
 555                 return true
 556             }
 557         }
 558     }
 559     else {
 560         if (architecture == 'x86' && os == 'Ubuntu') {
 561             return true
 562         }
 563     }
 564     return false
 565 }
 566
 567 def static getDockerImageName(def architecture, def os, def isBuild) {
 568     // We must change some docker private images to official later
 569     if (isBuild) {
 570         if (architecture == 'x86' && os == 'Ubuntu') {
 571             return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548"
 572         }
 573         else if (architecture == 'arm') {
 574             if (os == 'Ubuntu') {
 575                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239"
 576             }
 577             else if (os == 'Ubuntu16.04') {
 578                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548"
 579             }
 580             else if (os == 'Tizen') {
 581                 return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs"
 582             }
 583         }
 584     }
 585     else {
 586         if (architecture == 'x86' && os == 'Ubuntu') {
 587             return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test"
 588         }
 589     }
 590     println("Unknown architecture to use docker: ${architecture} ${os}");
 591     assert false
 592 }
 593
 594 // Calculates the name of the build job based on some typical parameters.
 595 //
 596 def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) {
 597     // If the architecture is x64, do not add that info into the build name.
 598     // Need to change around some systems and other builds to pick up the right builds
 599     // to do that.
 600
 601     def suffix = scenario != 'default' ? "_${scenario}" : '';
 602     if (isBuildOnly) {
 603         suffix += '_bld'
 604     }
 605     def baseName = ''
 606     switch (architecture) {
 607         case 'x64':
 608             if (scenario == 'default') {
 609                 // For now we leave x64 off of the name for compatibility with other jobs
 610                 baseName = configuration.toLowerCase() + '_' + os.toLowerCase()
 611             }
 612             else if (scenario == 'formatting') {
 613                 // we don't care about the configuration for the formatting job. It runs all configs
 614                 baseName = architecture.toLowerCase() + '_' + os.toLowerCase()
 615             }
 616             else {
 617                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 618             }
 619             break
 620         case 'arm64':
 621             if (os.toLowerCase() == "windows_nt") {
 622                 // These are cross builds
 623                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 624             }
 625             else {
 626                 // Defaults to a small page size set of machines.
 627                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
 628             }
 629             break
 630         case 'arm':
 631             // These are cross builds
 632             if (os == 'Tizen') {
 633                 // ABI: softfp
 634                 baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 635             }
 636             else {
 637                 baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 638             }
 639             break
 640         case 'armlb':
 641             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 642             break
 643         case 'x86':
 644         case 'x86_arm_altjit':
 645         case 'x64_arm64_altjit':
 646             baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
 647             break
 648         default:
 649             println("Unknown architecture: ${architecture}");
 650             assert false
 651             break
 652     }
 653
 654     return baseName + suffix
 655 }
 656
 657 def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) {
 658
 659     // Check scenario.
 660     switch (scenario) {
 661         case 'default':
 662             switch (architecture) {
 663                 case 'x64':
 664                 case 'x86':
 665                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
 666                         Utilities.addPeriodicTrigger(job, '@daily')
 667                     }
 668                     else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
 669                         Utilities.addGithubPushTrigger(job)
 670                     }
 671                     break
 672                 case 'arm':
 673                 case 'armlb':
 674                 case 'x86_arm_altjit':
 675                 case 'x64_arm64_altjit':
 676                     Utilities.addGithubPushTrigger(job)
 677                     break
 678                 case 'arm64':
 679                     // We would normally want a per-push trigger, but with limited hardware we can't keep up
 680                     Utilities.addPeriodicTrigger(job, "H H/4 * * *")
 681                     break
 682                 default:
 683                     println("Unknown architecture: ${architecture}");
 684                     assert false
 685                     break
 686             }
 687             break
 688         case 'r2r':
 689             assert !(os in bidailyCrossList)
 690             // r2r gets a push trigger for checked/release
 691             if (configuration == 'Checked' || configuration == 'Release') {
 692                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 693                 if (architecture == 'x64' && os != 'OSX10.12') {
 694                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
 695                     if (isFlowJob || os == 'Windows_NT') {
 696                         Utilities.addGithubPushTrigger(job)
 697                     }
 698                 // OSX10.12 r2r jobs should only run every 12 hours, not daily.
 699                 } else if (architecture == 'x64' && os == 'OSX10.12'){
 700                     if (isFlowJob) {
 701                         Utilities.addPeriodicTrigger(job, 'H H/12 * * *')
 702                     }
 703                 }
 704                 // For x86, only add per-commit jobs for Windows
 705                 else if (architecture == 'x86') {
 706                     if (os == 'Windows_NT') {
 707                         Utilities.addGithubPushTrigger(job)
 708                     }
 709                 }
 710                 // arm64 r2r jobs should only run daily.
 711                 else if (architecture == 'arm64') {
 712                     if (os == 'Windows_NT') {
 713                         Utilities.addPeriodicTrigger(job, '@daily')
 714                     }
 715                 }
 716             }
 717             break
 718         case 'r2r_jitstress1':
 719         case 'r2r_jitstress2':
 720         case 'r2r_jitstressregs1':
 721         case 'r2r_jitstressregs2':
 722         case 'r2r_jitstressregs3':
 723         case 'r2r_jitstressregs4':
 724         case 'r2r_jitstressregs8':
 725         case 'r2r_jitstressregs0x10':
 726         case 'r2r_jitstressregs0x80':
 727         case 'r2r_jitstressregs0x1000':
 728         case 'r2r_jitminopts':
 729         case 'r2r_jitforcerelocs':
 730         case 'r2r_gcstress15':
 731             assert !(os in bidailyCrossList)
 732
 733             // GCStress=C is currently not supported on OS X
 734             if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) {
 735                 break
 736             }
 737
 738             // GC Stress 15 r2r gets a push trigger for checked/release
 739             if (configuration == 'Checked' || configuration == 'Release') {
 740                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 741                 if (architecture == 'x64') {
 742                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
 743                     if (isFlowJob || os == 'Windows_NT') {
 744                         // Add a weekly periodic trigger
 745                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 746                     }
 747                 }
 748                 // For x86, only add per-commit jobs for Windows
 749                 else if (architecture == 'x86') {
 750                     if (os == 'Windows_NT') {
 751                         Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 752                     }
 753                 }
 754             }
 755             break
 756         case 'longgc':
 757             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 758             assert configuration == 'Release'
 759             assert architecture == 'x64'
 760             Utilities.addPeriodicTrigger(job, '@daily')
 761             // TODO: Add once external email sending is available again
 762             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 763             break
 764         case 'gcsimulator':
 765             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 766             assert configuration == 'Release'
 767             assert architecture == 'x64'
 768             Utilities.addPeriodicTrigger(job, 'H H * * 3,6') // some time every Wednesday and Saturday
 769             // TODO: Add once external email sending is available again
 770             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 771             break
 772         case 'standalone_gc':
 773             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 774             assert (configuration == 'Release' || configuration == 'Checked')
 775             // TODO: Add once external email sending is available again
 776             // addEmailPublisher(job, 'dotnetgctests@microsoft.com')
 777             Utilities.addPeriodicTrigger(job, '@daily')
 778             break
 779         case 'gc_reliability_framework':
 780             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 781             assert (configuration == 'Release' || configuration == 'Checked')
 782             // Only triggered by phrase.
 783             break
 784         case 'ilrt':
 785             assert !(os in bidailyCrossList)
 786             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
 787             if (architecture == 'x64' && configuration == 'Release') {
 788                 // We don't expect to see a job generated except in these scenarios
 789                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 790                 if (isFlowJob || os == 'Windows_NT') {
 791                     Utilities.addPeriodicTrigger(job, '@daily')
 792                 }
 793             }
 794             break
 795         case 'jitdiff':
 796             assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12')
 797             assert configuration == 'Checked'
 798             assert (architecture == 'x64' || architecture == 'x86')
 799             Utilities.addGithubPushTrigger(job)
 800             break
 801         case 'formatting':
 802             assert (os == 'Windows_NT' || os == "Ubuntu")
 803             assert architecture == 'x64'
 804             Utilities.addGithubPushTrigger(job)
 805             break
 806         case 'jitstressregs1':
 807         case 'jitstressregs2':
 808         case 'jitstressregs3':
 809         case 'jitstressregs4':
 810         case 'jitstressregs8':
 811         case 'jitstressregs0x10':
 812         case 'jitstressregs0x80':
 813         case 'jitstressregs0x1000':
 814         case 'minopts':
 815         case 'forcerelocs':
 816         case 'jitstress1':
 817         case 'jitstress2':
 818         case 'jitstress2_jitstressregs1':
 819         case 'jitstress2_jitstressregs2':
 820         case 'jitstress2_jitstressregs3':
 821         case 'jitstress2_jitstressregs4':
 822         case 'jitstress2_jitstressregs8':
 823         case 'jitstress2_jitstressregs0x10':
 824         case 'jitstress2_jitstressregs0x80':
 825         case 'jitstress2_jitstressregs0x1000':
 826         case 'tailcallstress':
 827         case 'jitsse2only':
 828         case 'jitnosimd':
 829         case 'corefx_baseline':
 830         case 'corefx_minopts':
 831         case 'corefx_jitstress1':
 832         case 'corefx_jitstress2':
 833         case 'corefx_jitstressregs1':
 834         case 'corefx_jitstressregs2':
 835         case 'corefx_jitstressregs3':
 836         case 'corefx_jitstressregs4':
 837         case 'corefx_jitstressregs8':
 838         case 'corefx_jitstressregs0x10':
 839         case 'corefx_jitstressregs0x80':
 840         case 'corefx_jitstressregs0x1000':
 841         case 'zapdisable':
 842             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 843                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 844                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 845                     if (os == 'Windows_NT') {
 846                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 847                         Utilities.addPeriodicTrigger(job, '@weekly')
 848                     }
 849                 }
 850                 else {
 851                     Utilities.addPeriodicTrigger(job, '@daily')
 852                 }
 853             }
 854             break
 855         case 'heapverify1':
 856         case 'gcstress0x3':
 857             if (os != 'CentOS7.1' && !(os in bidailyCrossList)) {
 858                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 859                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 860                     if (os == 'Windows_NT') {
 861                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 862                         Utilities.addPeriodicTrigger(job, '@weekly')
 863                     }
 864                     // TODO: Add once external email sending is available again
 865                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 866                 }
 867                 else {
 868                     Utilities.addPeriodicTrigger(job, '@weekly')
 869                 }
 870             }
 871             break
 872         case 'gcstress0xc':
 873         case 'gcstress0xc_zapdisable':
 874         case 'gcstress0xc_zapdisable_jitstress2':
 875         case 'gcstress0xc_zapdisable_heapverify1':
 876         case 'gcstress0xc_jitstress1':
 877         case 'gcstress0xc_jitstress2':
 878         case 'gcstress0xc_minopts_heapverify1':
 879             // GCStress=C is currently not supported on OS X
 880             if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) {
 881                 assert (os == 'Windows_NT') || (os in Constants.crossList)
 882                 if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) {
 883                     if (os == 'Windows_NT') {
 884                         // We don't have enough ARM64 machines to run these more frequently than weekly.
 885                         Utilities.addPeriodicTrigger(job, '@weekly')
 886                     }
 887                     // TODO: Add once external email sending is available again
 888                     // addEmailPublisher(job, 'dotnetonarm64@microsoft.com')
 889                 }
 890                 else {
 891                     Utilities.addPeriodicTrigger(job, '@weekly')
 892                 }
 893             }
 894             break
 895
 896         case 'illink':
 897             // Testing on other operating systems TBD
 898             assert (os == 'Windows_NT' || os == 'Ubuntu')
 899             if (architecture == 'x64' || architecture == 'x86') {
 900                 if (configuration == 'Checked') {
 901                     Utilities.addPeriodicTrigger(job, '@daily')
 902                 }
 903             }
 904             break
 905
 906         case 'tieredcompilation':
 907         case 'corefx_tieredcompilation':
 908             // No periodic jobs just yet, still testing
 909             break
 910
 911         default:
 912             println("Unknown scenario: ${scenario}");
 913             assert false
 914             break
 915     }
 916     return
 917 }
 918
 919 // **************************
 920 // Define the basic inner loop builds for PR and commit.  This is basically just the set
 921 // of coreclr builds over linux/osx 10.12/windows and debug/release/checked.  In addition, the windows
 922 // builds will do a couple extra steps.
 923 // **************************
 924
 925 // Adds a trigger for the PR build if one is needed.  If isFlowJob is true, then this is the
 926 // flow job that rolls up the build and test for non-windows OS's.  // If the job is a windows build only job,
 927 // it's just used for internal builds
 928 // If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md
 929 def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) {
 930     if (isWindowsBuildOnlyJob) {
 931         return
 932     }
 933
 934     def bidailyCrossList = ['RHEL7.2', 'Debian8.4']
 935     // Non pull request builds.
 936     if (!isPR) {
 937         addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList)
 938         return
 939     }
 940
 941      def arm64Users = [
 942         'AndyAyersMS',
 943         'briansull',
 944         'BruceForstall',
 945         'CarolEidt',
 946         'cmckinsey',
 947         'erozenfeld',
 948         'janvorli',
 949         'jashook',
 950         'JosephTremoulet',
 951         'pgodeq',
 952         'pgavlin',
 953         'rartemev',
 954         'russellhadley',
 955         'RussKeldorph',
 956         'sandreenko',
 957         'sdmaclea',
 958         'swaroop-sridhar',
 959         'jkotas',
 960         'markwilkie',
 961         'weshaggard'
 962     ]
 963
 964     // Pull request builds.  Generally these fall into two categories: default triggers and on-demand triggers
 965     // We generally only have a distinct set of default triggers but a bunch of on-demand ones.
 966     def osGroup = getOSGroup(os)
 967     switch (architecture) {
 968         case 'x64': // editor brace matching: {
 969             if (scenario == 'formatting') {
 970                 assert configuration == 'Checked'
 971                 if (os == 'Windows_NT' || os == 'Ubuntu') {
 972                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Innerloop Formatting")
 973                 }
 974                 break
 975             }
 976
 977             switch (os) {
 978                 // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test
 979                 case 'Debian8.4':
 980                 case 'RHEL7.2':
 981                     if (scenario == 'default') {
 982                         assert !isFlowJob
 983                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}.*")
 984                     }
 985                     break
 986                 case 'Ubuntu16.04':
 987                     assert !isFlowJob
 988                     assert scenario == 'default'
 989                     // Distinguish with the other architectures (arm and x86)
 990                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}.*")
 991                     break
 992                 case 'Fedora24':
 993                 case 'Ubuntu16.10':
 994                     assert !isFlowJob
 995                     assert scenario == 'default'
 996                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build", "(?i).*test\\W+${os}\\W+.*")
 997                     break
 998                 case 'Ubuntu':
 999                     if (scenario == 'illink') {
1000                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1001                         break
1002                     }
1003                     // fall through
1004                 case 'OSX10.12':
1005                     // Triggers on the non-flow jobs aren't necessary here
1006                     // Corefx testing uses non-flow jobs.
1007                     if (!isFlowJob && !isCoreFxScenario(scenario)) {
1008                         break
1009                     }
1010                     switch (scenario) {
1011                         case 'default':
1012                             // OSX uses checked for default PR tests
1013                             if (configuration == 'Checked') {
1014                                 // Default trigger
1015                                 assert !job.name.contains("centos")
1016                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1017                             }
1018                             break
1019                         case 'jitdiff':
1020                             if (configuration == 'Checked') {
1021                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1022                             }
1023                             break
1024                         case 'ilrt':
1025                             if (configuration == 'Release') {
1026                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1027                             }
1028                             break
1029                         case 'longgc':
1030                             if (configuration == 'Release') {
1031                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1032                             }
1033                             break
1034                         case 'gcsimulator':
1035                             if (configuration == 'Release') {
1036                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1037                             }
1038                             break
1039                         case 'standalone_gc':
1040                             if (configuration == 'Release' || configuration == 'Checked') {
1041                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1042                             }
1043                             break
1044                         case 'gc_reliability_framework':
1045                             if (configuration == 'Release' || configuration == 'Checked') {
1046                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1047                             }
1048                             break
1049                         default:
1050                             if (isJitStressScenario(scenario)) {
1051                                 def displayStr = getStressModeDisplayName(scenario)
1052                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1053                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1054                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1055                             }
1056                             else if (isR2RScenario(scenario)) {
1057                                 if (configuration == 'Release' || configuration == 'Checked') {
1058                                     def displayStr = getR2RDisplayName(scenario)
1059                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test",
1060                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1061                                 }
1062                             }
1063                             else {
1064                                 println("Unknown scenario: ${scenario}");
1065                                 assert false
1066                             }
1067                             break
1068                     }
1069                     break
1070
1071                 case 'CentOS7.1':
1072                     switch (scenario) {
1073                         case 'default':
1074                             // CentOS uses checked for default PR tests while debug is build only
1075                             if (configuration == 'Debug') {
1076                                 // Default trigger
1077                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build")
1078                             }
1079
1080                             // Make sure this is a flow job to get build and test.
1081                             if (configuration == 'Checked' && isFlowJob) {
1082                                 assert job.name.contains("flow")
1083                                 // Default trigger
1084                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1085                             }
1086                             break
1087                         default:
1088                             if (isR2RScenario(scenario)) {
1089                                 if (configuration == 'Release' || configuration == 'Checked') {
1090                                     def displayStr = getR2RDisplayName(scenario)
1091                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1092                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1093                                 }
1094                             }
1095                             break
1096                     }
1097                     break
1098
1099                 case 'Windows_NT':
1100                     switch (scenario) {
1101                         case 'default':
1102                             // Default trigger
1103                             if (configuration == 'Checked') {
1104                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1105                             }
1106                             break
1107                         case 'jitdiff':
1108                             if (configuration == 'Checked') {
1109                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Jit Diff Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1110                             }
1111                             break
1112                         case 'ilrt':
1113                             if (configuration == 'Release') {
1114                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test", "(?i).*test\\W+${os}\\W+${scenario}.*")
1115                             }
1116                             break
1117                         case 'longgc':
1118                             if (configuration == 'Release') {
1119                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1120                             }
1121                             break
1122                         case 'gcsimulator':
1123                             if (configuration == 'Release') {
1124                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1125                             }
1126                             break
1127                         case 'standalone_gc':
1128                             if (configuration == 'Release' || configuration == 'Checked') {
1129                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1130                             }
1131                             break
1132                         case 'gc_reliability_framework':
1133                             if (configuration == 'Release' || configuration == 'Checked') {
1134                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Reliability Framework", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*")
1135                             }
1136                             break
1137                         case 'illink':
1138                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1139                             break
1140                         default:
1141                             if (isJitStressScenario(scenario)) {
1142                                 def displayStr = getStressModeDisplayName(scenario)
1143                                 assert (os == 'Windows_NT') || (os in Constants.crossList)
1144                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1145                                    "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1146                             }
1147                             else if (isR2RScenario(scenario)) {
1148                                 if (configuration == 'Release' || configuration == 'Checked') {
1149                                     def displayStr = getR2RDisplayName(scenario)
1150                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1151                                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1152                                 }
1153                             }
1154                             else {
1155                                 println("Unknown scenario: ${scenario}");
1156                                 assert false
1157                             }
1158                             break
1159                     }
1160                     break
1161                 default:
1162                     println("Unknown os: ${os}");
1163                     assert false
1164                     break
1165             }
1166             break
1167         // editor brace matching: }
1168         case 'armlb':
1169         case 'arm': // editor brace matching: {
1170             switch (os) {
1171                 case 'Ubuntu':
1172                 case 'Ubuntu16.04':
1173                     if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu
1174                         break
1175                     }
1176                     assert scenario == 'default'
1177                     job.with {
1178                         publishers {
1179                             azureVMAgentPostBuildAction {
1180                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1181                             }
1182                         }
1183                     }
1184                     if (configuration == 'Debug') {
1185                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1186                     }
1187                     else {
1188                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1189                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1190                     }
1191                     break
1192                 case 'Tizen':
1193                     if (architecture == 'armlb') {  // No arm legacy backend testing for Tizen armel
1194                         break
1195                     }
1196                     architecture='armel'
1197                     job.with {
1198                         publishers {
1199                             azureVMAgentPostBuildAction {
1200                                 agentPostBuildAction('Delete agent if the build was not successful (when idle).')
1201                             }
1202                         }
1203                     }
1204                     if (configuration == 'Checked') {
1205                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build and Test")
1206                     }
1207                     else {
1208                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build",
1209                             "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*")
1210                     }
1211                     break
1212                 case 'Windows_NT':
1213                     // Triggers on the non-flow jobs aren't necessary here
1214                     if (!isFlowJob) {
1215                         break
1216                     }
1217
1218                     // Set up a private trigger
1219                     def contextString = "${os} ${architecture} Cross ${configuration}"
1220                     def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1221                     if (scenario == 'default') {
1222                         contextString += " Innerloop"
1223                         triggerString += "\\W+Innerloop"
1224                     }
1225                     else {
1226                         contextString += " ${scenario}"
1227                         triggerString += "\\W+${scenario}"
1228                     }
1229
1230                     if (configuration == 'Debug') {
1231                         contextString += " Build"
1232                         triggerString += "\\W+Build"
1233                     } else {
1234                         contextString += " Build and Test"
1235                         triggerString += "\\W+Build and Test"
1236                     }
1237
1238                     triggerString += ".*"
1239
1240                     switch (scenario) {
1241                         case 'default':
1242                             // Only Checked is a default trigger.
1243                             if (configuration == 'Checked')
1244                             {
1245                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1246                             }
1247                             else
1248                             {
1249                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1250                             }
1251                             break
1252                         default:
1253                             // Stress jobs will use this code path.
1254                             if (isArmWindowsScenario(scenario)) {
1255                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1256                             }
1257                             break
1258                     }
1259                     break
1260                 default:
1261                     println("NYI os: ${os}");
1262                     assert false
1263                     break
1264             }
1265             break
1266         // editor brace matching: }
1267         case 'arm64': // editor brace matching: {
1268             // Set up a private trigger
1269             def contextString = "${os} ${architecture} Cross ${configuration}"
1270             def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}"
1271             if (scenario == 'default') {
1272                 contextString += " Innerloop"
1273                 triggerString += "\\W+Innerloop"
1274             }
1275             else {
1276                 contextString += " ${scenario}"
1277                 triggerString += "\\W+${scenario}"
1278             }
1279
1280             if (configuration == 'Debug') {
1281                 contextString += " Build"
1282                 triggerString += "\\W+Build"
1283             } else {
1284                 contextString += " Build and Test"
1285                 triggerString += "\\W+Build and Test"
1286             }
1287
1288             triggerString += ".*"
1289
1290             switch (os) {
1291                 case 'Ubuntu':
1292                 case 'Ubuntu16.04':
1293                     switch (scenario) {
1294                         case 'default':
1295                             if (configuration == 'Debug' && !isFlowJob) {
1296                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Innerloop Build")
1297                             }
1298                             else {
1299                                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test", triggerString)
1300                             }
1301
1302                             break
1303                         default:
1304                             if (isR2RScenario(scenario)) {
1305                                 if (configuration == 'Checked' || configuration == 'Release') {
1306                                     def displayStr = getR2RDisplayName(scenario)
1307                                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build and Test", triggerString)
1308                                 }
1309                             }
1310                             break
1311                     }
1312                     break
1313                 case 'Windows_NT':
1314                     // Triggers on the non-flow jobs aren't necessary here
1315                     if (!isFlowJob) {
1316                         break
1317                     }
1318
1319                     assert isArmWindowsScenario(scenario)
1320                     switch (scenario) {
1321                         case 'default':
1322                             if (configuration == 'Checked') {
1323                                 Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users)
1324                             }
1325                             else {
1326                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1327                             }
1328                             break
1329                         default:
1330                             // Stress jobs will use this code path.
1331                             if (isArmWindowsScenario(scenario)) {
1332                                 Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users)
1333                             }
1334                             break
1335                     }
1336                     break
1337                 default:
1338                     println("NYI os: ${os}");
1339                     assert false
1340                     break
1341             }
1342             break
1343         // editor brace matching: }
1344         case 'x86': // editor brace matching: {
1345             assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && (scenario == 'default')))
1346             if (os == 'Ubuntu') {
1347                 // Triggers on the non-flow jobs aren't necessary here
1348                 if (!isFlowJob) {
1349                     break
1350                 }
1351                 // on-demand only for ubuntu x86
1352                 Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build",
1353                     "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1354                 break
1355             }
1356             switch (scenario) {
1357                 case 'default':
1358                     if (configuration == 'Checked') {
1359                         assert !job.name.contains("centos")
1360                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Innerloop Build and Test")
1361                     }
1362                     else {
1363                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1364                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*")
1365                     }
1366                     break
1367                 case 'ilrt':
1368                     if (configuration == 'Release') {
1369                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} IL RoundTrip Build and Test",
1370                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1371                     }
1372                     break
1373                 case 'longgc':
1374                     if (configuration == 'Release') {
1375                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test",
1376                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1377                     }
1378                     break
1379                 case 'gcsimulator':
1380                     if (configuration == 'Release') {
1381                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GC Simulator",
1382                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1383                     }
1384                     break
1385                 case 'standalone_gc':
1386                     if (configuration == 'Release' || configuration == 'Checked') {
1387                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Standalone GC",
1388                             "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1389                     }
1390                     break
1391                 case 'illink':
1392                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1393                     break
1394                 default:
1395                     if (isJitStressScenario(scenario)) {
1396                         def displayStr = getStressModeDisplayName(scenario)
1397                         Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - ${displayStr})",
1398                            "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1399                     }
1400                     else if (isR2RScenario(scenario)) {
1401                         if (configuration == 'Release' || configuration == 'Checked') {
1402                             def displayStr = getR2RDisplayName(scenario)
1403                             Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${displayStr} Build & Test",
1404                                 "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1405                         }
1406                     }
1407                     else {
1408                         println("Unknown scenario: ${os} ${architecture} ${scenario}");
1409                         assert false
1410                     }
1411                     break
1412             }
1413             break
1414          // editor brace matching: }
1415         case 'x64_arm64_altjit':
1416         case 'x86_arm_altjit': // editor brace matching: {
1417             assert (os == 'Windows_NT')
1418             switch (scenario) {
1419                 case 'default':
1420                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test",
1421                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build and Test.*")
1422                     break
1423                 default:
1424                     Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} ${scenario}",
1425                         "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*")
1426                     break
1427             }
1428             break
1429         // editor brace matching: }
1430         default:
1431             println("Unknown architecture: ${architecture}");
1432             assert false
1433             break
1434     }
1435 }
1436
1437 def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) {
1438     def buildCommands = [];
1439     def osGroup = getOSGroup(os)
1440     def lowerConfiguration = configuration.toLowerCase()
1441
1442     def priority = '1'
1443     if ((scenario == 'default' && isPR == true) || (scenario == 'default' && isBuildOnly == true)) {
1444         priority = '0'
1445     }
1446
1447     def enableCorefxTesting = isCoreFxScenario(scenario)
1448
1449     // Calculate the build steps, archival, and xunit results
1450     switch (os) {
1451         case 'Windows_NT': // editor brace matching: {
1452             switch (architecture) {
1453                 case 'x64':
1454                 case 'x86':
1455                 case 'x86_arm_altjit':
1456                 case 'x64_arm64_altjit':
1457                     def arch = architecture
1458                     def buildOpts = ''
1459                     if (architecture == 'x86_arm_altjit') {
1460                         arch = 'x86'
1461                     }
1462                     else if (architecture == 'x64_arm64_altjit') {
1463                         arch = 'x64'
1464                     }
1465
1466                     if (scenario == 'formatting') {
1467                         buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
1468                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1469                         break
1470                     }
1471
1472                     if (scenario == 'illink') {
1473                         buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}"
1474                     }
1475
1476                     // If it is a release build for windows, ensure PGO is used, else fail the build
1477                     if ((lowerConfiguration == 'release') &&
1478                         (scenario in Constants.basicScenarios) &&
1479                         (architecture != 'x86_arm_altjit') &&
1480                         (architecture != 'x64_arm64_altjit')) {
1481
1482                         buildOpts += ' -enforcepgo'
1483                     }
1484
1485                     if (enableCorefxTesting) {
1486                         buildOpts += ' skiptests';
1487                     } else {
1488                         buildOpts += " -priority=${priority}"
1489                     }
1490
1491                     // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will
1492                     // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about
1493                     // 35 characters long.
1494
1495                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}"
1496
1497                     if (!isBuildOnly) {
1498                         def runtestArguments = ''
1499                         def testOpts = 'collectdumps'
1500
1501                         if (isR2RScenario(scenario)) {
1502
1503                             // If this is a ReadyToRun scenario, pass 'crossgen' or 'crossgenaltjit'
1504                             // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests'
1505                             // to cause the tests to be crossgen'ed.
1506
1507                             if ((architecture == 'x86_arm_altjit') || (architecture == 'x64_arm64_altjit')) {
1508                                 testOpts += ' crossgenaltjit protononjit.dll'
1509                             } else {
1510                                 testOpts += ' crossgen'
1511                             }
1512
1513                             testOpts += ' runcrossgentests'
1514                         }
1515                         else if (scenario == 'jitdiff') {
1516                             testOpts += ' jitdisasm crossgen'
1517                         }
1518                         else if (scenario == 'ilrt') {
1519                             testOpts += ' ilasmroundtrip'
1520                         }
1521                         else if (isLongGc(scenario)) {
1522                             testOpts += " ${scenario} sequential"
1523                         }
1524                         else if (scenario == 'standalone_gc') {
1525                             testOpts += ' gcname clrgc.dll'
1526                         }
1527                         else if (scenario == 'illink') {
1528                             testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe"
1529                         }
1530
1531                         // If we are running a stress mode, we should write out the set of key
1532                         // value env pairs to a file at this point and then we'll pass that to runtest.cmd
1533
1534                         def envScriptPath = ''
1535                         if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
1536                             def buildCommandsStr = ''
1537                             envScriptPath = "%WORKSPACE%\\SetStressModes.bat"
1538                             buildCommandsStr += envScriptCreate(os, envScriptPath)
1539
1540                             if (isJitStressScenario(scenario)) {
1541                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath)
1542                             }
1543                             else if (isR2RStressScenario(scenario)) {
1544                                 buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath)
1545                             }
1546
1547                             if (architecture == 'x86_arm_altjit') {
1548                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x86_arm_altjit.cmd", envScriptPath)
1549                             }
1550                             else if (architecture == 'x64_arm64_altjit') {
1551                                 buildCommandsStr += envScriptAppendExistingScript(os, "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd", envScriptPath)
1552                             }
1553
1554                             envScriptFinalize(os, envScriptPath)
1555
1556                             // Note that buildCommands is an array of individually executed commands; we want all the commands used to
1557                             // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings
1558                             // into a single script.
1559                             buildCommands += buildCommandsStr
1560                         }
1561                         else if (architecture == 'x86_arm_altjit') {
1562                             envScriptPath = "%WORKSPACE%\\tests\\x86_arm_altjit.cmd"
1563                         }
1564                         else if (architecture == 'x64_arm64_altjit') {
1565                             envScriptPath = "%WORKSPACE%\\tests\\x64_arm64_altjit.cmd"
1566                         }
1567                         if (envScriptPath != '') {
1568                             testOpts += " TestEnv ${envScriptPath}"
1569                         }
1570
1571                         runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}"
1572
1573                         if (enableCorefxTesting) {
1574                             def workspaceRelativeFxRoot = "_/fx"
1575                             def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
1576
1577                             buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${envScriptPath}"
1578
1579                             setTestJobTimeOut(newJob, isPR, architecture, scenario)
1580
1581                             // Archive and process (only) the test results
1582                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1583                             Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1584
1585                             //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail
1586                             Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false)
1587                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/testhost/**", "", true, false)
1588                         }
1589                         else if (isGcReliabilityFramework(scenario)) {
1590                             buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly"
1591                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
1592                         }
1593                         else {
1594                             buildCommands += "tests\\runtest.cmd ${runtestArguments}"
1595                         }
1596                     }
1597
1598                     if (!enableCorefxTesting) {
1599                         // Run the rest of the build
1600                         // Build the mscorlib for the other OS's
1601                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib"
1602                         buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib"
1603
1604                         if (arch == "x64") {
1605                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
1606                         }
1607
1608                         // Zip up the tests directory so that we don't use so much space/time copying
1609                         // 10s of thousands of files around.
1610                         buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1611
1612                         if (!isJitStressScenario(scenario)) {
1613                             // For windows, pull full test results and test drops for x86/x64.
1614                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
1615                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1616                         }
1617
1618                         if (scenario == 'jitdiff') {
1619                             // retrive jit-dasm output for base commit, and run jit-diff
1620                             if (!isBuildOnly) {
1621                                 // if this is a build only job, we want to keep the default (build) artifacts for the flow job
1622                                 Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**")
1623                             }
1624                         }
1625
1626                         if (!isBuildOnly) {
1627                             Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
1628                             setTestJobTimeOut(newJob, isPR, architecture, scenario)
1629                         }
1630                     }
1631                     break
1632                 case 'armlb':
1633                 case 'arm':
1634                     assert isArmWindowsScenario(scenario)
1635
1636                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1637                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1638
1639                     setTestJobTimeOut(newJob, isPR, architecture, scenario)
1640
1641                     def buildArchitecture = 'arm'
1642
1643                     // For 'armlb' (the JIT LEGACY_BACKEND architecture for arm), tell build.cmd to use legacy backend for crossgen compilation.
1644                     // Legacy backend is not the default JIT; it is an aljit. So, this is a special case.
1645                     def armCrossgenOpt = ''
1646                     if (architecture == 'armlb') {
1647                         armCrossgenOpt = '-crossgenaltjit legacyjit.dll'
1648                     }
1649
1650                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1651                     priority = '1'
1652
1653                     // This is now a build only job. Do not run tests. Use the flow job.
1654                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} -priority=${priority} ${armCrossgenOpt}"
1655
1656                     // Zip up the tests directory so that we don't use so much space/time copying
1657                     // 10s of thousands of files around.
1658                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${buildArchitecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1659
1660                     // Add archival.
1661                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1662                     break
1663                 case 'arm64':
1664                     assert isArmWindowsScenario(scenario)
1665
1666                     def machineAffinityOptions = ['use_arm64_build_machine' : true]
1667                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
1668
1669                     setTestJobTimeOut(newJob, isPR, architecture, scenario)
1670
1671                     // Hack: build pri1 tests for arm/armlb/arm64 build job, until we have separate pri0 and pri1 builds for the flow job to use.
1672                     priority = '1'
1673
1674                     // This is now a build only job. Do not run tests. Use the flow job.
1675                     buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}"
1676
1677                     // Zip up the tests directory so that we don't use so much space/time copying
1678                     // 10s of thousands of files around.
1679                     buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
1680
1681                     // Add archival.
1682                     Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
1683                     break
1684                 default:
1685                     println("Unknown architecture: ${architecture}");
1686                     assert false
1687                     break
1688             }
1689             break
1690         // editor brace matching: }
1691         case 'Ubuntu':
1692         case 'Ubuntu16.04':
1693         case 'Ubuntu16.10':
1694         case 'Debian8.4':
1695         case 'OSX10.12':
1696         case 'CentOS7.1':
1697         case 'RHEL7.2':
1698         case 'Tizen':
1699         case 'Fedora24': // editor brace matching: {
1700             switch (architecture) {
1701                 case 'x64':
1702                 case 'x86':
1703                     if (architecture == 'x86' && os == 'Ubuntu') {
1704                         // build and PAL test
1705                         def dockerImage = getDockerImageName(architecture, os, true)
1706                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}"
1707                         dockerImage = getDockerImageName(architecture, os, false)
1708                         buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout"
1709                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1710                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1711                         break
1712                     }
1713
1714                     if (scenario == 'formatting') {
1715                         buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}"
1716                         Utilities.addArchival(newJob, "format.patch", "", true, false)
1717                         break
1718                     }
1719
1720                     if (scenario == 'illink') {
1721                         assert(os == 'Ubuntu')
1722                         buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}"
1723                     }
1724
1725                     if (!enableCorefxTesting) {
1726                         // We run pal tests on all OS but generate mscorlib (and thus, nuget packages)
1727                         // only on supported OS platforms.
1728                         def bootstrapRid = Utilities.getBoostrapPublishRid(os)
1729                         def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : ''
1730
1731                         buildCommands += "${bootstrapRidEnv}./build.sh verbose ${lowerConfiguration} ${architecture}"
1732                         buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout"
1733
1734                         setTestJobTimeOut(newJob, isPR, architecture, scenario)
1735
1736                         // Basic archiving of the build
1737                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1738                         // And pal tests
1739                         Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml')
1740                     }
1741                     else {
1742                         // Corefx stress testing
1743                         assert os == 'Ubuntu'
1744                         assert architecture == 'x64'
1745                         assert lowerConfiguration == 'checked'
1746                         assert isJitStressScenario(scenario)
1747
1748                         // Build coreclr
1749                         buildCommands += "./build.sh verbose ${lowerConfiguration} ${architecture}"
1750
1751                         def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
1752
1753                         def envScriptCmds = envScriptCreate(os, scriptFileName)
1754                         envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
1755                         envScriptCmds += envScriptFinalize(os, scriptFileName)
1756                         buildCommands += envScriptCmds
1757
1758                         // Build and text corefx
1759                         def workspaceRelativeFxRoot = "_/fx"
1760                         def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
1761
1762                         buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${branch} -env_script ${scriptFileName}"
1763
1764                         setTestJobTimeOut(newJob, isPR, architecture, scenario)
1765
1766                         // Archive and process (only) the test results
1767                         Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1768                         Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml")
1769                     }
1770                     break
1771                 case 'arm64':
1772                     if (!enableCorefxTesting) {
1773                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8"
1774
1775                         // HACK -- Arm64 does not have corefx jobs yet.
1776                         buildCommands += "git clone https://github.com/dotnet/corefx fx"
1777                         buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs-corefx ./fx/build-native.sh -release -buildArch=arm64 -- verbose cross clang3.8"
1778                         buildCommands += "mkdir ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1779                         buildCommands += "cp fx/bin/Linux.arm64.Release/native/* ./bin/Product/Linux.arm64.${configuration}/corefxNative"
1780
1781                         setTestJobTimeOut(newJob, isPR, architecture, scenario)
1782                         // Basic archiving of the build
1783                         Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1784                     }
1785                     break
1786                 case 'arm':
1787                     // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently
1788                     assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen')
1789
1790                     // default values for Ubuntu
1791                     def arm_abi="arm"
1792                     def linuxCodeName="trusty"
1793                     if (os == 'Ubuntu16.04') {
1794                         linuxCodeName="xenial"
1795                     }
1796                     else if (os == 'Tizen') {
1797                         arm_abi="armel"
1798                         linuxCodeName="tizen"
1799                     }
1800
1801                     // Unzip the Windows test binaries first. Exit with 0
1802                     buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
1803
1804                     // Unpack the corefx binaries
1805                     buildCommands += "mkdir ./bin/CoreFxBinDir"
1806                     buildCommands += "tar -xf ./bin/build.tar.gz -C ./bin/CoreFxBinDir"
1807                     if (os != 'Tizen') {
1808                         buildCommands += "chmod a+x ./bin/CoreFxBinDir/corerun"
1809                     }
1810                     // Test environment emulation using docker and qemu has some problem to use lttng library.
1811                     // We should remove libcoreclrtraceptprovider.so to avoid test hang.
1812                     if (os == 'Ubuntu') {
1813                         buildCommands += "rm -f -v ./bin/CoreFxBinDir/libcoreclrtraceptprovider.so"
1814                     }
1815
1816                     // Call the ARM CI script to cross build and test using docker
1817                     buildCommands += """./tests/scripts/arm32_ci_script.sh \\
1818                     --mode=docker \\
1819                     --${arm_abi} \\
1820                     --linuxCodeName=${linuxCodeName} \\
1821                     --buildConfig=${lowerConfiguration} \\
1822                     --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
1823                     --coreFxBinDir=./bin/CoreFxBinDir \\
1824                     --testDirFile=./tests/testsRunningInsideARM.txt"""
1825
1826                     // Basic archiving of the build, no pal tests
1827                     Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
1828                     break
1829                 default:
1830                     println("Unknown architecture: ${architecture}");
1831                     assert false
1832                     break
1833             }
1834             break
1835         // editor brace matching: }
1836         default:
1837             println("Unknown os: ${os}");
1838             assert false
1839             break
1840     } // os
1841
1842     return buildCommands
1843 }
1844
1845 Constants.allScenarios.each { scenario ->
1846     [true, false].each { isPR ->
1847         Constants.architectureList.each { architecture ->
1848             Constants.configurationList.each { configuration ->
1849                 Constants.osList.each { os ->
1850                     // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true
1851                     // and reset the os to Windows_NT
1852                     def isBuildOnly = false
1853                     if (os == 'Windows_NT_BuildOnly') {
1854                         isBuildOnly = true
1855                         os = 'Windows_NT'
1856                     }
1857
1858                     // Tizen is only supported for arm architecture
1859                     if (os == 'Tizen' && architecture != 'arm') {
1860                         return
1861                     }
1862
1863                     // Skip totally unimplemented (in CI) configurations.
1864                     switch (architecture) {
1865                         case 'arm64':
1866                             if (os == 'Ubuntu16.04') {
1867                                 os = 'Ubuntu'
1868                             }
1869
1870                             // Windows and Ubuntu only
1871                             if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) {
1872                                 return
1873                             }
1874                             break
1875                         case 'arm':
1876                             if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) {
1877                                 return
1878                             }
1879                             break
1880                         case 'armlb':
1881                             if (os != 'Windows_NT') {
1882                                 return
1883                             }
1884                             break
1885                         case 'x86':
1886                             if ((os != 'Ubuntu') && (os != 'Windows_NT')) {
1887                                 return
1888                             }
1889                             break
1890                         case 'x86_arm_altjit':
1891                         case 'x64_arm64_altjit':
1892                             if (os != 'Windows_NT') {
1893                                 return
1894                             }
1895                             break
1896                         case 'x64':
1897                             // Everything implemented
1898                             break
1899                         default:
1900                             println("Unknown architecture: ${architecture}")
1901                             assert false
1902                             break
1903                     }
1904
1905                     // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere
1906                     // with checked builds)
1907                     if (isJitStressScenario(scenario)) {
1908                         if (configuration != 'Checked') {
1909                             return
1910                         }
1911
1912                         // Since these are just execution time differences,
1913                         // skip platforms that don't execute the tests here (Windows_NT only)
1914                         def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario))
1915                         if (!isEnabledOS || isBuildOnly) {
1916                             return
1917                         }
1918
1919                         switch (architecture) {
1920                             case 'x64':
1921                             case 'x86':
1922                             case 'x86_arm_altjit':
1923                             case 'x64_arm64_altjit':
1924                                 // x86 ubuntu: default only
1925                                 if ((os == 'Ubuntu') && (architecture == 'x86')) {
1926                                     return
1927                                 }
1928                                 // Windows: Everything implemented
1929                                 break
1930
1931                             default:
1932                                 // arm, arm64, armlb: stress is handled through flow jobs.
1933                                 return
1934                         }
1935                     }
1936                     else if (isR2RScenario(scenario)) {
1937                         if (os != 'Windows_NT') {
1938                             return
1939                         }
1940                         // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow).
1941                         if ((configuration != 'Checked') && isR2RStressScenario(scenario)) {
1942                             return
1943                         }
1944                     }
1945                     else {
1946                         // Skip scenarios
1947                         switch (scenario) {
1948                             case 'ilrt':
1949                                 // The ilrt build isn't necessary except for Windows_NT2003.  Non-Windows NT uses
1950                                 // the default scenario build
1951                                 if (os != 'Windows_NT') {
1952                                     return
1953                                 }
1954                                 // Only x64 for now
1955                                 if (architecture != 'x64') {
1956                                     return
1957                                 }
1958                                 // Release only
1959                                 if (configuration != 'Release') {
1960                                     return
1961                                 }
1962                                 break
1963                             case 'jitdiff':
1964                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1965                                     return
1966                                 }
1967                                 if (architecture != 'x64') {
1968                                     return
1969                                 }
1970                                 if (configuration != 'Checked') {
1971                                     return
1972                                 }
1973                                 break
1974                             case 'longgc':
1975                             case 'gcsimulator':
1976                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1977                                     return
1978                                 }
1979                                 if (architecture != 'x64') {
1980                                     return
1981                                 }
1982                                 if (configuration != 'Release') {
1983                                     return
1984                                 }
1985                                 break
1986                             case 'gc_reliability_framework':
1987                             case 'standalone_gc':
1988                                 if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') {
1989                                     return
1990                                 }
1991
1992                                 if (architecture != 'x64') {
1993                                     return
1994                                 }
1995
1996                                 if (configuration != 'Release' && configuration != 'Checked') {
1997                                     return
1998                                 }
1999                                 break
2000                             // We only run Windows and Ubuntu x64 Checked for formatting right now
2001                             case 'formatting':
2002                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2003                                     return
2004                                 }
2005                                 if (architecture != 'x64') {
2006                                     return
2007                                 }
2008                                 if (configuration != 'Checked') {
2009                                     return
2010                                 }
2011                                 if (isBuildOnly) {
2012                                     return
2013                                 }
2014                                 break
2015                             case 'illink':
2016                                 if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) {
2017                                     return
2018                                 }
2019                                 if (architecture != 'x64' && architecture != 'x86') {
2020                                     return
2021                                 }
2022                                 if (isBuildOnly) {
2023                                     return
2024                                 }
2025                                 break
2026                             case 'default':
2027                                 // Nothing skipped
2028                                 break
2029                             default:
2030                                 println("Unknown scenario: ${scenario}")
2031                                 assert false
2032                                 break
2033                         }
2034                     }
2035
2036                     // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change
2037                     // runtime behavior, not compile-time behavior, are not interesting.
2038                     switch (architecture) {
2039                         case 'x86_arm_altjit':
2040                         case 'x64_arm64_altjit':
2041                             if (isGCStressRelatedTesting(scenario)) {
2042                                 return
2043                             }
2044                             break
2045                         default:
2046                             break
2047                     }
2048
2049                     // Calculate names
2050                     def lowerConfiguration = configuration.toLowerCase()
2051                     def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly)
2052                     def folderName = getJobFolder(scenario)
2053
2054                     // Create the new job
2055                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {}
2056
2057                     def machineAffinityOptions = null
2058
2059                     if (os != 'Windows_NT') {
2060                         machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null
2061                     }
2062                     else {
2063                         machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null
2064                     }
2065
2066                     setMachineAffinity(newJob, os, architecture, machineAffinityOptions)
2067
2068                     // Add all the standard options
2069                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2070                     addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false
2071
2072                     def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly)
2073                     def osGroup = getOSGroup(os)
2074
2075                     newJob.with {
2076                         steps {
2077                             if (os == 'Windows_NT') {
2078                                 buildCommands.each { buildCommand ->
2079                                     batchFile(buildCommand)
2080                                 }
2081                             }
2082                             else {
2083                                 // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2084                                 if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) {
2085                                     // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel
2086                                     // Define the Windows Tests and Corefx build job names
2087                                     def WindowTestsName = projectFolder + '/' +
2088                                                           Utilities.getFullJobName(project,
2089                                                                                    getJobName(lowerConfiguration,
2090                                                                                               'x64' ,
2091                                                                                               'windows_nt',
2092                                                                                               'default',
2093                                                                                               true),
2094                                                                                    false)
2095                                     def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
2096                                                        Utilities.getFolderName(branch)
2097
2098                                     // Copy the Windows test binaries and the Corefx build binaries
2099                                     copyArtifacts(WindowTestsName) {
2100                                         includePatterns('bin/tests/tests.zip')
2101                                         buildSelector {
2102                                             latestSuccessful(true)
2103                                         }
2104                                     }
2105
2106                                     def arm_abi = 'arm'
2107                                     def corefx_os = 'linux'
2108                                     if (os == 'Tizen') {
2109                                         arm_abi = 'armel'
2110                                         corefx_os = 'tizen'
2111                                     }
2112
2113                                     // Let's use release CoreFX to test checked CoreCLR,
2114                                     // because we do not generate checked CoreFX in CoreFX CI yet.
2115                                     def corefx_lowerConfiguration = lowerConfiguration
2116                                     if ( lowerConfiguration == 'checked' ) {
2117                                         corefx_lowerConfiguration='release'
2118                                     }
2119
2120                                     copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") {
2121                                         includePatterns('bin/build.tar.gz')
2122                                         buildSelector {
2123                                             latestSuccessful(true)
2124                                         }
2125                                     }
2126                                 }
2127
2128                                 buildCommands.each { buildCommand ->
2129                                     shell(buildCommand)
2130                                 }
2131                             }
2132                         }
2133                     } // newJob.with
2134
2135                 } // os
2136             } // configuration
2137         } // architecture
2138     } // isPR
2139 } // scenario
2140
2141
2142 // Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows.
2143 Constants.allScenarios.each { scenario ->
2144     [true, false].each { isPR ->
2145         ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture ->
2146             Constants.crossList.each { os ->
2147                 if (architecture == 'arm64') {
2148                     if (os != "Ubuntu" && os != "Windows_NT") {
2149                         return
2150                     }
2151                 } else if (architecture == 'arm' || architecture == 'armlb') {
2152                     if (os != 'Windows_NT') {
2153                         return
2154                     }
2155                 }
2156                 else if (architecture == 'x86') {
2157                     if (os != "Ubuntu") {
2158                         return
2159                     }
2160                 }
2161
2162                 def validWindowsNTCrossArches = ["arm", "armlb", "arm64"]
2163
2164                 if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) {
2165                     return
2166                 }
2167
2168                 Constants.configurationList.each { configuration ->
2169
2170                     // First, filter based on OS.
2171
2172                     if (os == 'Windows_NT') {
2173                         if (!isArmWindowsScenario(scenario)) {
2174                             return
2175                         }
2176                     }
2177                     else {
2178                         // Non-Windows
2179                         if (architecture == 'arm64') {
2180                             if (scenario != 'default' && scenario != 'r2r' && scenario != 'gcstress0x3' && scenario != 'gcstress0xc') {
2181                                 return
2182                             }
2183                         }
2184                         else if (architecture == 'x86') {
2185                             // Linux/x86 only want default test
2186                             if (scenario != 'default') {
2187                                 return
2188                             }
2189                         }
2190                     }
2191
2192                     // For CentOS, we only want Checked/Release builds.
2193                     if (os == 'CentOS7.1') {
2194                         if (configuration != 'Checked' && configuration != 'Release') {
2195                             return
2196                         }
2197                         if (scenario != 'default' && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) {
2198                             return
2199                         }
2200                     }
2201
2202                     // For RedHat and Debian, we only do Release builds.
2203                     else if (os == 'RHEL7.2' || os == 'Debian8.4') {
2204                         if (configuration != 'Release') {
2205                             return
2206                         }
2207                         if (scenario != 'default') {
2208                             return
2209                         }
2210                     }
2211
2212                     // Next, filter based on scenario.
2213
2214                     if (isJitStressScenario(scenario)) {
2215                         if (configuration != 'Checked') {
2216                             return
2217                         }
2218                         // CoreFx JIT stress tests currently not implemented for flow jobs.
2219                         if (isCoreFxScenario(scenario)) {
2220                             return
2221                         }
2222                     }
2223                     else if (isR2RBaselineScenario(scenario)) {
2224                         if (configuration != 'Checked' && configuration != 'Release') {
2225                             return
2226                         }
2227                     }
2228                     else if (isR2RStressScenario(scenario)) {
2229                         if (configuration != 'Checked') {
2230                             return
2231                         }
2232                     }
2233                     else {
2234                         // Skip scenarios
2235                         switch (scenario) {
2236                             case 'ilrt':
2237                             case 'longgc':
2238                             case 'gcsimulator':
2239                                 // Long GC tests take a long time on non-Release builds
2240                                 // ilrt is also Release only
2241                                 if (configuration != 'Release') {
2242                                     return
2243                                 }
2244                                 break
2245                             case 'jitdiff':
2246                                 if (configuration != 'Checked') {
2247                                     return;
2248                                 }
2249                                 break
2250                             case 'gc_reliability_framework':
2251                             case 'standalone_gc':
2252                                 if (configuration != 'Release' && configuration != 'Checked') {
2253                                     return
2254                                 }
2255                                 break
2256                             case 'formatting':
2257                                 return
2258                             case 'illink':
2259                                 if (os != 'Windows_NT' && os != 'Ubuntu') {
2260                                     return
2261                                 }
2262                                 break
2263                             case 'default':
2264                                 // Nothing skipped
2265                                 break
2266                             default:
2267                                 println("Unknown scenario: ${scenario}")
2268                                 assert false
2269                                 break
2270                         }
2271                     }
2272
2273                     // Done filtering. Now, create the jobs.
2274
2275                     def lowerConfiguration = configuration.toLowerCase()
2276                     def osGroup = getOSGroup(os)
2277                     def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
2278
2279                     def inputCoreCLRBuildName = projectFolder + '/' +
2280                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, 'default', false), isPR)
2281
2282                     // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only
2283                     // job for Windows_NT specific to that stress mode. Just copy from the default scenario.
2284                     def testBuildScenario = scenario
2285                     if (isJitStressScenario(testBuildScenario) || isR2RScenario(testBuildScenario) || isLongGc(testBuildScenario)) {
2286                         testBuildScenario = 'default'
2287                     }
2288
2289                     def inputWindowsTestBuildArch = architecture
2290                     if (architecture == "arm64" && os != "Windows_NT") {
2291                         // Use the x64 test build for arm64 unix
2292                         inputWindowsTestBuildArch = "x64"
2293                     }
2294
2295                     def inputWindowTestsBuildName = ""
2296
2297                     if (isJitStressScenario(scenario)) {
2298                         inputWindowTestsBuildName = projectFolder + '/' +
2299                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR)
2300                     } else {
2301                         inputWindowTestsBuildName = projectFolder + '/' +
2302                         Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR)
2303
2304                     }
2305
2306
2307                     // Enable Server GC for Ubuntu PR builds
2308                     def serverGCString = ''
2309                     if (os == 'Ubuntu' && isPR) {
2310                         serverGCString = '--useServerGC'
2311                     }
2312
2313                     def testOpts = ''
2314
2315                     if (isR2RScenario(scenario)) {
2316
2317                         testOpts += ' --crossgen --runcrossgentests'
2318
2319                         if (scenario == 'r2r_jitstress1') {
2320                             testOpts += ' --jitstress=1'
2321                         }
2322                         else if (scenario == 'r2r_jitstress2') {
2323                             testOpts += ' --jitstress=2'
2324                         }
2325                         else if (scenario == 'r2r_jitstressregs1') {
2326                             testOpts += ' --jitstressregs=1'
2327                         }
2328                         else if (scenario == 'r2r_jitstressregs2') {
2329                             testOpts += ' --jitstressregs=2'
2330                         }
2331                         else if (scenario == 'r2r_jitstressregs3') {
2332                             testOpts += ' --jitstressregs=3'
2333                         }
2334                         else if (scenario == 'r2r_jitstressregs4') {
2335                             testOpts += ' --jitstressregs=4'
2336                         }
2337                         else if (scenario == 'r2r_jitstressregs8') {
2338                             testOpts += ' --jitstressregs=8'
2339                         }
2340                         else if (scenario == 'r2r_jitstressregs0x10') {
2341                             testOpts += ' --jitstressregs=0x10'
2342                         }
2343                         else if (scenario == 'r2r_jitstressregs0x80') {
2344                             testOpts += ' --jitstressregs=0x80'
2345                         }
2346                         else if (scenario == 'r2r_jitstressregs0x1000') {
2347                             testOpts += ' --jitstressregs=0x1000'
2348                         }
2349                         else if (scenario == 'r2r_jitminopts') {
2350                             testOpts += ' --jitminopts'
2351                         }
2352                         else if (scenario == 'r2r_jitforcerelocs') {
2353                             testOpts += ' --jitforcerelocs'
2354                         }
2355                         else if (scenario == 'r2r_gcstress15') {
2356                             testOpts += ' --gcstresslevel=0xF'
2357                         }
2358                     }
2359                     else if (scenario == 'jitdiff') {
2360                         testOpts += ' --jitdisasm --crossgen'
2361                     }
2362                     else if (scenario == 'illink') {
2363                         testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink'
2364                     }
2365                     else if (isLongGc(scenario)) {
2366                         // Long GC tests behave very poorly when they are not
2367                         // the only test running (many of them allocate until OOM).
2368                         testOpts += ' --sequential'
2369
2370                         // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options
2371                         // for running long GC and GCSimulator tests, respectively. We don't use them
2372                         // here because using a playlist file produces much more readable output on the CI machines
2373                         // and reduces running time.
2374                         //
2375                         // The Long GC playlist contains all of the tests that are
2376                         // going to be run. The GCSimulator playlist contains all of
2377                         // the GC simulator tests.
2378                         if (scenario == 'longgc') {
2379                             testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt'
2380                         }
2381                         else if (scenario == 'gcsimulator') {
2382                             testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt'
2383                         }
2384                     }
2385                     else if (isGcReliabilityFramework(scenario)) {
2386                         testOpts += ' --build-overlay-only'
2387                     }
2388                     else if (scenario == 'standalone_gc') {
2389                         if (osGroup == 'OSX') {
2390                             testOpts += ' --gcname=libclrgc.dylib'
2391                         }
2392                         else if (osGroup == 'Linux') {
2393                             testOpts += ' --gcname=libclrgc.so'
2394                         }
2395                         else {
2396                             println("Unexpected OS group: ${osGroup} for os ${os}")
2397                             assert false
2398                         }
2399                     }
2400
2401                     def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches)
2402
2403                     def folder = getJobFolder(scenario)
2404                     def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) {
2405                         // Add parameters for the inputs
2406
2407                         if (windowsArmJob == true) {
2408                             parameters {
2409                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2410                             }
2411                         }
2412                         else {
2413                             parameters {
2414                                 stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR windows test binaries from')
2415                                 stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
2416                             }
2417                         }
2418
2419                         steps {
2420                             // Set up the copies
2421
2422                             // Coreclr build containing the tests and mscorlib
2423                             // pri1 jobs still need to copy windows_nt built tests
2424                             if (windowsArmJob != true) {
2425                                 copyArtifacts(inputWindowTestsBuildName) {
2426                                     excludePatterns('**/testResults.xml', '**/*.ni.dll')
2427                                     buildSelector {
2428                                         buildNumber('${CORECLR_WINDOWS_BUILD}')
2429                                     }
2430                                 }
2431                             }
2432
2433                             // Coreclr build we are trying to test
2434                             //
2435                             //  ** NOTE ** This will, correctly, overwrite over the CORE_ROOT from the windows test archive
2436
2437                             copyArtifacts(inputCoreCLRBuildName) {
2438                                 excludePatterns('**/testResults.xml', '**/*.ni.dll')
2439                                 buildSelector {
2440                                     buildNumber('${CORECLR_BUILD}')
2441                                 }
2442                             }
2443
2444                             // Windows CoreCLR Arm(64) will restore corefx
2445                             // packages correctly.
2446                             //
2447                             // In addition, test steps are entirely different
2448                             // because we do not have a unified runner
2449                             if (windowsArmJob != true) {
2450                                 def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch)
2451
2452                                 // HACK -- Arm64 does not have corefx jobs yet.
2453                                 // Clone corefx and build the native packages overwriting the x64 packages.
2454                                 if (architecture == 'arm64') {
2455                                     shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir")
2456                                     shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun")
2457                                 }
2458                                 else if (architecture == 'x86') {
2459                                     shell("mkdir ./bin/CoreFxNative")
2460
2461                                     copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") {
2462                                         includePatterns('bin/build.tar.gz')
2463                                         targetDirectory('bin/CoreFxNative')
2464                                         buildSelector {
2465                                             latestSuccessful(true)
2466                                         }
2467                                     }
2468
2469                                     shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir")
2470                                 }
2471
2472                                 // Unzip the tests first.  Exit with 0
2473                                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
2474                                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
2475
2476                                 shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
2477
2478                                 // Execute the tests
2479                                 def runDocker = isNeedDocker(architecture, os, false)
2480                                 def dockerPrefix = ""
2481                                 def dockerCmd = ""
2482                                 if (runDocker) {
2483                                     def dockerImage = getDockerImageName(architecture, os, false)
2484                                     dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} "
2485                                     dockerCmd = dockerPrefix + "${dockerImage} "
2486                                 }
2487
2488                                 // If we are running a stress mode, we'll set those variables first
2489                                 def testEnvOpt = ""
2490                                 if (isJitStressScenario(scenario)) {
2491                                     def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh"
2492                                     def envScriptCmds = envScriptCreate(os, scriptFileName)
2493                                     envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName)
2494                                     envScriptCmds += envScriptFinalize(os, scriptFileName)
2495                                     shell("${envScriptCmds}")
2496                                     testEnvOpt = "--test-env=" + scriptFileName
2497                                 }
2498
2499                                 if (isGCStressRelatedTesting(scenario)) {
2500                                     shell('./init-tools.sh')
2501                                 }
2502
2503                                 shell("""${dockerCmd}./tests/runtest.sh \\
2504                 --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
2505                 --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\
2506                 --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
2507                 --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""")
2508
2509                                 if (isGcReliabilityFramework(scenario)) {
2510                                     // runtest.sh doesn't actually execute the reliability framework - do it here.
2511                                     if (serverGCString != '') {
2512                                         if (runDocker) {
2513                                             dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} "
2514                                         }
2515                                         else {
2516                                             shell("export COMPlus_gcServer=1")
2517                                         }
2518                                     }
2519
2520                                     shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}")
2521                                 }
2522                             }
2523
2524                             else { // windowsArmJob == true
2525                                 // Unzip tests.
2526                                 batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')")
2527
2528                                 // Build the build commands
2529                                 def buildCommands = ""
2530
2531                                 def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root"
2532                                 def addEnvVariable =  { variable, value -> buildCommands += "set ${variable}=${value}\r\n"}
2533                                 def addCommand = { cmd -> buildCommands += "${cmd}\r\n"}
2534
2535                                 // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available.
2536                                 addCommand("SETLOCAL ENABLEEXTENSIONS")
2537
2538                                 // For all jobs
2539                                 addEnvVariable("CORE_ROOT", coreRootLocation)
2540
2541                                 addEnvVariable("COMPlus_NoGuiOnAssert", "1")
2542                                 addEnvVariable("COMPlus_ContinueOnAssert", "0")
2543
2544                                 // ARM legacy backend; this is an altjit.
2545                                 if (architecture == "armlb") {
2546                                     addEnvVariable("COMPlus_AltJit", "*")
2547                                     addEnvVariable("COMPlus_AltJitNgen", "*")
2548                                     addEnvVariable("COMPlus_AltJitName", "legacyjit.dll")
2549                                     addEnvVariable("COMPlus_AltJitAssertOnNYI", "1")
2550                                 }
2551
2552                                 // If we are running a stress mode, we'll set those variables as well
2553                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2554                                     def stressValues = null
2555                                     if (isJitStressScenario(scenario)) {
2556                                         stressValues = Constants.jitStressModeScenarios[scenario]
2557                                     }
2558                                     else {
2559                                         stressValues = Constants.r2rStressScenarios[scenario]
2560                                     }
2561
2562                                     stressValues.each { key, value ->
2563                                         addEnvVariable(key, value)
2564                                     }
2565                                 }
2566
2567                                 if (isR2RScenario(scenario)) {
2568                                     // Crossgen the framework assemblies.
2569                                     buildCommands += """
2570 @for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF
2571 @goto skip_PrecompileAssembly
2572
2573 :PrecompileAssembly
2574 @REM Skip mscorlib since it is already precompiled.
2575 @if /I "%3" == "mscorlib.dll" exit /b 0
2576 @if /I "%3" == "mscorlib.ni.dll" exit /b 0
2577
2578 "%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul
2579 @if "%errorlevel%" == "-2146230517" (
2580     echo %2 is not a managed assembly.
2581 ) else if "%errorlevel%" == "-2146234344" (
2582     echo %2 is not a managed assembly.
2583 ) else if %errorlevel% neq 0 (
2584     echo Unable to precompile %2
2585 ) else (
2586     echo Precompiled %2
2587 )
2588 @exit /b 0
2589
2590 :skip_PrecompileAssembly
2591 """
2592
2593                                     // Set RunCrossGen variable to cause test wrappers to invoke their logic to run
2594                                     // crossgen on tests before running them.
2595                                     addEnvVariable("RunCrossGen", "true")
2596                                 }
2597
2598                                 // Create the smarty command
2599                                 def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS "
2600                                 def addSmartyFlag = { flag -> smartyCommand += flag + " "}
2601                                 def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)}
2602
2603                                 def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } }
2604
2605                                 if (architecture == "armlb") {
2606                                     addExclude("LEGACYJIT_FAIL")
2607                                 }
2608
2609                                 if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) {
2610                                     def failTag = "JITSTRESS_FAIL"
2611                                     def excludeTag = "JITSTRESS_EXCLUDE"
2612
2613                                     if (scenario.contains('gc')) {
2614                                         failTag = "GCSTRESS_FAIL"
2615                                         excludeTag = "GCSTRESS_EXCLUDE"
2616                                     }
2617
2618                                     addArchSpecificExclude(architecture, failTag)
2619                                     addArchSpecificExclude(architecture, excludeTag)
2620                                 }
2621                                 else {
2622                                     addExclude("pri1")
2623                                 }
2624
2625                                 // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result.
2626                                 // TODO: We should create a "long running" job that runs these with a longer timeout.
2627                                 addExclude("LONG_RUNNING")
2628
2629                                 smartyCommand += "/lstFile Tests.lst"
2630
2631                                 def testListArch = [
2632                                     'arm64': 'arm64',
2633                                     'arm': 'arm',
2634                                     'armlb': 'arm'
2635                                 ]
2636
2637                                 def archLocation = testListArch[architecture]
2638
2639                                 addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}")
2640                                 addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}")
2641                                 addCommand("${smartyCommand}")
2642
2643                                 // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file.
2644                                 // However, we also need to remove all the variables that were set during this batch file, so we
2645                                 // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell
2646                                 // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well
2647                                 // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel%
2648                                 // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command.
2649                                 // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity.
2650                                 addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%")
2651
2652                                 // ZIP up the smarty output, no matter what the smarty result.
2653                                 addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"")
2654
2655                                 addCommand("echo %errorlevel%")
2656                                 addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}")
2657
2658                                 // Use the smarty errorlevel as the script errorlevel.
2659                                 addCommand("exit /b %__save_smarty_errorlevel%")
2660
2661                                 batchFile(buildCommands)
2662                             }
2663                         }
2664                     }
2665
2666                     if (scenario == 'jitdiff') {
2667                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**")
2668                     }
2669
2670                     // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links
2671                     if (os in ['Ubuntu']) {
2672                         SummaryBuilder summaries = new SummaryBuilder()
2673                         summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt')
2674                         summaries.emit(newJob)
2675                     }
2676
2677                     def affinityOptions = null
2678
2679                     if (windowsArmJob == true) {
2680                         affinityOptions = [
2681                             "use_arm64_build_machine" : false
2682                         ]
2683                     }
2684
2685                     else if (architecture == 'arm64' && os != 'Windows_NT') {
2686                         affinityOptions = [
2687                             "large_pages" : false
2688                         ]
2689                     }
2690
2691                     setMachineAffinity(newJob, os, architecture, affinityOptions)
2692                     Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
2693
2694                     setTestJobTimeOut(newJob, isPR, architecture, scenario)
2695
2696                     if (windowsArmJob != true) {
2697                         Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml')
2698                     }
2699                     else {
2700                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false)
2701
2702                         // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much,
2703                         // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get
2704                         // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large.
2705                         Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false)
2706                     }
2707
2708                     // Create a build flow to join together the build and tests required to run this test.
2709                     // Windows CoreCLR build and Linux CoreCLR build (in parallel) ->
2710                     // Linux CoreCLR test
2711                     def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow"
2712                     def fullTestJobName = projectFolder + '/' + newJob.name
2713                     // Add a reference to the input jobs for report purposes
2714                     JobReport.Report.addReference(inputCoreCLRBuildName)
2715                     JobReport.Report.addReference(inputWindowTestsBuildName)
2716                     JobReport.Report.addReference(fullTestJobName)
2717                     def newFlowJob = null
2718
2719                     if (os == 'RHEL7.2' || os == 'Debian8.4') {
2720                         // Do not create the flow job for RHEL jobs.
2721                         return
2722                     }
2723
2724                     // For pri0 jobs we can build tests on unix
2725                     if (windowsArmJob) {
2726                         // For Windows arm jobs there is no reason to build a parallel test job.
2727                         // The product build supports building and archiving the tests.
2728
2729                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2730                         buildFlow("""
2731 coreclrBuildJob = build(params, '${inputCoreCLRBuildName}')
2732
2733 // And then build the test build
2734 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}')
2735 """)
2736                         }
2737                     }
2738                     else {
2739                         newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) {
2740                         buildFlow("""
2741 // Build the input jobs in parallel
2742 parallel (
2743 { coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') },
2744 { windowsBuildJob = build(params, '${inputWindowTestsBuildName}') }
2745 )
2746
2747 // And then build the test build
2748 build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
2749             CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}')
2750 """)
2751                         }
2752                     }
2753
2754                     // For the flow jobs set the machine affinity as x64 if an armarch.
2755                     def flowArch = architecture
2756
2757                     if (flowArch in validWindowsNTCrossArches) {
2758                         flowArch = 'x64'
2759                         affinityOptions = null
2760                     }
2761
2762                     setMachineAffinity(newFlowJob, os, flowArch, affinityOptions)
2763                     Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
2764                     addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false
2765                 } // configuration
2766             } // os
2767         } // architecture
2768     } // isPR
2769 } // scenario
2770
2771 JobReport.Report.generateJobReport(out)
2772
2773 // Make the call to generate the help job
2774 Utilities.createHelperJob(this, project, branch,
2775     "Welcome to the ${project} Repository",  // This is prepended to the help message
2776     "Have a nice day!")  // This is appended to the help message.  You might put known issues here.
2777
2778 Utilities.addCROSSCheck(this, project, branch)